From bbe21a002ecece370e9305d58a69f75999a12d48 Mon Sep 17 00:00:00 2001 From: rsuplina Date: Tue, 16 Sep 2025 11:20:43 +0100 Subject: [PATCH 01/10] Update Grafana role with TLS/HTTPS Signed-off-by: rsuplina --- roles/grafana/README.md | 65 +++++++++++++++---- roles/grafana/defaults/main.yml | 12 +++- roles/grafana/handlers/main.yml | 21 ++++++ roles/grafana/meta/argument_specs.yml | 59 ++++++++++++++++- roles/grafana/tasks/main.yml | 29 +++++++-- .../templates/prometheus_source.yml.j2 | 2 +- 6 files changed, 165 insertions(+), 23 deletions(-) create mode 100644 roles/grafana/handlers/main.yml diff --git a/roles/grafana/README.md b/roles/grafana/README.md index b7c981e3..a9c93dfe 100644 --- a/roles/grafana/README.md +++ b/roles/grafana/README.md @@ -1,12 +1,27 @@ -# grafana_server -Set up Grafana server, connected to a Prometheus server. +# grafana -The role will: -- Install the Grafana server package(s). -- Configure Grafana data sources, primarily for Prometheus, based on the provided `prometheus_url`. -- Configure Grafana dashboard providers. -- Provision a default dashboard. +Automates the installation and configuration of a Grafana server, with Prometheus integration for monitoring and observability. + +## Features + +- Installs Grafana using OS-specific package management for major Linux distributions (Ubuntu, CentOS, RedHat, Rocky). +- Configures core Grafana server settings, including protocol (HTTP/HTTPS), port, domain, and root URL. +- Optionally enables HTTPS/TLS for secure access, with configurable certificate and key paths. +- Allows setting a custom admin password for the Grafana web interface. +- Provisions Prometheus as a data source, with the ability to specify a custom Prometheus endpoint. +- Configures dashboard providers and ensures dashboards are available at startup. +- Supports custom locations for data source and dashboard configuration files. +- Ensures idempotent and secure configuration changes, with sensitive values (like admin password) protected in logs. +- Designed for flexibility and easy extension to other monitoring backends or dashboard sources. + +## How it works + +1. Installs Grafana using the appropriate package manager for the detected OS. +2. Configures server and security settings in `grafana.ini`, including TLS and admin credentials if specified. +3. Provisions Prometheus as a data source and sets up dashboard providers using Jinja2 templates. +4. Ensures the dashboards directory exists and copies a default dashboard for immediate use. +5. Restarts or reloads the Grafana service as needed to apply configuration changes. ## Requirements @@ -20,10 +35,23 @@ None. | Parameter | Type | Default Value | Description | |----------------------------------|------|-------------------------------------------------|---------------------------------------------------------------------------| -| `grafana_datasource_directory` | `str`| `/etc/grafana/provisioning/datasources/automatic.yml`| Location of the Grafana data sources configuration file. | -| `grafana_providers_configuration`| `str`| `/etc/grafana/provisioning/dashboards/providers.yml` | Location of the Grafana dashboard provider configurations file. | -| `grafana_dashboard_directory` | `str`| `/var/lib/grafana/dashboards` | Location of the Grafana dashboard configurations directory. | -| `prometheus_url` | `str`| `localhost:9090` | URL (host:port) to the Prometheus server that Grafana will connect to. | +| `grafana_datasource_directory` | `str` | `/etc/grafana/provisioning/datasources/automatic.yml` | Location of the Grafana data sources configuration file. | +| `grafana_providers_configuration`| `str` | `/etc/grafana/provisioning/dashboards/providers.yml` | Location of the Grafana dashboard provider configurations file. | +| `grafana_dashboard_directory` | `str` | `/var/lib/grafana/dashboards` | Location of the Grafana dashboard configurations directory. | +| `prometheus_url` | `str` | `http://localhost:9090` | URL (host:port) to the Prometheus server that Grafana will connect to. | +| `tls_enabled` | `bool`| `false` | Enable or disable TLS/SSL for Grafana (HTTPS support). | +| `grafana_tls_cert_path` | `str` | `/etc/pki/tls/certs/grafana.crt` | Path to the TLS certificate file for Grafana. | +| `grafana_tls_key_path` | `str` | `/etc/pki/tls/private/grafana.key` | Path to the TLS private key file for Grafana. | +| `grafana_domain` | `str` | `localhost` | Domain name for the Grafana server (used in server configuration). | +| `grafana_root_url` | `str` | `http://localhost:3000` | The root URL for accessing Grafana (used in server configuration). | +| `grafana_config_file` | `str` | `/etc/grafana/grafana.ini` | Path to the main Grafana configuration file. | +| `grafana_http_port` | `int` | `3000` | HTTP port for Grafana to listen on. | +| `grafana_security_admin_password`| `str` | `admin` | Admin password for Grafana web interface. | + + +## TLS/HTTPS Support + +If `tls_enabled` is set to `true`, the role will configure Grafana to use HTTPS. You must provide valid certificate and key files at the specified paths (`grafana_tls_cert_path` and `grafana_tls_key_path`). ## Examples @@ -33,8 +61,6 @@ Basic installation connecting to a local Prometheus server: - name: Set up Grafana server with local Prometheus ansible.builtin.import_role: name: grafana_server - # No variables needed here as defaults will be used for local Prometheus - - name: Set up Grafana server for a specific Prometheus endpoint ansible.builtin.import_role: name: grafana_server @@ -49,6 +75,19 @@ Basic installation connecting to a local Prometheus server: grafana_providers_configuration: "/opt/grafana/configs/providers.yml" grafana_dashboard_directory: "/opt/grafana/dashboards_custom" prometheus_url: "http://monitoring-cluster.internal:9090" + +- name: Set up Grafana server with TLS/HTTPS enabled + ansible.builtin.import_role: + name: grafana_server + vars: + tls_enabled: true + grafana_security_admin_password: secretpassword + grafana_domain: "grafana.1.1.1.1.pvc.labs.com" + grafana_root_url: "https://grafana.1.1.1.1.pvc.labs.com:3000" + grafana_tls_cert_path: "/etc/pki/tls/certs/grafana.crt" + grafana_tls_key_path: "/etc/pki/tls/private/grafana.key" + prometheus_url: "https://prometheus.example.com:9090" + ``` ## License diff --git a/roles/grafana/defaults/main.yml b/roles/grafana/defaults/main.yml index b7ccddd6..b7c1d74f 100644 --- a/roles/grafana/defaults/main.yml +++ b/roles/grafana/defaults/main.yml @@ -14,8 +14,16 @@ --- +tls_enabled: false +grafana_http_port: 3000 +grafana_security_admin_password: admin +grafana_tls_cert_path: /etc/pki/tls/certs/grafana.crt +grafana_tls_key_path: /etc/pki/tls/private/grafana.key +grafana_domain: "localhost" +grafana_root_url: "http://localhost:3000" +grafana_config_file: /etc/grafana/grafana.ini + grafana_datasource_directory: /etc/grafana/provisioning/datasources/automatic.yml grafana_providers_configuration: /etc/grafana/provisioning/dashboards/providers.yml grafana_dashboard_directory: /var/lib/grafana/dashboards - -prometheus_url: localhost:9090 +prometheus_url: http://localhost:9090 diff --git a/roles/grafana/handlers/main.yml b/roles/grafana/handlers/main.yml new file mode 100644 index 00000000..7efb8d56 --- /dev/null +++ b/roles/grafana/handlers/main.yml @@ -0,0 +1,21 @@ +# Copyright 2025 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Manage Grafana Service + ansible.builtin.systemd: + name: grafana-server + daemon_reload: true + state: restarted + enabled: true diff --git a/roles/grafana/meta/argument_specs.yml b/roles/grafana/meta/argument_specs.yml index 04eb520e..4378c825 100644 --- a/roles/grafana/meta/argument_specs.yml +++ b/roles/grafana/meta/argument_specs.yml @@ -15,29 +15,82 @@ argument_specs: main: - short_description: Set up Grafana server + short_description: Install, configure, and provision Grafana server with Prometheus integration and optional TLS/HTTPS support description: | - Set up Grafana server, connected to a Prometheus server. + This role automates the installation and configuration of a Grafana server, supporting secure HTTPS/TLS access, custom admin credentials, and flexible provisioning of data sources and dashboards. + It is designed for seamless integration with Prometheus, but can be extended to other monitoring backends. The role handles OS-specific installation, server and security configuration, and dashboard/data source provisioning, making it suitable for a wide range of deployment scenarios. author: Cloudera Labs version_added: "2.4.0" options: + grafana_config_file: + description: + - Path to the main Grafana configuration file. + type: str + default: /etc/grafana/grafana.ini + required: false + grafana_http_port: + description: + - HTTP port for Grafana to listen on. + type: int + default: 3000 + required: false + grafana_security_admin_password: + description: + - Admin password for Grafana web interface. + type: str + default: admin + required: false + grafana_domain: + description: + - Domain name for the Grafana server (used in server configuration). + type: str + default: "localhost" + required: false + grafana_root_url: + description: + - The root URL for accessing Grafana (used in server configuration). + type: str + default: "http://localhost:3000" + required: false + tls_enabled: + description: + - Enable or disable TLS/SSL for Grafana (HTTPS support). + type: bool + default: false + required: false + grafana_tls_cert_path: + description: + - Path to the TLS certificate file for Grafana. + type: str + default: /etc/pki/tls/certs/grafana.crt + required: false + grafana_tls_key_path: + description: + - Path to the TLS private key file for Grafana. + type: str + default: /etc/pki/tls/private/grafana.key + required: false grafana_datasource_directory: description: - Location of the Grafana data sources. type: str default: /etc/grafana/provisioning/datasources/automatic.yml + required: false grafana_providers_configuration: description: - Location of the Grafana provider configurations. type: str default: /etc/grafana/provisioning/dashboards/providers.yml + required: false grafana_dashboard_directory: description: - Locattion of the Grafana dashboard configurations. type: str default: /var/lib/grafana/dashboards + required: false prometheus_url: description: - URL (host:port) to the Prometheus server. type: str - default: localhost:9090 + default: https://localhost:9090 + required: false diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml index 5cfcbe5f..065450d2 100644 --- a/roles/grafana/tasks/main.yml +++ b/roles/grafana/tasks/main.yml @@ -19,6 +19,29 @@ - name: Execute OS-specific Grafana installation ansible.builtin.include_tasks: "{{ ansible_facts['distribution'] }}.yml" +- name: Configure Grafana security settings (admin password) + ansible.builtin.command: + cmd: "grafana-cli admin reset-admin-password {{ grafana_security_admin_password }}" + no_log: true + notify: Manage Grafana Service + +- name: Configure Grafana server if TLS is enabled + when: tls_enabled + ansible.builtin.ini_file: + path: "{{ grafana_config_file }}" + section: server + option: "{{ item.option }}" + value: "{{ item.value }}" + mode: '0640' + loop: + - { option: 'protocol', value: 'https' } + - { option: 'http_port', value: '{{ grafana_http_port }}' } + - { option: 'domain', value: '{{ grafana_domain }}' } + - { option: 'root_url', value: '{{ grafana_root_url }}' } + - { option: 'cert_file', value: '{{ grafana_tls_cert_path }}' } + - { option: 'cert_key', value: '{{ grafana_tls_key_path }}' } + notify: Manage Grafana Service + - name: Connect Prometheus datasource ansible.builtin.template: src: prometheus_source.yml.j2 @@ -43,7 +66,5 @@ dest: "{{ grafana_dashboard_directory }}/base_dashboard.json" mode: "0755" -- name: Start Grafana service - ansible.builtin.systemd: - name: "{{ grafana_service }}" - state: started + + diff --git a/roles/grafana/templates/prometheus_source.yml.j2 b/roles/grafana/templates/prometheus_source.yml.j2 index d2beb1ce..39fc0753 100644 --- a/roles/grafana/templates/prometheus_source.yml.j2 +++ b/roles/grafana/templates/prometheus_source.yml.j2 @@ -3,7 +3,7 @@ datasources: - name: Prometheus type: prometheus access: proxy - url: http://{{ prometheus_url }} + url: {{ prometheus_url }} password: user: "" database: "" From 8231cc71070a08de150d84ca3cb028c09fdeb71c Mon Sep 17 00:00:00 2001 From: rsuplina Date: Tue, 16 Sep 2025 11:34:20 +0100 Subject: [PATCH 02/10] Update liting issues Signed-off-by: rsuplina --- roles/grafana/defaults/main.yml | 1 - roles/grafana/tasks/main.yml | 17 +++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/roles/grafana/defaults/main.yml b/roles/grafana/defaults/main.yml index b7c1d74f..2a0e56a1 100644 --- a/roles/grafana/defaults/main.yml +++ b/roles/grafana/defaults/main.yml @@ -13,7 +13,6 @@ # limitations under the License. --- - tls_enabled: false grafana_http_port: 3000 grafana_security_admin_password: admin diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml index 065450d2..13c49cc5 100644 --- a/roles/grafana/tasks/main.yml +++ b/roles/grafana/tasks/main.yml @@ -32,14 +32,14 @@ section: server option: "{{ item.option }}" value: "{{ item.value }}" - mode: '0640' + mode: "0640" loop: - - { option: 'protocol', value: 'https' } - - { option: 'http_port', value: '{{ grafana_http_port }}' } - - { option: 'domain', value: '{{ grafana_domain }}' } - - { option: 'root_url', value: '{{ grafana_root_url }}' } - - { option: 'cert_file', value: '{{ grafana_tls_cert_path }}' } - - { option: 'cert_key', value: '{{ grafana_tls_key_path }}' } + - { option: "protocol", value: "https" } + - { option: "http_port", value: "{{ grafana_http_port }}" } + - { option: "domain", value: "{{ grafana_domain }}" } + - { option: "root_url", value: "{{ grafana_root_url }}" } + - { option: "cert_file", value: "{{ grafana_tls_cert_path }}" } + - { option: "cert_key", value: "{{ grafana_tls_key_path }}" } notify: Manage Grafana Service - name: Connect Prometheus datasource @@ -65,6 +65,3 @@ src: files/base_dashboard.json dest: "{{ grafana_dashboard_directory }}/base_dashboard.json" mode: "0755" - - - From 7e238e1a40fdf09da9294ad5a989ec587d2b55a3 Mon Sep 17 00:00:00 2001 From: rsuplina Date: Tue, 16 Sep 2025 11:39:14 +0100 Subject: [PATCH 03/10] Fix formatting in Grafana README Signed-off-by: rsuplina --- roles/grafana/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/grafana/README.md b/roles/grafana/README.md index a9c93dfe..951d1018 100644 --- a/roles/grafana/README.md +++ b/roles/grafana/README.md @@ -1,7 +1,7 @@ # grafana -Automates the installation and configuration of a Grafana server, with Prometheus integration for monitoring and observability. +Automates the installation and configuration of a Grafana server, with Prometheus integration for monitoring and observability. ## Features From d48c52859d668113fcf97e435c6b9560a4378a99 Mon Sep 17 00:00:00 2001 From: rsuplina Date: Tue, 16 Sep 2025 16:30:06 +0100 Subject: [PATCH 04/10] - Add new variables for Prometheus TLS, adjusted datasource template to enable TLS authentication. Signed-off-by: rsuplina --- roles/grafana/README.md | 11 ++++++++--- roles/grafana/defaults/main.yml | 4 ++++ roles/grafana/meta/argument_specs.yml | 18 ++++++++++++++++++ roles/grafana/tasks/main.yml | 13 +++++++++++++ .../grafana/templates/prometheus_source.yml.j2 | 16 +++++----------- 5 files changed, 48 insertions(+), 14 deletions(-) diff --git a/roles/grafana/README.md b/roles/grafana/README.md index 951d1018..edb20782 100644 --- a/roles/grafana/README.md +++ b/roles/grafana/README.md @@ -38,7 +38,10 @@ None. | `grafana_datasource_directory` | `str` | `/etc/grafana/provisioning/datasources/automatic.yml` | Location of the Grafana data sources configuration file. | | `grafana_providers_configuration`| `str` | `/etc/grafana/provisioning/dashboards/providers.yml` | Location of the Grafana dashboard provider configurations file. | | `grafana_dashboard_directory` | `str` | `/var/lib/grafana/dashboards` | Location of the Grafana dashboard configurations directory. | -| `prometheus_url` | `str` | `http://localhost:9090` | URL (host:port) to the Prometheus server that Grafana will connect to. | +| `prometheus_url` | `str` | `http://localhost:9090` | URL (host:port) to the Prometheus server that Grafana will connect to. | +| `prometheus_hostname` | `str` | `localhost` | Hostname of the Prometheus server for TLS server name verification. | +| `prometheus_tls_cert_path` | `str` | `/etc/pki/tls/certs/prometheus.crt` | Path to the Prometheus server TLS certificate (for client auth, if needed). | +| `prometheus_tls_key_path` | `str` | `/etc/pki/tls/private/prometheus.key` | Path to the Prometheus server TLS key (for client auth, if needed). | | `tls_enabled` | `bool`| `false` | Enable or disable TLS/SSL for Grafana (HTTPS support). | | `grafana_tls_cert_path` | `str` | `/etc/pki/tls/certs/grafana.crt` | Path to the TLS certificate file for Grafana. | | `grafana_tls_key_path` | `str` | `/etc/pki/tls/private/grafana.key` | Path to the TLS private key file for Grafana. | @@ -84,8 +87,10 @@ Basic installation connecting to a local Prometheus server: grafana_security_admin_password: secretpassword grafana_domain: "grafana.1.1.1.1.pvc.labs.com" grafana_root_url: "https://grafana.1.1.1.1.pvc.labs.com:3000" - grafana_tls_cert_path: "/etc/pki/tls/certs/grafana.crt" - grafana_tls_key_path: "/etc/pki/tls/private/grafana.key" + grafana_tls_cert_path: "/etc/grafana/certs/grafana.crt" + grafana_tls_key_path: "/etc/grafana/private/grafana.key" + prometheus_tls_cert_path: /etc/prometheus/certs/prometheus.crt + prometheus_tls_key_path: /etc/prometheus/private/prometheus.key prometheus_url: "https://prometheus.example.com:9090" ``` diff --git a/roles/grafana/defaults/main.yml b/roles/grafana/defaults/main.yml index 2a0e56a1..98ba21a6 100644 --- a/roles/grafana/defaults/main.yml +++ b/roles/grafana/defaults/main.yml @@ -25,4 +25,8 @@ grafana_config_file: /etc/grafana/grafana.ini grafana_datasource_directory: /etc/grafana/provisioning/datasources/automatic.yml grafana_providers_configuration: /etc/grafana/provisioning/dashboards/providers.yml grafana_dashboard_directory: /var/lib/grafana/dashboards + prometheus_url: http://localhost:9090 +prometheus_hostname: localhost +prometheus_tls_cert_path: /etc/pki/tls/certs/prometheus.crt +prometheus_tls_key_path: /etc/pki/tls/private/prometheus.key diff --git a/roles/grafana/meta/argument_specs.yml b/roles/grafana/meta/argument_specs.yml index 4378c825..8decb331 100644 --- a/roles/grafana/meta/argument_specs.yml +++ b/roles/grafana/meta/argument_specs.yml @@ -22,6 +22,24 @@ argument_specs: author: Cloudera Labs version_added: "2.4.0" options: + prometheus_hostname: + description: + - Hostname of the Prometheus server for TLS server name verification. + type: str + default: localhost + required: false + prometheus_tls_cert_path: + description: + - Path to the Prometheus server TLS certificate (for client auth, if needed). + type: str + default: /etc/pki/tls/certs/prometheus.crt + required: false + prometheus_tls_key_path: + description: + - Path to the Prometheus server TLS key (for client auth, if needed). + type: str + default: /etc/pki/tls/private/prometheus.key + required: false grafana_config_file: description: - Path to the main Grafana configuration file. diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml index 13c49cc5..88e03cb4 100644 --- a/roles/grafana/tasks/main.yml +++ b/roles/grafana/tasks/main.yml @@ -42,6 +42,19 @@ - { option: "cert_key", value: "{{ grafana_tls_key_path }}" } notify: Manage Grafana Service +- name: Read remote Prometheus certificate content + ansible.builtin.slurp: + src: "{{ prometheus_tls_cert_path }}" + register: __prometheus_cert_content + when: tls_enabled + +- name: Read remote Prometheus key content + ansible.builtin.slurp: + src: "{{ prometheus_tls_key_path }}" + register: __prometheus_key_content + no_log: true + when: tls_enabled + - name: Connect Prometheus datasource ansible.builtin.template: src: prometheus_source.yml.j2 diff --git a/roles/grafana/templates/prometheus_source.yml.j2 b/roles/grafana/templates/prometheus_source.yml.j2 index 39fc0753..1bebcd78 100644 --- a/roles/grafana/templates/prometheus_source.yml.j2 +++ b/roles/grafana/templates/prometheus_source.yml.j2 @@ -4,20 +4,14 @@ datasources: type: prometheus access: proxy url: {{ prometheus_url }} - password: - user: "" - database: "" - basicAuth: false - basicAuthUser: "" - basicAuthPassword: "" - withCredentials: false isDefault: true jsonData: - tlsAuth: false + tlsAuth: true tlsAuthWithCACert: false + tlsSkipVerify: false + serverName: {{ prometheus_hostname }} secureJsonData: - tlsCACert: "" - tlsClientCert: "" - tlsClientKey: "" + tlsClientCert: "{{ (__prometheus_cert_content.content | b64decode) if tls_enabled | bool else '' }}" + tlsClientKey: "{{ (__prometheus_key_content.content | b64decode) if tls_enabled | bool else '' }}" version: 1 editable: true From 190948fcc4e952f947d2ebd545db9481d92499fb Mon Sep 17 00:00:00 2001 From: rsuplina Date: Wed, 17 Sep 2025 10:00:36 +0100 Subject: [PATCH 05/10] Add updates based on comments Signed-off-by: rsuplina --- roles/grafana/README.md | 6 ++--- roles/grafana/defaults/main.yml | 2 +- roles/grafana/handlers/main.yml | 2 +- roles/grafana/meta/argument_specs.yml | 2 +- roles/grafana/tasks/main.yml | 23 +++++++++++++++---- .../templates/prometheus_source.yml.j2 | 4 ++-- 6 files changed, 26 insertions(+), 13 deletions(-) diff --git a/roles/grafana/README.md b/roles/grafana/README.md index edb20782..2a9ba4f1 100644 --- a/roles/grafana/README.md +++ b/roles/grafana/README.md @@ -42,7 +42,7 @@ None. | `prometheus_hostname` | `str` | `localhost` | Hostname of the Prometheus server for TLS server name verification. | | `prometheus_tls_cert_path` | `str` | `/etc/pki/tls/certs/prometheus.crt` | Path to the Prometheus server TLS certificate (for client auth, if needed). | | `prometheus_tls_key_path` | `str` | `/etc/pki/tls/private/prometheus.key` | Path to the Prometheus server TLS key (for client auth, if needed). | -| `tls_enabled` | `bool`| `false` | Enable or disable TLS/SSL for Grafana (HTTPS support). | +| `grafana_tls_enabled` | `bool`| `false` | Enable or disable TLS/SSL for Grafana (HTTPS support). | | `grafana_tls_cert_path` | `str` | `/etc/pki/tls/certs/grafana.crt` | Path to the TLS certificate file for Grafana. | | `grafana_tls_key_path` | `str` | `/etc/pki/tls/private/grafana.key` | Path to the TLS private key file for Grafana. | | `grafana_domain` | `str` | `localhost` | Domain name for the Grafana server (used in server configuration). | @@ -54,7 +54,7 @@ None. ## TLS/HTTPS Support -If `tls_enabled` is set to `true`, the role will configure Grafana to use HTTPS. You must provide valid certificate and key files at the specified paths (`grafana_tls_cert_path` and `grafana_tls_key_path`). +If `grafana_tls_enabled` is set to `true`, the role will configure Grafana to use HTTPS. You must provide valid certificate and key files at the specified paths (`grafana_tls_cert_path` and `grafana_tls_key_path`). ## Examples @@ -83,7 +83,7 @@ Basic installation connecting to a local Prometheus server: ansible.builtin.import_role: name: grafana_server vars: - tls_enabled: true + grafana_tls_enabled: true grafana_security_admin_password: secretpassword grafana_domain: "grafana.1.1.1.1.pvc.labs.com" grafana_root_url: "https://grafana.1.1.1.1.pvc.labs.com:3000" diff --git a/roles/grafana/defaults/main.yml b/roles/grafana/defaults/main.yml index 98ba21a6..b6ed29e0 100644 --- a/roles/grafana/defaults/main.yml +++ b/roles/grafana/defaults/main.yml @@ -13,7 +13,7 @@ # limitations under the License. --- -tls_enabled: false +grafana_tls_enabled: false grafana_http_port: 3000 grafana_security_admin_password: admin grafana_tls_cert_path: /etc/pki/tls/certs/grafana.crt diff --git a/roles/grafana/handlers/main.yml b/roles/grafana/handlers/main.yml index 7efb8d56..6a95980d 100644 --- a/roles/grafana/handlers/main.yml +++ b/roles/grafana/handlers/main.yml @@ -13,7 +13,7 @@ # limitations under the License. --- -- name: Manage Grafana Service +- name: Restart Grafana Service ansible.builtin.systemd: name: grafana-server daemon_reload: true diff --git a/roles/grafana/meta/argument_specs.yml b/roles/grafana/meta/argument_specs.yml index 8decb331..3fa22d23 100644 --- a/roles/grafana/meta/argument_specs.yml +++ b/roles/grafana/meta/argument_specs.yml @@ -70,7 +70,7 @@ argument_specs: type: str default: "http://localhost:3000" required: false - tls_enabled: + grafana_tls_enabled: description: - Enable or disable TLS/SSL for Grafana (HTTPS support). type: bool diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml index 88e03cb4..1bcddd03 100644 --- a/roles/grafana/tasks/main.yml +++ b/roles/grafana/tasks/main.yml @@ -19,14 +19,27 @@ - name: Execute OS-specific Grafana installation ansible.builtin.include_tasks: "{{ ansible_facts['distribution'] }}.yml" +- name: Check if Grafana admin password has already been set + ansible.builtin.stat: + path: /etc/grafana/.admin_password_set + register: grafana_admin_password_marker + - name: Configure Grafana security settings (admin password) ansible.builtin.command: cmd: "grafana-cli admin reset-admin-password {{ grafana_security_admin_password }}" no_log: true - notify: Manage Grafana Service + notify: Restart Grafana Service + when: not grafana_admin_password_marker.stat.exists + +- name: Create marker file after setting admin password + ansible.builtin.file: + path: /etc/grafana/.admin_password_set + state: touch + mode: "0600" + when: not grafana_admin_password_marker.stat.exists - name: Configure Grafana server if TLS is enabled - when: tls_enabled + when: grafana_tls_enabled ansible.builtin.ini_file: path: "{{ grafana_config_file }}" section: server @@ -40,20 +53,20 @@ - { option: "root_url", value: "{{ grafana_root_url }}" } - { option: "cert_file", value: "{{ grafana_tls_cert_path }}" } - { option: "cert_key", value: "{{ grafana_tls_key_path }}" } - notify: Manage Grafana Service + notify: Restart Grafana Service - name: Read remote Prometheus certificate content ansible.builtin.slurp: src: "{{ prometheus_tls_cert_path }}" register: __prometheus_cert_content - when: tls_enabled + when: grafana_tls_enabled - name: Read remote Prometheus key content ansible.builtin.slurp: src: "{{ prometheus_tls_key_path }}" register: __prometheus_key_content no_log: true - when: tls_enabled + when: grafana_tls_enabled - name: Connect Prometheus datasource ansible.builtin.template: diff --git a/roles/grafana/templates/prometheus_source.yml.j2 b/roles/grafana/templates/prometheus_source.yml.j2 index 1bebcd78..df37ea1c 100644 --- a/roles/grafana/templates/prometheus_source.yml.j2 +++ b/roles/grafana/templates/prometheus_source.yml.j2 @@ -11,7 +11,7 @@ datasources: tlsSkipVerify: false serverName: {{ prometheus_hostname }} secureJsonData: - tlsClientCert: "{{ (__prometheus_cert_content.content | b64decode) if tls_enabled | bool else '' }}" - tlsClientKey: "{{ (__prometheus_key_content.content | b64decode) if tls_enabled | bool else '' }}" + tlsClientCert: "{{ (__prometheus_cert_content.content | b64decode) if grafana_tls_enabled | bool else '' }}" + tlsClientKey: "{{ (__prometheus_key_content.content | b64decode) if grafana_tls_enabled | bool else '' }}" version: 1 editable: true From 168f7ca7b428d6e2075990ac85de4ebe32f77fa1 Mon Sep 17 00:00:00 2001 From: rsuplina Date: Mon, 22 Sep 2025 09:59:33 +0100 Subject: [PATCH 06/10] Add grafana_protocol parameter and update admin password setup Signed-off-by: rsuplina --- roles/grafana/README.md | 1 + roles/grafana/defaults/main.yml | 1 + roles/grafana/meta/argument_specs.yml | 9 ++++++ roles/grafana/tasks/main.yml | 44 +++++++++++++++------------ 4 files changed, 36 insertions(+), 19 deletions(-) diff --git a/roles/grafana/README.md b/roles/grafana/README.md index 2a9ba4f1..c200e4f7 100644 --- a/roles/grafana/README.md +++ b/roles/grafana/README.md @@ -35,6 +35,7 @@ None. | Parameter | Type | Default Value | Description | |----------------------------------|------|-------------------------------------------------|---------------------------------------------------------------------------| +| `grafana_protocol` | `str` | `http` | Protocol to use for Grafana web interface and API (`http` or `https`). | | `grafana_datasource_directory` | `str` | `/etc/grafana/provisioning/datasources/automatic.yml` | Location of the Grafana data sources configuration file. | | `grafana_providers_configuration`| `str` | `/etc/grafana/provisioning/dashboards/providers.yml` | Location of the Grafana dashboard provider configurations file. | | `grafana_dashboard_directory` | `str` | `/var/lib/grafana/dashboards` | Location of the Grafana dashboard configurations directory. | diff --git a/roles/grafana/defaults/main.yml b/roles/grafana/defaults/main.yml index b6ed29e0..24452f2e 100644 --- a/roles/grafana/defaults/main.yml +++ b/roles/grafana/defaults/main.yml @@ -13,6 +13,7 @@ # limitations under the License. --- +grafana_protocol: http grafana_tls_enabled: false grafana_http_port: 3000 grafana_security_admin_password: admin diff --git a/roles/grafana/meta/argument_specs.yml b/roles/grafana/meta/argument_specs.yml index 3fa22d23..468ac9fa 100644 --- a/roles/grafana/meta/argument_specs.yml +++ b/roles/grafana/meta/argument_specs.yml @@ -22,6 +22,15 @@ argument_specs: author: Cloudera Labs version_added: "2.4.0" options: + grafana_protocol: + description: + - Protocol to use for Grafana web interface and API (http or https). + type: str + choices: + - http + - https + default: http + required: false prometheus_hostname: description: - Hostname of the Prometheus server for TLS server name verification. diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml index 1bcddd03..c64b23cc 100644 --- a/roles/grafana/tasks/main.yml +++ b/roles/grafana/tasks/main.yml @@ -19,25 +19,6 @@ - name: Execute OS-specific Grafana installation ansible.builtin.include_tasks: "{{ ansible_facts['distribution'] }}.yml" -- name: Check if Grafana admin password has already been set - ansible.builtin.stat: - path: /etc/grafana/.admin_password_set - register: grafana_admin_password_marker - -- name: Configure Grafana security settings (admin password) - ansible.builtin.command: - cmd: "grafana-cli admin reset-admin-password {{ grafana_security_admin_password }}" - no_log: true - notify: Restart Grafana Service - when: not grafana_admin_password_marker.stat.exists - -- name: Create marker file after setting admin password - ansible.builtin.file: - path: /etc/grafana/.admin_password_set - state: touch - mode: "0600" - when: not grafana_admin_password_marker.stat.exists - - name: Configure Grafana server if TLS is enabled when: grafana_tls_enabled ansible.builtin.ini_file: @@ -55,6 +36,31 @@ - { option: "cert_key", value: "{{ grafana_tls_key_path }}" } notify: Restart Grafana Service +- name: Ensure Grafana service is running + ansible.builtin.service: + name: grafana-server + state: started + enabled: true + +- name: Test Grafana admin password via API + ansible.builtin.uri: + url: "https://localhost:{{ grafana_http_port }}/api/user" + method: GET + user: admin + password: "{{ grafana_security_admin_password }}" + force_basic_auth: yes + status_code: 200 + validate_certs: false + register: grafana_admin_login + failed_when: false + +- name: Set Grafana admin password if API login fails + ansible.builtin.command: > + grafana-cli admin reset-admin-password {{ grafana_security_admin_password }} + no_log: true + notify: Restart Grafana Service + when: grafana_admin_login.status != 200 + - name: Read remote Prometheus certificate content ansible.builtin.slurp: src: "{{ prometheus_tls_cert_path }}" From 70e8bb779e39af683101b056152d9a2a183f087c Mon Sep 17 00:00:00 2001 From: rsuplina Date: Mon, 22 Sep 2025 14:28:22 +0100 Subject: [PATCH 07/10] Update to add Grafana Protocol Signed-off-by: rsuplina --- roles/grafana/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml index c64b23cc..053c8091 100644 --- a/roles/grafana/tasks/main.yml +++ b/roles/grafana/tasks/main.yml @@ -28,7 +28,7 @@ value: "{{ item.value }}" mode: "0640" loop: - - { option: "protocol", value: "https" } + - { option: "protocol", value: "{{ grafana_protocol }}" } - { option: "http_port", value: "{{ grafana_http_port }}" } - { option: "domain", value: "{{ grafana_domain }}" } - { option: "root_url", value: "{{ grafana_root_url }}" } @@ -44,7 +44,7 @@ - name: Test Grafana admin password via API ansible.builtin.uri: - url: "https://localhost:{{ grafana_http_port }}/api/user" + url: "{{ grafana_protocol }}://localhost:{{ grafana_http_port }}/api/user" method: GET user: admin password: "{{ grafana_security_admin_password }}" From c1498da6e9bdfceb55ed86505bef4e7714d415e7 Mon Sep 17 00:00:00 2001 From: rsuplina Date: Tue, 23 Sep 2025 12:31:49 +0100 Subject: [PATCH 08/10] Add requested changes Signed-off-by: rsuplina --- roles/grafana/README.md | 5 ----- roles/grafana/defaults/main.yml | 3 --- roles/grafana/meta/argument_specs.yml | 21 ------------------- roles/grafana/tasks/main.yml | 20 +++++------------- .../templates/prometheus_source.yml.j2 | 6 ++---- 5 files changed, 7 insertions(+), 48 deletions(-) diff --git a/roles/grafana/README.md b/roles/grafana/README.md index c200e4f7..3e03fb3e 100644 --- a/roles/grafana/README.md +++ b/roles/grafana/README.md @@ -35,14 +35,11 @@ None. | Parameter | Type | Default Value | Description | |----------------------------------|------|-------------------------------------------------|---------------------------------------------------------------------------| -| `grafana_protocol` | `str` | `http` | Protocol to use for Grafana web interface and API (`http` or `https`). | | `grafana_datasource_directory` | `str` | `/etc/grafana/provisioning/datasources/automatic.yml` | Location of the Grafana data sources configuration file. | | `grafana_providers_configuration`| `str` | `/etc/grafana/provisioning/dashboards/providers.yml` | Location of the Grafana dashboard provider configurations file. | | `grafana_dashboard_directory` | `str` | `/var/lib/grafana/dashboards` | Location of the Grafana dashboard configurations directory. | | `prometheus_url` | `str` | `http://localhost:9090` | URL (host:port) to the Prometheus server that Grafana will connect to. | | `prometheus_hostname` | `str` | `localhost` | Hostname of the Prometheus server for TLS server name verification. | -| `prometheus_tls_cert_path` | `str` | `/etc/pki/tls/certs/prometheus.crt` | Path to the Prometheus server TLS certificate (for client auth, if needed). | -| `prometheus_tls_key_path` | `str` | `/etc/pki/tls/private/prometheus.key` | Path to the Prometheus server TLS key (for client auth, if needed). | | `grafana_tls_enabled` | `bool`| `false` | Enable or disable TLS/SSL for Grafana (HTTPS support). | | `grafana_tls_cert_path` | `str` | `/etc/pki/tls/certs/grafana.crt` | Path to the TLS certificate file for Grafana. | | `grafana_tls_key_path` | `str` | `/etc/pki/tls/private/grafana.key` | Path to the TLS private key file for Grafana. | @@ -90,8 +87,6 @@ Basic installation connecting to a local Prometheus server: grafana_root_url: "https://grafana.1.1.1.1.pvc.labs.com:3000" grafana_tls_cert_path: "/etc/grafana/certs/grafana.crt" grafana_tls_key_path: "/etc/grafana/private/grafana.key" - prometheus_tls_cert_path: /etc/prometheus/certs/prometheus.crt - prometheus_tls_key_path: /etc/prometheus/private/prometheus.key prometheus_url: "https://prometheus.example.com:9090" ``` diff --git a/roles/grafana/defaults/main.yml b/roles/grafana/defaults/main.yml index 24452f2e..871a79e5 100644 --- a/roles/grafana/defaults/main.yml +++ b/roles/grafana/defaults/main.yml @@ -13,7 +13,6 @@ # limitations under the License. --- -grafana_protocol: http grafana_tls_enabled: false grafana_http_port: 3000 grafana_security_admin_password: admin @@ -29,5 +28,3 @@ grafana_dashboard_directory: /var/lib/grafana/dashboards prometheus_url: http://localhost:9090 prometheus_hostname: localhost -prometheus_tls_cert_path: /etc/pki/tls/certs/prometheus.crt -prometheus_tls_key_path: /etc/pki/tls/private/prometheus.key diff --git a/roles/grafana/meta/argument_specs.yml b/roles/grafana/meta/argument_specs.yml index 468ac9fa..52ece0c5 100644 --- a/roles/grafana/meta/argument_specs.yml +++ b/roles/grafana/meta/argument_specs.yml @@ -22,33 +22,12 @@ argument_specs: author: Cloudera Labs version_added: "2.4.0" options: - grafana_protocol: - description: - - Protocol to use for Grafana web interface and API (http or https). - type: str - choices: - - http - - https - default: http - required: false prometheus_hostname: description: - Hostname of the Prometheus server for TLS server name verification. type: str default: localhost required: false - prometheus_tls_cert_path: - description: - - Path to the Prometheus server TLS certificate (for client auth, if needed). - type: str - default: /etc/pki/tls/certs/prometheus.crt - required: false - prometheus_tls_key_path: - description: - - Path to the Prometheus server TLS key (for client auth, if needed). - type: str - default: /etc/pki/tls/private/prometheus.key - required: false grafana_config_file: description: - Path to the main Grafana configuration file. diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml index 053c8091..f9ed34fa 100644 --- a/roles/grafana/tasks/main.yml +++ b/roles/grafana/tasks/main.yml @@ -28,7 +28,10 @@ value: "{{ item.value }}" mode: "0640" loop: - - { option: "protocol", value: "{{ grafana_protocol }}" } + - { + option: "protocol", + value: "{{ 'https' if grafana_tls_enabled | default(false) else 'http' }}", + } - { option: "http_port", value: "{{ grafana_http_port }}" } - { option: "domain", value: "{{ grafana_domain }}" } - { option: "root_url", value: "{{ grafana_root_url }}" } @@ -44,7 +47,7 @@ - name: Test Grafana admin password via API ansible.builtin.uri: - url: "{{ grafana_protocol }}://localhost:{{ grafana_http_port }}/api/user" + url: "{{ 'https' if grafana_tls_enabled | default(false) else 'http' }}://localhost:{{ grafana_http_port }}/api/user" method: GET user: admin password: "{{ grafana_security_admin_password }}" @@ -61,19 +64,6 @@ notify: Restart Grafana Service when: grafana_admin_login.status != 200 -- name: Read remote Prometheus certificate content - ansible.builtin.slurp: - src: "{{ prometheus_tls_cert_path }}" - register: __prometheus_cert_content - when: grafana_tls_enabled - -- name: Read remote Prometheus key content - ansible.builtin.slurp: - src: "{{ prometheus_tls_key_path }}" - register: __prometheus_key_content - no_log: true - when: grafana_tls_enabled - - name: Connect Prometheus datasource ansible.builtin.template: src: prometheus_source.yml.j2 diff --git a/roles/grafana/templates/prometheus_source.yml.j2 b/roles/grafana/templates/prometheus_source.yml.j2 index df37ea1c..cb4e7f36 100644 --- a/roles/grafana/templates/prometheus_source.yml.j2 +++ b/roles/grafana/templates/prometheus_source.yml.j2 @@ -1,17 +1,15 @@ datasources: -- name: Prometheus +- name: prometheus type: prometheus access: proxy url: {{ prometheus_url }} isDefault: true + uid: PBFA97CFB590B2093 jsonData: tlsAuth: true tlsAuthWithCACert: false tlsSkipVerify: false serverName: {{ prometheus_hostname }} - secureJsonData: - tlsClientCert: "{{ (__prometheus_cert_content.content | b64decode) if grafana_tls_enabled | bool else '' }}" - tlsClientKey: "{{ (__prometheus_key_content.content | b64decode) if grafana_tls_enabled | bool else '' }}" version: 1 editable: true From 14d11a3f07b3c0339ca489877efa85be02e88131 Mon Sep 17 00:00:00 2001 From: rsuplina Date: Tue, 23 Sep 2025 12:44:31 +0100 Subject: [PATCH 09/10] Update to latest Exporter dashboard Signed-off-by: rsuplina --- roles/grafana/files/base_dashboard.json | 23199 ++++++---------- .../templates/prometheus_source.yml.j2 | 1 - 2 files changed, 7747 insertions(+), 15453 deletions(-) diff --git a/roles/grafana/files/base_dashboard.json b/roles/grafana/files/base_dashboard.json index 18d66f70..7409b5ef 100644 --- a/roles/grafana/files/base_dashboard.json +++ b/roles/grafana/files/base_dashboard.json @@ -1,8 +1,56 @@ { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.6.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], "annotations": { "list": [ { - "$$hashKey": "object:1058", "builtIn": 1, "datasource": { "type": "datasource", @@ -24,9 +72,8 @@ }, "editable": true, "fiscalYearStartMonth": 0, - "gnetId": 1860, "graphTooltip": 1, - "id": 1, + "id": null, "links": [ { "icon": "external link", @@ -45,14 +92,9 @@ "url": "https://grafana.com/grafana/dashboards/1860" } ], - "liveNow": false, "panels": [ { "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, @@ -61,75 +103,64 @@ }, "id": 261, "panels": [], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], "title": "Quick CPU / Mem / Disk", "type": "row" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "Busy state of all CPU cores together", + "description": "Resource pressure via PSI", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, + "links": [], + "mappings": [], + "max": 1, "min": 0, "thresholds": { - "mode": "absolute", + "mode": "percentage", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)", - "value": null + "color": "green" }, { - "color": "rgba(237, 129, 40, 0.89)", - "value": 85 + "color": "dark-yellow", + "value": 70 }, { - "color": "rgba(245, 54, 54, 0.9)", - "value": 95 + "color": "dark-red", + "value": 90 } ] }, - "unit": "percent" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { - "h": 7, + "h": 4, "w": 3, "x": 0, "y": 1 }, - "id": 20, - "links": [], + "id": 323, "options": { - "minVizHeight": 200, - "minVizWidth": 200, + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ @@ -138,38 +169,74 @@ "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" + "showUnfilled": true, + "sizing": "auto", + "text": {}, + "valueMode": "color" }, - "pluginVersion": "10.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", "exemplar": false, - "expr": "(sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))) * 100", - "hide": false, + "expr": "irate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", "instant": true, "intervalFactor": 1, - "legendFormat": "", + "legendFormat": "CPU", "range": false, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "Mem", + "range": false, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "I/O", + "range": false, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_pressure_irq_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "Irq", + "range": false, + "refId": "D", + "step": 240 } ], - "title": "CPU Busy", - "type": "gauge" + "title": "Pressure", + "type": "bargauge" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "Busy state of all CPU cores together (5 min average)", + "description": "Overall CPU busy percentage (averaged across all cores)", "fieldConfig": { "defaults": { "color": { @@ -193,8 +260,7 @@ "mode": "absolute", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)", - "value": null + "color": "rgba(50, 172, 45, 0.97)" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -211,17 +277,16 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 4, "w": 3, "x": 3, "y": 1 }, - "id": 155, - "links": [], + "id": 20, "options": { - "minVizHeight": 200, - "minVizWidth": 200, - "orientation": "horizontal", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -233,34 +298,30 @@ "showThresholdMarkers": true, "sizing": "auto" }, - "pluginVersion": "10.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", "exemplar": false, - "expr": "avg_over_time(node_load5{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))", - "format": "time_series", + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", instance=\"$node\"}[$__rate_interval])))", "hide": false, "instant": true, "intervalFactor": 1, + "legendFormat": "", "range": false, "refId": "A", "step": 240 } ], - "title": "Sys Load (5m avg)", + "title": "CPU Busy", "type": "gauge" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "Busy state of all CPU cores together (15 min average)", + "description": "System load over all CPU cores together", "fieldConfig": { "defaults": { "color": { @@ -284,8 +345,7 @@ "mode": "absolute", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)", - "value": null + "color": "rgba(50, 172, 45, 0.97)" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -302,17 +362,16 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 4, "w": 3, "x": 6, "y": 1 }, - "id": 19, - "links": [], + "id": 155, "options": { - "minVizHeight": 200, - "minVizWidth": 200, - "orientation": "horizontal", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -324,16 +383,13 @@ "showThresholdMarkers": true, "sizing": "auto" }, - "pluginVersion": "10.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", "exemplar": false, - "expr": "avg_over_time(node_load15{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100 / on(instance) group_left sum by (instance)(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))", + "expr": "scalar(node_load1{instance=\"$node\",job=\"$job\"}) * 100 / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "format": "time_series", "hide": false, "instant": true, "intervalFactor": 1, @@ -342,15 +398,15 @@ "step": 240 } ], - "title": "Sys Load (15m avg)", + "title": "Sys Load", "type": "gauge" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "Non available RAM memory", + "description": "Real RAM usage excluding cache and reclaimable memory", "fieldConfig": { "defaults": { "color": { @@ -364,8 +420,7 @@ "mode": "absolute", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)", - "value": null + "color": "rgba(50, 172, 45, 0.97)" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -382,18 +437,16 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 4, "w": 3, "x": 9, "y": 1 }, - "hideTimeOverride": false, "id": 16, - "links": [], "options": { - "minVizHeight": 200, - "minVizWidth": 200, - "orientation": "horizontal", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -405,32 +458,12 @@ "showThresholdMarkers": true, "sizing": "auto" }, - "pluginVersion": "10.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "((avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100", - "format": "time_series", - "hide": true, - "instant": true, - "intervalFactor": 1, - "range": false, - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", "exemplar": false, - "expr": "100 - ((avg_over_time(node_memory_MemAvailable_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) * 100) / avg_over_time(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]))", + "expr": "(1 - (node_memory_MemAvailable_bytes{instance=\"$node\", job=\"$job\"} / node_memory_MemTotal_bytes{instance=\"$node\", job=\"$job\"})) * 100", "format": "time_series", "hide": false, "instant": true, @@ -446,9 +479,9 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "Used Swap", + "description": "Percentage of swap space currently used by the system", "fieldConfig": { "defaults": { "color": { @@ -472,8 +505,7 @@ "mode": "absolute", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)", - "value": null + "color": "rgba(50, 172, 45, 0.97)" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -490,17 +522,16 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 4, "w": 3, "x": 12, "y": 1 }, "id": 21, - "links": [], "options": { - "minVizHeight": 200, - "minVizWidth": 200, - "orientation": "horizontal", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -512,16 +543,12 @@ "showThresholdMarkers": true, "sizing": "auto" }, - "pluginVersion": "10.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", "exemplar": false, - "expr": "((avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - avg_over_time(node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / (avg_over_time(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval]) )) * 100", + "expr": "((node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"})) * 100", "instant": true, "intervalFactor": 1, "range": false, @@ -535,7 +562,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "description": "Used Root FS", "fieldConfig": { @@ -561,8 +588,7 @@ "mode": "absolute", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)", - "value": null + "color": "rgba(50, 172, 45, 0.97)" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -579,17 +605,16 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 4, "w": 3, "x": 15, "y": 1 }, "id": 154, - "links": [], "options": { - "minVizHeight": 200, - "minVizWidth": 200, - "orientation": "horizontal", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -601,16 +626,12 @@ "showThresholdMarkers": true, "sizing": "auto" }, - "pluginVersion": "10.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", "exemplar": false, - "expr": "100 - ((avg_over_time(node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]) * 100) / avg_over_time(node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}[$__rate_interval]))", + "expr": "(\n (node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", mountpoint=\"/\", fstype!=\"rootfs\"}\n - node_filesystem_avail_bytes{instance=\"$node\", job=\"$job\", mountpoint=\"/\", fstype!=\"rootfs\"})\n / node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", mountpoint=\"/\", fstype!=\"rootfs\"}\n) * 100\n", "format": "time_series", "instant": true, "intervalFactor": 1, @@ -625,9 +646,9 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "Total number of CPU cores", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -648,12 +669,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] }, @@ -662,19 +678,19 @@ "overrides": [] }, "gridPos": { - "h": 3, + "h": 2, "w": 2, "x": 18, "y": 1 }, "id": 14, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -682,20 +698,19 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", + "exemplar": false, "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "instant": true, "legendFormat": "__auto", - "range": true, + "range": false, "refId": "A" } ], @@ -705,9 +720,85 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bool_yes_no" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 20, + "y": 1 + }, + "id": 328, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "node_reboot_required{instance=\"$node\",job=\"$job\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Reboot Required", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "description": "System uptime", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -729,8 +820,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -743,20 +833,19 @@ "overrides": [] }, "gridPos": { - "h": 3, - "w": 4, - "x": 20, + "h": 2, + "w": 2, + "x": 22, "y": 1 }, - "hideTimeOverride": true, "id": 15, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -764,16 +853,13 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", "exemplar": false, "expr": "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}", @@ -790,9 +876,9 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "Total RootFS", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -814,8 +900,7 @@ "mode": "absolute", "steps": [ { - "color": "rgba(50, 172, 45, 0.97)", - "value": null + "color": "rgba(50, 172, 45, 0.97)" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -832,19 +917,19 @@ "overrides": [] }, "gridPos": { - "h": 3, + "h": 2, "w": 2, "x": 18, - "y": 4 + "y": 3 }, "id": 23, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -852,16 +937,13 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", "exemplar": false, "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}", @@ -880,9 +962,9 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "Total RAM", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -904,8 +986,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -918,19 +999,19 @@ "overrides": [] }, "gridPos": { - "h": 3, + "h": 2, "w": 2, "x": 20, - "y": 4 + "y": 3 }, "id": 75, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -938,16 +1019,13 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", "exemplar": false, "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", @@ -964,9 +1042,9 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "Total SWAP", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -988,8 +1066,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1002,19 +1079,19 @@ "overrides": [] }, "gridPos": { - "h": 3, + "h": 2, "w": 2, "x": 22, - "y": 4 + "y": 3 }, "id": 18, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -1022,16 +1099,13 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", "exemplar": false, "expr": "node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}", @@ -1046,9210 +1120,745 @@ "type": "stat" }, { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 8 + "y": 5 }, "id": 263, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "panels": [], + "title": "Basic CPU / Mem / Net / Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "CPU time spent busy vs idle, split by activity type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "description": "Basic CPU info", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" }, - "overrides": [ + "properties": [ { - "matcher": { - "id": "byName", - "options": "Busy Iowait" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy System" + }, + "properties": [ { - "matcher": { - "id": "byName", - "options": "Idle" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy User" + }, + "properties": [ { - "matcher": { - "id": "byName", - "options": "Busy Iowait" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Other" + }, + "properties": [ { - "matcher": { - "id": "byName", - "options": "Idle" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Busy System", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Busy User", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy Iowait", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy IRQs", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy Other", + "range": true, + "refId": "E", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Idle", + "range": true, + "refId": "F", + "step": 240 + } + ], + "title": "CPU Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "RAM and swap usage overview, including caches", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "Busy System" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Swap used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } }, { - "matcher": { - "id": "byName", - "options": "Busy User" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] + "id": "custom.fillOpacity", + "value": 0 }, { - "matcher": { - "id": "byName", - "options": "Busy Other" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } } ] }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 77, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "width": 250 + { + "matcher": { + "id": "byName", + "options": "Cache + Buffer" }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Busy System", - "range": true, - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Busy User", - "range": true, - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Busy Iowait", - "range": true, - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Busy IRQs", - "range": true, - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Busy Other", - "range": true, - "refId": "E", - "step": 240 + { + "matcher": { + "id": "byName", + "options": "Free" }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Idle", - "range": true, - "refId": "F", - "step": 240 - } - ], - "title": "CPU Basic", - "type": "timeseries" + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 350 }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Total", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Used", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cache + Buffer", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Swap used", + "range": true, + "refId": "E", + "step": 240 + } + ], + "title": "Memory Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Per-interface network traffic (receive and transmit) in bits per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "description": "Basic memory usage", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, + "color": "green" + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Tx.*/" + }, + "properties": [ { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "SWAP Used" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap Used" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "custom.stacking", - "value": { - "group": false, - "mode": "normal" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM Cache + Buffer" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Available" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#DEDAF7", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "custom.stacking", - "value": { - "group": false, - "mode": "normal" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 78, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "RAM Total", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "RAM Used", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "RAM Cache + Buffer", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "RAM Free", - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SWAP Used", - "refId": "E", - "step": 240 - } - ], - "title": "Memory Basic", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Basic network info per interface", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Recv_bytes_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_drop_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_errs_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recv_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CCA300", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_bytes_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_drop_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_errs_eth2" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Trans_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CCA300", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_drop_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#967302", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_errs_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "recv_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_bytes_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_bytes_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_drop_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_drop_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#967302", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_errs_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "trans_errs_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 74, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "recv {{device}}", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "trans {{device}} ", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic Basic", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Disk space used of all filesystems mounted", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 152, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{mountpoint}}", - "refId": "A", - "step": 240 - } - ], - "title": "Disk Space Used Basic", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], - "title": "Basic CPU / Mem / Net / Disk", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 265, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "percentage", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 70, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Idle - Waiting for something to happen" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Iowait - Waiting for I/O to complete" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Irq - Servicing interrupts" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Nice - Niced processes executing in user mode" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Softirq - Servicing softirqs" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Steal - Time spent in other operating systems when running in a virtualized environment" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCE2DE", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "System - Processes executing in kernel mode" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "User - Normal processes executing in user mode" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#5195CE", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 3, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 250 - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "System - Processes executing in kernel mode", - "range": true, - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "User - Normal processes executing in user mode", - "range": true, - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Nice - Niced processes executing in user mode", - "range": true, - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Iowait - Waiting for I/O to complete", - "range": true, - "refId": "E", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Irq - Servicing interrupts", - "range": true, - "refId": "F", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"softirq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Softirq - Servicing softirqs", - "range": true, - "refId": "G", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"steal\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", - "range": true, - "refId": "H", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Idle - Waiting for something to happen", - "range": true, - "refId": "J", - "step": 240 - } - ], - "title": "CPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap - Swap memory usage" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused - Free memory unassigned" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Hardware Corrupted - *./" - }, - "properties": [ - { - "id": "custom.stacking", - "value": { - "group": false, - "mode": "normal" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 24, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Apps - Memory used by user-space applications", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)", - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Cache - Parked file data (file content) cache", - "refId": "E", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Buffers - Block device (e.g. harddisk) cache", - "refId": "F", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Unused - Free memory unassigned", - "refId": "G", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Swap - Swap space used", - "refId": "H", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working", - "refId": "I", - "step": 240 - } - ], - "title": "Memory Stack", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bits out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "receive_packets_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "receive_packets_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "transmit_packets_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "transmit_packets_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 22 - }, - "id": 84, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Receive", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit", - "refId": "B", - "step": 240 - } - ], - "title": "Network Traffic", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 22 - }, - "id": 156, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{mountpoint}}", - "refId": "A", - "step": 240 - } - ], - "title": "Disk Space Used", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "IO read (-) / write (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 34 - }, - "id": 229, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", - "intervalFactor": 4, - "legendFormat": "{{device}} - Reads completed", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", - "intervalFactor": 1, - "legendFormat": "{{device}} - Writes completed", - "refId": "B", - "step": 240 - } - ], - "title": "Disk IOps", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes read (-) / write (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "io time" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*read*./" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byType", - "options": "time" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "hidden" - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 34 - }, - "id": 42, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{device}} - Successfully read bytes", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{device}} - Successfully written bytes", - "refId": "B", - "step": 240 - } - ], - "title": "I/O Usage Read / Write", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "%util", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "io time" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byType", - "options": "time" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "hidden" - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 46 - }, - "id": 127, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [$__rate_interval])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{device}}", - "refId": "A", - "step": 240 - } - ], - "title": "I/O Utilization", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "percentage", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 70, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/^Guest - /" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#5195ce", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/^GuestNice - /" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#c15c17", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 46 - }, - "id": 319, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))", - "hide": false, - "legendFormat": "Guest - Time spent running a virtual CPU for a guest operating system", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\", mode=\"nice\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[1m])))", - "hide": false, - "legendFormat": "GuestNice - Time spent running a niced guest (virtual CPU for guest operating system)", - "range": true, - "refId": "B" - } - ], - "title": "CPU spent seconds in guests (VMs)", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], - "title": "CPU / Memory / Net / Disk", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 266, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 54 - }, - "id": 136, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Inactive - Memory which has been less recently used. It is more eligible to be reclaimed for other purposes", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Active / Inactive", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*CommitLimit - *./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 54 - }, - "id": 135, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Committed_AS - Amount of memory presently allocated on the system", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "CommitLimit - Amount of memory currently available to be allocated on the system", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Committed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 64 - }, - "id": 191, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Inactive_file - File-backed memory on inactive LRU list", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Active_file - File-backed memory on active LRU list", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs", - "refId": "D", - "step": 240 - } - ], - "title": "Memory Active / Inactive Detail", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 64 - }, - "id": 130, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Writeback - Memory which is actively being written back to disk", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "WritebackTmp - Memory used by FUSE for temporary writeback buffers", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Dirty - Memory which is waiting to get written back to the disk", - "refId": "C", - "step": 240 - } - ], - "title": "Memory Writeback and Dirty", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 74 - }, - "id": 138, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Mapped - Used memory in mapped pages files which have been mapped, such as libraries", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Shmem - Used shared memory (shared between several processes, thus including RAM disks)", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "ShmemPmdMapped - Amount of shared (shmem/tmpfs) memory backed by huge pages", - "refId": "D", - "step": 240 - } - ], - "title": "Memory Shared and Mapped", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 74 - }, - "id": 131, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SReclaimable - Part of Slab, that might be reclaimed, such as caches", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Slab", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 84 - }, - "id": 70, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "VmallocChunk - Largest contiguous block of vmalloc area which is free", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "VmallocTotal - Total size of vmalloc memory area", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "VmallocUsed - Amount of vmalloc area which is used", - "refId": "C", - "step": 240 - } - ], - "title": "Memory Vmalloc", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 84 - }, - "id": 159, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Bounce - Memory used for block device bounce buffers", - "refId": "A", - "step": 240 - } - ], - "title": "Memory Bounce", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Inactive *./" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 94 - }, - "id": 129, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "AnonHugePages - Memory in anonymous huge pages", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "AnonPages - Memory in user pages not backed by files", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Anonymous", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 94 - }, - "id": 160, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "KernelStack - Kernel memory stack. This is not reclaimable", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "PerCPU - Per CPU memory allocated dynamically by loadable modules", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Kernel / CPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "pages", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 104 - }, - "id": 140, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages_Free - Huge pages in the pool that are not yet allocated", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages", - "refId": "C", - "step": 240 - } - ], - "title": "Memory HugePages Counter", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 104 - }, - "id": 71, - "links": [], - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "HugePages - Total size of the pool of huge pages", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Hugepagesize - Huge Page size", - "refId": "B", - "step": 240 - } - ], - "title": "Memory HugePages Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM + Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "VmallocUsed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 114 - }, - "id": 128, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "DirectMap1G - Amount of pages mapped as this size", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "DirectMap2M - Amount of pages mapped as this size", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "DirectMap4K - Amount of pages mapped as this size", - "refId": "C", - "step": 240 - } - ], - "title": "Memory DirectMap", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Apps" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#629E51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A437C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#CFFAFF", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Rx {{device}}", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Tx {{device}} ", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Percentage of filesystem space used for each mounted device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "Unused" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] + "color": "green" } ] }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 114 - }, - "id": 137, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "MLocked - Size of pages locked to memory using the mlock() system call", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Unevictable and MLocked", - "type": "timeseries" + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 152, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] + "editorMode": "code", + "expr": "((node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", device!~\"rootfs\"} - node_filesystem_avail_bytes{instance=\"$node\", job=\"$job\", device!~\"rootfs\"}) / node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", device!~\"rootfs\"}) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Disk Space Used Basic", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 265, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "CPU time usage split by state, normalized across all CPU cores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byName", - "options": "Free" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 70, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] + "thresholdsStyle": { + "mode": "off" + } }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "percentunit" + }, + "overrides": [ { "matcher": { "id": "byName", - "options": "Page_Tables" + "options": "Idle - Waiting for something to happen" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#0A50A1", + "fixedColor": "#052B51", "mode": "fixed" } } @@ -10258,7 +1867,7 @@ { "matcher": { "id": "byName", - "options": "Slab_Cache" + "options": "Iowait - Waiting for I/O to complete" }, "properties": [ { @@ -10273,7 +1882,7 @@ { "matcher": { "id": "byName", - "options": "Swap" + "options": "Irq - Servicing interrupts" }, "properties": [ { @@ -10288,7 +1897,7 @@ { "matcher": { "id": "byName", - "options": "Swap_Cache" + "options": "Nice - Niced processes executing in user mode" }, "properties": [ { @@ -10303,13 +1912,13 @@ { "matcher": { "id": "byName", - "options": "Total" + "options": "Softirq - Servicing softirqs" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#511749", + "fixedColor": "#E24D42", "mode": "fixed" } } @@ -10318,13 +1927,13 @@ { "matcher": { "id": "byName", - "options": "Total RAM" + "options": "Steal - Time spent in other operating systems when running in a virtualized environment" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#052B51", + "fixedColor": "#FCE2DE", "mode": "fixed" } } @@ -10333,13 +1942,13 @@ { "matcher": { "id": "byName", - "options": "Total RAM + Swap" + "options": "System - Processes executing in kernel mode" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#052B51", + "fixedColor": "#508642", "mode": "fixed" } } @@ -10348,13 +1957,13 @@ { "matcher": { "id": "byName", - "options": "Total Swap" + "options": "User - Normal processes executing in user mode" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#614D93", + "fixedColor": "#5195CE", "mode": "fixed" } } @@ -10363,240 +1972,186 @@ { "matcher": { "id": "byName", - "options": "VmallocUsed" + "options": "Guest CPU usage" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 124 - }, - "id": 132, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "NFS Unstable - Memory in NFS pages sent to the server, but not yet committed to the storage", - "refId": "A", - "step": 240 - } - ], - "title": "Memory NFS", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], - "title": "Memory Meminfo", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 267, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "pages out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*out/" - }, - "properties": [ + "id": "custom.fillOpacity", + "value": 0 + }, { - "id": "custom.transform", - "value": "negative-Y" + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } } ] } ] }, "gridPos": { - "h": 10, + "h": 12, "w": 12, "x": 0, - "y": 41 + "y": 21 }, - "id": 176, - "links": [], + "id": 3, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "width": 250 }, "tooltip": { + "hideZeros": false, "mode": "multi", - "sort": "none" + "sort": "desc" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"system\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "Pagesin - Page in operations", + "legendFormat": "System - Processes executing in kernel mode", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"user\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "Pagesout - Page out operations", + "legendFormat": "User - Normal processes executing in user mode", + "range": true, "refId": "B", "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"nice\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nice - Niced processes executing in user mode", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"iowait\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Iowait - Waiting for I/O to complete", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"irq\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Irq - Servicing interrupts", + "range": true, + "refId": "E", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"softirq\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Softirq - Servicing softirqs", + "range": true, + "refId": "F", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"steal\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", + "range": true, + "refId": "G", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"idle\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Idle - Waiting for something to happen", + "range": true, + "refId": "H", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))) > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Guest CPU usage", + "range": true, + "refId": "I", + "step": 240 } ], - "title": "Memory Pages In / Out", + "title": "CPU", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Breakdown of physical memory and swap usage. Hardware-detected memory errors are also displayed", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "pages out (-) / in (+)", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 20, + "fillOpacity": 40, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10607,7 +2162,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -10615,158 +2170,163 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "bytes" }, "overrides": [ { "matcher": { - "id": "byRegexp", - "options": "/.*out/" + "id": "byName", + "options": "Apps" }, "properties": [ { - "id": "custom.transform", - "value": "negative-Y" + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } } ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 41 - }, - "id": 22, - "links": [], - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" }, - "expr": "irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pswpin - Pages swapped in", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] }, - "expr": "irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pswpout - Pages swapped out", - "refId": "B", - "step": 240 - } - ], - "title": "Memory Pages Swap In / Out", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "faults", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + { + "matcher": { + "id": "byName", + "options": "Cached" }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" }, - "thresholdsStyle": { - "mode": "off" - } + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ { - "color": "green" - }, + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } } ] }, - "unit": "short" - }, - "overrides": [ { "matcher": { "id": "byName", - "options": "Apps" + "options": "Page_Tables" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#629E51", + "fixedColor": "#0A50A1", "mode": "fixed" } } @@ -10775,13 +2335,13 @@ { "matcher": { "id": "byName", - "options": "Buffers" + "options": "RAM_Free" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#614D93", + "fixedColor": "#E0F9D7", "mode": "fixed" } } @@ -10790,13 +2350,13 @@ { "matcher": { "id": "byName", - "options": "Cache" + "options": "Slab" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#6D1F62", + "fixedColor": "#806EB7", "mode": "fixed" } } @@ -10805,13 +2365,13 @@ { "matcher": { "id": "byName", - "options": "Cached" + "options": "Slab_Cache" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#511749", + "fixedColor": "#E0752D", "mode": "fixed" } } @@ -10820,13 +2380,13 @@ { "matcher": { "id": "byName", - "options": "Committed" + "options": "Swap" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#508642", + "fixedColor": "#BF1B00", "mode": "fixed" } } @@ -10835,13 +2395,13 @@ { "matcher": { "id": "byName", - "options": "Free" + "options": "Swap - Swap memory usage" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#0A437C", + "fixedColor": "#BF1B00", "mode": "fixed" } } @@ -10850,13 +2410,13 @@ { "matcher": { "id": "byName", - "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + "options": "Swap_Cache" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#CFFAFF", + "fixedColor": "#C15C17", "mode": "fixed" } } @@ -10865,13 +2425,13 @@ { "matcher": { "id": "byName", - "options": "Inactive" + "options": "Swap_Free" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#584477", + "fixedColor": "#2F575E", "mode": "fixed" } } @@ -10880,13 +2440,13 @@ { "matcher": { "id": "byName", - "options": "PageTables" + "options": "Unused" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#0A50A1", + "fixedColor": "#EAB839", "mode": "fixed" } } @@ -10895,228 +2455,426 @@ { "matcher": { "id": "byName", - "options": "Page_Tables" + "options": "Unused - Free memory unassigned" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#0A50A1", + "fixedColor": "#052B51", "mode": "fixed" } } ] }, - { - "matcher": { - "id": "byName", - "options": "RAM_Free" + { + "matcher": { + "id": "byRegexp", + "options": "/.*Hardware Corrupted - *./" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Apps - Memory used by user-space applications", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Cache - Parked file data (file content) cache", + "range": true, + "refId": "E", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Buffers - Block device (e.g. harddisk) cache", + "range": true, + "refId": "F", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Unused - Free memory unassigned", + "range": true, + "refId": "G", + "step": 240 + }, + { + "editorMode": "code", + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Swap - Swap space used", + "range": true, + "refId": "H", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working", + "range": true, + "refId": "I", + "step": 240 + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Incoming and outgoing network traffic per interface", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#806EB7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] + "thresholdsStyle": { + "mode": "off" + } }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "bps" + }, + "overrides": [ { "matcher": { - "id": "byName", - "options": "Swap_Cache" + "id": "byRegexp", + "options": "/.*out.*/" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } + "id": "custom.transform", + "value": "negative-Y" } ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 303 + }, + "id": 84, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Network interface utilization as a percentage of its maximum capacity", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byName", - "options": "Swap_Free" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#2F575E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Unused" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "percentunit" + }, + "overrides": [ { "matcher": { - "id": "byName", - "options": "Pgfault - Page major and minor fault operations" + "id": "byRegexp", + "options": "/.*out.*/" }, "properties": [ { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "custom.stacking", - "value": { - "group": false, - "mode": "normal" - } + "id": "custom.transform", + "value": "negative-Y" } ] } ] }, "gridPos": { - "h": 10, + "h": 12, "w": 12, - "x": 0, - "y": 51 + "x": 12, + "y": 303 }, - "id": 175, - "links": [], + "id": 338, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true, - "width": 350 + "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "(rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])\n / ignoring(speed) node_network_speed_bytes{instance=\"$node\",job=\"$job\", speed!=\"-1\"}) * 100", "format": "time_series", "intervalFactor": 1, - "legendFormat": "Pgfault - Page major and minor fault operations", + "legendFormat": "{{device}} - Rx in", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Pgmajfault - Major page fault operations", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "(rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])\n / ignoring(speed) node_network_speed_bytes{instance=\"$node\",job=\"$job\", speed!=\"-1\"}) * 100", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "Pgminfault - Minor page fault operations", - "refId": "C", + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", "step": 240 } ], - "title": "Memory Page Faults", + "title": "Network Saturation", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Disk I/O operations per second for each device", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "read (-) / write (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -11125,6 +2883,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11143,397 +2902,344 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#99440A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Buffers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#58140C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6D1F62", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cached" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Committed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Dirty" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Free" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#B7DBAB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mapped" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PageTables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Page_Tables" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Slab_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Swap_Cache" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C15C17", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#511749", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RAM" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } + "mode": "absolute", + "steps": [ + { + "color": "green" } ] }, + "unit": "iops" + }, + "overrides": [ { "matcher": { - "id": "byName", - "options": "Total RAM + Swap" + "id": "byRegexp", + "options": "/.*Read.*/" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#052B51", - "mode": "fixed" - } + "id": "custom.transform", + "value": "negative-Y" } ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 315 + }, + "id": 229, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "intervalFactor": 4, + "legendFormat": "{{device}} - Read", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk IOps", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Disk I/O throughput per device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byName", - "options": "Total Swap" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "Bps" + }, + "overrides": [ { "matcher": { - "id": "byName", - "options": "VmallocUsed" + "id": "byRegexp", + "options": "/.*Read*./" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } + "id": "custom.transform", + "value": "negative-Y" } ] } ] }, "gridPos": { - "h": 10, + "h": 12, "w": 12, "x": 12, - "y": 51 + "y": 315 }, - "id": 307, - "links": [], + "id": 42, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Read", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Write", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Amount of available disk space per mounted filesystem, excluding rootfs. Based on block availability to non-root users", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 327 + }, + "id": 43, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "metric": "", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Free", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", "format": "time_series", - "interval": "", + "hide": true, "intervalFactor": 1, - "legendFormat": "oom killer invocations ", - "refId": "A", + "legendFormat": "{{mountpoint}} - Size", + "range": true, + "refId": "C", "step": 240 } ], - "title": "OOM Killer", + "title": "Filesystem Space Available", "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], - "title": "Memory Vmstat", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 293, - "panels": [ + }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "Disk usage (used = total - available) per mountpoint", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "seconds", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -11542,6 +3248,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11560,136 +3267,87 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "s" + "unit": "bytes" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Variation*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 10, + "h": 12, "w": 12, - "x": 0, - "y": 56 + "x": 12, + "y": 327 }, - "id": 260, - "links": [], + "id": 156, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", "format": "time_series", - "hide": false, - "interval": "", "intervalFactor": 1, - "legendFormat": "Estimated error in seconds", + "legendFormat": "{{mountpoint}}", + "range": true, "refId": "A", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Time offset in between local system and reference clock", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Maximum error in seconds", - "refId": "C", - "step": 240 } ], - "title": "Time Synchronized Drift", + "title": "Filesystem Used", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "Percentage of time the disk was actively processing I/O operations", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 20, + "fillOpacity": 40, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11708,91 +3366,89 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { - "h": 10, + "h": 12, "w": 12, - "x": 12, - "y": 56 + "x": 0, + "y": 339 }, - "id": 291, - "links": [], + "id": 127, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [$__rate_interval])", "format": "time_series", + "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "Phase-locked loop time adjust", + "legendFormat": "{{device}}", + "range": true, "refId": "A", "step": 240 } ], - "title": "Time PLL Adjust", + "title": "Disk I/O Utilization", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "How often tasks experience CPU, memory, or I/O delays. “Some” indicates partial slowdown; “Full” indicates all tasks are stalled. Based on Linux PSI metrics:\nhttps://docs.kernel.org/accounting/psi.html", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "some (-) / full (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 20, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11816,107 +3472,166 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*Variation*./" + "options": "/.*Some.*/" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Some.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" } ] } ] }, "gridPos": { - "h": 10, + "h": 12, "w": 12, - "x": 0, - "y": 66 + "x": 12, + "y": 339 }, - "id": 168, - "links": [], + "id": 322, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_timex_sync_status{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "rate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "Is clock synchronized to a reliable server (1 = yes, 0 = no)", - "refId": "A", + "legendFormat": "CPU - Some", + "range": true, + "refId": "CPU some", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "rate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "interval": "", + "hide": false, "intervalFactor": 1, - "legendFormat": "Local clock frequency adjustment", - "refId": "B", + "legendFormat": "Memory - Some", + "range": true, + "refId": "Memory some", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_pressure_memory_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Memory - Full", + "range": true, + "refId": "Memory full", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "I/O - Some", + "range": true, + "refId": "I/O some", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_pressure_io_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "I/O - Full", + "range": true, + "refId": "I/O full", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_pressure_irq_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "IRQ - Full", + "range": true, + "refId": "A", "step": 240 } ], - "title": "Time Synchronized Status", + "title": "Pressure Stall Information", "type": "timeseries" - }, + } + ], + "title": "CPU / Memory / Net / Disk", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 266, + "panels": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "Displays committed memory usage versus the system's commit limit. Exceeding the limit is allowed under Linux overcommit policies but may increase OOM risks under high load", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "seconds", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -11925,6 +3640,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11943,122 +3659,109 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*CommitLimit - *./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } }, { - "color": "red", - "value": 80 + "id": "custom.fillOpacity", + "value": 0 } ] - }, - "unit": "s" - }, - "overrides": [] + } + ] }, "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 66 + "x": 0, + "y": 602 }, - "id": 294, - "links": [], + "id": 135, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "width": 350 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "Seconds between clock ticks", + "legendFormat": "Committed_AS – Memory promised to processes (not necessarily used)", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "International Atomic Time (TAI) offset", + "legendFormat": "CommitLimit - Max allowable committed memory", + "range": true, "refId": "B", "step": 240 } ], - "title": "Time Misc", + "title": "Memory Committed", "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], - "title": "System Timesync", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 312, - "panels": [ + }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Memory currently dirty (modified but not yet written to disk), being actively written back, or held by writeback buffers. High dirty or writeback memory may indicate disk I/O pressure or delayed flushing", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -12067,6 +3770,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12091,88 +3795,103 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 43 + "x": 12, + "y": 602 }, - "id": 62, - "links": [], + "id": 130, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_procs_blocked{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", "intervalFactor": 1, - "legendFormat": "Processes blocked waiting for I/O to complete", + "legendFormat": "Writeback – Memory currently being flushed to disk", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_procs_running{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", "intervalFactor": 1, - "legendFormat": "Processes in runnable state", + "legendFormat": "WritebackTmp – FUSE temporary writeback buffers", + "range": true, "refId": "B", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Dirty – Memory marked dirty (pending write to disk)", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "NFS Unstable – Pages sent to NFS server, awaiting storage commit", + "range": true, + "refId": "D", + "step": 240 } ], - "title": "Processes Status", + "title": "Memory Writeback and Dirty", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Kernel slab memory usage, separated into reclaimable and non-reclaimable categories. Reclaimable memory can be freed under memory pressure (e.g., caches), while unreclaimable memory is locked by the kernel for core functions", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -12181,6 +3900,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12205,77 +3925,82 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 43 + "x": 0, + "y": 802 }, - "id": 315, - "links": [], + "id": 131, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_processes_state{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "{{ state }}", + "legendFormat": "SUnreclaim – Non-reclaimable slab memory (kernel objects)", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SReclaimable – Potentially reclaimable slab memory (e.g., inode cache)", + "range": true, + "refId": "B", + "step": 240 } ], - "title": "Processes State", + "title": "Memory Slab", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Memory used for mapped files (such as libraries) and shared memory (shmem and tmpfs), including variants backed by huge pages", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "forks / sec", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -12284,6 +4009,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12308,77 +4034,105 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 53 + "x": 12, + "y": 802 }, - "id": 148, - "links": [], + "id": 138, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "width": 350 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "Processes forks second", + "legendFormat": "Mapped – Memory mapped from files (e.g., libraries, mmap)", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Shmem – Shared memory used by processes and tmpfs", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ShmemHugePages – Shared memory (shmem/tmpfs) allocated with HugePages", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PMD Mapped – Shmem/tmpfs backed by Transparent HugePages (PMD)", + "range": true, + "refId": "D", + "step": 240 } ], - "title": "Processes Forks", + "title": "Memory Shared and Mapped", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Proportion of memory pages in the kernel's active and inactive LRU lists relative to total RAM. Active pages have been recently used, while inactive pages are less recently accessed but still resident in memory", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -12387,6 +4141,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12397,7 +4152,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -12411,25 +4166,39 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "decbytes" + "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*Max.*/" + "options": "/.*Active.*/" }, "properties": [ { - "id": "custom.fillOpacity", - "value": 0 + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Inactive.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } } ] } @@ -12438,102 +4207,73 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 53 + "x": 0, + "y": 812 }, - "id": 149, - "links": [], + "id": 136, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "width": 350 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, - "interval": "", + "editorMode": "code", + "expr": "(node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}) \n/ \n(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", "intervalFactor": 1, - "legendFormat": "Processes virtual memory size in bytes", + "legendFormat": "Inactive – Less recently used memory, more likely to be reclaimed", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "process_resident_memory_max_bytes{instance=\"$node\",job=\"$job\"}", - "hide": false, - "interval": "", + "editorMode": "code", + "expr": "(node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}) \n/ \n(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"})\n", + "format": "time_series", "intervalFactor": 1, - "legendFormat": "Maximum amount of virtual memory available in bytes", + "legendFormat": "Active – Recently used memory, retained unless under pressure", + "range": true, "refId": "B", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Processes virtual memory size in bytes", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Maximum amount of virtual memory available in bytes", - "refId": "D", - "step": 240 } ], - "title": "Processes Memory", + "title": "Memory LRU Active / Inactive (%)", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Breakdown of memory pages in the kernel's active and inactive LRU lists, separated by anonymous (heap, tmpfs) and file-backed (caches, mmap) pages.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -12542,6 +4282,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12552,7 +4293,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -12566,110 +4307,107 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "bytes" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "PIDs limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2495C", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 63 + "x": 12, + "y": 812 }, - "id": 313, - "links": [], + "id": 191, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "width": 350 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_processes_pids{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "interval": "", + "hide": false, "intervalFactor": 1, - "legendFormat": "Number of PIDs", + "legendFormat": "Inactive_file - File-backed memory on inactive LRU list", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_processes_max_processes{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "interval": "", + "hide": false, "intervalFactor": 1, - "legendFormat": "PIDs limit", + "legendFormat": "Inactive_anon – Anonymous memory on inactive LRU (incl. tmpfs & swap cache)", + "range": true, "refId": "B", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Active_file - File-backed memory on active LRU list", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Active_anon – Anonymous memory on active LRU (incl. tmpfs & swap cache)", + "range": true, + "refId": "D", + "step": 240 } ], - "title": "PIDs Number and Limit", + "title": "Memory LRU Active / Inactive Detail", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Tracks kernel memory used for CPU-local structures, per-thread stacks, and bounce buffers used for I/O on DMA-limited devices. These areas are typically small but critical for low-level operations", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "seconds", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -12678,6 +4416,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12696,108 +4435,102 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "s" + "unit": "bytes" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*waiting.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 63 + "x": 0, + "y": 822 }, - "id": 305, - "links": [], + "id": 160, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "width": 350 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "CPU {{ cpu }} - seconds spent running a process", + "legendFormat": "KernelStack – Kernel stack memory (per-thread, non-reclaimable)", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "CPU {{ cpu }} - seconds spent by processing waiting for this CPU", + "legendFormat": "PerCPU – Dynamically allocated per-CPU memory (used by kernel modules)", + "range": true, "refId": "B", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Bounce Memory – I/O buffer for DMA-limited devices", + "range": true, + "refId": "C", + "step": 240 } ], - "title": "Process schedule stats Running / Waiting", + "title": "Memory Kernel / CPU / IO", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Usage of the kernel's vmalloc area, which provides virtual memory allocations for kernel modules and drivers. Includes total, used, and largest free block sizes", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -12806,6 +4539,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12830,32 +4564,38 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "bytes" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "Threads limit" + "id": "byRegexp", + "options": "/.*Total.*/" }, "properties": [ { - "id": "color", + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", "value": { - "fixedColor": "#F2495C", - "mode": "fixed" + "dash": [ + 10, + 10 + ], + "fill": "dash" } }, { - "id": "custom.fillOpacity", - "value": 0 + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } } ] } @@ -12865,102 +4605,84 @@ "h": 10, "w": 12, "x": 12, - "y": 73 + "y": 822 }, - "id": 314, - "links": [], + "id": 70, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_processes_threads{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "interval": "", + "hide": false, "intervalFactor": 1, - "legendFormat": "Allocated threads", + "legendFormat": "Vmalloc Free Chunk – Largest available block in vmalloc area", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_processes_max_threads{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "interval": "", + "hide": false, "intervalFactor": 1, - "legendFormat": "Threads limit", + "legendFormat": "Vmalloc Total – Total size of the vmalloc memory area", + "range": true, "refId": "B", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Vmalloc Used – Portion of vmalloc area currently in use", + "range": true, + "refId": "C", + "step": 240 } ], - "title": "Threads Number and Limit", + "title": "Memory Vmalloc", "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], - "title": "System Processes", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 269, - "panels": [ + }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Memory used by anonymous pages (not backed by files), including standard and huge page allocations. Includes heap, stack, and memory-mapped anonymous regions", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -12969,6 +4691,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12993,14 +4716,10 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "bytes" }, "overrides": [] }, @@ -13008,74 +4727,71 @@ "h": 10, "w": 12, "x": 0, - "y": 26 + "y": 832 }, - "id": 8, - "links": [], + "id": 129, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", "intervalFactor": 1, - "legendFormat": "Context switches", + "legendFormat": "AnonHugePages – Anonymous memory using HugePages", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "Interrupts", + "legendFormat": "AnonPages – Anonymous memory (non-file-backed)", + "range": true, "refId": "B", "step": 240 } ], - "title": "Context Switches / Interrupts", + "title": "Memory Anonymous", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Memory that is locked in RAM and cannot be swapped out. Includes both kernel-unevictable memory and user-level memory locked with mlock()", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -13084,6 +4800,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -13108,102 +4825,101 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "bytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 10, "w": 12, "x": 12, - "y": 26 + "y": 832 }, - "id": 7, - "links": [], + "id": 137, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "width": 350 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_load1{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 4, - "legendFormat": "Load 1m", + "intervalFactor": 1, + "legendFormat": "Unevictable – Kernel-pinned memory (not swappable)", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_load5{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 4, - "legendFormat": "Load 5m", + "intervalFactor": 1, + "legendFormat": "Mlocked – Application-locked memory via mlock()", + "range": true, "refId": "B", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_load15{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 4, - "legendFormat": "Load 15m", - "refId": "C", - "step": 240 } ], - "title": "System Load", + "title": "Memory Unevictable and MLocked", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "How much memory is directly mapped in the kernel using different page sizes (4K, 2M, 1G). Helps monitor large page utilization in the direct map region", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, @@ -13229,6 +4945,7 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -13236,84 +4953,280 @@ "color": "green" }, { - "color": "red", - "value": 80 + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } } ] }, - "unit": "hertz" - }, - "overrides": [ { "matcher": { "id": "byName", - "options": "Max" + "options": "Swap_Cache" }, "properties": [ - { - "id": "custom.lineStyle", - "value": { - "dash": [ - 10, - 10 - ], - "fill": "dash" - } - }, { "id": "color", "value": { - "fixedColor": "blue", + "fixedColor": "#C15C17", "mode": "fixed" } - }, - { - "id": "custom.fillOpacity", - "value": 10 - }, + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ { - "id": "custom.hideFrom", + "id": "color", "value": { - "legend": true, - "tooltip": false, - "viz": false + "fixedColor": "#511749", + "mode": "fixed" } - }, - { - "id": "custom.fillBelowTo", - "value": "Min" } ] }, { "matcher": { "id": "byName", - "options": "Min" + "options": "Total RAM" }, "properties": [ { - "id": "custom.lineStyle", + "id": "color", "value": { - "dash": [ - 10, - 10 - ], - "fill": "dash" + "fixedColor": "#052B51", + "mode": "fixed" } - }, + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ { "id": "color", "value": { - "fixedColor": "blue", + "fixedColor": "#052B51", "mode": "fixed" } - }, + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ { - "id": "custom.hideFrom", + "id": "color", "value": { - "legend": true, - "tooltip": false, - "viz": false + "fixedColor": "#EA6460", + "mode": "fixed" } } ] @@ -13324,100 +5237,85 @@ "h": 10, "w": 12, "x": 0, - "y": 36 + "y": 842 }, - "id": 321, - "links": [], + "id": 128, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", - "sort": "desc" + "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", - "expr": "node_cpu_scaling_frequency_hertz{instance=\"$node\",job=\"$job\"}", + "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, - "interval": "", "intervalFactor": 1, - "legendFormat": "CPU {{ cpu }}", + "legendFormat": "DirectMap 1G – Memory mapped with 1GB pages", "range": true, - "refId": "B", + "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", - "expr": "avg(node_cpu_scaling_frequency_max_hertz{instance=\"$node\",job=\"$job\"})", + "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "Max", + "legendFormat": "DirectMap 2M – Memory mapped with 2MB pages", "range": true, - "refId": "A", + "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", - "expr": "avg(node_cpu_scaling_frequency_min_hertz{instance=\"$node\",job=\"$job\"})", + "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "Min", + "legendFormat": "DirectMap 4K – Memory mapped with 4KB pages", "range": true, "refId": "C", "step": 240 } ], - "title": "CPU Frequency Scaling", + "title": "Memory DirectMap", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "https://docs.kernel.org/accounting/psi.html", + "description": "Displays HugePages memory usage in bytes, including allocated, free, reserved, and surplus memory. All values are calculated based on the number of huge pages multiplied by their configured size", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, @@ -13449,199 +5347,117 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "percentunit" + "unit": "bytes" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Memory some" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory full" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "I/O some" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "I/O full" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-blue", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, - "y": 36 + "y": 842 }, - "id": 322, - "links": [], + "id": 140, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "rate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "CPU some", - "range": true, - "refId": "CPU some", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", - "expr": "rate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "expr": "node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "Memory some", + "legendFormat": "HugePages Used – Currently allocated", "range": true, - "refId": "Memory some", + "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", - "expr": "rate(node_pressure_memory_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "expr": "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "Memory full", + "legendFormat": "HugePages Reserved – Promised but unused", "range": true, - "refId": "Memory full", + "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", - "expr": "rate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "expr": "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "I/O some", + "legendFormat": "HugePages Surplus – Dynamic pool extension", "range": true, - "refId": "I/O some", + "refId": "C", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", - "expr": "rate(node_pressure_io_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "expr": "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", "hide": false, "intervalFactor": 1, - "legendFormat": "I/O full", + "legendFormat": "HugePages Total – Reserved memory", "range": true, - "refId": "I/O full", + "refId": "D", "step": 240 } ], - "title": "Pressure Stall Information", + "title": "Memory HugePages", "type": "timeseries" - }, + } + ], + "title": "Memory Meminfo", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 267, + "panels": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of memory pages being read from or written to disk (page-in and page-out operations). High page-out may indicate memory pressure or swapping activity", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -13650,6 +5466,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -13668,57 +5485,26 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "ops" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*Critical*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Max*./" + "options": "/.*out.*/" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 + "id": "custom.transform", + "value": "negative-Y" } ] } @@ -13728,62 +5514,71 @@ "h": 10, "w": 12, "x": 0, - "y": 46 + "y": 603 }, - "id": 259, - "links": [], + "id": 176, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "{{ type }} - {{ info }}", + "legendFormat": "Pagesin - Page in ops", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pagesout - Page out ops", + "range": true, + "refId": "B", + "step": 240 } ], - "title": "Interrupts Detail", + "title": "Memory Pages In / Out", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate at which memory pages are being swapped in from or out to disk. High swap-out activity may indicate memory pressure", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -13792,6 +5587,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -13815,77 +5611,95 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "ops" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] }, "gridPos": { "h": 10, "w": 12, "x": 12, - "y": 46 + "y": 603 }, - "id": 306, - "links": [], + "id": 22, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "CPU {{ cpu }}", + "legendFormat": "Pswpin - Pages swapped in", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pswpout - Pages swapped out", + "range": true, + "refId": "B", + "step": 240 } ], - "title": "Schedule timeslices executed by each cpu", + "title": "Memory Pages Swap In / Out", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of memory page faults, split into total, major (disk-backed), and derived minor (non-disk) faults. High major fault rates may indicate memory pressure or insufficient RAM", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -13894,6 +5708,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -13904,7 +5719,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -13918,76 +5733,133 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "ops" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pgfault - Page major and minor fault ops" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "none" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 10, "w": 12, "x": 0, - "y": 56 + "y": 783 }, - "id": 151, - "links": [], + "id": 175, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "width": 350 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_entropy_available_bits{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "Entropy available to random number generators", + "legendFormat": "Pgfault - Page major and minor fault ops", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Pgmajfault - Major page fault ops", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Pgminfault - Minor page fault ops", + "range": true, + "refId": "C", + "step": 240 } ], - "title": "Entropy", + "title": "Memory Page Faults", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of Out-of-Memory (OOM) kill events. A non-zero value indicates the kernel has terminated one or more processes due to memory exhaustion", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "seconds", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -13996,6 +5868,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14014,82 +5887,109 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "s" + "unit": "ops" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "OOM Kills" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 10, "w": 12, "x": 12, - "y": 56 + "y": 783 }, - "id": 308, - "links": [], + "id": 307, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "Time spent", + "legendFormat": "OOM Kills", + "range": true, "refId": "A", "step": 240 } ], - "title": "CPU time spent in user and system contexts", + "title": "OOM Killer", "type": "timeseries" - }, + } + ], + "title": "Memory Vmstat", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 293, + "panels": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Tracks the system clock's estimated and maximum error, as well as its offset from the reference clock (e.g., via NTP). Useful for detecting synchronization drift", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -14098,6 +5998,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14116,141 +6017,103 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "s" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Max*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, - "y": 66 + "y": 604 }, - "id": 64, - "links": [], + "id": 260, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "process_max_fds{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Estimated error", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "Maximum open file descriptors", - "refId": "A", + "legendFormat": "Offset local vs reference", + "range": true, + "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "process_open_fds{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "Open file descriptors", - "refId": "B", + "legendFormat": "Maximum error", + "range": true, + "refId": "C", "step": 240 } ], - "title": "File Descriptors", + "title": "Time Synchronized Drift", "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], - "title": "System Misc", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 304, - "panels": [ + }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "NTP phase-locked loop (PLL) time constant used by the kernel to control time adjustments. Lower values mean faster correction but less stability", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "temperature", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -14259,6 +6122,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14277,177 +6141,78 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "celsius" + "unit": "short" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Critical*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Max*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 59 + "x": 12, + "y": 604 }, - "id": 158, - "links": [], + "id": 291, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names", + "editorMode": "code", + "expr": "node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{ chip_name }} {{ sensor }} temp", + "legendFormat": "PLL Time Constant", + "range": true, "refId": "A", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ chip_name }} {{ sensor }} Critical Alarm", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ chip_name }} {{ sensor }} Critical", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ chip_name }} {{ sensor }} Critical Historical", - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ chip_name }} {{ sensor }} Max", - "refId": "E", - "step": 240 } ], - "title": "Hardware temperature monitor", + "title": "Time PLL Adjust", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Shows whether the system clock is synchronized to a reliable time source, and the current frequency correction ratio applied by the kernel to maintain synchronization", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -14456,6 +6221,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14479,111 +6245,108 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "short" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Max*./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 59 + "x": 0, + "y": 754 }, - "id": 300, - "links": [], + "id": 168, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_timex_sync_status{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "Current {{ name }} in {{ type }}", + "legendFormat": "Sync status (1 = ok)", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "Max {{ name }} in {{ type }}", + "legendFormat": "Frequency Adjustment", + "range": true, "refId": "B", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Tick Interval", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "TAI Offset", + "range": true, + "refId": "D", + "step": 240 } ], - "title": "Throttle cooling device", + "title": "Time Synchronized Status", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Displays the PPS signal's frequency offset and stability (jitter) in hertz. Useful for monitoring high-precision time sources like GPS or atomic clocks", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -14592,6 +6355,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14615,105 +6379,84 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "rothz" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 69 + "x": 12, + "y": 754 }, - "id": 302, - "links": [], + "id": 333, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_power_supply_online{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_timex_pps_frequency_hertz{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ power_supply }} online", - "refId": "A", - "step": 240 - } - ], - "title": "Power supply", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], - "title": "Hardware Misc", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 296, - "panels": [ + "intervalFactor": 1, + "legendFormat": "PPS Frequency Offset", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_timex_pps_stability_hertz{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PPS Frequency Stability", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "PPS Frequency / Stability", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Tracks PPS signal timing jitter and shift compared to system clock", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -14722,6 +6465,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14740,20 +6484,15 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "s" }, "overrides": [] }, @@ -14761,62 +6500,73 @@ "h": 10, "w": 12, "x": 0, - "y": 46 + "y": 764 }, - "id": 297, - "links": [], + "id": 334, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_timex_pps_jitter_seconds{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{ name }} Connections", + "legendFormat": "PPS Jitter", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_timex_pps_shift_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PPS Shift", + "range": true, + "refId": "B", + "step": 240 } ], - "title": "Systemd Sockets", + "title": "PPS Time Accuracy", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of PPS synchronization diagnostics including calibration events, jitter violations, errors, and frequency stability exceedances", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -14825,6 +6575,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14835,7 +6586,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -14855,226 +6606,232 @@ } ] }, - "unit": "short" + "unit": "ops" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Failed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2495C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inactive" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF9830", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#73BF69", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Deactivating" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFCB7D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Activating" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C8F2C2", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, - "y": 46 + "y": 764 }, - "id": 298, - "links": [], + "id": 335, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}", + "editorMode": "code", + "expr": "irate(node_timex_pps_calibration_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "Activating", + "legendFormat": "PPS Calibrations/sec", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}", + "editorMode": "code", + "expr": "irate(node_timex_pps_error_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "Active", + "legendFormat": "PPS Errors/sec", + "range": true, "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}", + "editorMode": "code", + "expr": "irate(node_timex_pps_stability_exceeded_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", + "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "Deactivating", + "legendFormat": "PPS Stability Exceeded/sec", + "range": true, "refId": "C", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}", + "editorMode": "code", + "expr": "irate(node_timex_pps_jitter_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", + "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "Failed", + "legendFormat": "PPS Jitter Events/sec", + "range": true, "refId": "D", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Inactive", - "refId": "E", - "step": 240 } ], - "title": "Systemd Units State", + "title": "PPS Sync Events", "type": "timeseries" } ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], - "title": "Systemd", + "title": "System Timesync", "type": "row" }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 17 + "y": 24 }, - "id": 270, + "id": 312, "panels": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" + }, + "description": "Processes currently in runnable or blocked states. Helps identify CPU contention or I/O wait bottlenecks.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 605 + }, + "id": 62, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_procs_blocked{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Blocked (I/O Wait)", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_procs_running{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Runnable (Ready for CPU)", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Processes Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "description": "The number (after merges) of I/O requests completed per second for the device", + "description": "Current number of processes in each state (e.g., running, sleeping, zombie). Requires --collector.processes to be enabled in node_exporter", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "IO read (-) / write (+)", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -15083,6 +6840,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -15093,7 +6851,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -15101,329 +6859,469 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "iops" + "unit": "short" }, "overrides": [ { "matcher": { - "id": "byRegexp", - "options": "/.*Read.*/" + "id": "byName", + "options": "D" }, "properties": [ { - "id": "custom.transform", - "value": "negative-Y" + "id": "displayName", + "value": "Uninterruptible Sleeping" } ] }, { "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" + "id": "byName", + "options": "I" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } + "id": "displayName", + "value": "Idle Kernel Thread" } ] }, { "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" + "id": "byName", + "options": "R" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } + "id": "displayName", + "value": "Running" } ] }, { "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" + "id": "byName", + "options": "S" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } + "id": "displayName", + "value": "Interruptible Sleeping" } ] }, { "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" + "id": "byName", + "options": "T" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } + "id": "displayName", + "value": "Stopped" } ] }, { "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" + "id": "byName", + "options": "X" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } + "id": "displayName", + "value": "Dead" } ] }, { "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" + "id": "byName", + "options": "Z" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } + "id": "displayName", + "value": "Zombie" } ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 605 + }, + "id": 315, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_processes_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ state }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Processes Detailed States", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of new processes being created on the system (forks/sec).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } + "color": "green" } ] }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 635 + }, + "id": 148, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Process Forks per second", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Processes Forks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shows CPU saturation per core, calculated as the proportion of time spent waiting to run relative to total time demanded (running + waiting).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] + "thresholdsStyle": { + "mode": "off" + } }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ + "color": "green" + }, { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } + "color": "red", + "value": 80 } ] }, + "unit": "percentunit" + }, + "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*sdd1.*/" + "options": "/.*waiting.*/" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } + "id": "custom.transform", + "value": "negative-Y" } ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 635 + }, + "id": 305, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }} - Running", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Waiting Queue", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])\n/\n(irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) + irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))\n", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}}", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "CPU Saturation per Core", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of active PIDs on the system and the configured maximum allowed. Useful for detecting PID exhaustion risk. Requires --collector.processes in node_exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "short" + }, + "overrides": [ { "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" + "id": "byName", + "options": "PIDs limit" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#FCEACA", + "fixedColor": "#F2495C", "mode": "fixed" } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ + }, { - "id": "color", + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" + "dash": [ + 10, + 10 + ], + "fill": "dash" } } ] @@ -15434,72 +7332,73 @@ "h": 10, "w": 12, "x": 0, - "y": 47 + "y": 645 }, - "id": 9, - "links": [], + "id": 313, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "intervalFactor": 4, - "legendFormat": "{{device}} - Reads completed", + "editorMode": "code", + "expr": "node_processes_pids{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Number of PIDs", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_processes_max_processes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{device}} - Writes completed", + "legendFormat": "PIDs limit", + "range": true, "refId": "B", "step": 240 } ], - "title": "Disk IOps Completed", + "title": "PIDs Number and Limit", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "The number of bytes read from or written to the device per second", + "description": "Number of active threads on the system and the configured thread limit. Useful for monitoring thread pressure. Requires --collector.processes in node_exporter", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes read (-) / write (+)", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -15508,6 +7407,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -15526,328 +7426,307 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } + "mode": "absolute", + "steps": [ + { + "color": "green" } ] }, + "unit": "short" + }, + "overrides": [ { "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" + "id": "byName", + "options": "Threads limit" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#F9934E", + "fixedColor": "#F2495C", "mode": "fixed" } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" - }, - "properties": [ + }, { - "id": "color", + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", "value": { - "fixedColor": "#EA6460", - "mode": "fixed" + "dash": [ + 10, + 10 + ], + "fill": "dash" } } ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 645 + }, + "id": 314, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_processes_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Allocated threads", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_processes_max_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Threads limit", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Threads Number and Limit", + "type": "timeseries" + } + ], + "title": "System Processes", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 269, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Per-second rate of context switches and hardware interrupts. High values may indicate intense CPU or I/O activity", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } + "color": "green" } ] }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 686 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Context switches", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Interrupts", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Context Switches / Interrupts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "System load average over 1, 5, and 15 minutes. Reflects the number of active or waiting processes. Values above CPU core count may indicate overload", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "short" + }, + "overrides": [ { "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" + "id": "byName", + "options": "CPU Core Count" }, "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, { "id": "color", "value": { - "fixedColor": "#F9E2D2", + "fixedColor": "dark-red", "mode": "fixed" } } @@ -15859,82 +7738,101 @@ "h": 10, "w": 12, "x": 12, - "y": 47 + "y": 686 }, - "id": 33, - "links": [], + "id": 7, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_load1{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "intervalFactor": 4, - "legendFormat": "{{device}} - Read bytes", + "intervalFactor": 1, + "legendFormat": "Load 1m", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_load5{instance=\"$node\",job=\"$job\"}", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{device}} - Written bytes", + "legendFormat": "Load 5m", + "range": true, "refId": "B", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_load15{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Load 15m", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "CPU Core Count", + "range": true, + "refId": "D", + "step": 240 } ], - "title": "Disk R/W Data", + "title": "System Load", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.", + "description": "Real-time CPU frequency scaling per core, including average minimum and maximum allowed scaling frequencies", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "time. read (-) / write (+)", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 30, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -15957,405 +7855,365 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "hertz" + }, + "overrides": [ { "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" + "id": "byName", + "options": "Max" }, "properties": [ { - "id": "color", + "id": "custom.lineStyle", "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" + "dash": [ + 10, + 10 + ], + "fill": "dash" } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ + }, { "id": "color", "value": { - "fixedColor": "#65C5DB", + "fixedColor": "dark-red", "mode": "fixed" } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ + }, { - "id": "color", + "id": "custom.hideFrom", "value": { - "fixedColor": "#F9934E", - "mode": "fixed" + "legend": true, + "tooltip": false, + "viz": false } } ] }, { "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" + "id": "byName", + "options": "Min" }, "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, { "id": "color", "value": { - "fixedColor": "#EA6460", + "fixedColor": "blue", "mode": "fixed" } + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } } ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 696 + }, + "id": 321, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_cpu_scaling_frequency_hertz{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "avg(node_cpu_scaling_frequency_max_hertz{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Max", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "avg(node_cpu_scaling_frequency_min_hertz{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Min", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "CPU Frequency Scaling", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of scheduling timeslices executed per CPU. Reflects how frequently the scheduler switches tasks on each core", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } + "color": "green" } ] }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 696 + }, + "id": 306, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "CPU Schedule Timeslices", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Breaks down hardware interrupts by type and device. Useful for diagnosing IRQ load on network, disk, or CPU interfaces. Requires --collector.interrupts to be enabled in node_exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } + "color": "green" } ] - } - ] + }, + "unit": "ops" + }, + "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, - "y": 57 + "y": 706 }, - "id": 37, - "links": [], + "id": 259, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}} - Read wait time avg", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "hide": false, + "editorMode": "code", + "expr": "irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{device}} - Write wait time avg", - "refId": "B", + "legendFormat": "{{ type }} - {{ info }}", + "range": true, + "refId": "A", "step": 240 } ], - "title": "Disk Average Wait Time", + "title": "IRQ Detail", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "The average queue length of the requests that were issued to the device", + "description": "Number of bits of entropy currently available to the system's random number generators (e.g., /dev/random). Low values may indicate that random number generation could block or degrade performance of cryptographic operations", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "aqu-sz", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -16364,6 +8222,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -16385,316 +8244,345 @@ "min": 0, "thresholds": { "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } + "steps": [ + { + "color": "green" } ] }, + "unit": "decbits" + }, + "overrides": [ { "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" + "id": "byName", + "options": "Entropy pool max" }, "properties": [ { - "id": "color", + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" + "dash": [ + 10, + 10 + ], + "fill": "dash" } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ + }, { "id": "color", "value": { - "fixedColor": "#65C5DB", + "fixedColor": "dark-red", "mode": "fixed" } } ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 706 + }, + "id": 151, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_entropy_available_bits{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Entropy available", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_entropy_pool_size_bits{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Entropy pool max", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Entropy", + "type": "timeseries" + } + ], + "title": "System Misc", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 304, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Monitors hardware sensor temperatures and critical thresholds as exposed by Linux hwmon. Includes CPU, GPU, and motherboard sensors where available", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "celsius" + }, + "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*sdd3.*/" + "options": "/.*Critical*./" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#EA6460", + "fixedColor": "#E24D42", "mode": "fixed" } + }, + { + "id": "custom.fillOpacity", + "value": 0 } ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 607 + }, + "id": 158, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }}", + "range": true, + "refId": "A", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Critical Alarm", + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Critical", + "range": true, + "refId": "C", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Critical Historical", + "refId": "D", + "step": 240 + }, + { + "expr": "node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Max", + "refId": "E", + "step": 240 + } + ], + "title": "Hardware Temperature Monitor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shows how hard each cooling device (fan/throttle) is working relative to its maximum capacity", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "percent" + }, + "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*sde3.*/" + "options": "/.*Max*./" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#F9E2D2", + "fixedColor": "#EF843C", "mode": "fixed" } + }, + { + "id": "custom.fillOpacity", + "value": 0 } ] } @@ -16704,62 +8592,63 @@ "h": 10, "w": 12, "x": 12, - "y": 57 + "y": 607 }, - "id": 35, - "links": [], + "id": 300, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "100 * node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"} / node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}}", + "intervalFactor": 1, + "legendFormat": "{{ name }} - {{ type }} ", + "range": true, "refId": "A", "step": 240 } ], - "title": "Average Queue Size", + "title": "Cooling Device Utilization", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "The number of read and write requests merged per second that were queued to the device", + "description": "Shows the online status of power supplies (e.g., AC, battery). A value of 1-Yes indicates the power supply is active/online", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "I/Os", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -16768,6 +8657,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -16798,91 +8688,245 @@ } ] }, - "unit": "iops" + "unit": "bool_yes_no" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Read.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 617 + }, + "id": 302, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_power_supply_online{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ power_supply }} online", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Power Supply", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Displays the current fan speeds (RPM) from hardware sensors via the hwmon interface", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } + "color": "green" } ] }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" + "unit": "rotrpm" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 617 + }, + "id": 325, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_hwmon_fan_rpm{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }}", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_hwmon_fan_min_rpm{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} rpm min", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Hardware Fan Speed", + "type": "timeseries" + } + ], + "title": "Hardware Misc", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 296, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Current number of systemd units in each operational state, such as active, failed, inactive, or transitioning", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] + "thresholdsStyle": { + "mode": "off" + } }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "short" + }, + "overrides": [ { "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" + "id": "byName", + "options": "Failed" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#E24D42", + "fixedColor": "#F2495C", "mode": "fixed" } } @@ -16890,14 +8934,14 @@ }, { "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" + "id": "byName", + "options": "Active" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#584477", + "fixedColor": "#73BF69", "mode": "fixed" } } @@ -16905,14 +8949,14 @@ }, { "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" + "id": "byName", + "options": "Activating" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#BA43A9", + "fixedColor": "#C8F2C2", "mode": "fixed" } } @@ -16920,14 +8964,14 @@ }, { "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" + "id": "byName", + "options": "Deactivating" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#F4D598", + "fixedColor": "orange", "mode": "fixed" } } @@ -16935,593 +8979,499 @@ }, { "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" + "id": "byName", + "options": "Inactive" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#0A50A1", + "fixedColor": "dark-blue", "mode": "fixed" } } ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 4098 + }, + "id": 298, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Activating", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Active", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Deactivating", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Failed", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Inactive", + "range": true, + "refId": "E", + "step": 240 + } + ], + "title": "Systemd Units State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Current number of active connections per systemd socket, as reported by the Node Exporter systemd collector", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } + "color": "green" } ] }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 4098 + }, + "id": 331, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_systemd_socket_current_connections{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ name }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Systemd Sockets Current", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of accepted connections per second for each systemd socket", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] + "thresholdsStyle": { + "mode": "off" + } }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } + "color": "green" } ] }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 4108 + }, + "id": 297, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ name }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Systemd Sockets Accepted", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of systemd socket connection refusals per second, typically due to service unavailability or backlog overflow", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] + "thresholdsStyle": { + "mode": "off" + } }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } + "color": "green" } ] }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9E2D2", - "mode": "fixed" - } - } - ] - } - ] + "unit": "eps" + }, + "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 67 + "x": 12, + "y": 4108 }, - "id": 133, - "links": [], + "id": 332, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_systemd_socket_refused_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{device}} - Read merged", + "legendFormat": "{{ name }}", + "range": true, "refId": "A", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "intervalFactor": 1, - "legendFormat": "{{device}} - Write merged", - "refId": "B", - "step": 240 } ], - "title": "Disk R/W Merged", + "title": "Systemd Sockets Refused", "type": "timeseries" - }, + } + ], + "title": "Systemd", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 270, + "panels": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially. But for devices serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.", + "description": "Number of I/O operations completed per second for the device (after merges), including both reads and writes", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "%util", + "axisLabel": "read (–) / write (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 30, + "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#962D82", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd3.*/" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EA6460", - "mode": "fixed" - } - } - ] + "thresholdsStyle": { + "mode": "off" + } }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" - }, - "properties": [ + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "iops" + }, + "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*sdd2.*/" + "options": "/.*Read.*/" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } + "id": "custom.transform", + "value": "negative-Y" } ] }, { "matcher": { "id": "byRegexp", - "options": "/.*sde3.*/" + "options": "/sda.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#F9E2D2", + "fixedColor": "orange", "mode": "fixed" } } @@ -17532,75 +9482,70 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 67 + "x": 0, + "y": 29 }, - "id": 36, - "links": [], + "id": 9, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}} - IO", + "editorMode": "code", + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Read", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}} - discard", + "editorMode": "code", + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write", + "range": true, "refId": "B", "step": 240 } ], - "title": "Time Spent Doing I/Os", + "title": "Disk Read/Write IOps", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.", + "description": "Number of bytes read from or written to the device per second", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "Outstanding req.", + "axisLabel": "read (–) / write (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -17609,6 +9554,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -17627,317 +9573,430 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "none" + "unit": "Bps" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" + "options": "/.*Read.*/" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } + "id": "custom.transform", + "value": "negative-Y" } ] }, { "matcher": { "id": "byRegexp", - "options": "/.*sda1.*/" + "options": "/sda.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#584477", + "fixedColor": "orange", "mode": "fixed" } } ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Read", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Write", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk Read/Write Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "read (–) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] + "thresholdsStyle": { + "mode": "off" + } }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" - }, - "properties": [ + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "s" + }, + "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*sdb3.*/" + "options": "/.*Read.*/" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } + "id": "custom.transform", + "value": "negative-Y" } ] }, { "matcher": { "id": "byRegexp", - "options": "/.*sdc1.*/" + "options": "/sda.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#962D82", + "fixedColor": "orange", "mode": "fixed" } } ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 259 + }, + "id": 37, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Read", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk Average Wait Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Average queue length of the requests that were issued to the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] + "thresholdsStyle": { + "mode": "off" + } }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "none" + }, + "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*sdd3.*/" + "options": "/sda_*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#EA6460", + "fixedColor": "#7EB26D", "mode": "fixed" } } ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 259 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Average Queue Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of read and write requests merged per second that were queued to the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "read (–) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "iops" + }, + "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*sdd2.*/" + "options": "/.*Read.*/" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } + "id": "custom.transform", + "value": "negative-Y" } ] }, { "matcher": { "id": "byRegexp", - "options": "/.*sde3.*/" + "options": "/sda.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#F9E2D2", + "fixedColor": "orange", "mode": "fixed" } } @@ -17949,62 +10008,69 @@ "h": 10, "w": 12, "x": 0, - "y": 77 + "y": 269 }, - "id": 34, - "links": [], + "id": 133, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_disk_io_now{instance=\"$node\",job=\"$job\"}", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}} - IO now", + "editorMode": "code", + "expr": "irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Read", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write", + "range": true, + "refId": "B", + "step": 240 } ], - "title": "Instantaneous Queue Size", + "title": "Disk R/W Merged", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "Percentage of time the disk spent actively processing I/O operations, including general I/O, discards (TRIM), and write cache flushes", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "IOs", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -18013,6 +10079,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -18031,316 +10098,413 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "iops" + "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*sda_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EAB839", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6ED0E0", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#EF843C", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda1.*/" + "options": "/sda.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#584477", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda2_.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BA43A9", + "fixedColor": "orange", "mode": "fixed" } } ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 269 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - General IO", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Discard/TRIM", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_flush_requests_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Flush (write cache)", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Time Spent Doing I/Os", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Per-second rate of discard (TRIM) and flush (write cache) operations. Useful for monitoring low-level disk activity on SSDs and advanced storage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sda3_.*/" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F4D598", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb1.*/" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0A50A1", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb2.*/" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] + "thresholdsStyle": { + "mode": "off" + } }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdb3.*/" - }, - "properties": [ + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#E0752D", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "ops" + }, + "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*sdc1.*/" + "options": "/sda.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#962D82", + "fixedColor": "orange", "mode": "fixed" } } ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 279 + }, + "id": 301, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Discards completed", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Discards merged", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_flush_requests_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Flush", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Disk Ops Discards / Flush", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shows how many disk sectors are discarded (TRIMed) per second. Useful for monitoring SSD behavior and storage efficiency", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc2.*/" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#614D93", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdc3.*/" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#9AC48A", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd1.*/" + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#65C5DB", - "mode": "fixed" - } - } - ] + "thresholdsStyle": { + "mode": "off" + } }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" - }, - "properties": [ + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#F9934E", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "short" + }, + "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*sdd3.*/" + "options": "/sda.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#EA6460", + "fixedColor": "orange", "mode": "fixed" } } ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 279 + }, + "id": 326, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_discarded_sectors_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Disk Sectors Discarded Successfully", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of in-progress I/O requests at the time of sampling (active requests in the disk queue)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sde1.*/" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E0F9D7", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*sdd2.*/" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "color", - "value": { - "fixedColor": "#FCEACA", - "mode": "fixed" - } + "color": "green" } ] }, + "unit": "none" + }, + "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*sde3.*/" + "options": "/sda.*/" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#F9E2D2", + "fixedColor": "orange", "mode": "fixed" } } @@ -18351,102 +10515,76 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 77 + "x": 0, + "y": 289 }, - "id": 301, - "links": [], + "id": 34, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{device}} - Discards completed", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_disk_io_now{instance=\"$node\",job=\"$job\"}", "interval": "", "intervalFactor": 1, - "legendFormat": "{{device}} - Discards merged", - "refId": "B", + "legendFormat": "{{device}}", + "range": true, + "refId": "A", "step": 240 } ], - "title": "Disk IOps Discards completed / merged", + "title": "Instantaneous Queue Size", "type": "timeseries" } ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], "title": "Storage Disk", "type": "row" }, { "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 18 + "y": 29 }, "id": 271, "panels": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "Number of file descriptors currently allocated system-wide versus the system limit. Important for detecting descriptor exhaustion risks", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -18455,6 +10593,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -18479,105 +10618,112 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "bytes" + "unit": "sishort" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max.*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 10, "w": 12, "x": 0, - "y": 62 + "y": 30 }, - "id": 43, - "links": [], + "id": 28, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "editorMode": "code", + "expr": "node_filefd_maximum{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - Available", - "metric": "", + "legendFormat": "Max open files", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "editorMode": "code", + "expr": "node_filefd_allocated{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": true, "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - Free", + "legendFormat": "Open files", + "range": true, "refId": "B", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - Size", - "refId": "C", - "step": 240 } ], - "title": "Filesystem space available", + "title": "File Descriptor", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "Number of free file nodes (inodes) available per mounted filesystem. A low count may prevent file creation even if disk space is available", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "file nodes", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -18586,6 +10732,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -18610,10 +10757,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -18625,39 +10768,36 @@ "h": 10, "w": 12, "x": 12, - "y": 62 + "y": 30 }, "id": 41, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "editorMode": "code", "expr": "node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}", "format": "time_series", "hide": false, "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - Free file nodes", + "legendFormat": "{{mountpoint}}", + "range": true, "refId": "A", "step": 240 } @@ -18668,20 +10808,22 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "Indicates filesystems mounted in read-only mode or reporting device-level I/O errors.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "files", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -18690,6 +10832,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -18700,7 +10843,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -18708,95 +10851,204 @@ }, "links": [], "mappings": [], + "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, + } + ] + }, + "unit": "bool_yes_no" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 240 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - ReadOnly", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Device error", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Filesystem in ReadOnly / Error", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of file nodes (inodes) available per mounted filesystem. Reflects maximum file capacity regardless of disk size", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ { - "color": "red", - "value": 80 + "color": "green" } ] }, - "unit": "short" + "unit": "sishort" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 72 + "x": 12, + "y": 240 }, - "id": 28, - "links": [], + "id": 219, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filefd_maximum{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "intervalFactor": 4, - "legendFormat": "Max open files", - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filefd_allocated{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "Open files", - "refId": "B", + "legendFormat": "{{mountpoint}}", + "range": true, + "refId": "A", "step": 240 } ], - "title": "File Descriptor", + "title": "File Nodes Size", "type": "timeseries" - }, + } + ], + "title": "Storage Filesystem", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 272, + "panels": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "Number of network packets received and transmitted per second, by interface.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "file Nodes", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -18805,6 +11057,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -18823,84 +11076,103 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "pps" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] }, "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 72 + "x": 0, + "y": 31 }, - "id": 219, - "links": [], + "id": 60, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "editorMode": "code", + "expr": "rate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "hide": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - File nodes total", + "legendFormat": "{{device}} - Rx in", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", + "step": 240 } ], - "title": "File Nodes Size", + "title": "Network Traffic by Packets", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "Rate of packet-level errors for each network interface. Receive errors may indicate physical or driver issues; transmit errors may reflect collisions or hardware faults", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -18909,6 +11181,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -18919,7 +11192,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -18927,35 +11200,26 @@ }, "links": [], "mappings": [], - "max": 1, - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "pps" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "/ ReadOnly" + "id": "byRegexp", + "options": "/.*out.*/" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } + "id": "custom.transform", + "value": "negative-Y" } ] } @@ -18964,102 +11228,73 @@ "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 82 + "x": 12, + "y": 31 }, - "id": 44, - "links": [], + "id": 142, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "editorMode": "code", + "expr": "rate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - ReadOnly", + "legendFormat": "{{device}} - Rx in", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}", + "editorMode": "code", + "expr": "rate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "{{mountpoint}} - Device error", + "legendFormat": "{{device}} - Tx out", + "range": true, "refId": "B", "step": 240 } ], - "title": "Filesystem in ReadOnly / Error", + "title": "Network Traffic Errors", "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], - "title": "Storage Filesystem", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 272, - "panels": [ + }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of dropped packets per network interface. Receive drops can indicate buffer overflow or driver issues; transmit drops may result from outbound congestion or queuing limits", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -19068,6 +11303,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -19091,80 +11327,16 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "pps" }, "overrides": [ - { - "matcher": { - "id": "byName", - "options": "receive_packets_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "receive_packets_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "transmit_packets_eth0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7EB26D", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "transmit_packets_lo" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E24D42", - "mode": "fixed" - } - } - ] - }, { "matcher": { "id": "byRegexp", - "options": "/.*Trans.*/" + "options": "/.*out.*/" }, "properties": [ { @@ -19179,17 +11351,15 @@ "h": 10, "w": 12, "x": 0, - "y": 47 + "y": 121 }, - "id": 60, - "links": [], + "id": 143, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -19197,58 +11367,56 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "rate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "{{device}} - Receive", + "legendFormat": "{{device}} - Rx in", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "rate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit", + "legendFormat": "{{device}} - Tx out", + "range": true, "refId": "B", "step": 240 } ], - "title": "Network Traffic by Packets", + "title": "Network Traffic Drop", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of compressed network packets received and transmitted per interface. These are common in low-bandwidth or special interfaces like PPP or SLIP", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -19257,6 +11425,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -19280,10 +11449,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -19293,7 +11458,7 @@ { "matcher": { "id": "byRegexp", - "options": "/.*Trans.*/" + "options": "/.*out.*/" }, "properties": [ { @@ -19308,17 +11473,15 @@ "h": 10, "w": 12, "x": 12, - "y": 47 + "y": 121 }, - "id": 142, - "links": [], + "id": 141, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -19326,56 +11489,56 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "rate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{device}} - Receive errors", + "legendFormat": "{{device}} - Rx in", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "rate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{device}} - Rransmit errors", + "legendFormat": "{{device}} - Tx out", + "range": true, "refId": "B", "step": 240 } ], - "title": "Network Traffic Errors", + "title": "Network Traffic Compressed", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of incoming multicast packets received per network interface. Multicast is used by protocols such as mDNS, SSDP, and some streaming or cluster services", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -19384,6 +11547,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -19406,11 +11570,7 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green" } ] }, @@ -19420,7 +11580,7 @@ { "matcher": { "id": "byRegexp", - "options": "/.*Trans.*/" + "options": "/.*out.*/" }, "properties": [ { @@ -19435,17 +11595,15 @@ "h": 10, "w": 12, "x": 0, - "y": 57 + "y": 131 }, - "id": 143, - "links": [], + "id": 146, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -19453,56 +11611,46 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "rate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{device}} - Receive drop", + "legendFormat": "{{device}} - Rx in", + "range": true, "refId": "A", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit drop", - "refId": "B", - "step": 240 } ], - "title": "Network Traffic Drop", + "title": "Network Traffic Multicast", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of received packets that could not be processed due to missing protocol or handler in the kernel. May indicate unsupported traffic or misconfiguration", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -19511,6 +11659,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -19534,10 +11683,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -19547,7 +11692,7 @@ { "matcher": { "id": "byRegexp", - "options": "/.*Trans.*/" + "options": "/.*out.*/" }, "properties": [ { @@ -19562,17 +11707,15 @@ "h": 10, "w": 12, "x": 12, - "y": 57 + "y": 131 }, - "id": 141, - "links": [], + "id": 327, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -19580,56 +11723,46 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "rate(node_network_receive_nohandler_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{device}} - Receive compressed", + "legendFormat": "{{device}} - Rx in", + "range": true, "refId": "A", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit compressed", - "refId": "B", - "step": 240 } ], - "title": "Network Traffic Compressed", + "title": "Network Traffic NoHandler", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of frame errors on received packets, typically caused by physical layer issues such as bad cables, duplex mismatches, or hardware problems", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -19638,6 +11771,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -19661,10 +11795,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -19674,7 +11804,7 @@ { "matcher": { "id": "byRegexp", - "options": "/.*Trans.*/" + "options": "/.*out.*/" }, "properties": [ { @@ -19689,17 +11819,15 @@ "h": 10, "w": 12, "x": 0, - "y": 67 + "y": 141 }, - "id": 146, - "links": [], + "id": 145, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -19707,44 +11835,47 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "rate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "{{device}} - Receive multicast", + "legendFormat": "{{device}} - Rx in", + "range": true, "refId": "A", "step": 240 } ], - "title": "Network Traffic Multicast", + "title": "Network Traffic Frame", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Tracks FIFO buffer overrun errors on network interfaces. These occur when incoming or outgoing packets are dropped due to queue or buffer overflows, often indicating congestion or hardware limits", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -19753,6 +11884,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -19776,10 +11908,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -19789,7 +11917,7 @@ { "matcher": { "id": "byRegexp", - "options": "/.*Trans.*/" + "options": "/.*out.*/" }, "properties": [ { @@ -19804,17 +11932,15 @@ "h": 10, "w": 12, "x": 12, - "y": 67 + "y": 141 }, "id": 144, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -19822,33 +11948,30 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "rate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{device}} - Receive fifo", + "legendFormat": "{{device}} - Rx in", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "rate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit fifo", + "legendFormat": "{{device}} - Tx out", + "range": true, "refId": "B", "step": 240 } @@ -19859,19 +11982,22 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of packet collisions detected during transmission. Mostly relevant on half-duplex or legacy Ethernet networks", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "packets out (-) / in (+)", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -19880,6 +12006,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -19903,10 +12030,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -19916,7 +12039,7 @@ { "matcher": { "id": "byRegexp", - "options": "/.*Trans.*/" + "options": "/.*out.*/" }, "properties": [ { @@ -19931,17 +12054,15 @@ "h": 10, "w": 12, "x": 0, - "y": 77 + "y": 151 }, - "id": 145, - "links": [], + "id": 232, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -19949,45 +12070,46 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "rate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "hide": false, "intervalFactor": 1, - "legendFormat": "{{device}} - Receive frame", + "legendFormat": "{{device}} - Tx out", + "range": true, "refId": "A", "step": 240 } ], - "title": "Network Traffic Frame", + "title": "Network Traffic Collision", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of carrier errors during transmission. These typically indicate physical layer issues like faulty cabling or duplex mismatches", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -19996,6 +12118,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -20019,14 +12142,10 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "pps" }, "overrides": [] }, @@ -20034,17 +12153,15 @@ "h": 10, "w": 12, "x": 12, - "y": 77 + "y": 151 }, "id": 231, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -20052,44 +12169,46 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "rate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{device}} - Statistic transmit_carrier", + "legendFormat": "{{device}} - Tx out", + "range": true, "refId": "A", "step": 240 } ], - "title": "Network Traffic Carrier", + "title": "Network Traffic Carrier Errors", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Number of ARP entries per interface. Useful for detecting excessive ARP traffic or table growth due to scanning or misconfiguration", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -20098,6 +12217,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -20116,95 +12236,78 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "short" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Trans.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, - "y": 87 + "y": 161 }, - "id": 232, - "links": [], + "id": 230, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true, - "width": 300 + "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_arp_entries{instance=\"$node\",job=\"$job\"}", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{device}} - Transmit colls", + "legendFormat": "{{ device }} ARP Table", + "range": true, "refId": "A", "step": 240 } ], - "title": "Network Traffic Colls", + "title": "ARP Entries", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Current and maximum connection tracking entries used by Netfilter (nf_conntrack). High usage approaching the limit may cause packet drops or connection issues", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "entries", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -20213,6 +12316,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -20237,10 +12341,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -20256,13 +12356,23 @@ { "id": "color", "value": { - "fixedColor": "#890F02", + "fixedColor": "dark-red", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } } ] } @@ -20272,73 +12382,71 @@ "h": 10, "w": 12, "x": 12, - "y": 87 + "y": 161 }, "id": 61, - "links": [], "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "editorMode": "code", "expr": "node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "NF conntrack entries", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "editorMode": "code", "expr": "node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "NF conntrack limit", + "range": true, "refId": "B", "step": 240 } ], - "title": "NF Contrack", + "title": "NF Conntrack", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Operational and physical link status of each network interface. Values are Yes for 'up' or link present, and No for 'down' or no carrier.\"", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "Entries", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -20347,6 +12455,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -20365,82 +12474,250 @@ }, "links": [], "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bool_yes_no" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 171 + }, + "id": 309, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{interface}} - Operational state UP", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_network_carrier{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{device}} - Physical link", + "refId": "B" + } + ], + "title": "Network Operational Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Maximum speed of each network interface as reported by the operating system. This is a static hardware capability, not current throughput", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "fieldMinMax": false, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 12, + "y": 171 + }, + "id": 280, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 30, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_network_speed_bytes{instance=\"$node\",job=\"$job\"} * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Speed", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "MTU (Maximum Transmission Unit) in bytes for each network interface. Affects packet size and transmission efficiency", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "links": [], + "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 10, - "w": 12, - "x": 0, - "y": 97 + "w": 6, + "x": 18, + "y": 171 }, - "id": 230, - "links": [], + "id": 288, "options": { + "displayMode": "basic", "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 30, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { "calcs": [ - "mean", - "lastNotNull", - "max", - "min" + "lastNotNull" ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_arp_entries{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{ device }} - ARP entries", + "legendFormat": "{{ device }}", + "range": true, "refId": "A", "step": 240 } ], - "title": "ARP Entries", - "type": "timeseries" - }, + "title": "MTU", + "type": "bargauge" + } + ], + "title": "Network Traffic", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 273, + "panels": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Tracks TCP socket usage and memory per node", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -20449,6 +12726,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -20465,7 +12743,6 @@ "mode": "off" } }, - "decimals": 0, "links": [], "mappings": [], "min": 0, @@ -20474,76 +12751,107 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "bytes" + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 97 + "x": 0, + "y": 32 }, - "id": 288, - "links": [], + "id": 63, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{ device }} - Bytes", + "legendFormat": "Allocated Sockets", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "In-Use Sockets", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Orphaned Sockets", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TIME_WAIT Sockets", + "range": true, + "refId": "D", + "step": 240 } ], - "title": "MTU", + "title": "Sockstat TCP", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Number of UDP and UDPLite sockets currently in use", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -20552,6 +12860,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -20568,7 +12877,6 @@ "mode": "off" } }, - "decimals": 0, "links": [], "mappings": [], "min": 0, @@ -20577,76 +12885,85 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "bytes" + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 107 + "x": 12, + "y": 32 }, - "id": 280, - "links": [], + "id": 124, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_network_speed_bytes{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{ device }} - Speed", + "legendFormat": "UDPLite - In-Use Sockets", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP - In-Use Sockets", + "range": true, + "refId": "B", + "step": 240 } ], - "title": "Speed", + "title": "Sockstat UDP", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Total number of sockets currently in use across all protocols (TCP, UDP, UNIX, etc.), as reported by /proc/net/sockstat", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "packets", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -20655,6 +12972,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -20671,7 +12989,6 @@ "mode": "off" } }, - "decimals": 0, "links": [], "mappings": [], "min": 0, @@ -20680,76 +12997,74 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "none" + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 107 + "x": 0, + "y": 122 }, - "id": 289, - "links": [], + "id": 126, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{ device }} - Interface transmit queue length", + "legendFormat": "Total sockets", + "range": true, "refId": "A", "step": 240 } ], - "title": "Queue Length", + "title": "Sockstat Used", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Number of FRAG and RAW sockets currently in use. RAW sockets are used for custom protocols or tools like ping; FRAG sockets are used internally for IP packet defragmentation", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "packetes drop (-) / process (+)", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -20758,6 +13073,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -20776,50 +13092,32 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "short" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Dropped.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 117 + "x": 12, + "y": 122 }, - "id": 290, - "links": [], + "id": 125, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -20827,58 +13125,58 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "CPU {{cpu}} - Processed", + "legendFormat": "FRAG - In-Use Sockets", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "CPU {{cpu}} - Dropped", - "refId": "B", + "legendFormat": "RAW - In-Use Sockets", + "range": true, + "refId": "C", "step": 240 } ], - "title": "Softnet Packets", + "title": "Sockstat FRAG / RAW", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "TCP/UDP socket memory usage in kernel (in pages)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -20887,6 +13185,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -20905,15 +13204,12 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -20924,18 +13220,16 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 117 + "x": 0, + "y": 132 }, - "id": 310, - "links": [], + "id": 336, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -20943,45 +13237,58 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "CPU {{cpu}} - Squeezed", + "legendFormat": "TCP", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP", + "range": true, + "refId": "B", + "step": 240 } ], - "title": "Softnet Out of Quota", + "title": "TCP/UDP Kernel Buffer Memory Pages", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Kernel memory used by TCP, UDP, and IP fragmentation buffers", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -20990,6 +13297,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -21008,37 +13316,32 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 127 + "x": 12, + "y": 132 }, - "id": 309, - "links": [], + "id": 220, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -21046,82 +13349,67 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{interface}} - Operational state UP", + "legendFormat": "TCP", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_network_carrier{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "instant": false, - "legendFormat": "{{device}} - Physical link state", - "refId": "B" + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Fragmentation", + "range": true, + "refId": "C" } ], - "title": "Network Operational Status", + "title": "Sockstat Memory Size", "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], - "title": "Network Traffic", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 273, - "panels": [ + }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Packets processed and dropped by the softnet network stack per CPU. Drops may indicate CPU saturation or network driver limitations", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "drop (-) / process (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -21130,6 +13418,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -21148,38 +13437,44 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "pps" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Dropped.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] }, "gridPos": { "h": 10, "w": 12, "x": 0, - "y": 48 + "y": 142 }, - "id": 63, - "links": [], + "id": 290, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -21187,98 +13482,58 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "TCP_alloc - Allocated sockets", + "legendFormat": "CPU {{cpu}} - Processed", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "TCP_inuse - Tcp sockets currently in use", + "legendFormat": "CPU {{cpu}} - Dropped", + "range": true, "refId": "B", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCP_mem - Used memory for tcp", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCP_orphan - Orphan sockets", - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCP_tw - Sockets waiting close", - "refId": "E", - "step": 240 } ], - "title": "Sockstat TCP", + "title": "Softnet Packets", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "How often the kernel was unable to process all packets in the softnet queue before time ran out. Frequent squeezes may indicate CPU contention or driver inefficiency", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -21287,6 +13542,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -21305,7 +13561,6 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -21318,7 +13573,7 @@ } ] }, - "unit": "short" + "unit": "eps" }, "overrides": [] }, @@ -21326,17 +13581,15 @@ "h": 10, "w": 12, "x": 12, - "y": 48 + "y": 142 }, - "id": 124, - "links": [], + "id": 310, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -21344,71 +13597,47 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "UDPLITE_inuse - Udplite sockets currently in use", + "legendFormat": "CPU {{cpu}} - Times Squeezed", + "range": true, "refId": "A", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "UDP_inuse - Udp sockets currently in use", - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "UDP_mem - Used memory for udp", - "refId": "C", - "step": 240 } ], - "title": "Sockstat UDP", + "title": "Softnet Out of Quota", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Tracks the number of packets processed or dropped by Receive Packet Steering (RPS), a mechanism to distribute packet processing across CPUs", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -21417,6 +13646,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -21435,38 +13665,51 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "pps" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Dropped.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 10, "w": 12, "x": 0, - "y": 58 + "y": 152 }, - "id": 125, - "links": [], + "id": 330, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -21474,58 +13717,73 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "irate(node_softnet_received_rps_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "FRAG_inuse - Frag sockets currently in use", + "legendFormat": "CPU {{cpu}} - Processed", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "irate(node_softnet_flow_limit_count_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", + "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "RAW_inuse - Raw sockets currently in use", - "refId": "C", + "legendFormat": "CPU {{cpu}} - Dropped", + "range": true, + "refId": "B", "step": 240 } ], - "title": "Sockstat FRAG / RAW", + "title": "Softnet RPS", "type": "timeseries" - }, + } + ], + "title": "Network Sockstat", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 274, + "panels": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of octets sent and received at the IP layer, as reported by /proc/net/netstat", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "bytes", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -21534,6 +13792,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -21552,38 +13811,44 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "bytes" + "unit": "Bps" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] }, "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 58 + "x": 0, + "y": 33 }, - "id": 220, - "links": [], + "id": 221, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", @@ -21591,69 +13856,57 @@ "width": 300 }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}", + { + "editorMode": "code", + "expr": "irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "mem_bytes - TCP sockets in that state", + "legendFormat": "IP Rx in", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "mem_bytes - UDP sockets in that state", + "legendFormat": "IP Tx out", + "range": true, "refId": "B", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}", - "interval": "", - "intervalFactor": 1, - "legendFormat": "FRAG_memory - Used memory for frag", - "refId": "C" } ], - "title": "Sockstat Memory Size", + "title": "Netstat IP In / Out Octets", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of TCP segments sent and received per second, including data and control segments", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "sockets", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -21662,6 +13915,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -21680,7 +13934,6 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -21693,98 +13946,101 @@ } ] }, - "unit": "short" + "unit": "pps" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Snd.*/" + }, + "properties": [] + } + ] }, "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 68 + "x": 12, + "y": 33 }, - "id": 126, - "links": [], + "id": 299, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true, - "width": 300 + "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", + "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "Sockets_used - Sockets currently in use", + "legendFormat": "TCP Rx in", "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP Tx out", + "range": true, + "refId": "B", + "step": 240 } ], - "title": "Sockstat Used", + "title": "TCP In / Out", "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], - "title": "Network Sockstat", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 274, - "panels": [ + }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of UDP datagrams sent and received per second, based on /proc/net/netstat", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "octets out (-) / in (+)", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -21793,6 +14049,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -21816,20 +14073,16 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "pps" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*Out.*/" + "options": "/.*out.*/" }, "properties": [ { @@ -21844,75 +14097,73 @@ "h": 10, "w": 12, "x": 0, - "y": 49 + "y": 63 }, - "id": 221, - "links": [], + "id": 55, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true, - "width": 300 + "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "InOctets - Received octets", + "legendFormat": "UDP Rx in", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "OutOctets - Sent octets", + "legendFormat": "UDP Tx out", + "range": true, "refId": "B", "step": 240 } ], - "title": "Netstat IP In / Out Octets", + "title": "UDP In / Out", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Number of ICMP messages sent and received per second, including error and control messages", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "datagrams", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -21921,6 +14172,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -21939,7 +14191,6 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -21952,71 +14203,94 @@ } ] }, - "unit": "short" + "unit": "pps" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] }, "gridPos": { "h": 10, "w": 12, "x": 12, - "y": 49 + "y": 63 }, - "id": 81, - "links": [], + "id": 115, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true, - "width": 300 + "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "Forwarding - IP forwarding", + "legendFormat": "ICMP Rx in", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ICMP Tx out", + "range": true, + "refId": "B", + "step": 240 } ], - "title": "Netstat IP Forwarding", + "title": "ICMP In / Out", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Tracks various TCP error and congestion-related events, including retransmissions, timeouts, dropped connections, and buffer issues", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "messages out (-) / in (+)", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -22025,6 +14299,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -22043,108 +14318,154 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "pps" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Out.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, - "y": 59 + "y": 73 }, - "id": 115, - "links": [], + "id": 104, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Listen Overflows", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Listen Drops", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "SYN Retransmits", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Segment Retransmits", + "range": true, + "refId": "D" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Receive Errors", + "range": true, + "refId": "E" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "RST Sent", + "range": true, + "refId": "F" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_TCPRcvQDrop{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "Receive Queue Drops", + "range": true, + "refId": "G" + }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_TCPOFOQueue{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "InMsgs - Messages which the entity received. Note that this counter includes all those counted by icmpInErrors", - "refId": "A", - "step": 240 + "legendFormat": "Out-of-order Queued", + "range": true, + "refId": "H" }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_TCPTimeouts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors", - "refId": "B", - "step": 240 + "legendFormat": "TCP Timeouts", + "range": true, + "refId": "I" } ], - "title": "ICMP In / Out", + "title": "TCP Errors", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of UDP and UDPLite datagram delivery errors, including missing listeners, buffer overflows, and protocol-specific issues", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "messages out (-) / in (+)", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -22153,6 +14474,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -22183,83 +14505,111 @@ } ] }, - "unit": "short" + "unit": "pps" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Out.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, - "y": 59 + "y": 73 }, - "id": 50, - "links": [], + "id": 109, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)", + "legendFormat": "UDP Rx in Errors", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP No Listener", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "UDPLite Rx in Errors", + "range": true, + "refId": "C" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP Rx in Buffer Errors", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP Tx out Buffer Errors", + "range": true, + "refId": "E", + "step": 240 } ], - "title": "ICMP Errors", + "title": "UDP Errors", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of incoming ICMP messages that contained protocol-specific errors, such as bad checksums or invalid lengths", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "datagrams out (-) / in (+)", + "axisLabel": "out (-) / in (+)", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -22268,6 +14618,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -22291,32 +14642,16 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "pps" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": "/.*Out.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Snd.*/" + "options": "/.*out.*/" }, "properties": [ { @@ -22331,75 +14666,62 @@ "h": 10, "w": 12, "x": 0, - "y": 69 + "y": 83 }, - "id": 55, - "links": [], + "id": 50, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "InDatagrams - Datagrams received", + "legendFormat": "ICMP Rx In", + "range": true, "refId": "A", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "OutDatagrams - Datagrams sent", - "refId": "B", - "step": 240 } ], - "title": "UDP In / Out", + "title": "ICMP Errors", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of TCP SYN cookies sent, validated, and failed. These are used to protect against SYN flood attacks and manage TCP handshake resources under load", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "datagrams", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -22408,6 +14730,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -22431,126 +14754,114 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "eps" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Failed.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 10, "w": 12, "x": 12, - "y": 69 + "y": 83 }, - "id": 109, - "links": [], + "id": 91, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", + "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "InErrors - UDP Datagrams that could not be delivered to an application", + "legendFormat": "SYN Cookies Failed", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", + "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "NoPorts - UDP Datagrams received on a port with no listener", + "legendFormat": "SYN Cookies Validated", + "range": true, "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "legendFormat": "InErrors Lite - UDPLite Datagrams that could not be delivered to an application", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "RcvbufErrors - UDP buffer errors received", - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", + "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "SndbufErrors - UDP buffer errors send", - "refId": "E", + "legendFormat": "SYN Cookies Sent", + "range": true, + "refId": "C", "step": 240 } ], - "title": "UDP Errors", + "title": "TCP SynCookie", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Number of currently established TCP connections and the system's max supported limit. On Linux, MaxConn may return -1 to indicate a dynamic/unlimited configuration", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "datagrams out (-) / in (+)", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -22559,6 +14870,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -22577,15 +14889,12 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -22595,24 +14904,29 @@ { "matcher": { "id": "byRegexp", - "options": "/.*Out.*/" + "options": "/.*Max*./" }, "properties": [ { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Snd.*/" - }, - "properties": [ + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, { - "id": "custom.transform", - "value": "negative-Y" + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } } ] } @@ -22622,77 +14936,75 @@ "h": 10, "w": 12, "x": 0, - "y": 79 + "y": 93 }, - "id": 299, - "links": [], + "id": 85, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}", "format": "time_series", - "instant": false, + "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "InSegs - Segments received, including those received in error. This count includes segments received on currently established connections", + "legendFormat": "Current Connections", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}", "format": "time_series", + "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets", + "legendFormat": "Max Connections", + "range": true, "refId": "B", "step": 240 } ], - "title": "TCP In / Out", + "title": "TCP Connections", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "Number of UDP packets currently queued in the receive (RX) and transmit (TX) buffers. A growing queue may indicate a bottleneck", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -22701,6 +15013,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -22719,16 +15032,11 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -22740,120 +15048,75 @@ "h": 10, "w": 12, "x": 12, - "y": 79 + "y": 93 }, - "id": 104, - "links": [], + "id": 337, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_udp_queues{instance=\"$node\",job=\"$job\",ip=\"v4\",queue=\"rx\"}", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "ListenOverflows - Times the listen queue of a socket overflowed", + "legendFormat": "UDP Rx in Queue", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_udp_queues{instance=\"$node\",job=\"$job\",ip=\"v4\",queue=\"tx\"}", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "ListenDrops - SYNs to LISTEN sockets ignored", + "legendFormat": "UDP Tx out Queue", + "range": true, "refId": "B", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits", - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "legendFormat": "RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "legendFormat": "InErrs - Segments received in error (e.g., bad TCP checksums)", - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "interval": "", - "legendFormat": "OutRsts - Segments sent with RST flag", - "refId": "F" } ], - "title": "TCP Errors", + "title": "UDP Queue", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of TCP connection initiations per second. 'Active' opens are initiated by this host. 'Passive' opens are accepted from incoming connections", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "connections", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -22862,6 +15125,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -22886,113 +15150,84 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "eps" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*MaxConn *./" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890F02", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, - "y": 89 + "y": 103 }, - "id": 85, - "links": [], + "id": 82, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT", + "legendFormat": "Active Opens", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "MaxConn - Limit on the total number of TCP connections the entity can support (Dynamic is \"-1\")", + "legendFormat": "Passive Opens", + "range": true, "refId": "B", "step": 240 } ], - "title": "TCP Connections", + "title": "TCP Direct Transition", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "Number of TCP sockets in key connection states. Requires the --collector.tcpstat flag on node_exporter", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter out (-) / in (+)", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -23001,6 +15236,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -23019,124 +15255,142 @@ }, "links": [], "mappings": [], + "min": 0, + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, "unit": "short" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Sent.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, - "y": 89 + "y": 103 }, - "id": 91, - "links": [], + "id": 320, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"established\",instance=\"$node\",job=\"$job\"}", "format": "time_series", - "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "SyncookiesFailed - Invalid SYN cookies received", + "legendFormat": "Established", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"fin_wait2\",instance=\"$node\",job=\"$job\"}", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "SyncookiesRecv - SYN cookies received", + "legendFormat": "FIN_WAIT2", + "range": true, "refId": "B", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"listen\",instance=\"$node\",job=\"$job\"}", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "SyncookiesSent - SYN cookies sent", + "legendFormat": "Listen", + "range": true, "refId": "C", "step": 240 + }, + { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"time_wait\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "TIME_WAIT", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"close_wait\", instance=\"$node\", job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CLOSE_WAIT", + "range": true, + "refId": "E", + "step": 240 } ], - "title": "TCP SynCookie", + "title": "TCP Stat", "type": "timeseries" - }, + } + ], + "title": "Network Netstat", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 279, + "panels": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Duration of each individual collector executed during a Node Exporter scrape. Useful for identifying slow or failing collectors", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "connections", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -23145,6 +15399,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -23155,7 +15410,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -23163,7 +15418,6 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -23176,7 +15430,7 @@ } ] }, - "unit": "short" + "unit": "s" }, "overrides": [] }, @@ -23184,76 +15438,63 @@ "h": 10, "w": 12, "x": 0, - "y": 99 + "y": 34 }, - "id": 82, - "links": [], + "id": 40, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "editorMode": "code", + "expr": "node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}", "format": "time_series", + "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state", + "legendFormat": "{{collector}}", + "range": true, "refId": "A", "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state", - "refId": "B", - "step": 240 } ], - "title": "TCP Direct Transition", + "title": "Node Exporter Scrape Time", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "Enable with --collector.tcpstat argument on node-exporter", + "description": "Rate of CPU time used by the process exposing this metric (user + system mode)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "connections", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -23262,6 +15503,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -23280,7 +15522,6 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -23289,7 +15530,7 @@ } ] }, - "unit": "short" + "unit": "percentunit" }, "overrides": [] }, @@ -23297,140 +15538,62 @@ "h": 10, "w": 12, "x": 12, - "y": 99 + "y": 34 }, - "id": 320, - "links": [], + "id": 308, "options": { "legend": { "calcs": [ - "mean", - "lastNotNull", - "max", - "min" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "node_tcp_connection_states{state=\"established\",instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "established - TCP sockets in established state", - "range": true, - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "node_tcp_connection_states{state=\"fin_wait2\",instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "fin_wait2 - TCP sockets in fin_wait2 state", - "range": true, - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "node_tcp_connection_states{state=\"listen\",instance=\"$node\",job=\"$job\"}", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "listen - TCP sockets in listen state", - "range": true, - "refId": "C", - "step": 240 + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, "editorMode": "code", - "expr": "node_tcp_connection_states{state=\"time_wait\",instance=\"$node\",job=\"$job\"}", + "expr": "irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", "format": "time_series", - "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "time_wait - TCP sockets in time_wait state", + "legendFormat": "Process CPU Usage", "range": true, - "refId": "D", + "refId": "A", "step": 240 } ], - "title": "TCP Stat", + "title": "Exporter Process CPU Usage", "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" - } - ], - "title": "Network Netstat", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 279, - "panels": [ + }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "Tracks the memory usage of the process exposing this metric (e.g., node_exporter), including current virtual memory and maximum virtual memory limit", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "seconds", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -23439,6 +15602,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -23449,7 +15613,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -23457,84 +15621,144 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Virtual Memory Limit" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 }, { - "color": "red", - "value": 80 + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } } ] }, - "unit": "s" - }, - "overrides": [] + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Virtual Memory" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] }, "gridPos": { "h": 10, - "w": 12, + "w": 10, "x": 0, - "y": 66 + "y": 44 }, - "id": 40, - "links": [], + "id": 149, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}", - "format": "time_series", + "editorMode": "code", + "expr": "process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{collector}} - Scrape duration", + "legendFormat": "Virtual Memory", + "range": true, "refId": "A", "step": 240 + }, + { + "editorMode": "code", + "expr": "process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Virtual Memory Limit", + "range": true, + "refId": "B", + "step": 240 } ], - "title": "Node Exporter Scrape Time", + "title": "Exporter Processes Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "Number of file descriptors used by the exporter process versus its configured limit", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "counter", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -23543,10 +15767,8 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -23564,15 +15786,12 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -23582,19 +15801,53 @@ { "matcher": { "id": "byRegexp", - "options": "/.*error.*/" + "options": "/.*Max*./" }, "properties": [ { "id": "color", "value": { - "fixedColor": "#F2495C", + "fixedColor": "#890F02", "mode": "fixed" } }, { - "id": "custom.transform", - "value": "negative-Y" + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Open file descriptors" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } } ] } @@ -23602,118 +15855,180 @@ }, "gridPos": { "h": 10, - "w": 12, - "x": 12, - "y": 66 + "w": 10, + "x": 10, + "y": 44 }, - "id": 157, - "links": [], + "id": 64, "options": { "legend": { "calcs": [ + "min", "mean", - "lastNotNull", - "max", - "min" + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.0", + "pluginVersion": "11.6.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "editorMode": "code", + "expr": "process_max_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum open file descriptors", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "process_open_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Open file descriptors", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Exporter File Descriptor Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shows whether each Node Exporter collector scraped successfully (1 = success, 0 = failure), and whether the textfile collector returned an error.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] }, + "unit": "bool" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 4, + "x": 20, + "y": 44 + }, + "id": 157, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", "expr": "node_scrape_collector_success{instance=\"$node\",job=\"$job\"}", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{collector}} - Scrape success", + "legendFormat": "{{collector}}", + "range": true, "refId": "A", "step": 240 }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}", + "editorMode": "code", + "expr": "1 - node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{collector}} - Scrape textfile error (1 = true)", + "legendFormat": "textfile", + "range": true, "refId": "B", "step": 240 } ], "title": "Node Exporter Scrape", - "type": "timeseries" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "000000001" - }, - "refId": "A" + "type": "bargauge" } ], "title": "Node Exporter", "type": "row" } ], - "refresh": "", - "revision": 1, - "schemaVersion": 39, + "refresh": "1m", + "schemaVersion": 41, "tags": [ "linux" ], "templating": { "list": [ { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, + "current": {}, "includeAll": false, - "label": "datasource", - "multi": false, + "label": "Datasource", "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" }, { - "current": { - "selected": false, - "text": "node_exporter", - "value": "node_exporter" - }, + "current": {}, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "definition": "", - "hide": 0, "includeAll": false, "label": "Job", - "multi": false, "name": "job", "options": [], "query": { @@ -23722,52 +16037,56 @@ }, "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { - "current": { - "selected": false, - "text": "localhost:9100", - "value": "localhost:9100" + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(node_uname_info{job=\"$job\"}, nodename)", + "includeAll": false, + "label": "Nodename", + "name": "nodename", + "options": [], + "query": { + "query": "label_values(node_uname_info{job=\"$job\"}, nodename)", + "refId": "Prometheus-nodename-Variable-Query" }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": {}, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values(node_uname_info{job=\"$job\"}, instance)", - "hide": 0, + "definition": "label_values(node_uname_info{job=\"$job\", nodename=\"$nodename\"}, instance)", "includeAll": false, - "label": "Host", - "multi": false, + "label": "Instance", "name": "node", "options": [], "query": { - "query": "label_values(node_uname_info{job=\"$job\"}, instance)", + "query": "label_values(node_uname_info{job=\"$job\", nodename=\"$nodename\"}, instance)", "refId": "Prometheus-node-Variable-Query" }, "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { "current": { - "selected": false, "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" }, "hide": 2, "includeAll": false, - "multi": false, "name": "diskdevices", "options": [ { @@ -23777,7 +16096,6 @@ } ], "query": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", - "skipUrlSync": false, "type": "custom" } ] @@ -23786,34 +16104,11 @@ "from": "now-24h", "to": "now" }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, + "timepicker": {}, "timezone": "browser", - "title": "Node Exporter Full2", - "uid": "rYdddlPWks", - "version": 2, - "weekStart": "" -} + "title": "Node Exporter Full", + "uid": "rYdddlPWk", + "version": 96, + "weekStart": "", + "gnetId": 1860 +} \ No newline at end of file diff --git a/roles/grafana/templates/prometheus_source.yml.j2 b/roles/grafana/templates/prometheus_source.yml.j2 index cb4e7f36..ed288dfa 100644 --- a/roles/grafana/templates/prometheus_source.yml.j2 +++ b/roles/grafana/templates/prometheus_source.yml.j2 @@ -5,7 +5,6 @@ datasources: access: proxy url: {{ prometheus_url }} isDefault: true - uid: PBFA97CFB590B2093 jsonData: tlsAuth: true tlsAuthWithCACert: false From ddd2b9a88bd8af3fae52bea2722e30455b122514 Mon Sep 17 00:00:00 2001 From: rsuplina Date: Tue, 23 Sep 2025 13:40:26 +0100 Subject: [PATCH 10/10] linting for dashboard Signed-off-by: rsuplina --- roles/grafana/files/base_dashboard.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/grafana/files/base_dashboard.json b/roles/grafana/files/base_dashboard.json index 7409b5ef..d47a37a4 100644 --- a/roles/grafana/files/base_dashboard.json +++ b/roles/grafana/files/base_dashboard.json @@ -16111,4 +16111,4 @@ "version": 96, "weekStart": "", "gnetId": 1860 -} \ No newline at end of file +}