diff --git a/.forgejo/workflows/push.yaml b/.forgejo/workflows/push.yaml index a2656dc..9de34f2 100644 --- a/.forgejo/workflows/push.yaml +++ b/.forgejo/workflows/push.yaml @@ -25,4 +25,17 @@ jobs: steps: - uses: actions/checkout@v4 - run: | + nix build --no-link '.#packages.x86_64-linux.terraform-provider-cloudflare' + nix build --no-link '.#packages.x86_64-linux.terraform-provider-hcloud' + nix build --no-link '.#packages.x86_64-linux.terraform-provider-openstack' nix build --no-link '.#packages.x86_64-linux.terraform-provider-unifi' + nix build --no-link '.#packages.x86_64-linux.terraform-provider-vault' + systems: + runs-on: cache.kaareskovgaard.net + steps: + - uses: actions/checkout@v4 + - run: | + nix build --no-link '.#nixosConfigurations."desktop.kaareskovgaard.net".config.system.build.toplevel' + nix build --no-link '.#nixosConfigurations."desktop.kaareskovgaard.net".config.system.build.vm' + nix build --no-link '.#nixosConfigurations."monitoring.kaareskovgaard.net".config.system.build.toplevel' + nix build --no-link '.#nixosConfigurations."security.kaareskovgaard.net".config.system.build.toplevel' diff --git a/.gitignore b/.gitignore index f18e503..aa10e55 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ result/ .DS_Store rust/target +*.qcow2 diff --git a/README.md b/README.md index bb691af..17b0fab 100644 --- a/README.md +++ b/README.md @@ -8,32 +8,26 @@ When running on a desktop machine, simply running `nixos-install` as per usual s ## Servers -To provision the cloud resources needed, the following can be run: +To provision the cloud resources needed, and install NixOS, the following can be run: ```bash nix run '.#create-instance' -- ``` -This will run the `provision.pre` terraform code to ensure the cloud resources are created as needed, on either hetzner or openstack. It should also select the appropriate secrets backend to fetch secrets from. In general every server should use `vault` (OpenBAO) as the backend, except for the server hosting OpenBAO. +This will run the `provision.pre` terraform code to ensure the cloud resources are created as needed, on either hetzner or openstack. It should also select the appropriate secrets backend to fetch secrets from. In general every server should use `vault` (OpenBAO) as the backend, except for the server hosting OpenBAO. Then it will install NixOS. -Once the instance has been created it will _not_ run NixOS, but rather something like Debian, which can then be provisioned into a NixOS installation. Run the following command to enroll NixOS on the instance: +When making changes to eg. the approle needed, and needing to provision the instance again (but not installing NixOS again, as that won't work), run: ```bash -nix run '.#inxos-install' -- +nix run '.#provision-instance' -- ``` -
-NOTE -If you're creating and destroying instances on the same host name and have DNS caching trouble, you can run the following to connect using an IP address: +To update the NixOS config on an instance: ```bash -nix run '.#nixos-install' -- +nix run '.#update-instance` -- ``` -
- -TODO: Here should be some guidance on how to transfer RoleID/SecretID to the server, as well as running the post provisioning scripts for the servers that need it. - To delete the resources again run: ```bash @@ -42,7 +36,7 @@ nix run '.#destroy-instance' -- ## Secrets -To transfer the secrets needed for OpenTofu from Bitwarden to OpenBAO run: +To transfer the secrets needed for OpenTofu from Bitwarden to OpenBAO/Vault run: ```bash nix run '.#bitwarden-to-vault' diff --git a/assets/wallpaper.jpg b/assets/wallpaper.jpg new file mode 100644 index 0000000..5cb1a5b Binary files /dev/null and b/assets/wallpaper.jpg differ diff --git a/flake.lock b/flake.lock index 7eecfc5..98a173c 100644 --- a/flake.lock +++ b/flake.lock @@ -16,35 +16,148 @@ "type": "github" } }, - "bats-assert": { - "flake": false, + "authentik-nix": { + "inputs": { + "authentik-src": "authentik-src", + "flake-compat": "flake-compat", + "flake-parts": [ + "flake-parts" + ], + "flake-utils": [ + "flake-utils" + ], + "napalm": "napalm", + "nixpkgs": [ + "nixpkgs" + ], + "pyproject-build-systems": "pyproject-build-systems", + "pyproject-nix": "pyproject-nix", + "systems": [ + "systems" + ], + "uv2nix": "uv2nix" + }, "locked": { - "lastModified": 1636059754, - "narHash": "sha256-ewME0l27ZqfmAwJO4h5biTALc9bDLv7Bl3ftBzBuZwk=", - "owner": "bats-core", - "repo": "bats-assert", - "rev": "34551b1d7f8c7b677c1a66fc0ac140d6223409e5", + "lastModified": 1751033152, + "narHash": "sha256-0ANu9OLQJszcEyvnfDB7G957uqskZwCrTzRXz/yfAmE=", + "owner": "nix-community", + "repo": "authentik-nix", + "rev": "1a4d6a5dd6fef39b99eb7ea4db79c5d5c7d7f1bf", "type": "github" }, "original": { - "owner": "bats-core", - "repo": "bats-assert", + "owner": "nix-community", + "repo": "authentik-nix", "type": "github" } }, - "bats-support": { + "authentik-src": { "flake": false, "locked": { - "lastModified": 1548869839, - "narHash": "sha256-Gr4ntadr42F2Ks8Pte2D4wNDbijhujuoJi4OPZnTAZU=", - "owner": "bats-core", - "repo": "bats-support", - "rev": "d140a65044b2d6810381935ae7f0c94c7023c8c3", + "lastModified": 1751031262, + "narHash": "sha256-SNgRMQUjL3DTlWkMyRMan+pY1FfIV+DMeq5BiTM0N0k=", + "owner": "goauthentik", + "repo": "authentik", + "rev": "b34665fabd8d938d81ce871a4e86ca528c5f253b", "type": "github" }, "original": { - "owner": "bats-core", - "repo": "bats-support", + "owner": "goauthentik", + "ref": "version/2025.4.3", + "repo": "authentik", + "type": "github" + } + }, + "base16": { + "inputs": { + "fromYaml": "fromYaml" + }, + "locked": { + "lastModified": 1746562888, + "narHash": "sha256-YgNJQyB5dQiwavdDFBMNKk1wyS77AtdgDk/VtU6wEaI=", + "owner": "SenchoPens", + "repo": "base16.nix", + "rev": "806a1777a5db2a1ef9d5d6f493ef2381047f2b89", + "type": "github" + }, + "original": { + "owner": "SenchoPens", + "repo": "base16.nix", + "type": "github" + } + }, + "base16-fish": { + "flake": false, + "locked": { + "lastModified": 1622559957, + "narHash": "sha256-PebymhVYbL8trDVVXxCvZgc0S5VxI7I1Hv4RMSquTpA=", + "owner": "tomyun", + "repo": "base16-fish", + "rev": "2f6dd973a9075dabccd26f1cded09508180bf5fe", + "type": "github" + }, + "original": { + "owner": "tomyun", + "repo": "base16-fish", + "type": "github" + } + }, + "base16-helix": { + "flake": false, + "locked": { + "lastModified": 1748408240, + "narHash": "sha256-9M2b1rMyMzJK0eusea0x3lyh3mu5nMeEDSc4RZkGm+g=", + "owner": "tinted-theming", + "repo": "base16-helix", + "rev": "6c711ab1a9db6f51e2f6887cc3345530b33e152e", + "type": "github" + }, + "original": { + "owner": "tinted-theming", + "repo": "base16-helix", + "type": "github" + } + }, + "base16-vim": { + "flake": false, + "locked": { + "lastModified": 1732806396, + "narHash": "sha256-e0bpPySdJf0F68Ndanwm+KWHgQiZ0s7liLhvJSWDNsA=", + "owner": "tinted-theming", + "repo": "base16-vim", + "rev": "577fe8125d74ff456cf942c733a85d769afe58b7", + "type": "github" + }, + "original": { + "owner": "tinted-theming", + "repo": "base16-vim", + "rev": "577fe8125d74ff456cf942c733a85d769afe58b7", + "type": "github" + } + }, + "cosmic-manager": { + "inputs": { + "flake-parts": [ + "flake-parts" + ], + "home-manager": [ + "home-manager" + ], + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1744387566, + "narHash": "sha256-O39zTv7LdRgr4Hw38d+eQG2LYpP75rw2XKqTGV5qzgs=", + "owner": "HeitorAugustoLN", + "repo": "cosmic-manager", + "rev": "52d3fdd080a9dd4639948687682a68282fbf0314", + "type": "github" + }, + "original": { + "owner": "HeitorAugustoLN", + "repo": "cosmic-manager", "type": "github" } }, @@ -70,11 +183,11 @@ ] }, "locked": { - "lastModified": 1751607816, - "narHash": "sha256-5PtrwjqCIJ4DKQhzYdm8RFePBuwb+yTzjV52wWoGSt4=", + "lastModified": 1751854533, + "narHash": "sha256-U/OQFplExOR1jazZY4KkaQkJqOl59xlh21HP9mI79Vc=", "owner": "nix-community", "repo": "disko", - "rev": "da6109c917b48abc1f76dd5c9bf3901c8c80f662", + "rev": "16b74a1e304197248a1bc663280f2548dbfcae3c", "type": "github" }, "original": { @@ -83,25 +196,19 @@ "type": "github" } }, - "disko_2": { - "inputs": { - "nixpkgs": [ - "nixos-anywhere", - "nixpkgs" - ] - }, + "firefox-gnome-theme": { + "flake": false, "locked": { - "lastModified": 1748225455, - "narHash": "sha256-AzlJCKaM4wbEyEpV3I/PUq5mHnib2ryEy32c+qfj6xk=", - "owner": "nix-community", - "repo": "disko", - "rev": "a894f2811e1ee8d10c50560551e50d6ab3c392ba", + "lastModified": 1748383148, + "narHash": "sha256-pGvD/RGuuPf/4oogsfeRaeMm6ipUIznI2QSILKjKzeA=", + "owner": "rafaelmardojai", + "repo": "firefox-gnome-theme", + "rev": "4eb2714fbed2b80e234312611a947d6cb7d70caf", "type": "github" }, "original": { - "owner": "nix-community", - "ref": "master", - "repo": "disko", + "owner": "rafaelmardojai", + "repo": "firefox-gnome-theme", "type": "github" } }, @@ -111,7 +218,9 @@ "nixpkgs" ], "snowfall-lib": "snowfall-lib", - "treefmt-nix": "treefmt-nix" + "treefmt-nix": [ + "treefmt-nix" + ] }, "locked": { "lastModified": 1751834884, @@ -128,6 +237,22 @@ } }, "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1747046372, + "narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_2": { "flake": false, "locked": { "lastModified": 1650374568, @@ -145,38 +270,14 @@ }, "flake-parts": { "inputs": { - "nixpkgs-lib": [ - "nixos-anywhere", - "nixpkgs" - ] + "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1743550720, - "narHash": "sha256-hIshGgKZCgWh6AYJpJmRgFdR3WUbkY04o82X05xqQiY=", + "lastModified": 1751413152, + "narHash": "sha256-Tyw1RjYEsp5scoigs1384gIg6e0GoBVjms4aXFfRssQ=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "c621e8422220273271f52058f618c94e405bb0f5", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" - } - }, - "flake-parts_2": { - "inputs": { - "nixpkgs-lib": [ - "terranix", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1736143030, - "narHash": "sha256-+hu54pAoLDEZT9pjHlqL9DNzWz0NbUn8NEAHP7PQPzU=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "b905f6fc23a9051a6e1b741e1438dbfc0634c6de", + "rev": "77826244401ea9de6e3bac47c2db46005e1f30b5", "type": "github" }, "original": { @@ -223,12 +324,17 @@ } }, "flake-utils_2": { + "inputs": { + "systems": [ + "systems" + ] + }, "locked": { - "lastModified": 1634851050, - "narHash": "sha256-N83GlSGPJJdcqhUxSCS/WwW5pksYf3VP1M13cDRTSVA=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "c91f3de5adaf1de973b797ef7485e441a65b8935", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -237,18 +343,36 @@ "type": "github" } }, - "flake-utils_3": { + "fromYaml": { + "flake": false, "locked": { - "lastModified": 1634851050, - "narHash": "sha256-N83GlSGPJJdcqhUxSCS/WwW5pksYf3VP1M13cDRTSVA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "c91f3de5adaf1de973b797ef7485e441a65b8935", + "lastModified": 1731966426, + "narHash": "sha256-lq95WydhbUTWig/JpqiB7oViTcHFP8Lv41IGtayokA8=", + "owner": "SenchoPens", + "repo": "fromYaml", + "rev": "106af9e2f715e2d828df706c386a685698f3223b", "type": "github" }, "original": { - "owner": "numtide", - "repo": "flake-utils", + "owner": "SenchoPens", + "repo": "fromYaml", + "type": "github" + } + }, + "gnome-shell": { + "flake": false, + "locked": { + "lastModified": 1744584021, + "narHash": "sha256-0RJ4mJzf+klKF4Fuoc8VN8dpQQtZnKksFmR2jhWE1Ew=", + "owner": "GNOME", + "repo": "gnome-shell", + "rev": "52c517c8f6c199a1d6f5118fae500ef69ea845ae", + "type": "github" + }, + "original": { + "owner": "GNOME", + "ref": "48.1", + "repo": "gnome-shell", "type": "github" } }, @@ -273,6 +397,32 @@ "type": "github" } }, + "napalm": { + "inputs": { + "flake-utils": [ + "authentik-nix", + "flake-utils" + ], + "nixpkgs": [ + "authentik-nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1725806412, + "narHash": "sha256-lGZjkjds0p924QEhm/r0BhAxbHBJE1xMOldB/HmQH04=", + "owner": "willibutz", + "repo": "napalm", + "rev": "b492440d9e64ae20736d3bec5c7715ffcbde83f5", + "type": "github" + }, + "original": { + "owner": "willibutz", + "ref": "avoid-foldl-stack-overflow", + "repo": "napalm", + "type": "github" + } + }, "nix-vm-test": { "inputs": { "nixpkgs": [ @@ -296,15 +446,21 @@ }, "nixos-anywhere": { "inputs": { - "disko": "disko_2", - "flake-parts": "flake-parts", + "disko": [ + "disko" + ], + "flake-parts": [ + "flake-parts" + ], "nix-vm-test": "nix-vm-test", "nixos-images": "nixos-images", "nixos-stable": "nixos-stable", "nixpkgs": [ "nixpkgs" ], - "treefmt-nix": "treefmt-nix_2" + "treefmt-nix": [ + "treefmt-nix" + ] }, "locked": { "lastModified": 1749105224, @@ -364,11 +520,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1751582995, - "narHash": "sha256-u7ubvtxdTnFPpV27AHpgoKn7qHuE7sgWgza/1oj5nzA=", + "lastModified": 1751943650, + "narHash": "sha256-7orTnNqkGGru8Je6Un6mq1T8YVVU/O5kyW4+f9C1mZQ=", "owner": "nixos", "repo": "nixpkgs", - "rev": "7a732ed41ca0dd64b4b71b563ab9805a80a7d693", + "rev": "88983d4b665fb491861005137ce2b11a9f89f203", "type": "github" }, "original": { @@ -378,33 +534,116 @@ "type": "github" } }, - "nixpkgs_2": { + "nixpkgs-lib": { "locked": { - "lastModified": 1636273007, - "narHash": "sha256-eb6HcZNacO9vIP/KcJ5CoCRYSGfD+VxzYs2cCafEo4Y=", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "c69c6533c820c55c3f1d924b399d8f6925a1e41a", + "lastModified": 1751159883, + "narHash": "sha256-urW/Ylk9FIfvXfliA1ywh75yszAbiTEVgpPeinFyVZo=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "14a40a1d7fb9afa4739275ac642ed7301a9ba1ab", "type": "github" }, "original": { - "owner": "nixos", - "repo": "nixpkgs", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "nur": { + "inputs": { + "flake-parts": [ + "stylix", + "flake-parts" + ], + "nixpkgs": [ + "stylix", + "nixpkgs" + ], + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1751320053, + "narHash": "sha256-3m6RMw0FbbaUUa01PNaMLoO7D99aBClmY5ed9V3vz+0=", + "owner": "nix-community", + "repo": "NUR", + "rev": "cbde1735782f9c2bb2c63d5e05fba171a14a4670", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "NUR", + "type": "github" + } + }, + "pyproject-build-systems": { + "inputs": { + "nixpkgs": [ + "authentik-nix", + "nixpkgs" + ], + "pyproject-nix": [ + "authentik-nix", + "pyproject-nix" + ], + "uv2nix": [ + "authentik-nix", + "uv2nix" + ] + }, + "locked": { + "lastModified": 1749519371, + "narHash": "sha256-UJONN7mA2stweZCoRcry2aa1XTTBL0AfUOY84Lmqhos=", + "owner": "pyproject-nix", + "repo": "build-system-pkgs", + "rev": "7c06967eca687f3482624250428cc12f43c92523", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "build-system-pkgs", + "type": "github" + } + }, + "pyproject-nix": { + "inputs": { + "nixpkgs": [ + "authentik-nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1750499893, + "narHash": "sha256-ThKBd8XSvITAh2JqU7enOp8AfKeQgf9u7zYC41cnBE4=", + "owner": "pyproject-nix", + "repo": "pyproject.nix", + "rev": "e824458bd917b44bf4c38795dea2650336b2f55d", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "pyproject.nix", "type": "github" } }, "root": { "inputs": { "advisory-db": "advisory-db", + "authentik-nix": "authentik-nix", + "cosmic-manager": "cosmic-manager", "crane": "crane", "disko": "disko", "flake-base": "flake-base", + "flake-parts": "flake-parts", + "flake-utils": "flake-utils_2", "home-manager": "home-manager", "nixos-anywhere": "nixos-anywhere", "nixpkgs": "nixpkgs", "rust-overlay": "rust-overlay", + "stylix": "stylix", + "systems": "systems_2", "terranix": "terranix", - "terranix-hcloud": "terranix-hcloud" + "terranix-hcloud": "terranix-hcloud", + "treefmt-nix": "treefmt-nix_2" } }, "rust-overlay": { @@ -414,11 +653,11 @@ ] }, "locked": { - "lastModified": 1751769931, - "narHash": "sha256-QR2Rp/41NkA5YxcpvZEKD1S2QE1Pb9U415aK8M/4tJc=", + "lastModified": 1752028888, + "narHash": "sha256-LRj3/PUpII6taWOrX1w/OeI6f1ncND02PP/kEHvPCqU=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "3ac4f630e375177ea8317e22f5c804156de177e8", + "rev": "a0f1c656e053463b47639234b151a05e4441bb19", "type": "github" }, "original": { @@ -429,7 +668,7 @@ }, "snowfall-lib": { "inputs": { - "flake-compat": "flake-compat", + "flake-compat": "flake-compat_2", "flake-utils-plus": "flake-utils-plus", "nixpkgs": [ "flake-base", @@ -450,6 +689,45 @@ "type": "github" } }, + "stylix": { + "inputs": { + "base16": "base16", + "base16-fish": "base16-fish", + "base16-helix": "base16-helix", + "base16-vim": "base16-vim", + "firefox-gnome-theme": "firefox-gnome-theme", + "flake-parts": [ + "flake-parts" + ], + "gnome-shell": "gnome-shell", + "nixpkgs": [ + "nixpkgs" + ], + "nur": "nur", + "systems": [ + "systems" + ], + "tinted-foot": "tinted-foot", + "tinted-kitty": "tinted-kitty", + "tinted-schemes": "tinted-schemes", + "tinted-tmux": "tinted-tmux", + "tinted-zed": "tinted-zed" + }, + "locked": { + "lastModified": 1752084754, + "narHash": "sha256-JorlZGCWxlYV01lXmUuDeKOZoLPdoN3fAKJv8YIuavs=", + "owner": "nix-community", + "repo": "stylix", + "rev": "2df042576646d012d15637f43d6075995e785ce3", + "type": "github" + }, + "original": { + "owner": "nix-community", + "ref": "release-25.05", + "repo": "stylix", + "type": "github" + } + }, "systems": { "locked": { "lastModified": 1681028828, @@ -480,13 +758,30 @@ "type": "github" } }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, "terranix": { "inputs": { - "flake-parts": "flake-parts_2", + "flake-parts": [ + "flake-parts" + ], "nixpkgs": [ "nixpkgs" ], - "systems": "systems_2" + "systems": "systems_3" }, "locked": { "lastModified": 1749381683, @@ -502,28 +797,17 @@ "type": "github" } }, - "terranix-examples": { - "locked": { - "lastModified": 1633465925, - "narHash": "sha256-BfXRW1ZHpK5jh5CVcw7eFpGsWE1CyVxL8R+V7uXemaU=", - "owner": "terranix", - "repo": "terranix-examples", - "rev": "70bf5d5a1ad4eabef1e4e71c1eb101021decd5a4", - "type": "github" - }, - "original": { - "owner": "terranix", - "repo": "terranix-examples", - "type": "github" - } - }, "terranix-hcloud": { "inputs": { - "flake-utils": "flake-utils_2", + "flake-utils": [ + "flake-utils" + ], "nixpkgs": [ "nixpkgs" ], - "terranix": "terranix_2" + "terranix": [ + "terranix" + ] }, "locked": { "lastModified": 1745572802, @@ -539,42 +823,101 @@ "type": "github" } }, - "terranix_2": { - "inputs": { - "bats-assert": "bats-assert", - "bats-support": "bats-support", - "flake-utils": "flake-utils_3", - "nixpkgs": "nixpkgs_2", - "terranix-examples": "terranix-examples" - }, + "tinted-foot": { + "flake": false, "locked": { - "lastModified": 1636274023, - "narHash": "sha256-HDiyJGgyDUoLnpL8N+wDm3cM/vEfYYc/p4N1kKH/kLk=", - "owner": "terranix", - "repo": "terranix", - "rev": "342ec8490bc948c8589414eb89f26b265cbfd62a", + "lastModified": 1726913040, + "narHash": "sha256-+eDZPkw7efMNUf3/Pv0EmsidqdwNJ1TaOum6k7lngDQ=", + "owner": "tinted-theming", + "repo": "tinted-foot", + "rev": "fd1b924b6c45c3e4465e8a849e67ea82933fcbe4", "type": "github" }, "original": { - "owner": "terranix", - "ref": "develop", - "repo": "terranix", + "owner": "tinted-theming", + "repo": "tinted-foot", + "rev": "fd1b924b6c45c3e4465e8a849e67ea82933fcbe4", + "type": "github" + } + }, + "tinted-kitty": { + "flake": false, + "locked": { + "lastModified": 1735730497, + "narHash": "sha256-4KtB+FiUzIeK/4aHCKce3V9HwRvYaxX+F1edUrfgzb8=", + "owner": "tinted-theming", + "repo": "tinted-kitty", + "rev": "de6f888497f2c6b2279361bfc790f164bfd0f3fa", + "type": "github" + }, + "original": { + "owner": "tinted-theming", + "repo": "tinted-kitty", + "type": "github" + } + }, + "tinted-schemes": { + "flake": false, + "locked": { + "lastModified": 1750770351, + "narHash": "sha256-LI+BnRoFNRa2ffbe3dcuIRYAUcGklBx0+EcFxlHj0SY=", + "owner": "tinted-theming", + "repo": "schemes", + "rev": "5a775c6ffd6e6125947b393872cde95867d85a2a", + "type": "github" + }, + "original": { + "owner": "tinted-theming", + "repo": "schemes", + "type": "github" + } + }, + "tinted-tmux": { + "flake": false, + "locked": { + "lastModified": 1751159871, + "narHash": "sha256-UOHBN1fgHIEzvPmdNMHaDvdRMgLmEJh2hNmDrp3d3LE=", + "owner": "tinted-theming", + "repo": "tinted-tmux", + "rev": "bded5e24407cec9d01bd47a317d15b9223a1546c", + "type": "github" + }, + "original": { + "owner": "tinted-theming", + "repo": "tinted-tmux", + "type": "github" + } + }, + "tinted-zed": { + "flake": false, + "locked": { + "lastModified": 1751158968, + "narHash": "sha256-ksOyv7D3SRRtebpXxgpG4TK8gZSKFc4TIZpR+C98jX8=", + "owner": "tinted-theming", + "repo": "base16-zed", + "rev": "86a470d94204f7652b906ab0d378e4231a5b3384", + "type": "github" + }, + "original": { + "owner": "tinted-theming", + "repo": "base16-zed", "type": "github" } }, "treefmt-nix": { "inputs": { "nixpkgs": [ - "flake-base", + "stylix", + "nur", "nixpkgs" ] }, "locked": { - "lastModified": 1750931469, - "narHash": "sha256-0IEdQB1nS+uViQw4k3VGUXntjkDp7aAlqcxdewb/hAc=", + "lastModified": 1733222881, + "narHash": "sha256-JIPcz1PrpXUCbaccEnrcUS8jjEb/1vJbZz5KkobyFdM=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "ac8e6f32e11e9c7f153823abc3ab007f2a65d3e1", + "rev": "49717b5af6f80172275d47a418c9719a31a78b53", "type": "github" }, "original": { @@ -586,16 +929,15 @@ "treefmt-nix_2": { "inputs": { "nixpkgs": [ - "nixos-anywhere", "nixpkgs" ] }, "locked": { - "lastModified": 1748243702, - "narHash": "sha256-9YzfeN8CB6SzNPyPm2XjRRqSixDopTapaRsnTpXUEY8=", + "lastModified": 1752055615, + "narHash": "sha256-19m7P4O/Aw/6+CzncWMAJu89JaKeMh3aMle1CNQSIwM=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "1f3f7b784643d488ba4bf315638b2b0a4c5fb007", + "rev": "c9d477b5d5bd7f26adddd3f96cfd6a904768d4f9", "type": "github" }, "original": { @@ -603,6 +945,31 @@ "repo": "treefmt-nix", "type": "github" } + }, + "uv2nix": { + "inputs": { + "nixpkgs": [ + "authentik-nix", + "nixpkgs" + ], + "pyproject-nix": [ + "authentik-nix", + "pyproject-nix" + ] + }, + "locked": { + "lastModified": 1750987094, + "narHash": "sha256-GujDElxLgYatnNvuL1U6qd18lcuG6anJMjpfYRScV08=", + "owner": "pyproject-nix", + "repo": "uv2nix", + "rev": "4b703d851b61e664a70238711a8ff0efa1aa2f52", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "uv2nix", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index b0f6305..c51f259 100644 --- a/flake.nix +++ b/flake.nix @@ -2,26 +2,61 @@ description = "A very basic flake"; inputs = { + authentik-nix = { + url = "github:nix-community/authentik-nix"; + inputs = { + flake-utils.follows = "flake-utils"; + nixpkgs.follows = "nixpkgs"; + flake-parts.follows = "flake-parts"; + systems.follows = "systems"; + }; + }; nixpkgs.url = "github:nixos/nixpkgs/nixos-25.05"; flake-base = { url = "git+https://khs.codes/nix/flake-base"; - inputs.nixpkgs.follows = "nixpkgs"; + inputs = { + nixpkgs.follows = "nixpkgs"; + treefmt-nix.follows = "treefmt-nix"; + }; + }; + flake-utils = { + url = "github:numtide/flake-utils"; + inputs = { + systems.follows = "systems"; + }; + }; + flake-parts = { + url = "github:hercules-ci/flake-parts"; }; disko = { url = "github:nix-community/disko"; - inputs.nixpkgs.follows = "nixpkgs"; + inputs = { + nixpkgs.follows = "nixpkgs"; + }; }; terranix = { url = "github:terranix/terranix"; - inputs.nixpkgs.follows = "nixpkgs"; + inputs = { + nixpkgs.follows = "nixpkgs"; + flake-parts.follows = "flake-parts"; + }; }; home-manager = { url = "github:nix-community/home-manager/release-25.05"; - inputs.nixpkgs.follows = "nixpkgs"; + inputs = { + nixpkgs.follows = "nixpkgs"; + }; }; terranix-hcloud = { url = "github:terranix/terranix-hcloud"; - inputs.nixpkgs.follows = "nixpkgs"; + inputs = { + flake-utils.follows = "flake-utils"; + nixpkgs.follows = "nixpkgs"; + terranix.follows = "terranix"; + }; + }; + systems = { + url = "github:nix-systems/default"; }; crane.url = "github:ipetkov/crane"; advisory-db = { @@ -34,10 +69,35 @@ nixpkgs.follows = "nixpkgs"; }; }; + treefmt-nix = { + url = "github:numtide/treefmt-nix"; + inputs = { + nixpkgs.follows = "nixpkgs"; + }; + }; nixos-anywhere = { url = "github:nix-community/nixos-anywhere/1.11.0"; inputs = { nixpkgs.follows = "nixpkgs"; + flake-parts.follows = "flake-parts"; + treefmt-nix.follows = "treefmt-nix"; + disko.follows = "disko"; + }; + }; + stylix = { + url = "github:nix-community/stylix/release-25.05"; + inputs = { + nixpkgs.follows = "nixpkgs"; + flake-parts.follows = "flake-parts"; + systems.follows = "systems"; + }; + }; + cosmic-manager = { + url = "github:HeitorAugustoLN/cosmic-manager"; + inputs = { + nixpkgs.follows = "nixpkgs"; + home-manager.follows = "home-manager"; + flake-parts.follows = "flake-parts"; }; }; }; @@ -45,22 +105,27 @@ outputs = inputs@{ self, ... }: let - dirsInPath = - path: - let - files = builtins.readDir path; - dirs = inputs.nixpkgs.lib.filterAttrs (name: kind: kind == "directory") files; - in - builtins.attrNames dirs; - profileArgs = { inherit self; }; - profileNames = dirsInPath ./nix/profiles; - nixosModules = dirsInPath ./nix/modules/nixos; - inputModules = [ inputs.disko.nixosModules.disko ]; + inputNixosModules = [ + inputs.disko.nixosModules.disko + inputs.stylix.nixosModules.stylix + inputs.authentik-nix.nixosModules.default + ]; + inputHomeModules = [ + inputs.cosmic-manager.homeManagerModules.cosmic-manager + ]; + allowUnfreePackages = [ + "spotify" + "google-chrome" + ]; in (inputs.flake-base.lib.mkFlake { inherit inputs; src = ./.; - systems.modules.nixos = inputModules; + channels-config = { + allowUnfreePredicate = pkg: builtins.elem (inputs.nixpkgs.lib.getName pkg) allowUnfreePackages; + }; + systems.modules.nixos = inputNixosModules; + homes.modules = inputHomeModules; snowfall = { root = ./nix; namespace = "khscodes"; @@ -69,44 +134,14 @@ name = "Machines"; }; }; - modules.nixos = { - default = - { - imports = builtins.map (m: self.nixosModules.${m}) nixosModules ++ inputModules; - } - // (builtins.listToAttrs ( - builtins.map (n: { - name = n; - value = (import ./nix/profiles/${n} profileArgs); - }) profileNames - )); - }; overlays = [ inputs.rust-overlay.overlays.default ]; }) // { - terranixModules.cloudflare = import ./nix/modules/terranix/cloudflare { - inherit inputs; - khscodesLib = inputs.self.lib; - }; - terranixModules.hcloud = import ./nix/modules/terranix/hcloud { - inherit inputs; - khscodesLib = inputs.self.lib; - }; - terranixModules.openbao = import ./nix/modules/terranix/openbao { - inherit inputs; - khscodesLib = inputs.self.lib; - }; - terranixModules.s3 = import ./nix/modules/terranix/s3 { - inherit inputs; - khscodesLib = inputs.self.lib; - }; - terranixModules.openstack = import ./nix/modules/terranix/openstack { - inherit inputs; - khscodesLib = inputs.self.lib; - }; - terranixModules.unifi = import ./nix/modules/terranix/unifi { - inherit inputs; - khscodesLib = inputs.self.lib; - }; + terranixModules.cloudflare = import ./nix/modules/terranix/cloudflare; + terranixModules.hcloud = import ./nix/modules/terranix/hcloud; + terranixModules.vault = import ./nix/modules/terranix/vault; + terranixModules.s3 = import ./nix/modules/terranix/s3; + terranixModules.openstack = import ./nix/modules/terranix/openstack; + terranixModules.unifi = import ./nix/modules/terranix/unifi; }; } diff --git a/nix/checks/hetzner-sets-ipv6/default.nix b/nix/checks/hetzner-sets-ipv6/default.nix index 68ce8ce..b4db636 100644 --- a/nix/checks/hetzner-sets-ipv6/default.nix +++ b/nix/checks/hetzner-sets-ipv6/default.nix @@ -12,7 +12,10 @@ pkgs.nixosTest { { ... }: { imports = [ - inputs.self.nixosModules.default + inputs.self.nixosModules.hetzner + inputs.self.nixosModules.systemd-boot + inputs.self.nixosModules."virtualisation/qemu-guest" + inputs.disko.nixosModules.disko sharedModule ]; khscodes.hetzner = { diff --git a/nix/homes/aarch64-linux/khs@khs.codes/default.nix b/nix/homes/aarch64-linux/khs@khs.codes/default.nix new file mode 100644 index 0000000..d214165 --- /dev/null +++ b/nix/homes/aarch64-linux/khs@khs.codes/default.nix @@ -0,0 +1,7 @@ +{ + ... +}: +{ + khscodes.khs.enable = true; + khscodes.khs.shell.oh-my-posh.enable = true; +} diff --git a/nix/homes/aarch64-linux/khs@security.kaareskovgaard.net/default.nix b/nix/homes/aarch64-linux/khs@security.kaareskovgaard.net/default.nix new file mode 100644 index 0000000..d214165 --- /dev/null +++ b/nix/homes/aarch64-linux/khs@security.kaareskovgaard.net/default.nix @@ -0,0 +1,7 @@ +{ + ... +}: +{ + khscodes.khs.enable = true; + khscodes.khs.shell.oh-my-posh.enable = true; +} diff --git a/nix/homes/x86_64-linux/khs/default.nix b/nix/homes/x86_64-linux/khs/default.nix new file mode 100644 index 0000000..1fd2522 --- /dev/null +++ b/nix/homes/x86_64-linux/khs/default.nix @@ -0,0 +1,8 @@ +{ + ... +}: +{ + khscodes.khs.enable = true; + khscodes.khs.shell.oh-my-posh.enable = true; + imports = [ ./desktop.nix ]; +} diff --git a/nix/homes/x86_64-linux/khs/desktop.nix b/nix/homes/x86_64-linux/khs/desktop.nix new file mode 100644 index 0000000..50c36bd --- /dev/null +++ b/nix/homes/x86_64-linux/khs/desktop.nix @@ -0,0 +1,14 @@ +{ + pkgs, + lib, + config, + ... +}: +{ + imports = [ ./linux-desktop.nix ]; + home.packages = lib.mkIf config.khscodes.desktop.enable [ + pkgs.bitwarden-cli + pkgs.nerd-fonts.inconsolata + pkgs.google-chrome + ]; +} diff --git a/nix/homes/x86_64-linux/khs/linux-desktop.nix b/nix/homes/x86_64-linux/khs/linux-desktop.nix new file mode 100644 index 0000000..9769a56 --- /dev/null +++ b/nix/homes/x86_64-linux/khs/linux-desktop.nix @@ -0,0 +1,9 @@ +{ + pkgs, + config, + lib, + ... +}: +{ + home.packages = lib.mkIf config.khscodes.desktop.enable [ pkgs.spotify ]; +} diff --git a/nix/homes/x86_64-linux/khs@test.kaareskovgaard.net/default.nix b/nix/homes/x86_64-linux/khs@test.kaareskovgaard.net/default.nix deleted file mode 100644 index 740142d..0000000 --- a/nix/homes/x86_64-linux/khs@test.kaareskovgaard.net/default.nix +++ /dev/null @@ -1,4 +0,0 @@ -{ - snowfallorg.user.name = "khs"; - home.stateVersion = "25.05"; -} diff --git a/nix/lib/disko-root-bios/default.nix b/nix/lib/disko-root-bios/default.nix new file mode 100644 index 0000000..a8423bb --- /dev/null +++ b/nix/lib/disko-root-bios/default.nix @@ -0,0 +1,35 @@ +{ ... }: +{ + disko-root-bios = + { + diskName, + device, + bootPartName ? "boot", + rootPartName ? "root", + }: + { + devices.disk = { + "${diskName}" = { + inherit device; + type = "disk"; + content = { + type = "gpt"; + partitions = { + ${bootPartName} = { + size = "1M"; + type = "EF02"; # for grub MBR + }; + ${rootPartName} = { + size = "100%"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/"; + }; + }; + }; + }; + }; + }; + }; +} diff --git a/nix/modules/nixos/infrastructure/default.nix b/nix/modules/home/applications/default.nix similarity index 100% rename from nix/modules/nixos/infrastructure/default.nix rename to nix/modules/home/applications/default.nix diff --git a/nix/modules/home/desktop/default.nix b/nix/modules/home/desktop/default.nix new file mode 100644 index 0000000..245f4b5 --- /dev/null +++ b/nix/modules/home/desktop/default.nix @@ -0,0 +1,6 @@ +{ lib, ... }: +{ + options.khscodes.desktop = { + enable = lib.mkEnableOption "Generic setting other modules can use to enable/disable stuff when used on desktops"; + }; +} diff --git a/nix/modules/home/khs/applications/alacritty/default.nix b/nix/modules/home/khs/applications/alacritty/default.nix new file mode 100644 index 0000000..7f66990 --- /dev/null +++ b/nix/modules/home/khs/applications/alacritty/default.nix @@ -0,0 +1,113 @@ +{ + lib, + pkgs, + config, + system, + ... +}: +let + isDarwin = lib.strings.hasSuffix "-darwin" system; + isLinux = lib.strings.hasSuffix "-darwin" system; + shell = pkgs.bashInteractive; + shellArgs = [ + "-c" + (lib.getExe pkgs.zellij) + ]; +in +{ + config = lib.mkIf (config.khscodes.khs.enable && config.khscodes.desktop.enable) { + programs.alacritty = { + enable = true; + settings = { + terminal = + { + shell = { + program = "${shell}${shell.shellPath}"; + args = shellArgs; + }; + } + // lib.attrsets.optionalAttrs isDarwin { + osc52 = "CopyPaste"; + }; + scrolling = { + history = 100000; + }; + window = { + padding = { + x = 2; + y = 0; + }; + }; + bell = { + animation = "EaseOutExpo"; + }; + hints = { + enabled = [ + { + regex = "(ipfs:|ipns:|magnet:|mailto:|gemini:|gopher:|https:|http:|news:|file:|git:|ssh:|ftp:)[^\\u0000-\\u001F\\u007F-\\u009F<>\"\\\\s{-}\\\\^⟨⟩`]+"; + command = if isLinux then "xdg-open" else "open"; + post_processing = true; + mouse = { + enabled = true; + mods = "Control"; + }; + } + ]; + }; + window = { + option_as_alt = "OnlyLeft"; + }; + env = { + TERM = "xterm-256color"; + }; + keyboard.bindings = + [ + { + key = "T"; + mods = "Control|Shift"; + action = "SpawnNewInstance"; + } + { + key = "W"; + mods = "Control|Shift"; + action = "Quit"; + } + { + key = "Plus"; + mods = "Control"; + action = "IncreaseFontSize"; + } + { + key = "Minus"; + mods = "Control"; + action = "DecreaseFontSize"; + } + { + key = "Key0"; + mods = "Control"; + action = "ResetFontSize"; + } + { + key = "C"; + mods = "Super"; + action = "None"; + } + ] + ++ lib.lists.optionals isDarwin [ + { + key = "N"; + mods = "Command"; + action = "SpawnNewInstance"; + } + # Allow zellij to receive the keys, to make copy/pasting work in darwin + { + key = "C"; + mods = "Command"; + action = "ReceiveChar"; + } + ]; + }; + }; + stylix.targets.alacritty.enable = true; + }; +} diff --git a/nix/modules/home/khs/applications/default.nix b/nix/modules/home/khs/applications/default.nix new file mode 100644 index 0000000..ffcd441 --- /dev/null +++ b/nix/modules/home/khs/applications/default.nix @@ -0,0 +1 @@ +{ } diff --git a/nix/modules/home/khs/default.nix b/nix/modules/home/khs/default.nix new file mode 100644 index 0000000..16400e2 --- /dev/null +++ b/nix/modules/home/khs/default.nix @@ -0,0 +1,17 @@ +{ lib, config, ... }: +let + cfg = config.khscodes.khs; +in +{ + options.khscodes.khs = { + enable = lib.mkEnableOption "Enables the settings for KHS"; + }; + + config = lib.mkIf cfg.enable { + snowfallorg.user.name = "khs"; + home.sessionVariables = { + EMAIL = "kaare@kaareskovgaard.net"; + }; + home.stateVersion = "25.05"; + }; +} diff --git a/nix/modules/home/khs/desktop/cosmic/default.nix b/nix/modules/home/khs/desktop/cosmic/default.nix new file mode 100644 index 0000000..e6ee668 --- /dev/null +++ b/nix/modules/home/khs/desktop/cosmic/default.nix @@ -0,0 +1,120 @@ +{ + config, + inputs, + lib, + ... +}: +let + cosmicLib = import "${inputs.cosmic-manager}/lib" { inherit lib; }; + mkRON = cosmicLib.mkRON; + accent = mkRON "optional" { + red = mkRON "raw" "0.3882353"; + green = mkRON "raw" "0.8156863"; + blue = mkRON "raw" "0.8745098"; + }; + active_hint = 1; + corner_radii = { + radius_0 = mkRON "tuple" [ + 0.0 + 0.0 + 0.0 + 0.0 + ]; + radius_xs = mkRON "tuple" [ + 2.0 + 2.0 + 2.0 + 2.0 + ]; + radius_s = mkRON "tuple" [ + 8.0 + 8.0 + 8.0 + 8.0 + ]; + radius_m = mkRON "tuple" [ + 8.0 + 8.0 + 8.0 + 8.0 + ]; + radius_l = mkRON "tuple" [ + 8.0 + 8.0 + 8.0 + 8.0 + ]; + radius_xl = mkRON "tuple" [ + 8.0 + 8.0 + 8.0 + 8.0 + ]; + }; + gaps = mkRON "tuple" [ + 0 + 1 + ]; +in +{ + config = lib.mkIf (config.khscodes.desktop.enable && config.khscodes.khs.enable) { + wayland.desktopManager.cosmic = { + enable = true; + applets = { + # This is the "dock" + app-list = { + settings = { + enable_drag_source = false; + favorites = [ + "com.system76.CosmicFiles" + "thunderbird" + "Google-chrome" + "Code" + "Alacritty" + "com.system76.CosmicSettings" + "Spotify" + "steam" + ]; + filter_top_levels = mkRON "optional" null; + }; + }; + }; + appearance = { + toolkit = { + interface_density = mkRON "enum" "Standard"; + monospace_font = { + family = config.stylix.fonts.monospace.name; + stretch = mkRON "enum" "Normal"; + style = mkRON "enum" "Normal"; + weight = mkRON "enum" "Normal"; + }; + }; + theme = { + light = { + inherit + accent + active_hint + corner_radii + gaps + ; + }; + dark = { + inherit + accent + active_hint + corner_radii + gaps + ; + }; + }; + }; + compositor = { + active_hint = true; + autotile = true; + autotile_behavior = mkRON "enum" "PerWorkspace"; + cursor_follows_focus = false; + focus_follows_cursor = false; + }; + }; + }; +} diff --git a/nix/modules/home/khs/shell/bash/default.nix b/nix/modules/home/khs/shell/bash/default.nix new file mode 100644 index 0000000..a711dc4 --- /dev/null +++ b/nix/modules/home/khs/shell/bash/default.nix @@ -0,0 +1,9 @@ +{ config, lib, ... }: +{ + config = lib.mkIf config.khscodes.khs.enable { + programs.bash = { + enable = true; + shellAliases = config.khscodes.khs.shell.aliases; + }; + }; +} diff --git a/nix/modules/home/khs/shell/carapace/default.nix b/nix/modules/home/khs/shell/carapace/default.nix new file mode 100644 index 0000000..c2af0bd --- /dev/null +++ b/nix/modules/home/khs/shell/carapace/default.nix @@ -0,0 +1,12 @@ +{ lib, config, ... }: +{ + config = lib.mkIf (config.khscodes.khs.enable && config.khscodes.desktop.enable) { + programs.carapace = { + enable = true; + enableBashIntegration = true; + enableZshIntegration = true; + enableFishIntegration = true; + enableNushellIntegration = true; + }; + }; +} diff --git a/nix/modules/home/khs/shell/default.nix b/nix/modules/home/khs/shell/default.nix new file mode 100644 index 0000000..05c1cf4 --- /dev/null +++ b/nix/modules/home/khs/shell/default.nix @@ -0,0 +1,18 @@ +{ lib, ... }: +{ + options.khscodes.khs.shell = { + aliases = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + description = "Shell aliases to be copied to different shells"; + default = { + add = "git add"; + commit = "git commit"; + st = "git status"; + push = "git push"; + puff = "git puff"; + pull = "git pull"; + purr = "git purr"; + }; + }; + }; +} diff --git a/nix/modules/home/khs/shell/fish/default.nix b/nix/modules/home/khs/shell/fish/default.nix new file mode 100644 index 0000000..ea46533 --- /dev/null +++ b/nix/modules/home/khs/shell/fish/default.nix @@ -0,0 +1,10 @@ +{ config, lib, ... }: +{ + config = lib.mkIf config.khscodes.khs.enable { + programs.fish = { + enable = true; + shellAliases = config.khscodes.khs.shell.aliases; + shellInit = "set fish_greeting"; + }; + }; +} diff --git a/nix/modules/home/khs/shell/nushell/default.nix b/nix/modules/home/khs/shell/nushell/default.nix new file mode 100644 index 0000000..e846379 --- /dev/null +++ b/nix/modules/home/khs/shell/nushell/default.nix @@ -0,0 +1,22 @@ +{ + lib, + config, + ... +}: +let + aliases = config.khscodes.khs.shell.aliases; +in +{ + config = lib.mkIf config.khscodes.khs.enable { + programs.nushell = { + enable = true; + shellAliases = aliases; + extraConfig = '' + $env.config = { + show_banner: false + } + ''; + environmentVariables = config.home.sessionVariables; + }; + }; +} diff --git a/nix/modules/home/khs/shell/oh-my-posh/default.nix b/nix/modules/home/khs/shell/oh-my-posh/default.nix new file mode 100644 index 0000000..c181680 --- /dev/null +++ b/nix/modules/home/khs/shell/oh-my-posh/default.nix @@ -0,0 +1,158 @@ +{ config, lib, ... }: +let + cfg = config.khscodes.khs.shell.oh-my-posh; + unicodeChar = code: builtins.fromJSON ''"\u${code}"''; + powerline_symbol = unicodeChar "e0b0"; + rpowerline_symbol = unicodeChar "e0b2"; + colors = config.lib.stylix.colors.withHashtag; + bright-yellow = config.lib.stylix.colors.yellow or config.lib.stylix.colors.base0a; + bright-yellow-hashtag = "#${bright-yellow}"; + segment_style = { + style = "diamond"; + leading_diamond = ""; + trailing_diamond = powerline_symbol; + }; + rsegment_style = { + style = "diamond"; + leading_diamond = rpowerline_symbol; + trailing_diamond = ""; + }; +in +{ + options.khscodes.khs.shell.oh-my-posh = { + enable = lib.mkEnableOption "Enables oh my posh khs setup"; + }; + config = lib.mkIf cfg.enable { + programs.oh-my-posh = { + enable = true; + enableBashIntegration = true; + enableZshIntegration = true; + enableFishIntegration = true; + enableNushellIntegration = true; + useTheme = null; + settings = { + "$schema" = "https://raw.githubusercontent.com/JanDeDobbeleer/oh-my-posh/main/themes/schema.json"; + "console_title_template" = "{{ .PWD }} @ {{ .HostName }}"; + "blocks" = [ + { + "alignment" = "left"; + "type" = "prompt"; + "segments" = [ + { + type = "shell"; + style = "diamond"; + leading_diamond = ""; + trailing_diamond = powerline_symbol; + "template" = "{{ .Name }}"; + "background" = colors.base00; + "foreground" = colors.blue; + "background_templates" = [ "{{ if eq .Name \"🐠\" }}${colors.base00}{{ end }}" ]; + properties = { + mapped_shell_names = { + "nushell" = "nu"; + "fish" = "🐟"; + "bash" = "$_"; + "zsh" = "%_"; + }; + }; + } + ( + { + "type" = "session"; + "background" = colors.red; + "foreground" = colors.base07; + "template" = "{{ if .SSHSession }} {{ .HostName }} {{ end }}"; + } + // segment_style + ) + ( + { + "type" = "path"; + "background" = colors.blue; + "foreground" = colors.base00; + "template" = " {{ .Path }} "; + "properties" = { + "style" = "full"; + }; + } + // segment_style + ) + ( + { + "type" = "git"; + "background" = colors.green; + "foreground" = colors.base00; + "background_templates" = [ + "{{ if or (.Working.Changed) (.Staging.Changed) }}${colors.yellow}{{ end }}" + ]; + "template" = + "{{ if .Detached }}➦ {{ trunc 7 .Commit.Sha }}{{ else }}{{ .UpstreamIcon }} {{ .Ref }}{{ end }}{{ if .Merge }}|merge{{ end }}{{ if .Rebase }}|rebase{{ end }}{{ if .CherryPick }}|cherrypick{{ end }}{{ if .Ahead}}↑{{ .Ahead }}{{ end }}{{ if .Behind }}↓{{ .Behind }}{{ end }}{{ if .Working.Changed}}●{{ end }}{{ if .Staging.Changed }}✚{{ end }}{{ if .StashCount }} 󰺿{{ end }} "; + "properties" = { + "fetch_status" = true; + "untracked_modes" = { + "/Users/user/Projects/oh-my-posh/" = "no"; + }; + fetch_upstream_icon = true; + upstream = { + git_icon = ""; + }; + "source" = "cli"; + }; + } + // segment_style + ) + ]; + } + { + alignment = "right"; + type = "rprompt"; + "segments" = [ + ( + { + type = "status"; + background = colors.base01; + background_templates = [ "{{ if .Error }}${colors.red}{{ end }}" ]; + foreground = colors.green; + foreground_templates = [ "{{ if .Error}}${bright-yellow-hashtag}{{ end }}" ]; + template = " {{ if .Error }}✘ {{ .Code }}{{ else }}✔{{ end }} "; + properties = { + always_enabled = true; + }; + } + // rsegment_style + ) + ( + { + type = "executiontime"; + foreground = colors.base00; + background = colors.yellow; + template = " ⌚{{ .FormattedMs }} "; + properties = { + always_enabled = false; + threshold = 3000; + style = "round"; + }; + } + // rsegment_style + ) + # Rendering this screws up spacing of the beginning of the prompt + ( + { + type = "nix-shell"; + background = colors.blue; + foreground = colors.base00; + template = " {{ .Type }} "; + } + // rsegment_style + ) + ]; + } + ]; + "terminal_background" = colors.base00; + "disable_notice" = true; + "final_space" = true; + "version" = 2; + }; + }; + }; +} diff --git a/nix/modules/home/khs/shell/zellij/default.nix b/nix/modules/home/khs/shell/zellij/default.nix new file mode 100644 index 0000000..d5bf5fe --- /dev/null +++ b/nix/modules/home/khs/shell/zellij/default.nix @@ -0,0 +1,505 @@ +{ + lib, + config, + pkgs, + system, + ... +}: +let + isDarwin = lib.strings.hasSuffix "-darwin" system; +in +{ + config = lib.mkIf (config.khscodes.khs.enable && config.khscodes.desktop.enable) { + # In built styles look off to me. And when alacritty is themed, + # this appears to not be needed. + stylix.targets.zellij.enable = false; + programs.zellij = { + enable = true; + settings = { + default_shell = lib.getExe pkgs.fish; + copy_on_select = false; + mouse_mode = true; + show_startup_tips = false; + scroll_buffer_size = 100000; + support_kitty_keyboard_protocol = true; + "keybinds clear-defaults=true" = { + normal = lib.attrsets.optionalAttrs isDarwin { + "bind \"Super c\"" = { + Copy = [ ]; + }; + }; + locked = { + "bind \"Ctrl g\"" = { + SwitchToMode = "Normal"; + }; + }; + resize = { + "bind \"Ctrl n\"" = { + SwitchToMode = "Normal"; + }; + "bind \"h\" \"Left\"" = { + Resize = "Increase Left"; + }; + "bind \"j\" \"Down\"" = { + Resize = "Increase Down"; + }; + "bind \"k\" \"Up\"" = { + Resize = "Increase Up"; + }; + "bind \"l\" \"Right\"" = { + Resize = "Increase Right"; + }; + "bind \"H\"" = { + Resize = "Decrease Left"; + }; + "bind \"J\"" = { + Resize = "Decrease Down"; + }; + "bind \"K\"" = { + Resize = "Decrease Up"; + }; + "bind \"L\"" = { + Resize = "Decrease Right"; + }; + "bind \"=\" \"+\"" = { + Resize = "Increase"; + }; + "bind \"-\"" = { + Resize = "Decrease"; + }; + }; + pane = { + "bind \"Ctrl\ p\"" = { + SwitchToMode = "Normal"; + }; + "bind \"h\" \"Left\"" = { + MoveFocus = "Left"; + }; + "bind \"l\" \"Right\"" = { + MoveFocus = "Right"; + }; + "bind \"j\" \"Down\"" = { + MoveFocus = "Down"; + }; + "bind \"k\" \"Up\"" = { + MoveFocus = "Up"; + }; + "bind \"p\"" = { + SwitchFocus = [ ]; + }; + "bind \"n\"" = { + NewPane = [ ]; + SwitchToMode = "Normal"; + }; + "bind \"d\"" = { + NewPane = "Down"; + SwitchToMode = "Normal"; + }; + "bind \"r\"" = { + NewPane = "Right"; + SwitchToMode = "Normal"; + }; + "bind \"x\"" = { + CloseFocus = [ ]; + SwitchToMode = "Normal"; + }; + "bind \"f\"" = { + ToggleFocusFullscreen = [ ]; + SwitchToMode = "Normal"; + }; + "bind \"z\"" = { + TogglePaneFrames = [ ]; + SwitchToMode = "Normal"; + }; + "bind \"w\"" = { + ToggleFloatingPanes = [ ]; + SwitchToMode = "Normal"; + }; + "bind \"e\"" = { + TogglePaneEmbedOrFloating = [ ]; + SwitchToMode = "Normal"; + }; + "bind \"c\"" = { + SwitchToMode = "RenamePane"; + PaneNameInput = 0; + }; + }; + move = { + "bind \"Ctrl h\"" = { + SwitchToMode = "Normal"; + }; + "bind \"n\" \"Tab\"" = { + MovePane = [ ]; + }; + "bind \"p\"" = { + MovePaneBackwards = [ ]; + }; + "bind \"h\" \"Left\"" = { + MovePane = "Left"; + }; + "bind \"j\" \"Down\"" = { + MovePane = "Down"; + }; + "bind \"k\" \"Up\"" = { + MovePane = "Up"; + }; + "bind \"l\" \"Right\"" = { + MovePane = "Right"; + }; + }; + tab = { + "bind \"Alt t\"" = { + SwitchToMode = "Normal"; + }; + "bind \"r\"" = { + SwitchToMode = "RenameTab"; + TabNameInput = 0; + }; + "bind \"h\" \"Left\" \"Up\" \"k\"" = { + GoToPreviousTab = [ ]; + }; + "bind \"l\" \"Right\" \"Down\" \"j\"" = { + GoToNextTab = [ ]; + }; + "bind \"n\"" = { + NewTab = [ ]; + SwitchToMode = "Normal"; + }; + "bind \"x\"" = { + CloseTab = [ ]; + SwitchToMode = "Normal"; + }; + "bind \"s\"" = { + ToggleActiveSyncTab = [ ]; + SwitchToMode = "Normal"; + }; + "bind \"1\"" = { + GoToTab = 1; + SwitchToMode = "Normal"; + }; + "bind \"2\"" = { + GoToTab = 2; + SwitchToMode = "Normal"; + }; + "bind \"3\"" = { + GoToTab = 3; + SwitchToMode = "Normal"; + }; + "bind \"4\"" = { + GoToTab = 4; + SwitchToMode = "Normal"; + }; + "bind \"5\"" = { + GoToTab = 5; + SwitchToMode = "Normal"; + }; + "bind \"6\"" = { + GoToTab = 6; + SwitchToMode = "Normal"; + }; + "bind \"7\"" = { + GoToTab = 7; + SwitchToMode = "Normal"; + }; + "bind \"8\"" = { + GoToTab = 8; + SwitchToMode = "Normal"; + }; + "bind \"9\"" = { + GoToTab = 9; + SwitchToMode = "Normal"; + }; + "bind \"Tab\"" = { + ToggleTab = [ ]; + }; + }; + scroll = { + "bind \"Ctrl s\"" = { + SwitchToMode = "Normal"; + }; + "bind \"e\"" = { + EditScrollback = [ ]; + SwitchToMode = "Normal"; + }; + "bind \"s\"" = { + SwitchToMode = "EnterSearch"; + SearchInput = 0; + }; + "bind \"Ctrl c\"" = { + ScrollToBottom = [ ]; + SwitchToMode = "Normal"; + }; + "bind \"j\" \"Down\"" = { + ScrollDown = [ ]; + }; + "bind \"k\" \"Up\"" = { + ScrollUp = [ ]; + }; + "bind \"Ctrl f\" \"PageDown\" \"Right\" \"l\"" = { + PageScrollDown = [ ]; + }; + "bind \"Ctrl b\" \"PageUp\" \"Left\" \"h\"" = { + PageScrollUp = [ ]; + }; + "bind \"d\"" = { + HalfPageScrollDown = [ ]; + }; + "bind \"u\"" = { + HalfPageScrollUp = [ ]; + }; + }; + search = { + "bind \"Ctrl s\"" = { + SwitchToMode = "Normal"; + }; + "bind \"Ctrl c\"" = { + ScrollToBottom = [ ]; + SwitchToMode = "Normal"; + }; + "bind \"j\" \"Down\"" = { + ScrollDown = [ ]; + }; + "bind \"k\" \"Up\"" = { + ScrollUp = [ ]; + }; + "bind \"Ctrl f\" \"PageDown\" \"Right\" \"l\"" = { + PageScrollDown = [ ]; + }; + "bind \"Ctrl b\" \"PageUp\" \"Left\" \"h\"" = { + PageScrollUp = [ ]; + }; + "bind \"d\"" = { + HalfPageScrollDown = [ ]; + }; + "bind \"u\"" = { + HalfPageScrollUp = [ ]; + }; + "bind \"n\"" = { + Search = "down"; + }; + "bind \"p\"" = { + Search = "up"; + }; + "bind \"c\"" = { + SearchToggleOption = "CaseSensitivity"; + }; + "bind \"w\"" = { + SearchToggleOption = "Wrap"; + }; + "bind \"o\"" = { + SearchToggleOption = "WholeWord"; + }; + }; + entersearch = { + "bind \"Ctrl c\" \"Esc\"" = { + SwitchToMode = "Scroll"; + }; + "bind \"Enter\"" = { + SwitchToMode = "Search"; + }; + }; + renametab = { + "bind \"Ctrl c\"" = { + SwitchToMode = "Normal"; + }; + "bind \"Esc\"" = { + UndoRenameTab = [ ]; + SwitchToMode = "Tab"; + }; + }; + renamepane = { + "bind \"Ctrl c\"" = { + SwitchToMode = "Normal"; + }; + "bind \"Esc\"" = { + UndoRenamePane = [ ]; + SwitchToMode = "Pane"; + }; + }; + session = { + "unbind \"Ctrl o\"" = [ ]; + "bind \"Ctrl q\"" = { + SwitchToMode = "Normal"; + }; + "bind \"Ctrl s\"" = { + SwitchToMode = "Scroll"; + }; + "bind \"d\"" = { + Detach = [ ]; + }; + }; + tmux = { + "bind \"[\"" = { + SwitchToMode = "Scroll"; + }; + "bind \"Ctrl b\"" = { + Write = 2; + SwitchToMode = "Normal"; + }; + "bind \"\\\"\"" = { + NewPane = "Down"; + SwitchToMode = "Normal"; + }; + "bind \"%\"" = { + NewPane = "Right"; + SwitchToMode = "Normal"; + }; + "bind \"z\"" = { + ToggleFocusFullscreen = [ ]; + SwitchToMode = "Normal"; + }; + "bind \"c\"" = { + NewTab = [ ]; + SwitchToMode = "Normal"; + }; + "bind \",\"" = { + SwitchToMode = "RenameTab"; + }; + "bind \"p\"" = { + GoToPreviousTab = [ ]; + SwitchToMode = "Normal"; + }; + "bind \"n\"" = { + GoToNextTab = [ ]; + SwitchToMode = "Normal"; + }; + "bind \"Left\"" = { + MoveFocus = "Left"; + SwitchToMode = "Normal"; + }; + "bind \"Right\"" = { + MoveFocus = "Right"; + SwitchToMode = "Normal"; + }; + "bind \"Down\"" = { + MoveFocus = "Down"; + SwitchToMode = "Normal"; + }; + "bind \"Up\"" = { + MoveFocus = "Up"; + SwitchToMode = "Normal"; + }; + "bind \"h\"" = { + MoveFocus = "Left"; + SwitchToMode = "Normal"; + }; + "bind \"l\"" = { + MoveFocus = "Right"; + SwitchToMode = "Normal"; + }; + "bind \"j\"" = { + MoveFocus = "Down"; + SwitchToMode = "Normal"; + }; + "bind \"k\"" = { + MoveFocus = "Up"; + SwitchToMode = "Normal"; + }; + "bind \"o\"" = { + FocusNextPane = [ ]; + }; + "bind \"d\"" = { + Detach = [ ]; + }; + "bind \"Space\"" = { + NextSwapLayout = [ ]; + }; + "bind \"x\"" = { + CloseFocus = [ ]; + SwitchToMode = "Normal"; + }; + }; + "shared_except \"locked\"" = { + "bind \"Ctrl g\"" = { + SwitchToMode = "Locked"; + }; + "bind \"Alt n\"" = { + NewPane = [ ]; + }; + "bind \"Alt h\" \"Alt Left\"" = { + MoveFocusOrTab = "Left"; + }; + "bind \"Alt l\" \"Alt Right\"" = { + MoveFocusOrTab = "Right"; + }; + "bind \"Alt j\" \"Alt Down\"" = { + MoveFocus = "Down"; + }; + "bind \"Alt k\" \"Alt Up\"" = { + MoveFocus = "Up"; + }; + "bind \"Alt =\" \"Alt +\"" = { + Resize = "Increase"; + }; + "bind \"Alt -\"" = { + Resize = "Decrease"; + }; + "bind \"Alt [\"" = { + PreviousSwapLayout = [ ]; + }; + "bind \"Alt ]\"" = { + NextSwapLayout = [ ]; + }; + }; + "shared_except \"normal\" \"locked\"" = { + "bind \"Enter\" \"Esc\"" = { + SwitchToMode = "Normal"; + }; + }; + "shared_except \"pane\" \"locked\"" = { + "bind \"Ctrl p\"" = { + SwitchToMode = "Pane"; + }; + }; + "shared_except \"resize\" \"locked\"" = { + "bind \"Ctrl n\"" = { + SwitchToMode = "Resize"; + }; + }; + "shared_except \"scroll\" \"locked\"" = { + "bind \"Ctrl s\"" = { + SwitchToMode = "Scroll"; + }; + }; + "shared_except \"session\" \"locked\"" = { + unbind = "Ctrl o"; + "bind \"Ctrl q\"" = { + SwitchToMode = "Session"; + }; + }; + "shared_except \"tab\" \"locked\"" = { + "bind \"Alt t\"" = { + SwitchToMode = "Tab"; + }; + }; + "shared_except \"move\" \"locked\"" = { + "bind \"Ctrl h\"" = { + SwitchToMode = "Move"; + }; + }; + "shared_except \"tmux\" \"locked\"" = { + "bind \"Ctrl b\"" = { + SwitchToMode = "Tmux"; + }; + }; + }; + plugins = { + tab-bar = { + path = "tab-bar"; + }; + status-bar = { + path = "status-bar"; + }; + strider = { + path = "strider"; + }; + compact-bar = { + path = "compact-bar"; + }; + }; + simplified-ui = false; + pane_frames = false; + }; + }; + }; +} diff --git a/nix/modules/home/khs/shell/zsh/default.nix b/nix/modules/home/khs/shell/zsh/default.nix new file mode 100644 index 0000000..1800055 --- /dev/null +++ b/nix/modules/home/khs/shell/zsh/default.nix @@ -0,0 +1,9 @@ +{ config, lib, ... }: +{ + config = lib.mkIf config.khscodes.khs.enable { + programs.zsh = { + enable = true; + shellAliases = config.khscodes.khs.shell.aliases; + }; + }; +} diff --git a/nix/modules/nixos/fqdn/default.nix b/nix/modules/nixos/fqdn/default.nix deleted file mode 100644 index 73cf099..0000000 --- a/nix/modules/nixos/fqdn/default.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ - config, - lib, - ... -}: -let - cfg = config.khscodes.fqdn; -in -{ - options.khscodes.fqdn = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - description = "Sets the FQDN of the machine. This is a prerequisite for many modules to be used"; - }; - - config = lib.mkIf (cfg != null) ( - let - hostname = builtins.head (lib.strings.splitString "." cfg); - domain = if hostname == cfg then null else (lib.strings.removePrefix "${hostname}." cfg); - in - { - networking.hostName = lib.mkForce hostname; - networking.domain = lib.mkForce domain; - boot.kernel.sysctl = { - "kernel.hostname" = cfg; - }; - } - ); -} diff --git a/nix/modules/nixos/infrastructure/hetzner-instance/default.nix b/nix/modules/nixos/infrastructure/hetzner-instance/default.nix index 27f9670..88cd2c8 100644 --- a/nix/modules/nixos/infrastructure/hetzner-instance/default.nix +++ b/nix/modules/nixos/infrastructure/hetzner-instance/default.nix @@ -6,7 +6,8 @@ }: let cfg = config.khscodes.infrastructure.hetzner-instance; - fqdn = config.khscodes.fqdn; + fqdn = config.khscodes.networking.fqdn; + provisioningUserData = config.khscodes.infrastructure.provisioning.instanceUserData; firewallTcpRules = lib.lists.map (p: { direction = "in"; protocol = "tcp"; @@ -52,7 +53,7 @@ in dnsNames = lib.mkOption { type = lib.types.listOf lib.types.str; description = "DNS names for the server"; - default = [ fqdn ]; + default = lib.lists.unique ([ fqdn ] ++ config.khscodes.networking.aliases); }; bucket = { key = lib.mkOption { @@ -61,14 +62,6 @@ in default = "${fqdn}.tfstate"; }; }; - secretsSource = lib.mkOption { - type = lib.types.enum [ - "bitwarden" - "vault" - ]; - description = "Whether to load opentofu secrets from Bitwarden or Vault"; - default = "vault"; - }; datacenter = lib.mkOption { type = lib.types.str; description = "The Hetzner datacenter to create a server in"; @@ -158,26 +151,23 @@ in inherit labels; name = fqdn; initial_image = "debian-12"; - rdns = fqdn; + rdns = lib.mkIf cfg.mapRdns fqdn; ssh_keys = [ config.khscodes.hcloud.output.data.ssh_key.khs.id ]; + user_data = provisioningUserData; }; khscodes.cloudflare = { enable = true; dns = { enable = true; zone_name = tldFromFqdn fqdn; - aRecords = [ - { - inherit fqdn; - content = config.khscodes.hcloud.output.server.compute.ipv4_address; - } - ]; - aaaaRecords = [ - { - inherit fqdn; - content = config.khscodes.hcloud.output.server.compute.ipv6_address; - } - ]; + aRecords = lib.lists.map (d: { + fqdn = d; + content = config.khscodes.hcloud.output.server.compute.ipv4_address; + }) cfg.dnsNames; + aaaaRecords = lib.lists.map (d: { + fqdn = d; + content = config.khscodes.hcloud.output.server.compute.ipv6_address; + }) cfg.dnsNames; }; }; resource.hcloud_firewall.fw = lib.mkIf firewallEnable { @@ -205,19 +195,14 @@ in { assertions = [ { - assertion = config.khscodes.fqdn != null; - message = "Must set config.khscodes.fqdn when using opentofu"; + assertion = config.khscodes.networking.fqdn != null; + message = "Must set config.khscodes.networking.fqdn when using opentofu"; } ]; + khscodes.services.read-vault-auth-from-userdata.url = "http://169.254.169.254/latest/user-data"; khscodes.infrastructure.provisioning.pre = { modules = modules; - secretsSource = cfg.secretsSource; - endpoints = [ - "aws" - "cloudflare" - "hcloud" - ]; }; } ); diff --git a/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix b/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix index 110b669..39fd74b 100644 --- a/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix +++ b/nix/modules/nixos/infrastructure/khs-openstack-instance/default.nix @@ -6,7 +6,8 @@ }: let cfg = config.khscodes.infrastructure.khs-openstack-instance; - fqdn = config.khscodes.fqdn; + fqdn = config.khscodes.networking.fqdn; + provisioningUserData = config.khscodes.infrastructure.provisioning.instanceUserData; firewallTcpRules = lib.lists.flatten ( lib.lists.map (p: [ { @@ -74,7 +75,9 @@ in dnsNames = lib.mkOption { type = lib.types.listOf lib.types.str; description = "DNS names for the instance"; - default = [ fqdn ]; + default = lib.lists.unique ( + [ config.khscodes.networking.fqdn ] ++ config.khscodes.networking.aliases + ); }; bucket = { key = lib.mkOption { @@ -83,14 +86,6 @@ in default = "${fqdn}.tfstate"; }; }; - secretsSource = lib.mkOption { - type = lib.types.enum [ - "bitwarden" - "vault" - ]; - description = "Whether to load opentofu secrets from Bitwarden or Vault"; - default = "vault"; - }; flavor = lib.mkOption { type = lib.types.nullOr lib.types.str; description = "The server type to create"; @@ -101,65 +96,17 @@ in description = "SSH key for the server (this only applies to the initial creation, deploying NixOS will render this key useless). Changing this will recreate the instance"; default = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCqY0FHnWFKfLG2yfgr4qka5sR9CK+EMAhzlHUkaQyWHTKD+G0/vC/fNPyL1VV3Dxc/ajxGuPzVE+mBMoyxazL3EtuCDOVvHJ5CR+MUSEckg/DDwcGHqy6rC8BvVVpTAVL04ByQdwFnpE1qNSBaQLkxaFVdtriGKkgMkc7+UNeYX/bv7yn+APqfP1a3xr6wdkSSdO8x4N2jsSygOIMx10hLyCV4Ueu7Kp8Ww4rGY8j5o7lKJhbgfItBfSOuQHdppHVF/GKYRhdnK6Y2fZVYbhq4KipUtclbZ6O/VYd8/sOO98+LMm7cOX+K35PQjUpYgcoNy5+Sw3CNS/NHn4JvOtTaUEYP7fK6c9LhMULOO3T7Cm6TMdiFjUKHkyG+s2Mu/LXJJoilw571zwuh6chkeitW8+Ht7k0aPV96kNEvTdoXwLhBifVEaChlAsLAzSUjUq+YYCiXVk0VIXCZQWKj8LoVNTmaqDksWwbcT64fw/FpVC0N18WHbKcFUEIW/O4spJMa30CQwf9FeqpoWoaF1oRClCSDPvX0AauCu0JcmRinz1/JmlXljnXWbSfm20/V+WyvktlI0wTD0cdpNuSasT9vS77YfJ8nutcWWZKSkCj4R4uHeCNpDTX5YXzapy7FxpM9ANCXLIvoGX7Yafba2Po+er7SSsUIY1AsnBBr8ZoDVw=="; }; + dns = { + mapIpv4Address = lib.mkEnableOption { + type = lib.types.bool; + description = "Also add the IPv4 address to DNS"; + default = false; + }; + }; extraFirewallRules = lib.mkOption { type = lib.types.listOf lib.types.attrs; description = "Extra firewall rules added to the instance"; - default = [ - { - direction = "egress"; - ethertype = "IPv4"; - protocol = "tcp"; - port = 80; - remote_subnet = "0.0.0.0/0"; - } - { - direction = "egress"; - ethertype = "IPv6"; - protocol = "tcp"; - port = 80; - remote_subnet = "::/0"; - } - { - direction = "egress"; - ethertype = "IPv4"; - protocol = "tcp"; - port = 443; - remote_subnet = "0.0.0.0/0"; - } - { - direction = "egress"; - ethertype = "IPv6"; - protocol = "tcp"; - port = 443; - remote_subnet = "::/0"; - } - { - direction = "egress"; - ethertype = "IPv4"; - protocol = "udp"; - port = 443; - remote_subnet = "0.0.0.0/0"; - } - { - direction = "egress"; - ethertype = "IPv6"; - protocol = "udp"; - port = 443; - remote_subnet = "::/0"; - } - { - direction = "egress"; - ethertype = "IPv4"; - protocol = "icmp"; - remote_subnet = "0.0.0.0/0"; - } - { - direction = "egress"; - ethertype = "IPv6"; - protocol = "icmp"; - remote_subnet = "::/0"; - } - ]; + default = [ ]; }; }; config = lib.mkIf cfg.enable ( @@ -188,6 +135,7 @@ in flavor = cfg.flavor; ssh_public_key = cfg.ssh_key; firewall_rules = firewallRules; + user_data = provisioningUserData; }; khscodes.unifi.enable = true; khscodes.unifi.static_route.compute = { @@ -201,18 +149,16 @@ in dns = { enable = true; zone_name = tldFromFqdn fqdn; - aRecords = [ - { - inherit fqdn; + aRecords = lib.mkIf cfg.dns.mapIpv4Address ( + lib.lists.map (d: { + fqdn = d; content = config.khscodes.openstack.output.compute_instance.compute.ipv4_address; - } - ]; - aaaaRecords = [ - { - inherit fqdn; - content = config.khscodes.openstack.output.compute_instance.compute.ipv6_address; - } - ]; + }) cfg.dnsNames + ); + aaaaRecords = lib.lists.map (d: { + fqdn = d; + content = config.khscodes.openstack.output.compute_instance.compute.ipv6_address; + }) cfg.dnsNames; }; }; output.ipv4_address = { @@ -232,21 +178,23 @@ in { assertions = [ { - assertion = config.khscodes.fqdn != null; - message = "Must set config.khscodes.fqdn when using opentofu"; + assertion = config.khscodes.networking.fqdn != null; + message = "Must set config.khscodes.networking.fqdn when using opentofu"; } ]; - + khscodes.services.openssh = { + enable = true; + hostCertificate = { + enable = true; + }; + }; + khscodes.services.read-vault-auth-from-userdata.url = "http://169.254.169.254/openstack/2012-08-10/user_data"; + # khs openstack hosted servers are cannot use http-01 challenges (or maybe they can through ipv6?) + # so enable dns-01. + khscodes.security.acme.dns01Enabled = true; khscodes.infrastructure.provisioning = { pre = { modules = modules; - secretsSource = cfg.secretsSource; - endpoints = [ - "aws" - "cloudflare" - "openstack" - "unifi" - ]; }; preImageUsername = "debian"; }; diff --git a/nix/modules/nixos/infrastructure/openbao/default.nix b/nix/modules/nixos/infrastructure/openbao/default.nix new file mode 100644 index 0000000..56e0850 --- /dev/null +++ b/nix/modules/nixos/infrastructure/openbao/default.nix @@ -0,0 +1,9 @@ +{ lib, ... }: +{ + options.khscodes.infrastructure.openbao = { + domain = lib.mkOption { + type = lib.types.str; + default = "vault.kaareskovgaard.net"; + }; + }; +} diff --git a/nix/modules/nixos/infrastructure/provisioning/default.nix b/nix/modules/nixos/infrastructure/provisioning/default.nix index afb8c28..b533d35 100644 --- a/nix/modules/nixos/infrastructure/provisioning/default.nix +++ b/nix/modules/nixos/infrastructure/provisioning/default.nix @@ -21,7 +21,73 @@ let description = "Where to get the secrets for the provisioning from"; default = "vault"; }; - endpoints = lib.mkOption { + }; + usesEndpoint = + search: endpoint: config: + if lib.strings.hasInfix search (builtins.readFile config) then [ endpoint ] else [ ]; + endpointsMaps = [ + { + search = "cloudflare/cloudflare"; + endpoint = "cloudflare"; + } + { + search = "terraform-provider-openstack/openstack"; + endpoint = "openstack"; + } + { + search = "paultyng/unifi"; + endpoint = "unifi"; + } + { + search = "hashicorp/vault"; + endpoint = "vault"; + } + { + search = ".r2.cloudflarestorage.com"; + endpoint = "aws"; + } + ]; + endpointsUsed = + config: + if config == null then + [ ] + else + lib.lists.flatten (lib.lists.map (c: usesEndpoint c.search c.endpoint config) endpointsMaps); + preConfig = + if lib.lists.length cfg.pre.modules > 0 then + inputs.terranix.lib.terranixConfiguration { + system = pkgs.hostPlatform.system; + modules = cfg.pre.modules; + extraArgs = { inherit lib inputs; }; + } + else + null; + preEndpoints = endpointsUsed preConfig; + postConfig = + if lib.lists.length cfg.post.modules > 0 then + inputs.terranix.lib.terranixConfiguration { + system = pkgs.hostPlatform.system; + modules = cfg.post.modules; + extraArgs = { inherit lib inputs; }; + } + else + null; + postEndpoints = endpointsUsed postConfig; +in +{ + options.khscodes.infrastructure.provisioning = { + pre = provisioning; + post = provisioning; + instanceUserData = lib.mkOption { + type = lib.types.str; + description = "User data that should be added to the instance during provisioning"; + default = ""; + }; + preConfig = lib.mkOption { + type = lib.types.nullOr lib.types.path; + description = "The generated config for the pre provisioning, if any was specified"; + }; + preEndpoints = lib.mkOption { type = lib.types.listOf ( lib.types.enum [ "openstack" @@ -29,21 +95,13 @@ let "unifi" "hcloud" "cloudflare" + "vault" + "authentik" ] ); description = "Needed endpoints to be used during provisioning"; default = [ ]; }; - }; -in -{ - options.khscodes.infrastructure.provisioning = { - pre = provisioning; - post = provisioning; - preConfig = lib.mkOption { - type = lib.types.nullOr lib.types.path; - description = "The generated config for the pre provisioning, if any was specified"; - }; preImageUsername = lib.mkOption { type = lib.types.str; description = "The username for the image being deployed before being swapped for NixOS"; @@ -53,24 +111,27 @@ in type = lib.types.nullOr lib.types.path; description = "The generated config for the post provisioning, if any was specified"; }; + postEndpoints = lib.mkOption { + type = lib.types.listOf ( + lib.types.enum [ + "openstack" + "aws" + "unifi" + "hcloud" + "cloudflare" + "vault" + "authentik" + ] + ); + description = "Needed endpoints to be used during provisioning"; + default = [ ]; + }; }; config = { - khscodes.infrastructure.provisioning.preConfig = - if lib.lists.length cfg.pre.modules > 0 then - inputs.terranix.lib.terranixConfiguration { - system = pkgs.hostPlatform.system; - modules = cfg.pre.modules; - } - else - null; - khscodes.infrastructure.provisioning.postConfig = - if lib.lists.length cfg.post.modules > 0 then - inputs.terranix.lib.terranixConfiguration { - system = pkgs.hostPlatform.system; - modules = cfg.post.modules; - } - else - null; + khscodes.infrastructure.provisioning.preConfig = preConfig; + khscodes.infrastructure.provisioning.preEndpoints = preEndpoints; + khscodes.infrastructure.provisioning.postConfig = postConfig; + khscodes.infrastructure.provisioning.postEndpoints = postEndpoints; }; } diff --git a/nix/modules/nixos/infrastructure/vault-loki-sender/default.nix b/nix/modules/nixos/infrastructure/vault-loki-sender/default.nix new file mode 100644 index 0000000..d6023a5 --- /dev/null +++ b/nix/modules/nixos/infrastructure/vault-loki-sender/default.nix @@ -0,0 +1,83 @@ +{ + config, + lib, + ... +}: +let + cfg = config.khscodes.infrastructure.vault-loki-sender; + fqdn = config.khscodes.networking.fqdn; + vaultRoleName = config.khscodes.infrastructure.vault-server-approle.role_name; + client_key = "/var/lib/alloy/loki.key"; + client_cert = "/var/lib/alloy/loki.cert"; +in +{ + options.khscodes.infrastructure.vault-loki-sender = { + enable = lib.mkEnableOption "Configures the server approle to allow sending data to loki"; + terranixBackendName = lib.mkOption { + type = lib.types.str; + description = "This should only be configured for the server hosting loki, to allow setting up dependencies in terraform"; + default = "loki-mtls"; + }; + }; + + config = lib.mkIf cfg.enable { + khscodes.infrastructure.vault-server-approle = { + enable = true; + policy = { + "loki-mtls" = { + capabilities = [ "read" ]; + }; + "loki-mtls/issue/${fqdn}" = { + capabilities = [ + "create" + "update" + ]; + }; + }; + stageModules = [ + ( + { ... }: + { + khscodes.vault.pki_secret_backend_role."${vaultRoleName}-loki" = { + name = vaultRoleName; + backend = cfg.terranixBackendName; + allowed_domains = [ fqdn ]; + allow_bare_domains = true; + enforce_hostnames = true; + server_flag = false; + client_flag = true; + }; + } + ) + ]; + }; + khscodes.services.vault-agent.templates = [ + { + contents = '' + {{- with pkiCert "loki-mtls/issue/${fqdn}" "common_name=${fqdn}" -}} + {{ .Key }} + {{ .Cert }} + {{ .CA }} + {{ .Key | writeToFile "${client_key}" "${config.khscodes.services.alloy.user}" "${config.khscodes.services.alloy.group}" "0600" }} + {{ .Cert | writeToFile "${client_cert}" "${config.khscodes.services.alloy.user}" "${config.khscodes.services.alloy.group}" "0644" }} + {{- end -}} + ''; + destination = "/var/lib/alloy/cache.key"; + owner = "alloy"; + group = "alloy"; + perms = "0600"; + reloadOrRestartUnits = [ "alloy.service" ]; + } + ]; + khscodes.services.alloy = { + enable = true; + environment = { + LOKI_CLIENT_KEY = client_key; + LOKI_CLIENT_CERT = client_cert; + }; + }; + environment.etc."alloy/loki.alloy" = { + source = ./loki.alloy; + }; + }; +} diff --git a/nix/modules/nixos/infrastructure/vault-loki-sender/loki.alloy b/nix/modules/nixos/infrastructure/vault-loki-sender/loki.alloy new file mode 100644 index 0000000..5deef96 --- /dev/null +++ b/nix/modules/nixos/infrastructure/vault-loki-sender/loki.alloy @@ -0,0 +1,78 @@ +// Collect logs from systemd journal for node_exporter integration +loki.source.journal "logs_integrations_integrations_node_exporter_journal_scrape" { + // Only collect logs from the last 24 hours + max_age = "24h0m0s" + // Apply relabeling rules to the logs + relabel_rules = discovery.relabel.logs_integrations_integrations_node_exporter_journal_scrape.rules + // Send logs to the local Loki instance + forward_to = [loki.write.local.receiver] +} + +// Define which log files to collect for node_exporter +local.file_match "logs_integrations_integrations_node_exporter_direct_scrape" { + path_targets = [{ + // Target localhost for log collection + __address__ = "localhost", + // Collect standard system logs + __path__ = "/var/log/{syslog,messages,*.log}", + // Add instance label with hostname + instance = constants.hostname, + // Add job label for logs + job = "integrations/node_exporter", + }] +} + +// Define relabeling rules for systemd journal logs +discovery.relabel "logs_integrations_integrations_node_exporter_journal_scrape" { + targets = [] + + rule { + // Extract systemd unit information into a label + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + + rule { + // Extract boot ID information into a label + source_labels = ["__journal__boot_id"] + target_label = "boot_id" + } + + rule { + // Extract transport information into a label + source_labels = ["__journal__transport"] + target_label = "transport" + } + + rule { + // Extract log priority into a level label + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } + + rule { + // Set the instance label to the hostname of the machine + target_label = "instance" + replacement = constants.hostname + } +} + +// Collect logs from files for node_exporter +loki.source.file "logs_integrations_integrations_node_exporter_direct_scrape" { + // Use targets defined in local.file_match + targets = local.file_match.logs_integrations_integrations_node_exporter_direct_scrape.targets + // Send logs to the local Loki instance + forward_to = [loki.write.local.receiver] +} + +// Define where to send logs for storage +loki.write "local" { + endpoint { + // Send logs to a locally running Loki instance + url = "https://loki.kaareskovgaard.net/loki/api/v1/push" + tls_config { + cert_file = sys.env("LOKI_CLIENT_CERT") + key_file = sys.env("LOKI_CLIENT_KEY") + } + } +} diff --git a/nix/modules/nixos/infrastructure/vault-prometheus-sender/default.nix b/nix/modules/nixos/infrastructure/vault-prometheus-sender/default.nix new file mode 100644 index 0000000..b4237b6 --- /dev/null +++ b/nix/modules/nixos/infrastructure/vault-prometheus-sender/default.nix @@ -0,0 +1,83 @@ +{ + config, + lib, + ... +}: +let + cfg = config.khscodes.infrastructure.vault-prometheus-sender; + fqdn = config.khscodes.networking.fqdn; + vaultRoleName = config.khscodes.infrastructure.vault-server-approle.role_name; + client_key = "/var/lib/alloy/prometheus.key"; + client_cert = "/var/lib/alloy/prometheus.cert"; +in +{ + options.khscodes.infrastructure.vault-prometheus-sender = { + enable = lib.mkEnableOption "Configures the server approle to allow sending data to prometheus"; + terranixBackendName = lib.mkOption { + type = lib.types.str; + description = "This should only be configured for the server hosting vault, to allow setting up dependencies in terraform"; + default = "prometheus-mtls"; + }; + }; + + config = lib.mkIf cfg.enable { + khscodes.infrastructure.vault-server-approle = { + enable = true; + policy = { + "prometheus-mtls" = { + capabilities = [ "read" ]; + }; + "prometheus-mtls/issue/${fqdn}" = { + capabilities = [ + "create" + "update" + ]; + }; + }; + stageModules = [ + ( + { ... }: + { + khscodes.vault.pki_secret_backend_role."${vaultRoleName}-prometheus" = { + name = vaultRoleName; + backend = cfg.terranixBackendName; + allowed_domains = [ fqdn ]; + allow_bare_domains = true; + enforce_hostnames = true; + server_flag = false; + client_flag = true; + }; + } + ) + ]; + }; + khscodes.services.vault-agent.templates = [ + { + contents = '' + {{- with pkiCert "prometheus-mtls/issue/${fqdn}" "common_name=${fqdn}" -}} + {{ .Key }} + {{ .Cert }} + {{ .CA }} + {{ .Key | writeToFile "${client_key}" "${config.khscodes.services.alloy.user}" "${config.khscodes.services.alloy.group}" "0600" }} + {{ .Cert | writeToFile "${client_cert}" "${config.khscodes.services.alloy.user}" "${config.khscodes.services.alloy.group}" "0644" }} + {{- end -}} + ''; + destination = "/var/lib/alloy/cache.key"; + owner = "alloy"; + group = "alloy"; + perms = "0600"; + reloadOrRestartUnits = [ "alloy.service" ]; + } + ]; + khscodes.services.alloy = { + enable = true; + environment = { + PROMETHEUS_CLIENT_KEY = client_key; + PROMETHEUS_CLIENT_CERT = client_cert; + }; + }; + environment.etc."alloy/prometheus.alloy" = { + source = ./prometheus.alloy; + }; + }; +} diff --git a/nix/modules/nixos/infrastructure/vault-prometheus-sender/prometheus.alloy b/nix/modules/nixos/infrastructure/vault-prometheus-sender/prometheus.alloy new file mode 100644 index 0000000..0513a68 --- /dev/null +++ b/nix/modules/nixos/infrastructure/vault-prometheus-sender/prometheus.alloy @@ -0,0 +1,72 @@ +// This block relabels metrics coming from node_exporter to add standard labels +discovery.relabel "integrations_node_exporter" { + targets = prometheus.exporter.unix.integrations_node_exporter.targets + + rule { + // Set the instance label to the hostname of the machine + target_label = "instance" + replacement = constants.hostname + } + + rule { + // Set a standard job name for all node_exporter metrics + target_label = "job" + replacement = "integrations/node_exporter" + } +} +// +// Configure the node_exporter integration to collect system metrics +prometheus.exporter.unix "integrations_node_exporter" { + // Disable unnecessary collectors to reduce overhead + disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs", "zfs"] + enable_collectors = ["meminfo"] + + filesystem { + // Exclude filesystem types that aren't relevant for monitoring + fs_types_exclude = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$" + // Exclude mount points that aren't relevant for monitoring + mount_points_exclude = "^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)" + // Timeout for filesystem operations + mount_timeout = "5s" + } + + netclass { + // Ignore virtual and container network interfaces + ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$" + } + + netdev { + // Exclude virtual and container network interfaces from device metrics + device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$" + } + + +} + +// Define how to scrape metrics from the node_exporter +prometheus.scrape "integrations_node_exporter" { +scrape_interval = "15s" + // Use the targets with labels from the discovery.relabel component + targets = discovery.relabel.integrations_node_exporter.output + // Send the scraped metrics to the relabeling component + forward_to = [otelcol.receiver.prometheus.default.receiver] +} + +otelcol.receiver.prometheus "default" { + output { + metrics = [otelcol.exporter.otlphttp.default.input] + } +} + + +// Define where to send the metrics for storage +otelcol.exporter.otlphttp "default" { + client { + endpoint = "https://prometheus.kaareskovgaard.net/api/v1/otlp/" + tls { + cert_file = sys.env("PROMETHEUS_CLIENT_CERT") + key_file = sys.env("PROMETHEUS_CLIENT_KEY") + } + } + encoding = "proto" +} diff --git a/nix/modules/nixos/infrastructure/vault-server-approle/default.nix b/nix/modules/nixos/infrastructure/vault-server-approle/default.nix new file mode 100644 index 0000000..4f6b3ba --- /dev/null +++ b/nix/modules/nixos/infrastructure/vault-server-approle/default.nix @@ -0,0 +1,141 @@ +{ + config, + lib, + inputs, + ... +}: +let + cfg = config.khscodes.infrastructure.vault-server-approle; + vaultDomain = config.khscodes.infrastructure.openbao.domain; +in +{ + options.khscodes.infrastructure.vault-server-approle = { + enable = lib.mkEnableOption "Enables creating an OpenBAO role for the server"; + stage = lib.mkOption { + type = lib.types.enum [ + "pre" + "post" + ]; + description = "The provisioning stage that should include the provisioning. This should be pre for every server except the OpenBAO server itself"; + default = "pre"; + }; + path = lib.mkOption { + type = lib.types.str; + default = "approle"; + description = "Sets the path, as a terraform expression, for the approle to get created in. Not useful for most instances, but useful when doing bootstrapping, to establish a dependency."; + }; + role_name = lib.mkOption { + type = lib.types.str; + description = "Name of the role being created"; + default = config.networking.fqdnOrHostName; + }; + policy = lib.mkOption { + type = lib.types.attrsOf ( + lib.khscodes.mkSubmodule { + options = { + capabilities = lib.mkOption { + type = lib.types.listOf ( + lib.types.enum [ + "create" + "update" + "patch" + "read" + "delete" + "list" + ] + ); + }; + }; + description = "Vault role policy"; + } + ); + }; + stageModules = lib.mkOption { + type = lib.types.listOf lib.types.anything; + description = "Extra modules to add to the configured stage"; + default = [ ]; + }; + }; + + config = lib.mkIf cfg.enable { + khscodes.services.read-vault-auth-from-userdata.enable = cfg.stage == "pre"; + khscodes.services.vault-agent.enable = true; + khscodes.infrastructure.provisioning.${cfg.stage} = { + modules = [ + ( + { config, lib, ... }: + { + imports = [ inputs.self.terranixModules.vault ]; + output = lib.mkIf (cfg.stage == "post") { + role-id = { + value = config.khscodes.vault.output.approle_auth_backend_role.${cfg.role_name}.role_id; + sensitive = false; + }; + secret-id-wrapped = { + value = + config.khscodes.vault.output.approle_auth_backend_role_secret_id.${cfg.role_name}.wrapping_token; + sensitive = true; + }; + }; + khscodes.vault = { + enable = true; + domain = vaultDomain; + approle_auth_backend_role.${cfg.role_name} = { + backend = cfg.path; + role_name = cfg.role_name; + # Secret IDs never expire, to allow vault agent to restart without issues. + # TODO: Look into doing this in a better way going forward, such that this won't + # be an issue under normal circumstances, but vault-agents (or instances) + # being offline for long periods of time should invalidate the secret id's. + secret_id_ttl = 0; + secret_id_num_uses = 0; + token_ttl = 20 * 60; + token_max_ttl = 30 * 60; + token_policies = [ cfg.role_name ]; + }; + approle_auth_backend_role_secret_id.${cfg.role_name} = { + backend = cfg.path; + # Not hardcoding the role name here, as reading it like this will create a dependency + # on the role being created first, which is needed. + role_name = config.khscodes.vault.output.approle_auth_backend_role.${cfg.role_name}.role_name; + # Should only be 5-10 mins once done testing + wrapping_ttl = 5 * 60; + + # This should simply mean that we never attempt to recreate the secret id, as we don't want a rerun of the + # provisioning to invalidate the existing secret id, nor recreate the entire server. + with_wrapped_accessor = true; + lifecycle = { + ignore_changes = [ + "num_uses" + "ttl" + ]; + }; + }; + policy.${cfg.role_name} = { + name = cfg.role_name; + policy = lib.strings.concatStringsSep "\n\n" ( + lib.lists.map ( + { name, value }: + '' + path "${name}" { + capabilities = ${builtins.toJSON value.capabilities} + } + '' + ) (lib.attrsToList cfg.policy) + ); + }; + }; + } + ) + ] ++ cfg.stageModules; + }; + # I can only provide the user data if the stage is pre (along with the instance creation) + # Also I should probably find a way of injecting this in a nicer way than this mess. + khscodes.infrastructure.provisioning.instanceUserData = lib.mkIf (cfg.stage == "pre") '' + { + "VAULT_ROLE_ID": "''${ vault_approle_auth_backend_role.${lib.khscodes.sanitize-terraform-name cfg.role_name}.role_id }", + "VAULT_SECRET_ID_WRAPPED": "''${ vault_approle_auth_backend_role_secret_id.${lib.khscodes.sanitize-terraform-name cfg.role_name}.wrapping_token }" + } + ''; + }; +} diff --git a/nix/modules/nixos/machine/default.nix b/nix/modules/nixos/machine/default.nix new file mode 100644 index 0000000..e68cece --- /dev/null +++ b/nix/modules/nixos/machine/default.nix @@ -0,0 +1,25 @@ +{ config, lib, ... }: +let + cfg = config.khscodes.machine; +in +rec { + options.khscodes.machine = { + type = lib.mkOption { + type = lib.types.enum [ + "server" + "desktop" + ]; + description = "The kind of machine that is running"; + }; + }; + config = { + home-manager.sharedModules = [ + { + inherit options; + config = { + khscodes.desktop.enable = cfg.type == "desktop"; + }; + } + ]; + }; +} diff --git a/nix/modules/nixos/networking/fqdn/default.nix b/nix/modules/nixos/networking/fqdn/default.nix new file mode 100644 index 0000000..d846c3a --- /dev/null +++ b/nix/modules/nixos/networking/fqdn/default.nix @@ -0,0 +1,39 @@ +{ + config, + lib, + ... +}: +let + cfg = config.khscodes.networking; +in +{ + options.khscodes.networking = { + fqdn = lib.mkOption { + type = lib.types.str; + default = null; + description = "Sets the FQDN of the machine. This is a prerequisite for many modules to be used"; + }; + aliases = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + }; + }; + + config = + let + hostname = builtins.head (lib.strings.splitString "." cfg.fqdn); + domain = if hostname == cfg then null else (lib.strings.removePrefix "${hostname}." cfg.fqdn); + in + { + networking.hostName = lib.mkForce hostname; + networking.domain = lib.mkForce domain; + networking.fqdn = cfg.fqdn; + # Add the name of the server to the ssh host certificate domains, but let other configs enable getting the host certificates. + khscodes.services.openssh.hostCertificate.hostNames = lib.lists.unique ( + [ cfg.fqdn ] ++ cfg.aliases + ); + boot.kernel.sysctl = { + "kernel.hostname" = cfg.fqdn; + }; + }; +} diff --git a/nix/modules/nixos/nix/default.nix b/nix/modules/nixos/nix/default.nix new file mode 100644 index 0000000..ed3b7a2 --- /dev/null +++ b/nix/modules/nixos/nix/default.nix @@ -0,0 +1,15 @@ +{ config, lib, ... }: +let + cfg = config.khscodes.nix; +in +{ + options.khscodes.nix = { + nix-community.enable = lib.mkEnableOption "Enables nix-community substituters"; + }; + config = { + nix.settings = lib.mkIf cfg.nix-community.enable { + substituters = [ "https://nix-community.cachix.org" ]; + trusted-public-keys = [ "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" ]; + }; + }; +} diff --git a/nix/modules/nixos/openstack/default.nix b/nix/modules/nixos/openstack/default.nix index d372a03..9a54b5b 100644 --- a/nix/modules/nixos/openstack/default.nix +++ b/nix/modules/nixos/openstack/default.nix @@ -16,10 +16,14 @@ in }; }; config = lib.mkIf cfg.enable { - disko = lib.khscodes.disko-root-lvm-bios { - device = "/dev/sda"; - diskName = cfg.diskName; - }; + disko = lib.mkDefault ( + lib.khscodes.disko-root-lvm-bios { + device = "/dev/sda"; + diskName = cfg.diskName; + } + ); + # When this is set as the default, outbound ipv6 doesn't work on the instance. + networking.tempAddresses = "disabled"; boot.loader.grub.efiSupport = false; boot.loader.timeout = 1; khscodes.virtualisation.qemu-guest.enable = true; diff --git a/nix/modules/nixos/opentofu-openbao/default.nix b/nix/modules/nixos/opentofu-openbao/default.nix deleted file mode 100644 index 32e84c8..0000000 --- a/nix/modules/nixos/opentofu-openbao/default.nix +++ /dev/null @@ -1 +0,0 @@ -{ pkgs, ... }: { } diff --git a/nix/modules/nixos/os/auto-update/default.nix b/nix/modules/nixos/os/auto-update/default.nix new file mode 100644 index 0000000..a0dcb4c --- /dev/null +++ b/nix/modules/nixos/os/auto-update/default.nix @@ -0,0 +1,55 @@ +{ + config, + lib, + inputs, + pkgs, + ... +}: +let + cfg = config.khscodes.os.auto-update; + upgradePath = "/var/lib/system-upgrade"; + upgradeVersion = "/var/lib/system-upgrade.version"; + prepareUpgrade = pkgs.writeShellApplication { + runtimeInputs = [ + pkgs.uutils-coreutils-noprefix + pkgs.nix + ]; + name = "nixos-prepare-upgrade"; + text = '' + + + current_version="" + if [[ -f ${upgradeVersion} ]]; then + current_version="$(cat ${upgradeVersion})" + fi + if [[ "$current_version" != "${inputs.self.outPath}" ]]; then + rm -rf ${upgradePath} + cp -r ${inputs.self.outPath} ${upgradePath} + echo -n ${inputs.self.outPath} > ${upgradeVersion} + fi + cd ${upgradePath} + NIX_CONFIG="extra-experimental-features=flake nix-command" nix flake update + ''; + }; +in +{ + options.khscodes.os.auto-update = { + enable = lib.mkEnableOption "Enables automatic OS updates"; + dates = "02:00"; + randomizedDelaySec = "45min"; + }; + config = lib.mkIf cfg.enable { + system.autoUpgrade = { + enable = true; + flake = upgradePath; + }; + systemd.services.nixos-upgrade-prepare-flake = { + wantedBy = [ "nixos-upgrade.service" ]; + before = [ "nixos-upgrade.service" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = lib.getExe prepareUpgrade; + }; + }; + }; +} diff --git a/nix/modules/nixos/security/acme/default.nix b/nix/modules/nixos/security/acme/default.nix new file mode 100644 index 0000000..d4955d0 --- /dev/null +++ b/nix/modules/nixos/security/acme/default.nix @@ -0,0 +1,59 @@ +{ config, lib, ... }: +let + cfg = config.khscodes.security.acme; + vaultAgentCredentialsFile = "/var/lib/vault-agent/acme/cloudflare-api-token"; + cloudflareSecret = "opentofu/data/cloudflare"; + acmeServicesToRestart = lib.lists.map (a: "acme-${a}.service") ( + lib.attrsets.attrNames config.security.acme.certs + ); +in +{ + options.khscodes.security.acme = { + enable = lib.mkEnableOption "Enables acme"; + dns01Enabled = lib.mkOption { + type = lib.types.bool; + description = "Whether to use DNS01 instead of http-01 challenges. This will make the approle gain policy to retrieve the needed cloudflare secrets to manage dns."; + default = config.khscodes.infrastructure.khs-openstack-instance.enable; + }; + }; + config = lib.mkIf cfg.enable { + security.acme = { + acceptTerms = true; + defaults = + { + email = "kaare@kaareskovgaard.net"; + } + // lib.attrsets.optionalAttrs cfg.dns01Enabled { + dnsProvider = "cloudflare"; + dnsResolver = null; + credentialsFile = vaultAgentCredentialsFile; + }; + }; + khscodes.infrastructure.vault-server-approle = { + enable = true; + policy = { + "${cloudflareSecret}" = { + capabilities = [ "read" ]; + }; + }; + }; + khscodes.services.vault-agent = lib.mkIf (cfg.dns01Enabled && acmeServicesToRestart != [ ]) { + enable = true; + templates = [ + { + contents = '' + {{- with secret "${cloudflareSecret}" -}} + CLOUDFLARE_DNS_API_TOKEN={{ .Data.data.TF_VAR_cloudflare_token }} + CLOUDFLARE_DNS_EMAIL={{ .Data.data.TF_VAR_cloudflare_email }} + {{- end -}} + ''; + destination = vaultAgentCredentialsFile; + perms = "0600"; + owner = "acme"; + group = "acme"; + restartUnits = acmeServicesToRestart; + } + ]; + }; + }; +} diff --git a/nix/modules/nixos/security/yubikey/default.nix b/nix/modules/nixos/security/yubikey/default.nix new file mode 100644 index 0000000..889c91e --- /dev/null +++ b/nix/modules/nixos/security/yubikey/default.nix @@ -0,0 +1,31 @@ +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.khscodes.security.yubikey; +in +{ + options.khscodes.security.yubikey = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + }; + }; + config = lib.mkIf cfg.enable { + services.pcscd.enable = true; + services.udev.packages = [ pkgs.yubikey-personalization ]; + + environment.systemPackages = [ + pkgs.yubikey-manager + pkgs.yubico-piv-tool + ]; + + programs.gnupg.agent = { + enable = true; + enableSSHSupport = true; + }; + }; +} diff --git a/nix/modules/nixos/services/alloy/default.nix b/nix/modules/nixos/services/alloy/default.nix new file mode 100644 index 0000000..a306d74 --- /dev/null +++ b/nix/modules/nixos/services/alloy/default.nix @@ -0,0 +1,39 @@ +{ config, lib, ... }: +let + cfg = config.khscodes.services.alloy; +in +{ + options.khscodes.services.alloy = { + enable = lib.mkEnableOption "Enables alloy"; + user = lib.mkOption { + type = lib.types.str; + default = "alloy"; + }; + group = lib.mkOption { + type = lib.types.str; + default = "alloy"; + }; + environment = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + }; + }; + + config = lib.mkIf cfg.enable { + services.alloy.enable = true; + systemd.services.alloy = { + serviceConfig = { + DynamicUser = lib.mkForce false; + User = "${cfg.user}"; + Group = "${cfg.group}"; + }; + environment = cfg.environment; + }; + users.users.${cfg.user} = { + description = "Alloy service user"; + isSystemUser = true; + group = cfg.group; + }; + users.groups.${cfg.group} = { }; + }; +} diff --git a/nix/modules/nixos/services/default.nix b/nix/modules/nixos/services/default.nix deleted file mode 100644 index 0e9f32f..0000000 --- a/nix/modules/nixos/services/default.nix +++ /dev/null @@ -1,8 +0,0 @@ -{ ... }: -{ } -# let -# modules = lib.khscodes.dirsInPath ./.; -# in -# { -# imports = lib.lists.map (d: import d args) modules; -# } diff --git a/nix/modules/nixos/services/nginx/default.nix b/nix/modules/nixos/services/nginx/default.nix new file mode 100644 index 0000000..fe6eb26 --- /dev/null +++ b/nix/modules/nixos/services/nginx/default.nix @@ -0,0 +1,223 @@ +{ + config, + lib, + pkgs, + modulesPath, + ... +}: +let + cfg = config.khscodes.services.nginx; + locationOptions = import "${modulesPath}/services/web-servers/nginx/location-options.nix" { + inherit lib config; + }; + vhostOption = lib.khscodes.mkSubmodule { + description = "nginx vhost"; + options = { + acme = lib.mkOption { + description = "If a simple certificate for the virtual host name itself is not desired auto configured, then set this option. If set to a string it will be used as `useAcmeHost` from NixOS nginx service configuration. Otherwise set to the acme submodule and configure the desired certificate that way"; + type = lib.types.nullOr ( + lib.types.oneOf [ + lib.types.str + (lib.khscodes.mkSubmodule { + description = "acme certificate"; + options = { + domains = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "Domain names the certificate should be requested for, should include the virtual host itself"; + }; + }; + }) + ] + ); + default = null; + }; + globalRedirect = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "If set, all requests for this host are redirected (defaults to 301, configurable with redirectCode) to the given hostname."; + }; + redirectCode = lib.mkOption { + type = lib.types.int; + default = 301; + description = "HTTP status used by globalRedirect and forceSSL. Possible usecases include temporary (302, 307) redirects, keeping the request method and body (307, 308), or explicitly resetting the method to GET (303). See https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections."; + }; + mtls = lib.mkOption { + type = lib.types.nullOr ( + lib.khscodes.mkSubmodule { + options = { + verify = lib.mkOption { + type = lib.types.enum [ + "optional" + "on" + ]; + default = "on"; + }; + certificate = lib.mkOption { + type = lib.types.str; + description = "Path to the certificate to verify client certificates against"; + }; + }; + description = "Nginx MTLS settings"; + } + ); + default = null; + }; + extraConfig = lib.mkOption { + type = lib.types.lines; + description = "Extra configuration to inject into the generated nginx config"; + default = ''''; + }; + locations = lib.mkOption { + type = lib.types.attrsOf ( + lib.khscodes.mkSubmodule { + description = "nginx virtual host location"; + options = locationOptions.options; + } + ); + default = { }; + }; + }; + }; + dns01Enabled = config.khscodes.security.acme.dns01Enabled; + useAcmeConfiguration = lib.attrsets.foldlAttrs ( + acc: name: item: + acc || (item.acme != null && !lib.attrsets.isAttrs item.acme) + ) false cfg.virtualHosts; + modernSslAppendedHttpConfig = + if cfg.sslConfiguration == "modern" then + '' + ssl_ecdh_curve X25519:prime256v1:secp384r1; + '' + else + ''''; +in +{ + options.khscodes.services.nginx = { + enable = lib.mkEnableOption "Enables nginx"; + sslConfiguration = lib.mkOption { + type = lib.types.enum [ + "modern" + "intermediate" + ]; + description = '' + Which sort of ssl configuration following https://ssl-config.mozilla.org/#server=nginx&version=1.28.0&config=modern&openssl=3.4.1&guideline=5.7 as a baseline to generate. + The generated config is not guarenteed to follow this template specifically. In general, modern is preferred, intermediate should only be used if there's a specific reason to do so. + Do note that intermediate requires generating dhparams of large size, which can take hours to complete. + + TODO: Look into OCSP stapling. + ''; + default = "modern"; + }; + virtualHosts = lib.mkOption { + type = lib.types.attrsOf vhostOption; + description = "Virtual hosts settings"; + default = { }; + }; + }; + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = !useAcmeConfiguration || dns01Enabled; + message = "Cannot use `config.khscodes.services.nginx.virtualHosts..acme = {}` without setting config.khscodes.security.acme.dns01Enabled"; + } + ]; + khscodes.networking.aliases = lib.attrsets.attrNames cfg.virtualHosts; + khscodes.security.acme.enable = true; + security.dhparams.enable = lib.mkIf (cfg.sslConfiguration == "intermediate") { + enable = true; + params."nginx" = { + bits = 4096; + }; + }; + services.nginx = { + enable = true; + package = lib.mkDefault pkgs.nginxStable; + sslDhparam = lib.mkIf ( + cfg.sslConfiguration == "intermediate" + ) "${config.security.dhparams.params."nginx".path}"; # DHParams only used when using the ciphers of intermediate + sslProtocols = lib.mkIf (cfg.sslConfiguration == "modern") "TLSv1.3"; # The default matches intermediate + sslCiphers = lib.mkIf (cfg.sslConfiguration == "modern") null; + recommendedTlsSettings = lib.mkDefault true; + recommendedGzipSettings = lib.mkDefault true; + recommendedOptimisation = lib.mkDefault true; + recommendedZstdSettings = lib.mkDefault true; + recommendedProxySettings = lib.mkDefault true; + appendHttpConfig = '' + map $scheme $hsts_header { + https "max-age=63072000; preload"; + } + add_header Strict-Transport-Security $hsts_header; + + add_header X-Frame-Options DENY; + add_header X-Content-Type-Options nosniff; + + ${modernSslAppendedHttpConfig} + ''; + virtualHosts = lib.attrsets.mapAttrs ( + name: value: + let + mtls = + if value.mtls != null then + '' + ssl_client_certificate ${value.mtls.certificate}; + ssl_verify_client ${value.mtls.verify}; + '' + else + ''''; + extraConfig = '' + ${mtls} + ${value.extraConfig} + ''; + in + { + inherit (value) + locations + globalRedirect + redirectCode + ; + inherit extraConfig; + forceSSL = true; + enableACME = value.acme == null && !dns01Enabled; + useACMEHost = + if lib.strings.isString value.acme then + value.acme + else if lib.attrsets.isAttrs value.acme || dns01Enabled then + name + else + null; + } + ) cfg.virtualHosts; + }; + networking.firewall.allowedTCPPorts = [ + 80 + 443 + ]; + networking.firewall.allowedUDPPorts = [ 443 ]; + users.users.nginx.extraGroups = lib.lists.optional dns01Enabled "acme"; + security.acme.certs = lib.mkIf dns01Enabled ( + lib.attrsets.foldlAttrs ( + acc: name: value: + ( + acc + // (lib.attrsets.optionalAttrs + (lib.attrsets.isAttrs value.acme || (dns01Enabled && !lib.strings.isString value.acme)) + { + "${name}" = + if value.acme == null then + { + domain = name; + reloadServices = [ "nginx" ]; + } + else + { + domain = lib.lists.head value.acme.domains; + extraDomainNames = lib.lists.tail value.acme.domains; + reloadServices = [ "nginx" ]; + }; + } + ) + ) + ) { } cfg.virtualHosts + ); + }; +} diff --git a/nix/modules/nixos/services/openssh/default.nix b/nix/modules/nixos/services/openssh/default.nix index a5090ba..948cd1f 100644 --- a/nix/modules/nixos/services/openssh/default.nix +++ b/nix/modules/nixos/services/openssh/default.nix @@ -5,16 +5,85 @@ in { options.khscodes.services.openssh = { enable = lib.mkEnableOption "Enables openssh service for the instance"; - }; - - config = lib.mkIf cfg.enable { - services.openssh = { - enable = true; - settings = { - PasswordAuthentication = false; - PermitRootLogin = "no"; - KbdInteractiveAuthentication = false; + hostCertificate = { + enable = lib.mkEnableOption "Enables getting host certificates from OpenBAO"; + path = lib.mkOption { + type = lib.types.str; + default = "ssh-host"; + }; + hostNames = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "The list of host names to get certificates for"; + default = [ ]; }; }; }; + + config = lib.mkIf cfg.enable ( + let + certificateNames = lib.lists.unique cfg.hostCertificate.hostNames; + hostCertificatEnable = cfg.hostCertificate.enable && cfg.hostCertificate.hostNames != [ ]; + vaultRoleName = config.khscodes.infrastructure.vault-server-approle.role_name; + fqdn = config.networking.fqdnOrHostName; + sshHostBackend = "ssh-host"; + in + { + services.openssh = { + enable = true; + settings = { + PasswordAuthentication = false; + PermitRootLogin = "no"; + KbdInteractiveAuthentication = false; + }; + extraConfig = lib.mkIf hostCertificatEnable '' + HostCertificate /etc/ssh/ssh_host_ed25519_key-cert.pub + ''; + }; + khscodes.infrastructure.vault-server-approle = { + enable = true; + policy."${sshHostBackend}/sign/${vaultRoleName}" = { + capabilities = [ + "read" + "update" + "create" + ]; + }; + stageModules = [ + { + khscodes.vault.ssh_secret_backend_role.${vaultRoleName} = { + name = fqdn; + backend = cfg.hostCertificate.path; + key_type = "ca"; + allow_host_certificates = true; + allow_bare_domains = true; + allowed_domains = certificateNames; + allowed_user_key_config = [ + { + type = "ed25519"; + lengths = [ 0 ]; + } + ]; + }; + } + ]; + }; + khscodes.services.vault-agent = lib.mkIf hostCertificatEnable { + enable = true; + templates = [ + { + contents = '' + {{- $public_key := file "/etc/ssh/ssh_host_ed25519_key.pub" -}} + {{- $public_key = printf "public_key=%s" $public_key -}} + {{- with secret "ssh-host/sign/${fqdn}" "cert_type=host" $public_key "valid_principals=${lib.strings.concatStringsSep "," certificateNames}" -}} + {{ .Data.signed_key }} + {{- end -}} + ''; + destination = "/etc/ssh/ssh_host_ed25519_key-cert.pub"; + perms = "0644"; + restartUnits = [ "sshd.service" ]; + } + ]; + }; + } + ); } diff --git a/nix/modules/nixos/services/read-vault-auth-from-userdata/default.nix b/nix/modules/nixos/services/read-vault-auth-from-userdata/default.nix new file mode 100644 index 0000000..e150534 --- /dev/null +++ b/nix/modules/nixos/services/read-vault-auth-from-userdata/default.nix @@ -0,0 +1,77 @@ +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.khscodes.services.read-vault-auth-from-userdata; +in +{ + options.khscodes.services.read-vault-auth-from-userdata = { + enable = lib.mkEnableOption "Enables reading vault auth information from instance userdata"; + url = lib.mkOption { + type = lib.types.str; + description = "URL to retrieve instance metadata from"; + }; + }; + + config = lib.mkIf (cfg.enable && config.khscodes.services.vault-agent.enable) ( + let + vault_addr = lib.escapeShellArg config.khscodes.services.vault-agent.vault.address; + secretIdFilePath = lib.escapeShellArg config.khscodes.services.vault-agent.vault.secretIdFilePath; + roleIdFilePath = lib.escapeShellArg config.khscodes.services.vault-agent.vault.roleIdFilePath; + cacheFilePath = lib.escapeShellArg "${config.khscodes.services.vault-agent.vault.secretIdFilePath}.wrapped"; + in + { + systemd.services."read-vault-auth-from-userdata" = { + enable = true; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = lib.getExe ( + pkgs.writeShellApplication { + name = "read-vault-auth-from-userdata"; + runtimeInputs = [ + pkgs.curl + pkgs.jq + pkgs.openbao + pkgs.getent + pkgs.systemd + ]; + text = '' + userdata="$(curl ${lib.escapeShellArg cfg.url})" + role_id="$(echo "$userdata" | jq --raw-output '.VAULT_ROLE_ID')" + secret_id_wrapped="$(echo "$userdata" | jq --raw-output '.VAULT_SECRET_ID_WRAPPED')" + if [[ -f ${cacheFilePath} ]]; then + cache_key="$(cat ${cacheFilePath})" + if [[ "$secret_id_wrapped" == "$cache_key" ]]; then + echo "Secret id matched last used value, exiting program" + exit 0 + fi + fi + secret_id="$(BAO_ADDR=${vault_addr} bao unwrap -field=secret_id "$secret_id_wrapped")" + mkdir -p "$(dirname ${secretIdFilePath})" + mkdir -p "$(dirname ${roleIdFilePath})" + echo -n "$role_id" > ${roleIdFilePath} + echo -n "$secret_id" > ${secretIdFilePath} + chown root:root ${secretIdFilePath} + chmod 0600 ${secretIdFilePath} + chown root:root ${roleIdFilePath} + chmod 0600 ${roleIdFilePath} + echo -n "$secret_id_wrapped" > ${cacheFilePath} + chmod 0600 ${cacheFilePath} + chown root:root ${cacheFilePath} + echo "Role id and secret id copied, restarting vault-agent" + systemctl restart vault-agent-openbao.service + ''; + } + ); + }; + }; + } + ); +} diff --git a/nix/modules/nixos/services/vault-agent/default.nix b/nix/modules/nixos/services/vault-agent/default.nix new file mode 100644 index 0000000..cb5c881 --- /dev/null +++ b/nix/modules/nixos/services/vault-agent/default.nix @@ -0,0 +1,176 @@ +{ + lib, + config, + pkgs, + ... +}: +let + cfg = config.khscodes.services.vault-agent; + mkSubmodule = + { + options, + description, + }: + lib.types.submoduleWith { + description = description; + shorthandOnlyDefinesConfig = true; + modules = lib.toList { inherit options; }; + }; + restartUnits = + svcs: + lib.strings.concatStringsSep "\n" ( + lib.lists.map (svc: "systemctl restart ${lib.escapeShellArg svc}") svcs + ); + reloadOrRestartUnits = + svcs: + lib.strings.concatStringsSep "\n" ( + lib.lists.map (svc: "systemctl reload-or-restart ${lib.escapeShellArg svc}") svcs + ); + mapTemplate = + template: + let + command = lib.getExe ( + pkgs.writeShellApplication { + name = "restart-command"; + runtimeInputs = [ pkgs.systemd ]; + text = '' + chown ${lib.escapeShellArg template.owner}:${lib.escapeShellArg template.group} ${lib.escapeShellArg template.destination} + ${restartUnits template.restartUnits} + ${reloadOrRestartUnits template.reloadOrRestartUnits} + ${template.exec} + ''; + meta = { + mainProgram = "restart-command"; + }; + } + ); + in + { + inherit (template) destination perms contents; + exec = { + command = command; + }; + }; + settings = { + vault = { + address = cfg.vault.address; + }; + auto_auth = { + method = [ + { + type = "approle"; + config = { + mount_path = "auth/approle"; + role_id_file_path = cfg.vault.roleIdFilePath; + secret_id_file_path = cfg.vault.secretIdFilePath; + remove_secret_id_file_after_reading = false; + }; + } + ]; + }; + + template_config = { + exit_on_retry_failure = true; + static_secret_render_interval = "60m"; + max_connections_per_host = 10; + leases_renewal_threshold = 0.5; + }; + template = lib.mkIf (cfg.templates != [ ]) (lib.lists.map mapTemplate cfg.templates); + }; + unitsDependsOnAgent = lib.lists.unique ( + lib.lists.flatten (lib.lists.map (t: t.restartUnits ++ t.reloadOrRestartUnits) cfg.templates) + ); +in +{ + options.khscodes.services.vault-agent = { + enable = lib.mkEnableOption "Enables the OpenBAO agent"; + package = lib.mkOption { + type = lib.types.package; + default = pkgs.openbao; + defaultText = "pkgs.openbao"; + }; + vault = { + address = lib.mkOption { + type = lib.types.str; + description = "Address of the Vault/OpenBAO service"; + default = "https://${config.khscodes.infrastructure.openbao.domain}"; + }; + roleIdFilePath = lib.mkOption { + type = lib.types.str; + description = "Location of the role id"; + default = "/var/lib/vault-agent/role-id"; + }; + secretIdFilePath = lib.mkOption { + type = lib.types.str; + description = "Location of the secret id"; + default = "/var/lib/vault-agent/secret-id"; + }; + }; + templates = lib.mkOption { + default = [ ]; + type = lib.types.listOf (mkSubmodule { + description = "List of templates to render"; + options = { + contents = lib.mkOption { + type = lib.types.str; + description = "Contents of the template (.ctmpl)"; + }; + destination = lib.mkOption { + type = lib.types.str; + description = "Destination file for the template"; + }; + restartUnits = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "List of systemd units to restart when template changes"; + default = [ ]; + }; + reloadOrRestartUnits = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "List of systemd units to reload-or-restart when template changes"; + default = [ ]; + }; + perms = lib.mkOption { + type = lib.types.str; + description = "Permissions of the generated file, by default will only be readable by root"; + default = "0600"; + }; + owner = lib.mkOption { + type = lib.types.str; + description = "Owner (user) of the generated file"; + default = "root"; + }; + group = lib.mkOption { + type = lib.types.str; + description = "Group of the generated file"; + default = "root"; + }; + exec = lib.mkOption { + type = lib.types.lines; + default = ''''; + description = "Command to execute when template renders new data"; + }; + }; + }); + }; + }; + + config = lib.mkIf cfg.enable { + services.vault-agent.instances.openbao = { + inherit settings; + enable = true; + package = cfg.package; + user = "root"; + group = "root"; + }; + systemd.services."vault-agent-openbao" = { + before = unitsDependsOnAgent; + wantedBy = unitsDependsOnAgent; + unitConfig = { + ConditionPathExists = [ + cfg.vault.secretIdFilePath + cfg.vault.roleIdFilePath + ]; + }; + }; + }; +} diff --git a/nix/modules/nixos/users/khs/default.nix b/nix/modules/nixos/users/khs/default.nix new file mode 100644 index 0000000..527be3e --- /dev/null +++ b/nix/modules/nixos/users/khs/default.nix @@ -0,0 +1,21 @@ +{ lib, config, ... }: +let + cfg = config.khscodes.users.khs; +in +{ + options.khscodes.users.khs = { + enable = lib.mkEnableOption "Enables settings for the khs user. This should be used in conjunction with homes"; + }; + + config = lib.mkIf cfg.enable { + snowfallorg.users.khs.admin = true; + users.users.khs = { + # TODO: What should I do wrt. ensuring the passwords are consistent? + # Maybe set them through OpenBAO and some service? + initialPassword = "changeme"; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCqY0FHnWFKfLG2yfgr4qka5sR9CK+EMAhzlHUkaQyWHTKD+G0/vC/fNPyL1VV3Dxc/ajxGuPzVE+mBMoyxazL3EtuCDOVvHJ5CR+MUSEckg/DDwcGHqy6rC8BvVVpTAVL04ByQdwFnpE1qNSBaQLkxaFVdtriGKkgMkc7+UNeYX/bv7yn+APqfP1a3xr6wdkSSdO8x4N2jsSygOIMx10hLyCV4Ueu7Kp8Ww4rGY8j5o7lKJhbgfItBfSOuQHdppHVF/GKYRhdnK6Y2fZVYbhq4KipUtclbZ6O/VYd8/sOO98+LMm7cOX+K35PQjUpYgcoNy5+Sw3CNS/NHn4JvOtTaUEYP7fK6c9LhMULOO3T7Cm6TMdiFjUKHkyG+s2Mu/LXJJoilw571zwuh6chkeitW8+Ht7k0aPV96kNEvTdoXwLhBifVEaChlAsLAzSUjUq+YYCiXVk0VIXCZQWKj8LoVNTmaqDksWwbcT64fw/FpVC0N18WHbKcFUEIW/O4spJMa30CQwf9FeqpoWoaF1oRClCSDPvX0AauCu0JcmRinz1/JmlXljnXWbSfm20/V+WyvktlI0wTD0cdpNuSasT9vS77YfJ8nutcWWZKSkCj4R4uHeCNpDTX5YXzapy7FxpM9ANCXLIvoGX7Yafba2Po+er7SSsUIY1AsnBBr8ZoDVw==" + ]; + }; + }; +} diff --git a/nix/modules/nixos/virtualisation/default.nix b/nix/modules/nixos/virtualisation/default.nix deleted file mode 100644 index 7817ed4..0000000 --- a/nix/modules/nixos/virtualisation/default.nix +++ /dev/null @@ -1,10 +0,0 @@ -# TODO: Why is this needed just for this directory? -# In the other directories this will create the modules twice. -# Perhaps because there's only a single sub directory here? -args@{ lib, ... }: -let - modules = lib.khscodes.dirsInPath ./.; -in -{ - imports = lib.lists.map (d: import d args) modules; -} diff --git a/nix/modules/nixos/virtualisation/qemu-guest/default.nix b/nix/modules/nixos/virtualisation/qemu-guest/default.nix index a902f0e..f40f946 100644 --- a/nix/modules/nixos/virtualisation/qemu-guest/default.nix +++ b/nix/modules/nixos/virtualisation/qemu-guest/default.nix @@ -1,16 +1,47 @@ { config, lib, - modulesPath, ... }: let cfg = config.khscodes.virtualisation.qemu-guest; + rng = "-device virtio-rng-pci,rng=rng0 -object rng-random,id=rng0,filename=/dev/urandom"; + spice = [ + "-spice disable-ticketing=on,gl=on,unix=on,addr=/tmp/spice.sock" + "-device virtio-serial-pci" + "-chardev socket,id=agent0,path=vm.sock,server=on,wait=off" + "-device virtserialport,chardev=agent0,name=org.qemu.guest_agent.0" + "-chardev spicevmc,id=vdagent0,name=vdagent" + "-device virtserialport,chardev=vdagent0,name=com.redhat.spice.0" + "-chardev spiceport,id=webdav0,name=org.spice-space.webdav.0" + "-device virtserialport,chardev=webdav0,name=org.spice-space.webdav.0" + ]; in { options.khscodes.virtualisation.qemu-guest = { enable = lib.mkEnableOption "Configures machine with NixOS profile for qemu guest"; + enableWhenVmTarget = lib.mkEnableOption "Enables some enhancement settings when building as a vm"; }; - config = lib.mkIf cfg.enable (import "${modulesPath}/profiles/qemu-guest.nix" { }); + imports = [ ./profile.nix ]; + + config = lib.mkIf cfg.enable { + services.qemuGuest.enable = true; + virtualisation = lib.mkIf cfg.enableWhenVmTarget { + vmVariant = { + khscodes.virtualisation.qemu-guest.enable = true; + services.spice-vdagentd.enable = true; + virtualisation = { + memorySize = 1024 * 8; + qemu = { + options = [ + "-smp 8" + "-vga none -device virtio-gpu-gl,hostmem=2G,blob=true,venus=true" + rng + ] ++ spice; + }; + }; + }; + }; + }; } diff --git a/nix/modules/nixos/virtualisation/qemu-guest/profile.nix b/nix/modules/nixos/virtualisation/qemu-guest/profile.nix new file mode 100644 index 0000000..77560ee --- /dev/null +++ b/nix/modules/nixos/virtualisation/qemu-guest/profile.nix @@ -0,0 +1,12 @@ +{ + config, + lib, + modulesPath, + ... +}: +let + cfg = config.khscodes.virtualisation.qemu-guest; +in +{ + config = lib.mkIf cfg.enable (import "${modulesPath}/profiles/qemu-guest.nix" { }); +} diff --git a/nix/modules/terranix/cloudflare/default.nix b/nix/modules/terranix/cloudflare/default.nix index 8835e4d..54fbf96 100644 --- a/nix/modules/terranix/cloudflare/default.nix +++ b/nix/modules/terranix/cloudflare/default.nix @@ -1,4 +1,3 @@ -{ inputs, khscodesLib }: { config, lib, ... }: let cfg = config.khscodes.cloudflare; @@ -13,7 +12,7 @@ let "@" else fqdn; - dnsARecordModule = khscodesLib.mkSubmodule { + dnsARecordModule = lib.khscodes.mkSubmodule { description = "Module for defining dns A/AAAA record"; options = { fqdn = lib.mkOption { @@ -36,7 +35,7 @@ let }; }; }; - dnsTxtRecordModule = khscodesLib.mkSubmodule { + dnsTxtRecordModule = lib.khscodes.mkSubmodule { description = "Module for defining dns TXT record"; options = { fqdn = lib.mkOption { @@ -54,7 +53,7 @@ let }; }; }; - dnsMxRecordModule = khscodesLib.mkSubmodule { + dnsMxRecordModule = lib.khscodes.mkSubmodule { description = "Module for defining dns MX record"; options = { fqdn = lib.mkOption { @@ -126,7 +125,7 @@ in resource.cloudflare_record = lib.attrsets.optionalAttrs cfg.dns.enable ( lib.listToAttrs ( (lib.lists.map (record: { - name = "${khscodesLib.sanitize-terraform-name record.fqdn}_a"; + name = "${lib.khscodes.sanitize-terraform-name record.fqdn}_a"; value = { inherit (record) content ttl proxied; name = nameFromFQDNAndZone record.fqdn cfg.dns.zone_name; @@ -136,7 +135,7 @@ in }; }) cfg.dns.aRecords) ++ (lib.lists.map (record: { - name = "${khscodesLib.sanitize-terraform-name record.fqdn}_aaaa"; + name = "${lib.khscodes.sanitize-terraform-name record.fqdn}_aaaa"; value = { inherit (record) content ttl proxied; name = nameFromFQDNAndZone record.fqdn cfg.dns.zone_name; @@ -146,7 +145,7 @@ in }; }) cfg.dns.aaaaRecords) ++ (lib.lists.map (record: { - name = "${khscodesLib.sanitize-terraform-name record.fqdn}_txt"; + name = "${lib.khscodes.sanitize-terraform-name record.fqdn}_txt"; value = { inherit (record) content ttl; name = nameFromFQDNAndZone record.fqdn cfg.dns.zone_name; @@ -156,7 +155,7 @@ in }; }) cfg.dns.txtRecords) ++ (lib.lists.map (record: { - name = "${khscodesLib.sanitize-terraform-name record.fqdn}_mx"; + name = "${lib.khscodes.sanitize-terraform-name record.fqdn}_mx"; value = { inherit (record) content priority; name = nameFromFQDNAndZone record.fqdn cfg.dns.zone_name; diff --git a/nix/modules/terranix/hcloud/default.nix b/nix/modules/terranix/hcloud/default.nix index 039c649..af515a6 100644 --- a/nix/modules/terranix/hcloud/default.nix +++ b/nix/modules/terranix/hcloud/default.nix @@ -1,5 +1,9 @@ -{ inputs, khscodesLib }: -{ config, lib, ... }: +{ + config, + lib, + inputs, + ... +}: let cfg = config.khscodes.hcloud; serversWithRdns = lib.filterAttrs (_: value: value.rdns != null) cfg.server; @@ -9,7 +13,7 @@ let lib.map ( { name, value }: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = sanitizedName; @@ -20,7 +24,7 @@ let } ) (lib.attrsToList list) ); - hcloudServerModule = khscodesLib.mkSubmodule { + hcloudServerModule = lib.khscodes.mkSubmodule { description = "Module for defining hcloud server"; options = { name = lib.mkOption { @@ -51,9 +55,14 @@ let default = null; description = "FQDN to map rDNS to"; }; + user_data = lib.mkOption { + type = lib.types.str; + default = ""; + description = "User data for the instance"; + }; }; }; - hcloudDataSshKeys = khscodesLib.mkSubmodule { + hcloudDataSshKeys = lib.khscodes.mkSubmodule { description = "SSH Keys"; options = { name = lib.mkOption { @@ -83,7 +92,7 @@ in }; imports = [ inputs.terranix-hcloud.terranixModules.hcloud - (import ./output.nix { inherit inputs khscodesLib; }) + ./output.nix ]; config = lib.mkIf cfg.enable { @@ -106,6 +115,7 @@ in ipv6_enabled = true; ipv6 = "\${ hcloud_primary_ip.${name}_ipv6.id }"; }; + user_data = builtins.toJSON value.user_data; lifecycle = { ignore_changes = [ "ssh_keys" @@ -119,7 +129,7 @@ in (lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = "${sanitizedName}_ipv4"; @@ -136,7 +146,7 @@ in // (lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = "${sanitizedName}_ipv6"; @@ -154,7 +164,7 @@ in (lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = "${sanitizedName}_ipv4"; @@ -168,7 +178,7 @@ in // (lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = "${sanitizedName}_ipv6"; diff --git a/nix/modules/terranix/hcloud/output.nix b/nix/modules/terranix/hcloud/output.nix index 36ae028..da9c437 100644 --- a/nix/modules/terranix/hcloud/output.nix +++ b/nix/modules/terranix/hcloud/output.nix @@ -1,8 +1,7 @@ -{ khscodesLib, ... }: { config, lib, ... }: let cfg = config.khscodes.hcloud; - hcloudOutputServerModule = khscodesLib.mkSubmodule { + hcloudOutputServerModule = lib.khscodes.mkSubmodule { description = "Module defined when a corresponding server has been defined"; options = { id = lib.mkOption { @@ -19,7 +18,7 @@ let }; }; }; - hcloudDataOutputSshKeyModule = khscodesLib.mkSubmodule { + hcloudDataOutputSshKeyModule = lib.khscodes.mkSubmodule { description = "Module defined when a corresponding ssh key has ben retrieved"; options = { id = lib.mkOption { @@ -47,7 +46,7 @@ in name: value: ( let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { id = "\${ hcloud_server.${sanitizedName}.id }"; @@ -59,7 +58,7 @@ in khscodes.hcloud.output.data.ssh_key = lib.attrsets.mapAttrs ( name: _: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { id = "\${ data.hcloud_ssh_key.${sanitizedName}.id }"; diff --git a/nix/modules/terranix/openbao/default.nix b/nix/modules/terranix/openbao/default.nix deleted file mode 100644 index adfe0a6..0000000 --- a/nix/modules/terranix/openbao/default.nix +++ /dev/null @@ -1,32 +0,0 @@ -{ khscodesLib, inputs }: -{ lib, config, ... }: -let - cfg = config.khscodes.openbao; - modules = [ - ./output.nix - ./vault_mount.nix - ]; -in -{ - options.khscodes.openbao = { - enable = lib.mkEnableOption "Enables the openbao provider"; - }; - - imports = lib.lists.map (m: import m { inherit khscodesLib inputs; }) modules; - - config = lib.mkIf cfg.enable { - provider.vault = { - address = "https://auth.kaareskovgaard.net"; - }; - terraform.required_providers.vault = { - source = "hashicorp/vault"; - version = "5.0.0"; - }; - resource.vault_mount = lib.mapAttrs' ( - name: value: { - name = khscodesLib.sanitize-terraform-name name; - value = value; - } - ); - }; -} diff --git a/nix/modules/terranix/openbao/output.nix b/nix/modules/terranix/openbao/output.nix deleted file mode 100644 index 9e92755..0000000 --- a/nix/modules/terranix/openbao/output.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ khscodesLib, ... }: -{ config, lib, ... }: -let - cfg = config.khscodes.openbao; -in -{ - options.khscodes.openbao = { }; - config = { - }; -} diff --git a/nix/modules/terranix/openbao/ssh_secret_backend_ca.nix b/nix/modules/terranix/openbao/ssh_secret_backend_ca.nix deleted file mode 100644 index 4521947..0000000 --- a/nix/modules/terranix/openbao/ssh_secret_backend_ca.nix +++ /dev/null @@ -1,45 +0,0 @@ -{ khscodesLib, ... }: -{ lib, config, ... }: -let - cfg = config.khscodes.openbao; -in -{ - options.khscodes.openbao = { - vault_ssh_secret_backend_ca = lib.mkOption { - type = lib.types.attrsOf ( - khscodesLib.mkSubmodule { - options = { - backend = lib.mkOption { - type = lib.types.str; - description = "Path of the backend mount"; - }; - generate_signing_key = lib.mkOption { - type = lib.types.bool; - description = "Generate a signing key on the server"; - }; - key_type = lib.mkOption { - type = lib.types.str; - description = "The type of the signing key to use/generate"; - }; - }; - description = "vault_ssh_secret_backend_ca"; - } - ); - }; - }; - config = lib.mkIf cfg.enable { - provider.vault = { - address = "https://auth.kaareskovgaard.net"; - }; - terraform.required_providers.vault = { - source = "hashicorp/vault"; - version = "5.0.0"; - }; - resource.vault_ssh_secret_backend_ca = lib.mapAttrs' ( - name: value: { - name = khscodesLib.sanitize-terraform-name name; - value = value; - } - ); - }; -} diff --git a/nix/modules/terranix/openbao/vault_mount.nix b/nix/modules/terranix/openbao/vault_mount.nix deleted file mode 100644 index 4f4be60..0000000 --- a/nix/modules/terranix/openbao/vault_mount.nix +++ /dev/null @@ -1,52 +0,0 @@ -{ khscodesLib, ... }: -{ lib, config, ... }: -let - cfg = config.khscodes.openbao; -in -{ - options.khscodes.openbao = { - vault_mount = lib.mkOption { - type = lib.types.attrsOf ( - khscodesLib.mkSubmodule { - options = { - type = lib.mkOption { - type = lib.types.str; - description = "Type of mount"; - }; - path = lib.mkOption { - type = lib.types.str; - description = "Path of the mount"; - default = null; - }; - default_lease_ttl_seconds = lib.mkOption { - type = lib.types.int; - description = "Default lease ttl in seconds"; - default = null; - }; - max_lease_ttl_seconds = lib.mkOption { - type = lib.types.int; - description = "Max lease ttl in seconds"; - default = null; - }; - }; - description = "vault_mount"; - } - ); - }; - }; - config = lib.mkIf cfg.enable { - provider.vault = { - address = "https://auth.kaareskovgaard.net"; - }; - terraform.required_providers.vault = { - source = "hashicorp/vault"; - version = "5.0.0"; - }; - resource.vault_mount = lib.mapAttrs' ( - name: value: { - name = khscodesLib.sanitize-terraform-name name; - value = value; - } - ); - }; -} diff --git a/nix/modules/terranix/openstack/default.nix b/nix/modules/terranix/openstack/default.nix index ba13dba..0941c2f 100644 --- a/nix/modules/terranix/openstack/default.nix +++ b/nix/modules/terranix/openstack/default.nix @@ -1,11 +1,11 @@ -{ khscodesLib, inputs }: -{ lib, config, ... }: +{ + lib, + config, + ... +}: let cfg = config.khscodes.openstack; - modules = [ - ./output.nix - ]; - firewallRuleModule = khscodesLib.mkSubmodule { + firewallRuleModule = lib.khscodes.mkSubmodule { description = "Firewall rule"; options = { direction = lib.mkOption { @@ -53,7 +53,7 @@ let port_range_min = rule.port; port_range_max = rule.port; }); - openstackComputeInstance = khscodesLib.mkSubmodule { + openstackComputeInstance = lib.khscodes.mkSubmodule { description = "Openstack compute instance"; options = { name = lib.mkOption { @@ -85,6 +85,11 @@ let "1.0.0.1" ]; }; + user_data = lib.mkOption { + type = lib.types.str; + default = ""; + description = "User data for the instance"; + }; volume_size = lib.mkOption { type = lib.types.int; description = "Size of the root volume, in gigabytes"; @@ -127,7 +132,7 @@ in }; }; - imports = lib.lists.map (m: import m { inherit khscodesLib inputs; }) modules; + imports = [ ./output.nix ]; config = lib.mkIf cfg.enable { terraform.required_providers.openstack = { @@ -169,7 +174,7 @@ in data.openstack_compute_flavor_v2 = lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = sanitizedName; @@ -183,7 +188,7 @@ in data.openstack_images_image_v2 = lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = sanitizedName; @@ -209,12 +214,12 @@ in resource.openstack_compute_keypair_v2 = lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = sanitizedName; value = { - name = khscodesLib.sanitize-terraform-name value.name; + name = lib.khscodes.sanitize-terraform-name value.name; public_key = value.ssh_public_key; }; } @@ -224,7 +229,7 @@ in resource.openstack_networking_router_v2 = lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = sanitizedName; @@ -240,7 +245,7 @@ in resource.openstack_networking_network_v2 = lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = sanitizedName; @@ -256,7 +261,7 @@ in (lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = "${sanitizedName}_ip4"; @@ -273,7 +278,7 @@ in // (lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = "${sanitizedName}_ip6"; @@ -295,7 +300,7 @@ in (lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = "${sanitizedName}_ip4"; @@ -308,7 +313,7 @@ in // (lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = "${sanitizedName}_ip6"; @@ -323,7 +328,7 @@ in resource.openstack_networking_floatingip_v2 = lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = sanitizedName; @@ -338,7 +343,7 @@ in resource.openstack_blockstorage_volume_v3 = lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = sanitizedName; @@ -355,7 +360,7 @@ in resource.openstack_networking_secgroup_v2 = lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = sanitizedName; @@ -372,7 +377,7 @@ in lib.lists.map ( { name, value }: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in lib.listToAttrs ( lib.lists.map ( @@ -382,7 +387,7 @@ in if rule.protocol == "icmp" then "icmp" else "${rule.protocol}_${builtins.toString rule.port}"; in { - name = "${sanitizedName}_${rule.direction}_${rule.ethertype}_${protocol}_${khscodesLib.sanitize-terraform-name rule.remote_subnet}"; + name = "${sanitizedName}_${rule.direction}_${rule.ethertype}_${protocol}_${lib.khscodes.sanitize-terraform-name rule.remote_subnet}"; value = mapFirewallRule "\${ resource.openstack_networking_secgroup_v2.${sanitizedName}.id }" rule; } ) value.firewall_rules @@ -395,7 +400,7 @@ in data.openstack_networking_port_v2 = lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = sanitizedName; @@ -408,7 +413,7 @@ in resource.openstack_compute_instance_v2 = lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = sanitizedName; @@ -432,6 +437,7 @@ in uuid = "\${ openstack_networking_network_v2.${sanitizedName}.id }"; } ]; + user_data = value.user_data; }; } ) cfg.compute_instance; @@ -440,7 +446,7 @@ in resource.openstack_networking_floatingip_associate_v2 = lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = sanitizedName; diff --git a/nix/modules/terranix/openstack/output.nix b/nix/modules/terranix/openstack/output.nix index 4f501b0..b1aedeb 100644 --- a/nix/modules/terranix/openstack/output.nix +++ b/nix/modules/terranix/openstack/output.nix @@ -1,8 +1,7 @@ -{ khscodesLib, ... }: { config, lib, ... }: let cfg = config.khscodes.openstack; - openstackOutputInstanceModule = khscodesLib.mkSubmodule { + openstackOutputInstanceModule = lib.khscodes.mkSubmodule { description = "Instance output"; options = { id = lib.mkOption { @@ -41,7 +40,7 @@ in name: value: ( let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { id = "\${ openstack_compute_instance_v2.${sanitizedName}.id }"; diff --git a/nix/modules/terranix/s3/default.nix b/nix/modules/terranix/s3/default.nix index c11c9f5..c4518a9 100644 --- a/nix/modules/terranix/s3/default.nix +++ b/nix/modules/terranix/s3/default.nix @@ -1,4 +1,3 @@ -{ ... }: { lib, config, ... }: let cfg = config.khscodes.s3; diff --git a/nix/modules/terranix/unifi/default.nix b/nix/modules/terranix/unifi/default.nix index e7a9f5d..8497021 100644 --- a/nix/modules/terranix/unifi/default.nix +++ b/nix/modules/terranix/unifi/default.nix @@ -1,11 +1,7 @@ -{ khscodesLib, inputs }: { lib, config, ... }: let cfg = config.khscodes.unifi; - modules = [ - ./output.nix - ]; - unifiStaticRouteModule = khscodesLib.mkSubmodule { + unifiStaticRouteModule = lib.khscodes.mkSubmodule { description = "Unifi static route"; options = { network = lib.mkOption { @@ -36,7 +32,7 @@ in }; }; - imports = lib.lists.map (m: import m { inherit khscodesLib inputs; }) modules; + imports = [ ./output.nix ]; config = lib.mkIf cfg.enable { terraform.required_providers.unifi = { @@ -50,7 +46,7 @@ in resource.unifi_static_route = lib.mapAttrs' ( name: value: let - sanitizedName = khscodesLib.sanitize-terraform-name name; + sanitizedName = lib.khscodes.sanitize-terraform-name name; in { name = sanitizedName; diff --git a/nix/modules/terranix/unifi/output.nix b/nix/modules/terranix/unifi/output.nix index 8288c92..2f06cfc 100644 --- a/nix/modules/terranix/unifi/output.nix +++ b/nix/modules/terranix/unifi/output.nix @@ -1,5 +1,4 @@ -{ khscodesLib, ... }: -{ config, lib, ... }: +{ config, ... }: let cfg = config.khscodes.unifi; in diff --git a/nix/modules/terranix/vault/approle_auth_backend.nix b/nix/modules/terranix/vault/approle_auth_backend.nix new file mode 100644 index 0000000..5d5fb46 --- /dev/null +++ b/nix/modules/terranix/vault/approle_auth_backend.nix @@ -0,0 +1,119 @@ +{ lib, config, ... }: +let + cfg = config.khscodes.vault; +in +{ + options.khscodes.vault = { + approle_auth_backend_role = lib.mkOption { + type = lib.types.attrsOf ( + lib.khscodes.mkSubmodule { + options = { + backend = lib.mkOption { + type = lib.types.str; + description = "Path of the backend"; + default = "approle"; + }; + role_name = lib.mkOption { + type = lib.types.str; + description = "Name of the role"; + }; + secret_id_ttl = lib.mkOption { + type = lib.types.int; + description = "TTL for the secret id, in seconds"; + }; + secret_id_num_uses = lib.mkOption { + type = lib.types.int; + description = "Maximum number of uses per secret id"; + }; + token_ttl = lib.mkOption { + type = lib.types.int; + description = "TTL for the tokens issued, in seconds"; + }; + token_max_ttl = lib.mkOption { + type = lib.types.int; + description = "Max TTL for the tokens issued, in seconds"; + }; + token_policies = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "Policies attached to the backend role"; + }; + }; + description = "vault_approle_auth_backend_role"; + } + ); + description = "Defines an app backend role"; + default = { }; + }; + approle_auth_backend_role_secret_id = lib.mkOption { + type = lib.types.attrsOf ( + lib.khscodes.mkSubmodule { + options = { + backend = lib.mkOption { + type = lib.types.str; + description = "Path of the backend"; + default = "approle"; + }; + role_name = lib.mkOption { + type = lib.types.str; + description = "NThe name of the role to create the SecretID for"; + }; + cidr_list = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "If set, specifies blocks of IP addresses which can perform the login operation using this SecretID"; + default = [ ]; + }; + metadata = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + description = "Metadata associated with tokens issued by this secret"; + default = { }; + }; + num_uses = lib.mkOption { + type = lib.types.int; + description = "Number of uses for the secret id"; + default = 300; + }; + wrapping_ttl = lib.mkOption { + type = lib.types.nullOr lib.types.int; + description = "If set, the SecretID response will be response-wrapped and available for the duration specified. Only a single unwrapping of the token is allowed."; + default = null; + }; + with_wrapped_accessor = lib.mkOption { + type = lib.types.bool; + description = "Set to `true` to use the wrapped secret-id accessor as the resource ID. If `false` (default value), a fresh secret ID will be regenerated whenever the wrapping token is expired or invalidated through unwrapping."; + default = false; + }; + lifecycle.ignore_changes = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "Ignores changes to the following properties when rerunning the terraform script"; + default = [ ]; + }; + }; + description = "vault_approle_auth_backend_role_secret_id"; + } + ); + description = "Defines an app backend role secret id"; + default = { }; + }; + }; + config = lib.mkIf cfg.enable { + resource.vault_approle_auth_backend_role = lib.mapAttrs' (name: value: { + name = lib.khscodes.sanitize-terraform-name name; + value = value; + }) cfg.approle_auth_backend_role; + resource.vault_approle_auth_backend_role_secret_id = lib.mapAttrs' (name: value: { + name = lib.khscodes.sanitize-terraform-name name; + value = { + inherit (value) + backend + role_name + cidr_list + wrapping_ttl + num_uses + with_wrapped_accessor + lifecycle + ; + metadata = if value.metadata != null then builtins.toJSON value.metadata else null; + }; + }) cfg.approle_auth_backend_role_secret_id; + }; +} diff --git a/nix/modules/terranix/vault/default.nix b/nix/modules/terranix/vault/default.nix new file mode 100644 index 0000000..af666fe --- /dev/null +++ b/nix/modules/terranix/vault/default.nix @@ -0,0 +1,51 @@ +{ lib, config, ... }: +let + cfg = config.khscodes.vault; +in +{ + options.khscodes.vault = { + enable = lib.mkEnableOption "Enables the openbao provider"; + domain = lib.mkOption { + type = lib.types.str; + }; + policy = lib.mkOption { + type = lib.types.attrsOf ( + lib.khscodes.mkSubmodule { + options = { + name = lib.mkOption { + type = lib.types.str; + description = "Name of the policy"; + }; + policy = lib.mkOption { + type = lib.types.lines; + description = "The policy"; + }; + }; + description = "vault_policy"; + } + ); + }; + }; + + imports = [ + ./approle_auth_backend.nix + ./output.nix + ./mount.nix + ./ssh_secret_backend.nix + ./pki_secret_backend.nix + ]; + + config = lib.mkIf cfg.enable { + provider.vault = { + address = "https://${cfg.domain}"; + }; + terraform.required_providers.vault = { + source = "hashicorp/vault"; + version = "5.0.0"; + }; + resource.vault_policy = lib.mapAttrs' (name: value: { + name = lib.khscodes.sanitize-terraform-name name; + value = value; + }) cfg.policy; + }; +} diff --git a/nix/modules/terranix/vault/mount.nix b/nix/modules/terranix/vault/mount.nix new file mode 100644 index 0000000..b96b522 --- /dev/null +++ b/nix/modules/terranix/vault/mount.nix @@ -0,0 +1,54 @@ +{ lib, config, ... }: +let + cfg = config.khscodes.vault; +in +{ + options.khscodes.vault = { + mount = lib.mkOption { + type = lib.types.attrsOf ( + lib.khscodes.mkSubmodule { + options = { + type = lib.mkOption { + type = lib.types.str; + description = "Type of mount"; + }; + path = lib.mkOption { + type = lib.types.str; + description = "Path of the mount"; + default = null; + }; + default_lease_ttl_seconds = lib.mkOption { + type = lib.types.nullOr lib.types.int; + description = "Default lease ttl in seconds"; + default = null; + }; + max_lease_ttl_seconds = lib.mkOption { + type = lib.types.nullOr lib.types.int; + description = "Max lease ttl in seconds"; + default = null; + }; + options = lib.mkOption { + type = lib.types.nullOr lib.types.attrs; + description = "Options for the mount"; + default = null; + }; + description = lib.mkOption { + type = lib.types.nullOr lib.types.str; + description = "Usage description for the mount"; + default = null; + }; + }; + description = "vault_mount"; + } + ); + description = "Defines a vault mount"; + default = { }; + }; + }; + config = lib.mkIf cfg.enable { + resource.vault_mount = lib.mapAttrs' (name: value: { + name = lib.khscodes.sanitize-terraform-name name; + value = value; + }) cfg.mount; + }; +} diff --git a/nix/modules/terranix/vault/output.nix b/nix/modules/terranix/vault/output.nix new file mode 100644 index 0000000..561a060 --- /dev/null +++ b/nix/modules/terranix/vault/output.nix @@ -0,0 +1,83 @@ +{ config, lib, ... }: +let + cfg = config.khscodes.vault; +in +{ + options.khscodes.vault = { + output = { + approle_auth_backend_role = lib.mkOption { + type = lib.types.attrsOf ( + lib.khscodes.mkSubmodule { + options = { + role_name = lib.mkOption { + type = lib.types.str; + description = "The name of the role. Can be used instead of hardcoding the role, to create a dependency in OpenTofu"; + }; + role_id = lib.mkOption { + type = lib.types.str; + description = "ID of the role"; + }; + }; + description = "vault_approle_auth_backend_role output"; + } + ); + }; + approle_auth_backend_role_secret_id = lib.mkOption { + type = lib.types.attrsOf ( + lib.khscodes.mkSubmodule { + options = { + wrapping_token = lib.mkOption { + type = lib.types.str; + description = "The generated wrapping token"; + }; + }; + description = "vault_approle_auth_backend_role_secret_id"; + } + ); + }; + mount = lib.mkOption { + type = lib.types.attrsOf ( + lib.khscodes.mkSubmodule { + options = { + path = lib.mkOption { + type = lib.types.str; + description = "The path of the mount, this is here mainly to set up dependencies"; + }; + }; + description = "vault_mount output"; + } + ); + }; + }; + }; + config = { + khscodes.vault.output.approle_auth_backend_role = lib.mapAttrs ( + name: value: + let + sanitizedName = lib.khscodes.sanitize-terraform-name name; + in + { + role_name = "\${ vault_approle_auth_backend_role.${sanitizedName}.role_name }"; + role_id = "\${ vault_approle_auth_backend_role.${sanitizedName}.role_id }"; + } + ) cfg.approle_auth_backend_role; + khscodes.vault.output.approle_auth_backend_role_secret_id = lib.mapAttrs ( + name: value: + let + sanitizedName = lib.khscodes.sanitize-terraform-name name; + in + { + wrapping_token = "\${ vault_approle_auth_backend_role_secret_id.${sanitizedName}.wrapping_token }"; + } + ) cfg.approle_auth_backend_role_secret_id; + khscodes.vault.output.mount = lib.mapAttrs ( + name: value: + let + sanitizedName = lib.khscodes.sanitize-terraform-name name; + in + { + path = "\${ vault_mount.${sanitizedName}.path }"; + } + ) cfg.mount; + }; +} diff --git a/nix/modules/terranix/vault/pki_secret_backend.nix b/nix/modules/terranix/vault/pki_secret_backend.nix new file mode 100644 index 0000000..24c5cb1 --- /dev/null +++ b/nix/modules/terranix/vault/pki_secret_backend.nix @@ -0,0 +1,117 @@ +{ lib, config, ... }: +let + cfg = config.khscodes.vault; +in +{ + options.khscodes.vault = { + pki_secret_backend_root_cert = lib.mkOption { + type = lib.types.attrsOf ( + lib.khscodes.mkSubmodule { + options = { + backend = lib.mkOption { + type = lib.types.str; + description = "Path of the backend"; + default = "pki"; + }; + type = lib.mkOption { + type = lib.types.enum [ + "exported" + "internal" + "kms" + ]; + description = "Type of intermediate to create. Must be either \"exported\", \"internal\" or \"kms\""; + }; + common_name = lib.mkOption { + type = lib.types.str; + description = "CN of intermediate to create"; + }; + ttl = lib.mkOption { + type = lib.types.str; + description = "TTL for the root certificate, in seconds"; + default = "315360000"; + }; + key_type = lib.mkOption { + type = lib.types.enum [ + "rsa" + "ed25519" + "ec" + ]; + description = "Specifies the desired key type; must be rsa, ed25519 or ec."; + default = "ed25519"; + }; + issuer_name = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Name's the issuer when signing new certificates"; + }; + }; + description = "vault_pki_secret_backend_root_cert"; + } + ); + description = "Generates a new self-signed CA certificate and private keys for the PKI Secret Backend."; + default = { }; + }; + pki_secret_backend_role = lib.mkOption { + type = lib.types.attrsOf ( + lib.khscodes.mkSubmodule { + options = { + backend = lib.mkOption { + type = lib.types.str; + description = "Path of the backend"; + default = "pki"; + }; + name = lib.mkOption { + type = lib.types.str; + description = "The name to identify this role within the backend. Must be unique within the backend."; + }; + allowed_domains = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "List of allowed domains for certificates"; + }; + enforce_hostnames = lib.mkOption { + type = lib.types.nullOr lib.types.bool; + default = null; + description = "Flag to allow only valid host names"; + }; + allow_bare_domains = lib.mkOption { + type = lib.types.nullOr lib.types.bool; + default = null; + description = "Flag to allow certificates matching the actual domain"; + }; + server_flag = lib.mkOption { + type = lib.types.nullOr lib.types.bool; + default = null; + description = "Flag to specify certificates for server use"; + }; + client_flag = lib.mkOption { + type = lib.types.nullOr lib.types.bool; + default = null; + description = "Flag to specify certificates for client use"; + }; + key_type = lib.mkOption { + type = lib.types.enum [ + "rsa" + "ed25519" + "ec" + ]; + description = "Specifies the desired key type; must be rsa, ed25519 or ec."; + default = "ed25519"; + }; + }; + description = "vault_pki_secret_backend_role"; + } + ); + default = { }; + }; + }; + config = lib.mkIf cfg.enable { + resource.vault_pki_secret_backend_root_cert = lib.mapAttrs' (name: value: { + name = lib.khscodes.sanitize-terraform-name name; + value = value; + }) cfg.pki_secret_backend_root_cert; + resource.vault_pki_secret_backend_role = lib.mapAttrs' (name: value: { + name = lib.khscodes.sanitize-terraform-name name; + value = value; + }) cfg.pki_secret_backend_role; + }; +} diff --git a/nix/modules/terranix/vault/ssh_secret_backend.nix b/nix/modules/terranix/vault/ssh_secret_backend.nix new file mode 100644 index 0000000..4ed0bc4 --- /dev/null +++ b/nix/modules/terranix/vault/ssh_secret_backend.nix @@ -0,0 +1,128 @@ +{ lib, config, ... }: +let + cfg = config.khscodes.vault; +in +{ + options.khscodes.vault = { + ssh_secret_backend_role = lib.mkOption { + type = lib.types.attrsOf ( + lib.khscodes.mkSubmodule { + options = { + name = lib.mkOption { + type = lib.types.str; + description = "Specifies the name of the role to create."; + }; + backend = lib.mkOption { + type = lib.types.str; + description = "The path where the SSH secret backend is mounted."; + }; + key_type = lib.mkOption { + type = lib.types.enum [ + "otp" + "dynamic" + "ca" + ]; + description = "Specifies the type of credentials generated by this role."; + }; + allow_bare_domains = lib.mkOption { + type = lib.types.nullOr (lib.types.bool); + description = "Specifies if host certificates that are requested are allowed to use the base domains listed in allowed_domains."; + default = false; + }; + allow_host_certificates = lib.mkOption { + type = lib.types.nullOr (lib.types.bool); + description = "Specifies if certificates are allowed to be signed for use as a 'host'."; + default = false; + }; + allow_subdomains = lib.mkOption { + type = lib.types.nullOr (lib.types.bool); + description = "Specifies if host certificates that are requested are allowed to be subdomains of those listed in allowed_domains."; + default = false; + }; + allow_user_certificates = lib.mkOption { + type = lib.types.nullOr (lib.types.bool); + description = "Specifies if certificates are allowed to be signed for use as a 'user'."; + default = false; + }; + allowed_critical_options = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "Specifies a list of critical options that certificates can have when signed."; + default = [ ]; + }; + allowed_users = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "Specifies a list of usernames that are to be allowed, only if certain usernames are to be allowed."; + default = [ ]; + }; + default_user = lib.mkOption { + type = lib.types.nullOr lib.types.str; + description = "Specifies the default username for which a credential will be generated."; + default = null; + }; + allowed_domains = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "The list of domains for which a client can request a host certificate"; + default = [ ]; + }; + allowed_user_key_config = lib.mkOption { + type = lib.types.listOf ( + lib.khscodes.mkSubmodule { + options = { + type = lib.mkOption { + type = lib.types.enum [ + "rsa" + "ecdsa" + "ec" + "dsa" + "ed25519" + "ssh-rsa" + "ssh-dss" + "ssh-ed25519" + "ecdsa-sha2-nistp256" + "ecdsa-sha2-nistp384" + "ecdsa-sha2-nistp521" + ]; + description = "The SSH public key type."; + }; + lengths = lib.mkOption { + type = lib.types.listOf lib.types.int; + description = "A list of allowed key lengths as integers. For key types that do not support setting the length a value of [0] should be used."; + }; + }; + description = "allowed_user_key_config"; + } + ); + description = "Set of configuration blocks to define allowed user key configuration, like key type and their lengths."; + }; + }; + description = "vault_ssh_secret_backend_role"; + } + ); + description = "Defines an ssh secret backend"; + default = { }; + }; + }; + config = lib.mkIf cfg.enable { + resource.vault_ssh_secret_backend_role = lib.mapAttrs' (name: value: { + name = lib.khscodes.sanitize-terraform-name name; + value = { + inherit (value) + name + backend + key_type + allow_bare_domains + allow_host_certificates + allow_subdomains + allow_user_certificates + default_user + allowed_user_key_config + ; + allowed_critical_options = lib.strings.concatStringsSep "," ( + lib.lists.unique value.allowed_critical_options + ); + allowed_domains = lib.strings.concatStringsSep "," (lib.lists.unique value.allowed_domains); + allowed_users = lib.strings.concatStringsSep "," (lib.lists.unique value.allowed_users); + }; + }) cfg.ssh_secret_backend_role; + }; +} diff --git a/nix/packages/bw-opentofu/secrets-map.nix b/nix/packages/bw-opentofu/secrets-map.nix index 05d18df..8109b39 100644 --- a/nix/packages/bw-opentofu/secrets-map.nix +++ b/nix/packages/bw-opentofu/secrets-map.nix @@ -21,4 +21,11 @@ UNIFI_PASSWORD = "Terraform password"; UNIFI_API = "Terraform URL"; }; + "auth.kaareskovgaard.net" = { + "AUTHENTIK_TOKEN" = "Admin API Token"; + "TF_VAR_authentik_username" = "login.username"; + }; + "vault.kaareskovgaard.net" = { + "VAULT_TOKEN" = "Initial root token"; + }; } diff --git a/nix/packages/create-instance/default.nix b/nix/packages/create-instance/default.nix index b3e1b38..6474d9d 100644 --- a/nix/packages/create-instance/default.nix +++ b/nix/packages/create-instance/default.nix @@ -1,9 +1,17 @@ -{ pkgs, ... }: +{ pkgs, inputs, ... }: pkgs.writeShellApplication { name = "create-instance"; - runtimeInputs = [ pkgs.khscodes.pre-provisioning ]; + runtimeInputs = [ + pkgs.khscodes.provision-instance + pkgs.khscodes.nixos-install + pkgs.jq + ]; text = '' - instance="''${1:-}" - pre-provisioning "$instance" apply + hostname="$1" + # Build the configuration to ensure it doesn't fail when trying to install it on the host + nix build --no-link '${inputs.self}#nixosConfigurations."'"$hostname"'".config.system.build.toplevel' + output="$(provision-instance "$hostname")" + ipv4_addr="$(echo "$output" | jq --raw-output '.ipv4_address.value')" + nixos-install "$hostname" "$ipv4_addr" "no" ''; } diff --git a/nix/packages/instance-opentofu/default.nix b/nix/packages/instance-opentofu/default.nix index a959fae..35a2438 100644 --- a/nix/packages/instance-opentofu/default.nix +++ b/nix/packages/instance-opentofu/default.nix @@ -17,6 +17,11 @@ pkgs.writeShellApplication { cat "''${config}" > "$dir/config.tf.json" tofu -chdir="$dir" init > /dev/null - tofu -chdir="$dir" "$cmd" + if [[ "$cmd" == "apply" ]]; then + tofu -chdir="$dir" "$cmd" >&2 + tofu -chdir="$dir" output -json + else + tofu -chdir="$dir" "$cmd" + fi ''; } diff --git a/nix/packages/nixos-install/default.nix b/nix/packages/nixos-install/default.nix index 7edf965..ac2ac5b 100644 --- a/nix/packages/nixos-install/default.nix +++ b/nix/packages/nixos-install/default.nix @@ -11,10 +11,13 @@ pkgs.writeShellApplication { # TODO: Use secret source and required secrets to set up the correct env variables text = '' hostname="$1" - # Build the configuration to ensure it doesn't fail when trying to install it on the host - nix build --no-link '${inputs.self}#nixosConfigurations."'"$hostname"'".config.system.build.toplevel' # Allow overriding the host to connec tto, this is useful when testing and the DNS entries are stale with older IPs. host="''${2:-$1}" + verify="''${3:-yes}" + if [[ "$verify" == "yes" ]]; then + # Build the configuration to ensure it doesn't fail when trying to install it on the host + nix build --no-link '${inputs.self}#nixosConfigurations."'"$hostname"'".config.system.build.toplevel' + fi baseAttr='${inputs.self}#nixosConfigurations."'"$hostname"'".config.khscodes.infrastructure.provisioning' config="$(nix build --no-link --print-out-paths "''${baseAttr}.preConfig")" username="$(nix eval --raw "''${baseAttr}.preImageUsername")" diff --git a/nix/packages/opentofu/default.nix b/nix/packages/opentofu/default.nix index 2d18324..4212c13 100644 --- a/nix/packages/opentofu/default.nix +++ b/nix/packages/opentofu/default.nix @@ -4,4 +4,5 @@ pkgs.opentofu.withPlugins (p: [ pkgs.khscodes.terraform-provider-cloudflare pkgs.khscodes.terraform-provider-hcloud pkgs.khscodes.terraform-provider-openstack + pkgs.khscodes.terraform-provider-vault ]) diff --git a/nix/packages/post-provisioning/default.nix b/nix/packages/post-provisioning/default.nix new file mode 100644 index 0000000..e686976 --- /dev/null +++ b/nix/packages/post-provisioning/default.nix @@ -0,0 +1,33 @@ +{ + inputs, + pkgs, +}: +pkgs.writeShellApplication { + name = "pre-provisioning"; + runtimeInputs = [ + pkgs.nix + pkgs.khscodes.bw-opentofu + pkgs.khscodes.instance-opentofu + pkgs.khscodes.openbao-helper + pkgs.jq + ]; + # TODO: Use secret source and required secrets to set up the correct env variables + text = '' + hostname="$1" + cmd="''${2:-apply}" + baseAttr='${inputs.self}#nixosConfigurations."'"$hostname"'".config.khscodes.infrastructure.provisioning' + config="$(nix build --no-link --print-out-paths "''${baseAttr}.postConfig")" + secretsSource="$(nix eval --raw "''${baseAttr}.post.secretsSource")" + endpoints="$(nix eval --show-trace --json "''${baseAttr}.postEndpoints")" + if [[ "$config" == "null" ]]; then + echo "No preprovisioning needed" + exit 0 + fi + if [[ "$secretsSource" == "vault" ]]; then + readarray -t endpoints_args < <(echo "$endpoints" | jq -cr 'map(["-e", .])[][]') + openbao-helper wrap-program "''${endpoints_args[@]}" -- instance-opentofu "$hostname" "$config" "$cmd" + exit 0 + fi + bw-opentofu "$hostname" "$config" "$cmd" + ''; +} diff --git a/nix/packages/pre-provisioning/default.nix b/nix/packages/pre-provisioning/default.nix index 04232a1..f3eaf63 100644 --- a/nix/packages/pre-provisioning/default.nix +++ b/nix/packages/pre-provisioning/default.nix @@ -15,10 +15,10 @@ pkgs.writeShellApplication { text = '' hostname="$1" cmd="''${2:-apply}" - baseAttr='${inputs.self}#nixosConfigurations."'"$hostname"'".config.khscodes.infrastructue.provisioning' + baseAttr='${inputs.self}#nixosConfigurations."'"$hostname"'".config.khscodes.infrastructure.provisioning' config="$(nix build --no-link --print-out-paths "''${baseAttr}.preConfig")" secretsSource="$(nix eval --raw "''${baseAttr}.pre.secretsSource")" - endpoints="$(nix eval --json "''${baseAttr}.pre.endpoints")" + endpoints="$(nix eval --show-trace --json "''${baseAttr}.preEndpoints")" if [[ "$config" == "null" ]]; then echo "No preprovisioning needed" exit 0 diff --git a/nix/packages/provision-instance/default.nix b/nix/packages/provision-instance/default.nix new file mode 100644 index 0000000..0acf0d5 --- /dev/null +++ b/nix/packages/provision-instance/default.nix @@ -0,0 +1,9 @@ +{ pkgs, ... }: +pkgs.writeShellApplication { + name = "provision-instance"; + runtimeInputs = [ pkgs.khscodes.pre-provisioning ]; + text = '' + instance="''${1:-}" + pre-provisioning "$instance" apply + ''; +} diff --git a/nix/packages/start-vm/default.nix b/nix/packages/start-vm/default.nix new file mode 100644 index 0000000..7cd65c4 --- /dev/null +++ b/nix/packages/start-vm/default.nix @@ -0,0 +1,24 @@ +{ pkgs, inputs, ... }: +pkgs.writeShellApplication { + name = "start-vm"; + runtimeInputs = [ + pkgs.spice-gtk + pkgs.uutils-findutils + ]; + text = '' + host="''${1:-}" + clean="''${2:-no}" + if [[ "$clean" == "clean" ]]; then + find . -type f -name '*.qcow2' -delete + fi + run_vm="$(nix build --no-link --print-out-paths '${inputs.self}#nixosConfigurations."'"$host"'".config.system.build.vm' --show-trace)" + + # shellcheck disable=SC2211 + # shellcheck disable=SC2086 + $run_vm/bin/* & + pid=$! + trap 'kill $pid' EXIT + sleep 2 + spicy --title "$host" --uri=spice+unix:///tmp/spice.sock + ''; +} diff --git a/nix/packages/terraform-provider-vault/default.nix b/nix/packages/terraform-provider-vault/default.nix new file mode 100644 index 0000000..580112e --- /dev/null +++ b/nix/packages/terraform-provider-vault/default.nix @@ -0,0 +1,10 @@ +{ pkgs }: +pkgs.terraform-providers.mkProvider { + hash = "sha256-Vqnmw69fktBQhSkj/W0legJ4sHOQP9Moqqi6Z5qYFy4="; + homepage = "https://registry.terraform.io/providers/hashicorp/vault"; + owner = "hashicorp"; + repo = "terraform-provider-vault"; + rev = "v5.0.0"; + spdx = "MPL-2.0"; + vendorHash = "sha256-6gWw4ypQZWPX7VC9cZxHiU/KhTYEdMTZ276B9neGAiI="; +} diff --git a/nix/packages/update-instance/default.nix b/nix/packages/update-instance/default.nix index a18794d..4cd2786 100644 --- a/nix/packages/update-instance/default.nix +++ b/nix/packages/update-instance/default.nix @@ -5,6 +5,6 @@ pkgs.writeShellApplication { text = '' instance="''${1:-}" connect_host="''${2:-$1}" - nixos-rebuild switch --flake "${inputs.self}#$instance" --target-host "$connect_host" --build-host "localhost" + nixos-rebuild switch --flake "${inputs.self}#$instance" --target-host "$connect_host" --build-host "$connect_host" --use-remote-sudo ''; } diff --git a/nix/profiles/hetzner-server.nix b/nix/profiles/nixos/hetzner-server.nix similarity index 65% rename from nix/profiles/hetzner-server.nix rename to nix/profiles/nixos/hetzner-server.nix index da9068b..f284144 100644 --- a/nix/profiles/hetzner-server.nix +++ b/nix/profiles/nixos/hetzner-server.nix @@ -1,7 +1,7 @@ { ... }: { + imports = [ ./khs-server.nix ]; config.khscodes = { hetzner.enable = true; - services.openssh.enable = true; }; } diff --git a/nix/profiles/nixos/khs-base.nix b/nix/profiles/nixos/khs-base.nix new file mode 100644 index 0000000..ade4c17 --- /dev/null +++ b/nix/profiles/nixos/khs-base.nix @@ -0,0 +1,18 @@ +{ pkgs, config, ... }: +{ + imports = [ ./nix-base.nix ]; + snowfallorg.users.khs.admin = true; + users.users.khs = { + # TODO: Figure out how to provision password changes to servers from VAULT + initialPassword = "changeme"; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCqY0FHnWFKfLG2yfgr4qka5sR9CK+EMAhzlHUkaQyWHTKD+G0/vC/fNPyL1VV3Dxc/ajxGuPzVE+mBMoyxazL3EtuCDOVvHJ5CR+MUSEckg/DDwcGHqy6rC8BvVVpTAVL04ByQdwFnpE1qNSBaQLkxaFVdtriGKkgMkc7+UNeYX/bv7yn+APqfP1a3xr6wdkSSdO8x4N2jsSygOIMx10hLyCV4Ueu7Kp8Ww4rGY8j5o7lKJhbgfItBfSOuQHdppHVF/GKYRhdnK6Y2fZVYbhq4KipUtclbZ6O/VYd8/sOO98+LMm7cOX+K35PQjUpYgcoNy5+Sw3CNS/NHn4JvOtTaUEYP7fK6c9LhMULOO3T7Cm6TMdiFjUKHkyG+s2Mu/LXJJoilw571zwuh6chkeitW8+Ht7k0aPV96kNEvTdoXwLhBifVEaChlAsLAzSUjUq+YYCiXVk0VIXCZQWKj8LoVNTmaqDksWwbcT64fw/FpVC0N18WHbKcFUEIW/O4spJMa30CQwf9FeqpoWoaF1oRClCSDPvX0AauCu0JcmRinz1/JmlXljnXWbSfm20/V+WyvktlI0wTD0cdpNuSasT9vS77YfJ8nutcWWZKSkCj4R4uHeCNpDTX5YXzapy7FxpM9ANCXLIvoGX7Yafba2Po+er7SSsUIY1AsnBBr8ZoDVw==" + ]; + }; + environment = { + systemPackages = [ pkgs.openbao ]; + variables = { + BAO_ADDR = "https://${config.khscodes.infrastructure.openbao.domain}"; + }; + }; +} diff --git a/nix/profiles/nixos/khs-desktop.nix b/nix/profiles/nixos/khs-desktop.nix new file mode 100644 index 0000000..0a7c7d9 --- /dev/null +++ b/nix/profiles/nixos/khs-desktop.nix @@ -0,0 +1,39 @@ +{ + lib, + pkgs, + inputs, + ... +}: +{ + imports = [ ./khs-base.nix ]; + khscodes.virtualisation.qemu-guest.enableWhenVmTarget = true; + khscodes.machine.type = "desktop"; + + services.desktopManager.cosmic.enable = true; + services.displayManager.cosmic-greeter.enable = true; + + stylix = { + enable = true; + autoEnable = false; + image = "${inputs.self}/assets/khs-desktop-wallpaper.jpg"; + base16Scheme = lib.mkDefault "${pkgs.base16-schemes}/share/themes/solarized-dark.yaml"; + targets.console.enable = true; + fonts = { + monospace = { + package = pkgs.nerd-fonts.inconsolata; + name = "Inconsolata Nerd Font"; + }; + sizes = { + terminal = 14; + }; + }; + }; + console = { + enable = true; + font = "${pkgs.powerline-fonts}/share/consolefonts/ter-powerline-v24b.psf.gz"; + packages = [ + pkgs.terminus_font + pkgs.powerline-fonts + ]; + }; +} diff --git a/nix/profiles/khs-openstack-server.nix b/nix/profiles/nixos/khs-openstack-server.nix similarity index 65% rename from nix/profiles/khs-openstack-server.nix rename to nix/profiles/nixos/khs-openstack-server.nix index dafafd2..6c4bd48 100644 --- a/nix/profiles/khs-openstack-server.nix +++ b/nix/profiles/nixos/khs-openstack-server.nix @@ -1,7 +1,7 @@ { ... }: { + imports = [ ./khs-server.nix ]; config.khscodes = { openstack.enable = true; - services.openssh.enable = true; }; } diff --git a/nix/profiles/nixos/khs-server.nix b/nix/profiles/nixos/khs-server.nix new file mode 100644 index 0000000..92c64f9 --- /dev/null +++ b/nix/profiles/nixos/khs-server.nix @@ -0,0 +1,21 @@ +{ lib, pkgs, ... }: +{ + imports = [ ./nix-base.nix ]; + config = { + khscodes = { + services.openssh.enable = true; + machine.type = "server"; + os.auto-update.enable = true; + infrastructure = { + vault-server-approle.enable = lib.mkDefault true; + vault-loki-sender.enable = lib.mkDefault true; + }; + }; + stylix = { + enable = true; + autoEnable = false; + base16Scheme = lib.mkDefault "${pkgs.base16-schemes}/share/themes/solarized-dark.yaml"; + targets.console.enable = true; + }; + }; +} diff --git a/nix/profiles/nix-base.nix b/nix/profiles/nixos/nix-base.nix similarity index 100% rename from nix/profiles/nix-base.nix rename to nix/profiles/nixos/nix-base.nix diff --git a/nix/systems/aarch64-linux/khs.codes/default.nix b/nix/systems/aarch64-linux/khs.codes/default.nix index 0d8a456..6e096ac 100644 --- a/nix/systems/aarch64-linux/khs.codes/default.nix +++ b/nix/systems/aarch64-linux/khs.codes/default.nix @@ -3,13 +3,12 @@ ... }: { - imports = [ "${inputs.self}/nix/profiles/hetzner-server.nix" ]; + imports = [ "${inputs.self}/nix/profiles/nixos/hetzner-server.nix" ]; khscodes.infrastructure.hetzner-instance = { enable = true; mapRdns = true; server_type = "cax11"; - secretsSource = "bitwarden"; }; - khscodes.fqdn = "khs.codes"; + khscodes.networking.fqdn = "khs.codes"; system.stateVersion = "25.05"; } diff --git a/nix/systems/aarch64-linux/security.kaareskovgaard.net/README.md b/nix/systems/aarch64-linux/security.kaareskovgaard.net/README.md new file mode 100644 index 0000000..a16ab41 --- /dev/null +++ b/nix/systems/aarch64-linux/security.kaareskovgaard.net/README.md @@ -0,0 +1,7 @@ +# After creating the instance + +Open https://vault.kaareskovgaard.net and initialize OpenBAO. Remember to get some sort of auto unsealing set up afterwards, currently this is implemented with a cronjob on TrueNAS. Doing it this way allows various certificates to continue getting issued, even as OpenBAO gets sealed (due to auto updates). + +After this, run the post provisioning script to initialize the various OpenBAO parts needed. +In order for `security.kaareskovgaard.net` to authenticate itself with OpenBAO +Then `nix run '.#bitwarden-to-vault` can transfer the needed Bitwarden secrets to vault. diff --git a/nix/systems/aarch64-linux/security.kaareskovgaard.net/authentik.nix b/nix/systems/aarch64-linux/security.kaareskovgaard.net/authentik.nix new file mode 100644 index 0000000..75b7484 --- /dev/null +++ b/nix/systems/aarch64-linux/security.kaareskovgaard.net/authentik.nix @@ -0,0 +1,60 @@ +{ config, ... }: +let + secretsFile = "/var/lib/authentik/authentik-env"; + domain = "auth-test.kaareskovgaard.net"; +in +{ + config = { + khscodes.nix.nix-community.enable = true; + services.authentik = { + enable = true; + environmentFile = secretsFile; + settings = { + email = { + host = "smtp.soverin.net"; + port = 587; + username = "kaare@kaareskovgaard.net"; + use_tls = true; + use_ssl = false; + from = "kaare@kaareskovgaard.net"; + }; + disable_startup_analytics = true; + avatars = "initials"; + }; + }; + khscodes.services.nginx.virtualHosts.${domain} = { + locations."/" = { + proxyPass = "https://localhost:9443"; + recommendedProxySettings = true; + }; + }; + services.postgresqlBackup = { + enable = true; + databases = [ "authentik" ]; + }; + + systemd.services = { + authentik-migrate = { + unitConfig = { + ConditionPathExists = secretsFile; + }; + }; + authentik-worker = { + unitConfig = { + ConditionPathExists = secretsFile; + }; + serviceConfig = { + LoadCredential = [ + "${domain}.pem:${config.security.acme.certs.${domain}.directory}/fullchain.pem" + "${domain}.key:${config.security.acme.certs.${domain}.directory}/key.pem" + ]; + }; + }; + authentik = { + unitConfig = { + ConditionPathExists = secretsFile; + }; + }; + }; + }; +} diff --git a/nix/systems/aarch64-linux/security.kaareskovgaard.net/default.nix b/nix/systems/aarch64-linux/security.kaareskovgaard.net/default.nix new file mode 100644 index 0000000..db4fe80 --- /dev/null +++ b/nix/systems/aarch64-linux/security.kaareskovgaard.net/default.nix @@ -0,0 +1,30 @@ +{ + inputs, + ... +}: +{ + imports = [ + "${inputs.self}/nix/profiles/nixos/hetzner-server.nix" + ./authentik.nix + ./openbao.nix + ./post/openbao + ]; + khscodes.services.nginx.enable = true; + khscodes.infrastructure.hetzner-instance = { + enable = true; + server_type = "cax11"; + }; + # Cannot use vault for secrets source, as this is the server containing vault. + khscodes.infrastructure.provisioning.pre.secretsSource = "bitwarden"; + khscodes.infrastructure.provisioning.post.secretsSource = "bitwarden"; + khscodes.infrastructure.vault-server-approle.stage = "post"; + khscodes.networking.fqdn = "security.kaareskovgaard.net"; + users.users.khs = { + initialPassword = "changeme"; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCqY0FHnWFKfLG2yfgr4qka5sR9CK+EMAhzlHUkaQyWHTKD+G0/vC/fNPyL1VV3Dxc/ajxGuPzVE+mBMoyxazL3EtuCDOVvHJ5CR+MUSEckg/DDwcGHqy6rC8BvVVpTAVL04ByQdwFnpE1qNSBaQLkxaFVdtriGKkgMkc7+UNeYX/bv7yn+APqfP1a3xr6wdkSSdO8x4N2jsSygOIMx10hLyCV4Ueu7Kp8Ww4rGY8j5o7lKJhbgfItBfSOuQHdppHVF/GKYRhdnK6Y2fZVYbhq4KipUtclbZ6O/VYd8/sOO98+LMm7cOX+K35PQjUpYgcoNy5+Sw3CNS/NHn4JvOtTaUEYP7fK6c9LhMULOO3T7Cm6TMdiFjUKHkyG+s2Mu/LXJJoilw571zwuh6chkeitW8+Ht7k0aPV96kNEvTdoXwLhBifVEaChlAsLAzSUjUq+YYCiXVk0VIXCZQWKj8LoVNTmaqDksWwbcT64fw/FpVC0N18WHbKcFUEIW/O4spJMa30CQwf9FeqpoWoaF1oRClCSDPvX0AauCu0JcmRinz1/JmlXljnXWbSfm20/V+WyvktlI0wTD0cdpNuSasT9vS77YfJ8nutcWWZKSkCj4R4uHeCNpDTX5YXzapy7FxpM9ANCXLIvoGX7Yafba2Po+er7SSsUIY1AsnBBr8ZoDVw==" + ]; + }; + khscodes.infrastructure.openbao.domain = "vault-test.kaareskovgaard.net"; + system.stateVersion = "25.05"; +} diff --git a/nix/systems/aarch64-linux/security.kaareskovgaard.net/openbao.nix b/nix/systems/aarch64-linux/security.kaareskovgaard.net/openbao.nix new file mode 100644 index 0000000..ca2e99d --- /dev/null +++ b/nix/systems/aarch64-linux/security.kaareskovgaard.net/openbao.nix @@ -0,0 +1,51 @@ +{ pkgs, config, ... }: +let + domain = config.khscodes.infrastructure.openbao.domain; +in +{ + config = { + services.openbao = { + enable = true; + package = pkgs.openbao; + settings = { + ui = true; + listener.tcp = { + type = "tcp"; + tls_cert_file = "${config.security.acme.certs.${domain}.directory}/fullchain.pem"; + tls_key_file = "${config.security.acme.certs.${domain}.directory}/key.pem"; + }; + api_addr = "https://${domain}"; + storage.postgresql.connection_url = "postgres://openbao?host=/run/postgresql"; + }; + }; + security.acme.certs.${domain}.reloadServices = [ "openbao.service" ]; + + systemd.services.openbao.after = [ "postgresql.service" ]; + + # Allow openbao to read the certificate file + users.groups.nginx.members = [ "openbao" ]; + + services.postgresql = { + enable = true; + ensureDatabases = [ "openbao" ]; + ensureUsers = [ + { + name = "openbao"; + ensureDBOwnership = true; + } + ]; + }; + + services.postgresqlBackup = { + enable = true; + databases = [ "openbao" ]; + }; + + khscodes.services.nginx.virtualHosts.${domain} = { + locations."/" = { + proxyPass = "https://${config.services.openbao.settings.listener.tcp.address}/"; + recommendedProxySettings = true; + }; + }; + }; +} diff --git a/nix/systems/aarch64-linux/security.kaareskovgaard.net/post/openbao/approle.nix b/nix/systems/aarch64-linux/security.kaareskovgaard.net/post/openbao/approle.nix new file mode 100644 index 0000000..5d2a867 --- /dev/null +++ b/nix/systems/aarch64-linux/security.kaareskovgaard.net/post/openbao/approle.nix @@ -0,0 +1,10 @@ +{ + khscodes.infrastructure.vault-server-approle.path = "\${ vault_auth_backend.approle.path }"; + khscodes.infrastructure.provisioning.post.modules = [ + { + resource.vault_auth_backend.approle = { + type = "approle"; + }; + } + ]; +} diff --git a/nix/systems/aarch64-linux/security.kaareskovgaard.net/post/openbao/default.nix b/nix/systems/aarch64-linux/security.kaareskovgaard.net/post/openbao/default.nix new file mode 100644 index 0000000..bbeb301 --- /dev/null +++ b/nix/systems/aarch64-linux/security.kaareskovgaard.net/post/openbao/default.nix @@ -0,0 +1,29 @@ +{ + imports = [ + ./approle.nix + ./ssh-host.nix + ./loki-mtls.nix + ./prometheus-mtls.nix + ]; + khscodes.infrastructure.vault-server-approle.path = "\${ vault_auth_backend.approle.path }"; + khscodes.infrastructure.provisioning.post.modules = [ + ( + { config, ... }: + { + khscodes.vault.mount.opentofu = { + path = "opentofu"; + type = "kv"; + options = { + version = "2"; + }; + description = "Secrets used during provisioning"; + }; + resource.vault_kv_secret_backend_v2.opentofu = { + mount = config.khscodes.vault.output.mount.opentofu.path; + max_versions = 5; + cas_required = false; + }; + } + ) + ]; +} diff --git a/nix/systems/aarch64-linux/security.kaareskovgaard.net/post/openbao/loki-mtls.nix b/nix/systems/aarch64-linux/security.kaareskovgaard.net/post/openbao/loki-mtls.nix new file mode 100644 index 0000000..31e2ebd --- /dev/null +++ b/nix/systems/aarch64-linux/security.kaareskovgaard.net/post/openbao/loki-mtls.nix @@ -0,0 +1,26 @@ +# This should go into the setup of the vault server itself, as the vault server also needs stuff that depends on this. +{ + khscodes.infrastructure.vault-loki-sender = { + terranixBackendName = "\${ vault_mount.loki-mtls.path }"; + }; + khscodes.infrastructure.provisioning.post.modules = [ + ( + { config, ... }: + { + khscodes.vault.enable = true; + khscodes.vault.mount.loki-mtls = { + type = "pki"; + path = "loki-mtls"; + max_lease_ttl_seconds = 10 * 365 * 24 * 60 * 60; + default_lease_ttl_seconds = 60 * 60; + }; + khscodes.vault.pki_secret_backend_root_cert.loki-mtls = { + backend = config.khscodes.vault.output.mount.loki-mtls.path; + type = "internal"; + common_name = "loki.kaareskovgaard.net"; + issuer_name = "loki-mtls-root-ca"; + }; + } + ) + ]; +} diff --git a/nix/systems/aarch64-linux/security.kaareskovgaard.net/post/openbao/prometheus-mtls.nix b/nix/systems/aarch64-linux/security.kaareskovgaard.net/post/openbao/prometheus-mtls.nix new file mode 100644 index 0000000..ed4d52b --- /dev/null +++ b/nix/systems/aarch64-linux/security.kaareskovgaard.net/post/openbao/prometheus-mtls.nix @@ -0,0 +1,27 @@ +# This should go into the setup of the vault server itself, as the vault server also needs stuff that depends on this. +{ + khscodes.infrastructure.vault-prometheus-sender = { + # This is quite ugly, but should get the job done. Sadly I cannot reference the output from here. + terranixBackendName = "\${ vault_mount.prometheus-mtls.path }"; + }; + khscodes.infrastructure.provisioning.post.modules = [ + ( + { config, ... }: + { + khscodes.vault.enable = true; + khscodes.vault.mount.prometheus-mtls = { + type = "pki"; + path = "prometheus-mtls"; + max_lease_ttl_seconds = 10 * 365 * 24 * 60 * 60; + default_lease_ttl_seconds = 60 * 60; + }; + khscodes.vault.pki_secret_backend_root_cert.prometheus-mtls = { + backend = config.khscodes.vault.output.mount.prometheus-mtls.path; + type = "internal"; + common_name = "prometheus.kaareskovgaard.net"; + issuer_name = "prometheus-mtls-root-ca"; + }; + } + ) + ]; +} diff --git a/nix/systems/aarch64-linux/security.kaareskovgaard.net/post/openbao/ssh-host.nix b/nix/systems/aarch64-linux/security.kaareskovgaard.net/post/openbao/ssh-host.nix new file mode 100644 index 0000000..d43f969 --- /dev/null +++ b/nix/systems/aarch64-linux/security.kaareskovgaard.net/post/openbao/ssh-host.nix @@ -0,0 +1,22 @@ +{ + khscodes.services.openssh.hostCertificate.path = "\${ vault_mount.ssh-host.path }"; + khscodes.infrastructure.provisioning.post.modules = [ + ( + { config, ... }: + { + khscodes.vault.mount.ssh-host = { + type = "ssh"; + path = "ssh-host"; + default_lease_ttl_seconds = 24 * 60 * 60; + max_lease_ttl_seconds = 24 * 60 * 60; + }; + + resource.vault_ssh_secret_backend_ca.ssh-host = { + backend = config.khscodes.vault.output.mount.ssh-host.path; + generate_signing_key = true; + key_type = "ed25519"; + }; + } + ) + ]; +} diff --git a/nix/systems/x86_64-linux/test.kaareskovgaard.net/default.nix b/nix/systems/x86_64-linux/desktop.kaareskovgaard.net/default.nix similarity index 73% rename from nix/systems/x86_64-linux/test.kaareskovgaard.net/default.nix rename to nix/systems/x86_64-linux/desktop.kaareskovgaard.net/default.nix index ddfcb62..b66eb59 100644 --- a/nix/systems/x86_64-linux/test.kaareskovgaard.net/default.nix +++ b/nix/systems/x86_64-linux/desktop.kaareskovgaard.net/default.nix @@ -1,21 +1,20 @@ { inputs, + lib, ... }: { - imports = [ "${inputs.self}/nix/profiles/khs-openstack-server.nix" ]; - khscodes.infrastructure.khs-openstack-instance = { - enable = true; - flavor = "m.medium"; - secretsSource = "vault"; + imports = [ "${inputs.self}/nix/profiles/nixos/khs-desktop.nix" ]; + khscodes.networking.fqdn = "desktop.kaareskovgaard.net"; + disko = lib.khscodes.disko-root-bios { + device = "/dev/sda"; + diskName = "nixos"; }; - snowfallorg.users.khs.admin = true; users.users.khs = { - initialPassword = "test"; + initialPassword = "changeme"; openssh.authorizedKeys.keys = [ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCqY0FHnWFKfLG2yfgr4qka5sR9CK+EMAhzlHUkaQyWHTKD+G0/vC/fNPyL1VV3Dxc/ajxGuPzVE+mBMoyxazL3EtuCDOVvHJ5CR+MUSEckg/DDwcGHqy6rC8BvVVpTAVL04ByQdwFnpE1qNSBaQLkxaFVdtriGKkgMkc7+UNeYX/bv7yn+APqfP1a3xr6wdkSSdO8x4N2jsSygOIMx10hLyCV4Ueu7Kp8Ww4rGY8j5o7lKJhbgfItBfSOuQHdppHVF/GKYRhdnK6Y2fZVYbhq4KipUtclbZ6O/VYd8/sOO98+LMm7cOX+K35PQjUpYgcoNy5+Sw3CNS/NHn4JvOtTaUEYP7fK6c9LhMULOO3T7Cm6TMdiFjUKHkyG+s2Mu/LXJJoilw571zwuh6chkeitW8+Ht7k0aPV96kNEvTdoXwLhBifVEaChlAsLAzSUjUq+YYCiXVk0VIXCZQWKj8LoVNTmaqDksWwbcT64fw/FpVC0N18WHbKcFUEIW/O4spJMa30CQwf9FeqpoWoaF1oRClCSDPvX0AauCu0JcmRinz1/JmlXljnXWbSfm20/V+WyvktlI0wTD0cdpNuSasT9vS77YfJ8nutcWWZKSkCj4R4uHeCNpDTX5YXzapy7FxpM9ANCXLIvoGX7Yafba2Po+er7SSsUIY1AsnBBr8ZoDVw==" ]; }; - khscodes.fqdn = "test.kaareskovgaard.net"; system.stateVersion = "25.05"; } diff --git a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/default.nix b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/default.nix new file mode 100644 index 0000000..a4eba2d --- /dev/null +++ b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/default.nix @@ -0,0 +1,181 @@ +{ + inputs, + config, + ... +}: +let + grafana = config.services.grafana; + loki = config.services.loki; + prometheus = config.services.prometheus; +in +{ + imports = [ + "${inputs.self}/nix/profiles/nixos/khs-openstack-server.nix" + ./vault_loki.nix + ./vault_prometheus.nix + ]; + services.grafana = { + enable = true; + settings = { + server = { + http_addr = "127.0.0.1"; + http_port = 3000; + domain = "monitoring.kaareskovgaard.net"; + root_url = "https://monitoring.kaareskovgaard.net"; + serve_from_sub_path = false; + }; + }; + provision = { + enable = true; + datasources.settings.datasources = [ + { + url = "http://${loki.configuration.server.http_listen_address}:${toString loki.configuration.server.http_listen_port}"; + type = "loki"; + name = "Logs"; + } + { + url = "http://${prometheus.listenAddress}:${toString prometheus.port}"; + type = "prometheus"; + name = "Metrics"; + } + ]; + dashboards.settings.providers = [ + { + name = "Node Exporter"; + options.path = ./grafana/dashboards/node_exporter; + } + ]; + }; + }; + services.prometheus = { + enable = true; + listenAddress = "127.0.0.1"; + extraFlags = [ "--web.enable-otlp-receiver" ]; + }; + services.loki = { + enable = true; + configuration = { + auth_enabled = false; + server = { + http_listen_port = 3100; + http_listen_address = "127.0.0.1"; + }; + common = { + ring = { + instance_addr = "127.0.0.1"; + kvstore = { + store = "inmemory"; + }; + }; + replication_factor = 1; + path_prefix = "${config.services.loki.dataDir}/common"; + }; + schema_config = { + configs = [ + { + from = "2025-07-11"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + }; + storage_config = { + tsdb_shipper = { + active_index_directory = "${config.services.loki.dataDir}/index"; + cache_location = "${config.services.loki.dataDir}/index_cache"; + }; + filesystem = { + directory = "${config.services.loki.dataDir}/chunks"; + }; + }; + pattern_ingester = { + enabled = true; + }; + compactor = { + retention_enabled = true; + compaction_interval = "24h"; + retention_delete_delay = "24h"; + delete_request_store = "filesystem"; + working_directory = "${config.services.loki.dataDir}/retention"; + }; + }; + }; + khscodes = { + infrastructure.khs-openstack-instance = { + enable = true; + flavor = "m.medium"; + }; + services.nginx = { + enable = true; + virtualHosts."monitoring.kaareskovgaard.net" = { + locations."/" = { + proxyPass = "http://${grafana.settings.server.http_addr}:${toString grafana.settings.server.http_port}"; + proxyWebsockets = true; + recommendedProxySettings = true; + }; + }; + virtualHosts."loki.kaareskovgaard.net" = { + mtls = { + verify = "on"; + certificate = "/etc/loki/client-signer.pem"; + }; + locations."/" = { + proxyPass = "http://${loki.configuration.server.http_listen_address}:${toString loki.configuration.server.http_listen_port}"; + proxyWebsockets = true; + recommendedProxySettings = true; + }; + }; + virtualHosts."prometheus.kaareskovgaard.net" = { + mtls = { + verify = "on"; + certificate = "/etc/prometheus/client-signer.pem"; + }; + locations."/" = { + proxyPass = "http://${prometheus.listenAddress}:${toString prometheus.port}"; + proxyWebsockets = true; + recommendedProxySettings = true; + }; + }; + }; + services.vault-agent.templates = [ + { + contents = '' + {{- with secret "loki-mtls/cert/ca_chain" -}} + {{ .Data.certificate }} + {{- end -}} + ''; + destination = "/etc/loki/client-signer.pem"; + owner = "loki"; + group = "loki"; + perms = "0644"; + reloadOrRestartUnits = [ "nginx.service" ]; + } + { + contents = '' + {{- with secret "prometheus-mtls/cert/ca_chain" -}} + {{ .Data.certificate }} + {{- end -}} + ''; + destination = "/etc/prometheus/client-signer.pem"; + owner = "prometheus"; + group = "prometheus"; + perms = "0644"; + reloadOrRestartUnits = [ "nginx.service" ]; + } + ]; + }; + snowfallorg.users.khs.admin = true; + users.users.khs = { + initialPassword = "changeme"; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCqY0FHnWFKfLG2yfgr4qka5sR9CK+EMAhzlHUkaQyWHTKD+G0/vC/fNPyL1VV3Dxc/ajxGuPzVE+mBMoyxazL3EtuCDOVvHJ5CR+MUSEckg/DDwcGHqy6rC8BvVVpTAVL04ByQdwFnpE1qNSBaQLkxaFVdtriGKkgMkc7+UNeYX/bv7yn+APqfP1a3xr6wdkSSdO8x4N2jsSygOIMx10hLyCV4Ueu7Kp8Ww4rGY8j5o7lKJhbgfItBfSOuQHdppHVF/GKYRhdnK6Y2fZVYbhq4KipUtclbZ6O/VYd8/sOO98+LMm7cOX+K35PQjUpYgcoNy5+Sw3CNS/NHn4JvOtTaUEYP7fK6c9LhMULOO3T7Cm6TMdiFjUKHkyG+s2Mu/LXJJoilw571zwuh6chkeitW8+Ht7k0aPV96kNEvTdoXwLhBifVEaChlAsLAzSUjUq+YYCiXVk0VIXCZQWKj8LoVNTmaqDksWwbcT64fw/FpVC0N18WHbKcFUEIW/O4spJMa30CQwf9FeqpoWoaF1oRClCSDPvX0AauCu0JcmRinz1/JmlXljnXWbSfm20/V+WyvktlI0wTD0cdpNuSasT9vS77YfJ8nutcWWZKSkCj4R4uHeCNpDTX5YXzapy7FxpM9ANCXLIvoGX7Yafba2Po+er7SSsUIY1AsnBBr8ZoDVw==" + ]; + }; + khscodes.networking.fqdn = "monitoring.kaareskovgaard.net"; + system.stateVersion = "25.05"; +} diff --git a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/grafana/dashboards/node_exporter/node_exporter_full_r41.json b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/grafana/dashboards/node_exporter/node_exporter_full_r41.json new file mode 100644 index 0000000..7409b5e --- /dev/null +++ b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/grafana/dashboards/node_exporter/node_exporter_full_r41.json @@ -0,0 +1,16114 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.6.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "GitHub", + "type": "link", + "url": "https://github.com/rfmoz/grafana-dashboards" + }, + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "Grafana", + "type": "link", + "url": "https://grafana.com/grafana/dashboards/1860" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 261, + "panels": [], + "title": "Quick CPU / Mem / Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Resource pressure via PSI", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + }, + { + "color": "dark-yellow", + "value": 70 + }, + { + "color": "dark-red", + "value": 90 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 323, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "CPU", + "range": false, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "Mem", + "range": false, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "I/O", + "range": false, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_pressure_irq_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "Irq", + "range": false, + "refId": "D", + "step": 240 + } + ], + "title": "Pressure", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Overall CPU busy percentage (averaged across all cores)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 20, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", instance=\"$node\"}[$__rate_interval])))", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "CPU Busy", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "System load over all CPU cores together", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 155, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "scalar(node_load1{instance=\"$node\",job=\"$job\"}) * 100 / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Sys Load", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Real RAM usage excluding cache and reclaimable memory", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 16, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "(1 - (node_memory_MemAvailable_bytes{instance=\"$node\", job=\"$job\"} / node_memory_MemTotal_bytes{instance=\"$node\", job=\"$job\"})) * 100", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "B", + "step": 240 + } + ], + "title": "RAM Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Percentage of swap space currently used by the system", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 25 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 21, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "((node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"})) * 100", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "SWAP Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Used Root FS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 154, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "(\n (node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", mountpoint=\"/\", fstype!=\"rootfs\"}\n - node_filesystem_avail_bytes{instance=\"$node\", job=\"$job\", mountpoint=\"/\", fstype!=\"rootfs\"})\n / node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", mountpoint=\"/\", fstype!=\"rootfs\"}\n) * 100\n", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Root FS Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 18, + "y": 1 + }, + "id": 14, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bool_yes_no" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 20, + "y": 1 + }, + "id": 328, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "node_reboot_required{instance=\"$node\",job=\"$job\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Reboot Required", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 22, + "y": 1 + }, + "id": 15, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 18, + "y": 3 + }, + "id": 23, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "RootFS Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 20, + "y": 3 + }, + "id": 75, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "RAM Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 22, + "y": 3 + }, + "id": 18, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "SWAP Total", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 263, + "panels": [], + "title": "Basic CPU / Mem / Net / Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "CPU time spent busy vs idle, split by activity type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy System" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy User" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Other" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Busy System", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Busy User", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy Iowait", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy IRQs", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy Other", + "range": true, + "refId": "E", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Idle", + "range": true, + "refId": "F", + "step": 240 + } + ], + "title": "CPU Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "RAM and swap usage overview, including caches", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Swap used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache + Buffer" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Total", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Used", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cache + Buffer", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Swap used", + "range": true, + "refId": "E", + "step": 240 + } + ], + "title": "Memory Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Per-interface network traffic (receive and transmit) in bits per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Tx.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Rx {{device}}", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Tx {{device}} ", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Percentage of filesystem space used for each mounted device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 152, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "((node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", device!~\"rootfs\"} - node_filesystem_avail_bytes{instance=\"$node\", job=\"$job\", device!~\"rootfs\"}) / node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", device!~\"rootfs\"}) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Disk Space Used Basic", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 265, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "CPU time usage split by state, normalized across all CPU cores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 70, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Idle - Waiting for something to happen" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Iowait - Waiting for I/O to complete" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Irq - Servicing interrupts" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nice - Niced processes executing in user mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Softirq - Servicing softirqs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Steal - Time spent in other operating systems when running in a virtualized environment" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCE2DE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "System - Processes executing in kernel mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "User - Normal processes executing in user mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5195CE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Guest CPU usage" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"system\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "System - Processes executing in kernel mode", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"user\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "User - Normal processes executing in user mode", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"nice\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nice - Niced processes executing in user mode", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"iowait\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Iowait - Waiting for I/O to complete", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"irq\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Irq - Servicing interrupts", + "range": true, + "refId": "E", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"softirq\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Softirq - Servicing softirqs", + "range": true, + "refId": "F", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"steal\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", + "range": true, + "refId": "G", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum(irate(node_cpu_seconds_total{mode=\"idle\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Idle - Waiting for something to happen", + "range": true, + "refId": "H", + "step": 240 + }, + { + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))) > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Guest CPU usage", + "range": true, + "refId": "I", + "step": 240 + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Breakdown of physical memory and swap usage. Hardware-detected memory errors are also displayed", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap - Swap memory usage" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused - Free memory unassigned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Hardware Corrupted - *./" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Apps - Memory used by user-space applications", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Cache - Parked file data (file content) cache", + "range": true, + "refId": "E", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Buffers - Block device (e.g. harddisk) cache", + "range": true, + "refId": "F", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Unused - Free memory unassigned", + "range": true, + "refId": "G", + "step": 240 + }, + { + "editorMode": "code", + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Swap - Swap space used", + "range": true, + "refId": "H", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working", + "range": true, + "refId": "I", + "step": 240 + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Incoming and outgoing network traffic per interface", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 303 + }, + "id": 84, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Network interface utilization as a percentage of its maximum capacity", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 303 + }, + "id": 338, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "(rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])\n / ignoring(speed) node_network_speed_bytes{instance=\"$node\",job=\"$job\", speed!=\"-1\"}) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "(rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])\n / ignoring(speed) node_network_speed_bytes{instance=\"$node\",job=\"$job\", speed!=\"-1\"}) * 100", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Saturation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Disk I/O operations per second for each device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 315 + }, + "id": 229, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "intervalFactor": 4, + "legendFormat": "{{device}} - Read", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk IOps", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Disk I/O throughput per device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read*./" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 315 + }, + "id": 42, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Read", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Write", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Amount of available disk space per mounted filesystem, excluding rootfs. Based on block availability to non-root users", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 327 + }, + "id": 43, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "metric": "", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Free", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Size", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Filesystem Space Available", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Disk usage (used = total - available) per mountpoint", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 327 + }, + "id": 156, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Filesystem Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Percentage of time the disk was actively processing I/O operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 339 + }, + "id": 127, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Disk I/O Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "How often tasks experience CPU, memory, or I/O delays. “Some” indicates partial slowdown; “Full” indicates all tasks are stalled. Based on Linux PSI metrics:\nhttps://docs.kernel.org/accounting/psi.html", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "some (-) / full (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Some.*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Some.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 339 + }, + "id": 322, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "CPU - Some", + "range": true, + "refId": "CPU some", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Memory - Some", + "range": true, + "refId": "Memory some", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_pressure_memory_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Memory - Full", + "range": true, + "refId": "Memory full", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "I/O - Some", + "range": true, + "refId": "I/O some", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_pressure_io_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "I/O - Full", + "range": true, + "refId": "I/O full", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_pressure_irq_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "IRQ - Full", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Pressure Stall Information", + "type": "timeseries" + } + ], + "title": "CPU / Memory / Net / Disk", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 266, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Displays committed memory usage versus the system's commit limit. Exceeding the limit is allowed under Linux overcommit policies but may increase OOM risks under high load", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*CommitLimit - *./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 602 + }, + "id": 135, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Committed_AS – Memory promised to processes (not necessarily used)", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "CommitLimit - Max allowable committed memory", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Memory Committed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Memory currently dirty (modified but not yet written to disk), being actively written back, or held by writeback buffers. High dirty or writeback memory may indicate disk I/O pressure or delayed flushing", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 602 + }, + "id": 130, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Writeback – Memory currently being flushed to disk", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "WritebackTmp – FUSE temporary writeback buffers", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Dirty – Memory marked dirty (pending write to disk)", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "NFS Unstable – Pages sent to NFS server, awaiting storage commit", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Memory Writeback and Dirty", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Kernel slab memory usage, separated into reclaimable and non-reclaimable categories. Reclaimable memory can be freed under memory pressure (e.g., caches), while unreclaimable memory is locked by the kernel for core functions", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 802 + }, + "id": 131, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SUnreclaim – Non-reclaimable slab memory (kernel objects)", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SReclaimable – Potentially reclaimable slab memory (e.g., inode cache)", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Memory Slab", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Memory used for mapped files (such as libraries) and shared memory (shmem and tmpfs), including variants backed by huge pages", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 802 + }, + "id": 138, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Mapped – Memory mapped from files (e.g., libraries, mmap)", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Shmem – Shared memory used by processes and tmpfs", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ShmemHugePages – Shared memory (shmem/tmpfs) allocated with HugePages", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PMD Mapped – Shmem/tmpfs backed by Transparent HugePages (PMD)", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Memory Shared and Mapped", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Proportion of memory pages in the kernel's active and inactive LRU lists relative to total RAM. Active pages have been recently used, while inactive pages are less recently accessed but still resident in memory", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Active.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Inactive.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 812 + }, + "id": 136, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "(node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}) \n/ \n(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Inactive – Less recently used memory, more likely to be reclaimed", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "(node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}) \n/ \n(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"})\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active – Recently used memory, retained unless under pressure", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Memory LRU Active / Inactive (%)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Breakdown of memory pages in the kernel's active and inactive LRU lists, separated by anonymous (heap, tmpfs) and file-backed (caches, mmap) pages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 812 + }, + "id": 191, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Inactive_file - File-backed memory on inactive LRU list", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Inactive_anon – Anonymous memory on inactive LRU (incl. tmpfs & swap cache)", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Active_file - File-backed memory on active LRU list", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Active_anon – Anonymous memory on active LRU (incl. tmpfs & swap cache)", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Memory LRU Active / Inactive Detail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Tracks kernel memory used for CPU-local structures, per-thread stacks, and bounce buffers used for I/O on DMA-limited devices. These areas are typically small but critical for low-level operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 822 + }, + "id": 160, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "KernelStack – Kernel stack memory (per-thread, non-reclaimable)", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PerCPU – Dynamically allocated per-CPU memory (used by kernel modules)", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Bounce Memory – I/O buffer for DMA-limited devices", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Memory Kernel / CPU / IO", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Usage of the kernel's vmalloc area, which provides virtual memory allocations for kernel modules and drivers. Includes total, used, and largest free block sizes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Total.*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 822 + }, + "id": 70, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Vmalloc Free Chunk – Largest available block in vmalloc area", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Vmalloc Total – Total size of the vmalloc memory area", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Vmalloc Used – Portion of vmalloc area currently in use", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Memory Vmalloc", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Memory used by anonymous pages (not backed by files), including standard and huge page allocations. Includes heap, stack, and memory-mapped anonymous regions", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 832 + }, + "id": 129, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "AnonHugePages – Anonymous memory using HugePages", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "AnonPages – Anonymous memory (non-file-backed)", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Memory Anonymous", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Memory that is locked in RAM and cannot be swapped out. Includes both kernel-unevictable memory and user-level memory locked with mlock()", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 832 + }, + "id": 137, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Unevictable – Kernel-pinned memory (not swappable)", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Mlocked – Application-locked memory via mlock()", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Memory Unevictable and MLocked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "How much memory is directly mapped in the kernel using different page sizes (4K, 2M, 1G). Helps monitor large page utilization in the direct map region", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 842 + }, + "id": 128, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "DirectMap 1G – Memory mapped with 1GB pages", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "DirectMap 2M – Memory mapped with 2MB pages", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "DirectMap 4K – Memory mapped with 4KB pages", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Memory DirectMap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Displays HugePages memory usage in bytes, including allocated, free, reserved, and surplus memory. All values are calculated based on the number of huge pages multiplied by their configured size", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 842 + }, + "id": 140, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages Used – Currently allocated", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages Reserved – Promised but unused", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages Surplus – Dynamic pool extension", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "HugePages Total – Reserved memory", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Memory HugePages", + "type": "timeseries" + } + ], + "title": "Memory Meminfo", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 267, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of memory pages being read from or written to disk (page-in and page-out operations). High page-out may indicate memory pressure or swapping activity", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 603 + }, + "id": 176, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pagesin - Page in ops", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pagesout - Page out ops", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Memory Pages In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate at which memory pages are being swapped in from or out to disk. High swap-out activity may indicate memory pressure", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 603 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pswpin - Pages swapped in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pswpout - Pages swapped out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Memory Pages Swap In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of memory page faults, split into total, major (disk-backed), and derived minor (non-disk) faults. High major fault rates may indicate memory pressure or insufficient RAM", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pgfault - Page major and minor fault ops" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "none" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 783 + }, + "id": 175, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Pgfault - Page major and minor fault ops", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Pgmajfault - Major page fault ops", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Pgminfault - Minor page fault ops", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Memory Page Faults", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of Out-of-Memory (OOM) kill events. A non-zero value indicates the kernel has terminated one or more processes due to memory exhaustion", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "OOM Kills" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 783 + }, + "id": 307, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OOM Kills", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "OOM Killer", + "type": "timeseries" + } + ], + "title": "Memory Vmstat", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 293, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Tracks the system clock's estimated and maximum error, as well as its offset from the reference clock (e.g., via NTP). Useful for detecting synchronization drift", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 604 + }, + "id": 260, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Estimated error", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Offset local vs reference", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum error", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Time Synchronized Drift", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "NTP phase-locked loop (PLL) time constant used by the kernel to control time adjustments. Lower values mean faster correction but less stability", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 604 + }, + "id": 291, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PLL Time Constant", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Time PLL Adjust", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shows whether the system clock is synchronized to a reliable time source, and the current frequency correction ratio applied by the kernel to maintain synchronization", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 754 + }, + "id": 168, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_timex_sync_status{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Sync status (1 = ok)", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Frequency Adjustment", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Tick Interval", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "TAI Offset", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Time Synchronized Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Displays the PPS signal's frequency offset and stability (jitter) in hertz. Useful for monitoring high-precision time sources like GPS or atomic clocks", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "rothz" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 754 + }, + "id": 333, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_timex_pps_frequency_hertz{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PPS Frequency Offset", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_timex_pps_stability_hertz{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PPS Frequency Stability", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "PPS Frequency / Stability", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Tracks PPS signal timing jitter and shift compared to system clock", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 764 + }, + "id": 334, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_timex_pps_jitter_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PPS Jitter", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_timex_pps_shift_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PPS Shift", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "PPS Time Accuracy", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of PPS synchronization diagnostics including calibration events, jitter violations, errors, and frequency stability exceedances", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 764 + }, + "id": 335, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_timex_pps_calibration_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PPS Calibrations/sec", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_timex_pps_error_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PPS Errors/sec", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_timex_pps_stability_exceeded_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "PPS Stability Exceeded/sec", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_timex_pps_jitter_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "PPS Jitter Events/sec", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "PPS Sync Events", + "type": "timeseries" + } + ], + "title": "System Timesync", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 312, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Processes currently in runnable or blocked states. Helps identify CPU contention or I/O wait bottlenecks.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 605 + }, + "id": 62, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_procs_blocked{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Blocked (I/O Wait)", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_procs_running{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Runnable (Ready for CPU)", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Processes Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Current number of processes in each state (e.g., running, sleeping, zombie). Requires --collector.processes to be enabled in node_exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "D" + }, + "properties": [ + { + "id": "displayName", + "value": "Uninterruptible Sleeping" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "I" + }, + "properties": [ + { + "id": "displayName", + "value": "Idle Kernel Thread" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "R" + }, + "properties": [ + { + "id": "displayName", + "value": "Running" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S" + }, + "properties": [ + { + "id": "displayName", + "value": "Interruptible Sleeping" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "T" + }, + "properties": [ + { + "id": "displayName", + "value": "Stopped" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "X" + }, + "properties": [ + { + "id": "displayName", + "value": "Dead" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Z" + }, + "properties": [ + { + "id": "displayName", + "value": "Zombie" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 605 + }, + "id": 315, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_processes_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ state }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Processes Detailed States", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of new processes being created on the system (forks/sec).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 635 + }, + "id": 148, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Process Forks per second", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Processes Forks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shows CPU saturation per core, calculated as the proportion of time spent waiting to run relative to total time demanded (running + waiting).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*waiting.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 635 + }, + "id": 305, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }} - Running", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Waiting Queue", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])\n/\n(irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) + irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))\n", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}}", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "CPU Saturation per Core", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of active PIDs on the system and the configured maximum allowed. Useful for detecting PID exhaustion risk. Requires --collector.processes in node_exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PIDs limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 645 + }, + "id": 313, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_processes_pids{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Number of PIDs", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_processes_max_processes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PIDs limit", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "PIDs Number and Limit", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of active threads on the system and the configured thread limit. Useful for monitoring thread pressure. Requires --collector.processes in node_exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Threads limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 645 + }, + "id": 314, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_processes_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Allocated threads", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_processes_max_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Threads limit", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Threads Number and Limit", + "type": "timeseries" + } + ], + "title": "System Processes", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 269, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Per-second rate of context switches and hardware interrupts. High values may indicate intense CPU or I/O activity", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 686 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Context switches", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Interrupts", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Context Switches / Interrupts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "System load average over 1, 5, and 15 minutes. Reflects the number of active or waiting processes. Values above CPU core count may indicate overload", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "CPU Core Count" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 686 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_load1{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Load 1m", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_load5{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Load 5m", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_load15{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Load 15m", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "CPU Core Count", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "System Load", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Real-time CPU frequency scaling per core, including average minimum and maximum allowed scaling frequencies", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "hertz" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Max" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Min" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 696 + }, + "id": 321, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_cpu_scaling_frequency_hertz{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "avg(node_cpu_scaling_frequency_max_hertz{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Max", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "avg(node_cpu_scaling_frequency_min_hertz{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Min", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "CPU Frequency Scaling", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of scheduling timeslices executed per CPU. Reflects how frequently the scheduler switches tasks on each core", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 696 + }, + "id": 306, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "CPU Schedule Timeslices", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Breaks down hardware interrupts by type and device. Useful for diagnosing IRQ load on network, disk, or CPU interfaces. Requires --collector.interrupts to be enabled in node_exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 706 + }, + "id": 259, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ type }} - {{ info }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "IRQ Detail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of bits of entropy currently available to the system's random number generators (e.g., /dev/random). Low values may indicate that random number generation could block or degrade performance of cryptographic operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "decbits" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Entropy pool max" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 706 + }, + "id": 151, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_entropy_available_bits{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Entropy available", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_entropy_pool_size_bits{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Entropy pool max", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Entropy", + "type": "timeseries" + } + ], + "title": "System Misc", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 304, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Monitors hardware sensor temperatures and critical thresholds as exposed by Linux hwmon. Includes CPU, GPU, and motherboard sensors where available", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "celsius" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Critical*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 607 + }, + "id": 158, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }}", + "range": true, + "refId": "A", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Critical Alarm", + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Critical", + "range": true, + "refId": "C", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Critical Historical", + "refId": "D", + "step": 240 + }, + { + "expr": "node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} Max", + "refId": "E", + "step": 240 + } + ], + "title": "Hardware Temperature Monitor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shows how hard each cooling device (fan/throttle) is working relative to its maximum capacity", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 607 + }, + "id": 300, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "100 * node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"} / node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ name }} - {{ type }} ", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Cooling Device Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shows the online status of power supplies (e.g., AC, battery). A value of 1-Yes indicates the power supply is active/online", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bool_yes_no" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 617 + }, + "id": 302, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_power_supply_online{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ power_supply }} online", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Power Supply", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Displays the current fan speeds (RPM) from hardware sensors via the hwmon interface", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "rotrpm" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 617 + }, + "id": 325, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_hwmon_fan_rpm{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }}", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_hwmon_fan_min_rpm{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ chip_name }} {{ sensor }} rpm min", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Hardware Fan Speed", + "type": "timeseries" + } + ], + "title": "Hardware Misc", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 296, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Current number of systemd units in each operational state, such as active, failed, inactive, or transitioning", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Activating" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C8F2C2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Deactivating" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 4098 + }, + "id": 298, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Activating", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Active", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Deactivating", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Failed", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Inactive", + "range": true, + "refId": "E", + "step": 240 + } + ], + "title": "Systemd Units State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Current number of active connections per systemd socket, as reported by the Node Exporter systemd collector", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 4098 + }, + "id": 331, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_systemd_socket_current_connections{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ name }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Systemd Sockets Current", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of accepted connections per second for each systemd socket", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 4108 + }, + "id": 297, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ name }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Systemd Sockets Accepted", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of systemd socket connection refusals per second, typically due to service unavailability or backlog overflow", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 4108 + }, + "id": 332, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_systemd_socket_refused_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ name }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Systemd Sockets Refused", + "type": "timeseries" + } + ], + "title": "Systemd", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 270, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of I/O operations completed per second for the device (after merges), including both reads and writes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "read (–) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Read", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk Read/Write IOps", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of bytes read from or written to the device per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "read (–) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Read", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Write", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk Read/Write Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "read (–) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 259 + }, + "id": 37, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Read", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk Average Wait Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Average queue length of the requests that were issued to the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/sda_*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 259 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Average Queue Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of read and write requests merged per second that were queued to the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "read (–) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 269 + }, + "id": 133, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Read", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk R/W Merged", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Percentage of time the disk spent actively processing I/O operations, including general I/O, discards (TRIM), and write cache flushes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 269 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - General IO", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Discard/TRIM", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_flush_requests_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Flush (write cache)", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Time Spent Doing I/Os", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Per-second rate of discard (TRIM) and flush (write cache) operations. Useful for monitoring low-level disk activity on SSDs and advanced storage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 279 + }, + "id": 301, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Discards completed", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Discards merged", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_disk_flush_requests_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Flush", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Disk Ops Discards / Flush", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shows how many disk sectors are discarded (TRIMed) per second. Useful for monitoring SSD behavior and storage efficiency", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 279 + }, + "id": 326, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_disk_discarded_sectors_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Disk Sectors Discarded Successfully", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of in-progress I/O requests at the time of sampling (active requests in the disk queue)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 289 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_disk_io_now{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Instantaneous Queue Size", + "type": "timeseries" + } + ], + "title": "Storage Disk", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 271, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of file descriptors currently allocated system-wide versus the system limit. Important for detecting descriptor exhaustion risks", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "sishort" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max.*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_filefd_maximum{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Max open files", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_filefd_allocated{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Open files", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "File Descriptor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of free file nodes (inodes) available per mounted filesystem. A low count may prevent file creation even if disk space is available", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 41, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "File Nodes Free", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Indicates filesystems mounted in read-only mode or reporting device-level I/O errors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bool_yes_no" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 240 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - ReadOnly", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Device error", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Filesystem in ReadOnly / Error", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of file nodes (inodes) available per mounted filesystem. Reflects maximum file capacity regardless of disk size", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "sishort" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 240 + }, + "id": 219, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "File Nodes Size", + "type": "timeseries" + } + ], + "title": "Storage Filesystem", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 272, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of network packets received and transmitted per second, by interface.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 60, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic by Packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of packet-level errors for each network interface. Receive errors may indicate physical or driver issues; transmit errors may reflect collisions or hardware faults", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 142, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of dropped packets per network interface. Receive drops can indicate buffer overflow or driver issues; transmit drops may result from outbound congestion or queuing limits", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 121 + }, + "id": 143, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Drop", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of compressed network packets received and transmitted per interface. These are common in low-bandwidth or special interfaces like PPP or SLIP", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 121 + }, + "id": 141, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Compressed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of incoming multicast packets received per network interface. Multicast is used by protocols such as mDNS, SSDP, and some streaming or cluster services", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 131 + }, + "id": 146, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Multicast", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of received packets that could not be processed due to missing protocol or handler in the kernel. May indicate unsupported traffic or misconfiguration", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 131 + }, + "id": 327, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_nohandler_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic NoHandler", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of frame errors on received packets, typically caused by physical layer issues such as bad cables, duplex mismatches, or hardware problems", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 141 + }, + "id": 145, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Frame", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Tracks FIFO buffer overrun errors on network interfaces. These occur when incoming or outgoing packets are dropped due to queue or buffer overflows, often indicating congestion or hardware limits", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 141 + }, + "id": 144, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "rate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Fifo", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of packet collisions detected during transmission. Mostly relevant on half-duplex or legacy Ethernet networks", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 151 + }, + "id": 232, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Collision", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of carrier errors during transmission. These typically indicate physical layer issues like faulty cabling or duplex mismatches", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 151 + }, + "id": 231, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Tx out", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Carrier Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of ARP entries per interface. Useful for detecting excessive ARP traffic or table growth due to scanning or misconfiguration", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 161 + }, + "id": 230, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_arp_entries{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} ARP Table", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "ARP Entries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Current and maximum connection tracking entries used by Netfilter (nf_conntrack). High usage approaching the limit may cause packet drops or connection issues", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "NF conntrack limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 161 + }, + "id": 61, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "NF conntrack entries", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "NF conntrack limit", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "NF Conntrack", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Operational and physical link status of each network interface. Values are Yes for 'up' or link present, and No for 'down' or no carrier.\"", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bool_yes_no" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 171 + }, + "id": 309, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{interface}} - Operational state UP", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_network_carrier{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{device}} - Physical link", + "refId": "B" + } + ], + "title": "Network Operational Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Maximum speed of each network interface as reported by the operating system. This is a static hardware capability, not current throughput", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "fieldMinMax": false, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 12, + "y": 171 + }, + "id": 280, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 30, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_network_speed_bytes{instance=\"$node\",job=\"$job\"} * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Speed", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "MTU (Maximum Transmission Unit) in bytes for each network interface. Affects packet size and transmission efficiency", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 171 + }, + "id": 288, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 30, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "MTU", + "type": "bargauge" + } + ], + "title": "Network Traffic", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 273, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Tracks TCP socket usage and memory per node", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 63, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Allocated Sockets", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "In-Use Sockets", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Orphaned Sockets", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TIME_WAIT Sockets", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Sockstat TCP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of UDP and UDPLite sockets currently in use", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 124, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDPLite - In-Use Sockets", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP - In-Use Sockets", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Sockstat UDP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of sockets currently in use across all protocols (TCP, UDP, UNIX, etc.), as reported by /proc/net/sockstat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 122 + }, + "id": 126, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Total sockets", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Sockstat Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of FRAG and RAW sockets currently in use. RAW sockets are used for custom protocols or tools like ping; FRAG sockets are used internally for IP packet defragmentation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 122 + }, + "id": 125, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "FRAG - In-Use Sockets", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RAW - In-Use Sockets", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Sockstat FRAG / RAW", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "TCP/UDP socket memory usage in kernel (in pages)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 132 + }, + "id": 336, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "TCP/UDP Kernel Buffer Memory Pages", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Kernel memory used by TCP, UDP, and IP fragmentation buffers", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 132 + }, + "id": 220, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Fragmentation", + "range": true, + "refId": "C" + } + ], + "title": "Sockstat Memory Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Packets processed and dropped by the softnet network stack per CPU. Drops may indicate CPU saturation or network driver limitations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "drop (-) / process (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Dropped.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 142 + }, + "id": 290, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Processed", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Dropped", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Softnet Packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "How often the kernel was unable to process all packets in the softnet queue before time ran out. Frequent squeezes may indicate CPU contention or driver inefficiency", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 142 + }, + "id": 310, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Times Squeezed", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Softnet Out of Quota", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Tracks the number of packets processed or dropped by Receive Packet Steering (RPS), a mechanism to distribute packet processing across CPUs", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Dropped.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 152 + }, + "id": 330, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_softnet_received_rps_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Processed", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_softnet_flow_limit_count_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Dropped", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Softnet RPS", + "type": "timeseries" + } + ], + "title": "Network Sockstat", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 274, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of octets sent and received at the IP layer, as reported by /proc/net/netstat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 221, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "IP Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "IP Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Netstat IP In / Out Octets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of TCP segments sent and received per second, including data and control segments", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Snd.*/" + }, + "properties": [] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 299, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP Rx in", + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "TCP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of UDP datagrams sent and received per second, based on /proc/net/netstat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 55, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "UDP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of ICMP messages sent and received per second, including error and control messages", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 115, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ICMP Rx in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ICMP Tx out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "ICMP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Tracks various TCP error and congestion-related events, including retransmissions, timeouts, dropped connections, and buffer issues", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 73 + }, + "id": 104, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Listen Overflows", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Listen Drops", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "SYN Retransmits", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Segment Retransmits", + "range": true, + "refId": "D" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Receive Errors", + "range": true, + "refId": "E" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "RST Sent", + "range": true, + "refId": "F" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_TCPRcvQDrop{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "Receive Queue Drops", + "range": true, + "refId": "G" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_TCPOFOQueue{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "Out-of-order Queued", + "range": true, + "refId": "H" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_TCPTimeouts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "TCP Timeouts", + "range": true, + "refId": "I" + } + ], + "title": "TCP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of UDP and UDPLite datagram delivery errors, including missing listeners, buffer overflows, and protocol-specific issues", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 73 + }, + "id": 109, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP Rx in Errors", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP No Listener", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "UDPLite Rx in Errors", + "range": true, + "refId": "C" + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP Rx in Buffer Errors", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP Tx out Buffer Errors", + "range": true, + "refId": "E", + "step": 240 + } + ], + "title": "UDP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of incoming ICMP messages that contained protocol-specific errors, such as bad checksums or invalid lengths", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 50, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ICMP Rx In", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "ICMP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of TCP SYN cookies sent, validated, and failed. These are used to protect against SYN flood attacks and manage TCP handshake resources under load", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "eps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Failed.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 91, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SYN Cookies Failed", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SYN Cookies Validated", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SYN Cookies Sent", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "TCP SynCookie", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of currently established TCP connections and the system's max supported limit. On Linux, MaxConn may return -1 to indicate a dynamic/unlimited configuration", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 93 + }, + "id": 85, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Current Connections", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Max Connections", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of UDP packets currently queued in the receive (RX) and transmit (TX) buffers. A growing queue may indicate a bottleneck", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 93 + }, + "id": 337, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_udp_queues{instance=\"$node\",job=\"$job\",ip=\"v4\",queue=\"rx\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP Rx in Queue", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_udp_queues{instance=\"$node\",job=\"$job\",ip=\"v4\",queue=\"tx\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP Tx out Queue", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "UDP Queue", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of TCP connection initiations per second. 'Active' opens are initiated by this host. 'Passive' opens are accepted from incoming connections", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 103 + }, + "id": 82, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Active Opens", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Passive Opens", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "TCP Direct Transition", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of TCP sockets in key connection states. Requires the --collector.tcpstat flag on node_exporter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 103 + }, + "id": 320, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"established\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Established", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"fin_wait2\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "FIN_WAIT2", + "range": true, + "refId": "B", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"listen\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Listen", + "range": true, + "refId": "C", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"time_wait\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "TIME_WAIT", + "range": true, + "refId": "D", + "step": 240 + }, + { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"close_wait\", instance=\"$node\", job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CLOSE_WAIT", + "range": true, + "refId": "E", + "step": 240 + } + ], + "title": "TCP Stat", + "type": "timeseries" + } + ], + "title": "Network Netstat", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 279, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Duration of each individual collector executed during a Node Exporter scrape. Useful for identifying slow or failing collectors", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 40, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{collector}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Node Exporter Scrape Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of CPU time used by the process exposing this metric (user + system mode)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 308, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Process CPU Usage", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Exporter Process CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Tracks the memory usage of the process exposing this metric (e.g., node_exporter), including current virtual memory and maximum virtual memory limit", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Virtual Memory Limit" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Virtual Memory" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 10, + "x": 0, + "y": 44 + }, + "id": 149, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Virtual Memory", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Virtual Memory Limit", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Exporter Processes Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of file descriptors used by the exporter process versus its configured limit", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Open file descriptors" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 10, + "x": 10, + "y": 44 + }, + "id": 64, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "process_max_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum open file descriptors", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "process_open_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Open file descriptors", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Exporter File Descriptor Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shows whether each Node Exporter collector scraped successfully (1 = success, 0 = failure), and whether the textfile collector returned an error.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "bool" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 4, + "x": 20, + "y": 44 + }, + "id": 157, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.6.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_scrape_collector_success{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{collector}}", + "range": true, + "refId": "A", + "step": 240 + }, + { + "editorMode": "code", + "expr": "1 - node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "textfile", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Node Exporter Scrape", + "type": "bargauge" + } + ], + "title": "Node Exporter", + "type": "row" + } + ], + "refresh": "1m", + "schemaVersion": 41, + "tags": [ + "linux" + ], + "templating": { + "list": [ + { + "current": {}, + "includeAll": false, + "label": "Datasource", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "", + "includeAll": false, + "label": "Job", + "name": "job", + "options": [], + "query": { + "query": "label_values(node_uname_info, job)", + "refId": "Prometheus-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(node_uname_info{job=\"$job\"}, nodename)", + "includeAll": false, + "label": "Nodename", + "name": "nodename", + "options": [], + "query": { + "query": "label_values(node_uname_info{job=\"$job\"}, nodename)", + "refId": "Prometheus-nodename-Variable-Query" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(node_uname_info{job=\"$job\", nodename=\"$nodename\"}, instance)", + "includeAll": false, + "label": "Instance", + "name": "node", + "options": [], + "query": { + "query": "label_values(node_uname_info{job=\"$job\", nodename=\"$nodename\"}, instance)", + "refId": "Prometheus-node-Variable-Query" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" + }, + "hide": 2, + "includeAll": false, + "name": "diskdevices", + "options": [ + { + "selected": true, + "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" + } + ], + "query": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "type": "custom" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Node Exporter Full", + "uid": "rYdddlPWk", + "version": 96, + "weekStart": "", + "gnetId": 1860 +} \ No newline at end of file diff --git a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/vault_loki.nix b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/vault_loki.nix new file mode 100644 index 0000000..d5f034b --- /dev/null +++ b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/vault_loki.nix @@ -0,0 +1,28 @@ +# This should go into the setup of the vault server itself, as the vault server also needs stuff that depends on this. +{ + khscodes.infrastructure.vault-loki-sender = { + enable = true; + # This is quite ugly, but should get the job done. Sadly I cannot reference the output from here. + terranixBackendName = "\${ vault_mount.loki-mtls.path }"; + }; + khscodes.infrastructure.provisioning.pre.modules = [ + ( + { config, ... }: + { + khscodes.vault.enable = true; + khscodes.vault.mount.loki-mtls = { + type = "pki"; + path = "loki-mtls"; + max_lease_ttl_seconds = 10 * 365 * 24 * 60 * 60; + default_lease_ttl_seconds = 60 * 60; + }; + khscodes.vault.pki_secret_backend_root_cert.loki-mtls = { + backend = config.khscodes.vault.output.mount.loki-mtls.path; + type = "internal"; + common_name = "loki.kaareskovgaard.net"; + issuer_name = "loki-mtls-root-ca"; + }; + } + ) + ]; +} diff --git a/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/vault_prometheus.nix b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/vault_prometheus.nix new file mode 100644 index 0000000..cf989de --- /dev/null +++ b/nix/systems/x86_64-linux/monitoring.kaareskovgaard.net/vault_prometheus.nix @@ -0,0 +1,28 @@ +# This should go into the setup of the vault server itself, as the vault server also needs stuff that depends on this. +{ + khscodes.infrastructure.vault-prometheus-sender = { + enable = true; + # This is quite ugly, but should get the job done. Sadly I cannot reference the output from here. + terranixBackendName = "\${ vault_mount.prometheus-mtls.path }"; + }; + khscodes.infrastructure.provisioning.pre.modules = [ + ( + { config, ... }: + { + khscodes.vault.enable = true; + khscodes.vault.mount.prometheus-mtls = { + type = "pki"; + path = "prometheus-mtls"; + max_lease_ttl_seconds = 10 * 365 * 24 * 60 * 60; + default_lease_ttl_seconds = 60 * 60; + }; + khscodes.vault.pki_secret_backend_root_cert.prometheus-mtls = { + backend = config.khscodes.vault.output.mount.prometheus-mtls.path; + type = "internal"; + common_name = "prometheus.kaareskovgaard.net"; + issuer_name = "prometheus-mtls-root-ca"; + }; + } + ) + ]; +} diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 7db53b4..a4d112d 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -21,7 +21,7 @@ clap = { version = "4.5.39", default-features = false, features = [ "derive", ] } log = { version = "0.4.27", default-features = false, features = ["std"] } -nix = { version = "0.30.1", default-features = false, features = ["process"] } +nix = { version = "0.30.1", default-features = false } serde = { version = "1.0.219", default-features = false, features = [ "derive", "std", diff --git a/rust/program/openbao-helper/Cargo.toml b/rust/program/openbao-helper/Cargo.toml index b85f031..b73ed62 100644 --- a/rust/program/openbao-helper/Cargo.toml +++ b/rust/program/openbao-helper/Cargo.toml @@ -9,6 +9,6 @@ anyhow = { workspace = true } clap = { workspace = true } common = { path = "../../lib/common" } log = { workspace = true } -nix = { workspace = true } +nix = { workspace = true, features = ["env", "process"] } serde = { workspace = true } hakari = { version = "0.1", path = "../../lib/hakari" } diff --git a/rust/program/openbao-helper/src/enventry.rs b/rust/program/openbao-helper/src/enventry.rs new file mode 100644 index 0000000..0a38665 --- /dev/null +++ b/rust/program/openbao-helper/src/enventry.rs @@ -0,0 +1,108 @@ +use std::{collections::BTreeMap, marker::PhantomData, vec::IntoIter}; + +use serde::Deserialize; + +pub trait EnvEntryConfig { + const SECRETS: &'static [&'static str]; + const BAO_KEY: &'static str; +} + +pub struct EnvEntry(Vec<(&'static str, String)>, PhantomData); + +impl EnvEntry { + pub fn try_new_from_env() -> anyhow::Result { + let mut result = Vec::with_capacity(T::SECRETS.len()); + for key in T::SECRETS { + let value = common::env::read_env(key)?; + result.push((*key, value)); + } + Ok(Self(result, PhantomData)) + } + + fn new_from_values(values: Vec<(&'static str, String)>) -> Self { + Self(values, PhantomData) + } + + pub fn read_from_bao() -> anyhow::Result { + read_bao_data() + } +} + +impl From> for Vec<(&'static str, String)> { + fn from(value: EnvEntry) -> Self { + value.0 + } +} + +impl IntoIterator for EnvEntry { + type Item = (&'static str, String); + + type IntoIter = IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +impl<'de, T: EnvEntryConfig> serde::Deserialize<'de> for EnvEntry { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_map(EnvEntryVisitor(PhantomData)) + } +} +struct EnvEntryVisitor(PhantomData); + +impl<'de, T: EnvEntryConfig> serde::de::Visitor<'de> for EnvEntryVisitor { + type Value = EnvEntry; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_fmt(format_args!( + "a map with unique keys {} with string values", + T::SECRETS.join(", "), + )) + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let mut values = BTreeMap::<&'static str, String>::new(); + while let Some((key, value)) = map.next_entry::<&'de str, String>()? { + let mapped_key = T::SECRETS.iter().find(|n| **n == key).copied(); + let Some(key) = mapped_key else { + return Err(serde::de::Error::unknown_field(key, T::SECRETS)); + }; + if values.contains_key(key) { + return Err(serde::de::Error::duplicate_field(key)); + } + values.insert(key, value); + } + for key in T::SECRETS { + if !values.contains_key(key) { + return Err(serde::de::Error::missing_field(key)); + } + } + let values = values.into_iter().collect(); + let entry = EnvEntry::::new_from_values(values); + Ok(entry) + } +} + +#[derive(Debug, Deserialize)] +struct OpenBaoKvEntry { + data: OpenBaoKvEntryData, +} + +#[derive(Debug, Deserialize)] +struct OpenBaoKvEntryData { + data: T, +} + +fn read_bao_data() -> anyhow::Result> { + let mut cmd = common::proc::Command::new("bao"); + cmd.args(["kv", "get", "-format=json", "-mount=opentofu", T::BAO_KEY]); + let result: OpenBaoKvEntry> = cmd.try_spawn_to_json()?; + Ok(result.data.data) +} diff --git a/rust/program/openbao-helper/src/main.rs b/rust/program/openbao-helper/src/main.rs index 9f32de2..f76dd52 100644 --- a/rust/program/openbao-helper/src/main.rs +++ b/rust/program/openbao-helper/src/main.rs @@ -1,8 +1,15 @@ -use std::{collections::BTreeSet, ffi::CString}; +use std::{ + collections::BTreeSet, + convert::Infallible, + ffi::{CStr, CString, OsStr, OsString}, +}; use anyhow::Context as _; use clap::{Parser, Subcommand}; -use serde::Deserialize; + +mod enventry; + +use enventry::*; fn main() { common::entrypoint(program); @@ -45,6 +52,10 @@ pub enum Endpoint { Hcloud, #[value(name = "unifi")] Unifi, + #[value(name = "vault")] + Vault, + #[value(name = "authentik")] + Authentik, } impl Endpoint { @@ -52,23 +63,31 @@ impl Endpoint { match self { Self::Openstack => { let data = OpenstackData::read_from_bao()?; - Ok(data.into_env_data()) + Ok(data.into()) } Self::Aws => { let data = AwsData::read_from_bao()?; - Ok(data.into_env_data()) + Ok(data.into()) } Self::Hcloud => { let data = HcloudData::read_from_bao()?; - Ok(data.into_env_data()) + Ok(data.into()) } Self::Cloudflare => { let data = CloudflareData::read_from_bao()?; - Ok(data.into_env_data()) + Ok(data.into()) } Self::Unifi => { let data = UnifiData::read_from_bao()?; - Ok(data.into_env_data()) + Ok(data.into()) + } + Self::Authentik => { + let data = AuthentikData::read_from_bao()?; + Ok(data.into()) + } + Self::Vault => { + let data = VaultData::read_from_bao()?; + Ok(data.into()) } } } @@ -82,267 +101,88 @@ fn program() -> anyhow::Result<()> { } } -#[derive(Debug, Deserialize)] -struct OpenBaoKvEntry { - data: OpenBaoKvEntryData, +macro_rules! entry_definition { + ($config_id:ident, $id: ident, $bao_key: expr, $secrets: expr) => { + struct $config_id; + + impl EnvEntryConfig for $config_id { + const SECRETS: &'static [&'static str] = $secrets; + const BAO_KEY: &'static str = $bao_key; + } + + type $id = EnvEntry<$config_id>; + }; } -#[derive(Debug, Deserialize)] -struct OpenBaoKvEntryData { - data: T, -} - -#[derive(Debug, Deserialize)] -struct OpenstackData { - username: String, - password: String, - tenant_name: String, - auth_url: String, - endpoint_type: String, - region: String, -} - -fn read_bao_data Deserialize<'de>>(key: &str) -> anyhow::Result { - let mut cmd = common::proc::Command::new("bao"); - cmd.args(["kv", "get", "-format=json", "-mount=opentofu", key]); - let result: OpenBaoKvEntry = cmd.try_spawn_to_json()?; - Ok(result.data.data) -} - -impl OpenstackData { - pub fn read_from_env() -> anyhow::Result { - let username = common::env::read_env("TF_VAR_openstack_username")?; - let password = common::env::read_env("TF_VAR_openstack_password")?; - let tenant_name = common::env::read_env("TF_VAR_openstack_tenant_name")?; - let auth_url = common::env::read_env("TF_VAR_openstack_auth_url")?; - let endpoint_type = common::env::read_env("TF_VAR_openstack_endpoint_type")?; - let region = common::env::read_env("TF_VAR_openstack_region")?; - Ok(Self { - username, - password, - tenant_name, - auth_url, - endpoint_type, - region, - }) - } - - pub fn read_from_bao() -> anyhow::Result { - let data = read_bao_data("openstack")?; - Ok(data) - } - - pub fn into_env_data(self) -> Vec<(&'static str, String)> { - vec![ - ("TF_VAR_openstack_username", self.username), - ("TF_VAR_openstack_password", self.password), - ("TF_VAR_openstack_tenant_name", self.tenant_name), - ("TF_VAR_openstack_auth_url", self.auth_url), - ("TF_VAR_openstack_endpoint_type", self.endpoint_type), - ("TF_VAR_openstack_region", self.region), - ] - } -} - -impl IntoIterator for OpenstackData { - type Item = (&'static str, String); - - type IntoIter = as IntoIterator>::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - vec![ - ("username", self.username), - ("password", self.password), - ("tenant_name", self.tenant_name), - ("auth_url", self.auth_url), - ("endpoint_type", self.endpoint_type), - ("region", self.region), - ] - .into_iter() - } -} - -#[derive(Debug, Deserialize)] -struct CloudflareData { - token: String, - email: String, -} - -impl CloudflareData { - pub fn read_from_env() -> anyhow::Result { - let token = common::env::read_env("TF_VAR_cloudflare_token")?; - let email = common::env::read_env("TF_VAR_cloudflare_email")?; - Ok(Self { token, email }) - } - - pub fn read_from_bao() -> anyhow::Result { - let data = read_bao_data("cloudflare")?; - Ok(data) - } - - pub fn into_env_data(self) -> Vec<(&'static str, String)> { - vec![ - ("TF_VAR_cloudflare_token", self.token), - ("TF_VAR_cloudflare_email", self.email), - ] - } -} - -impl IntoIterator for CloudflareData { - type Item = (&'static str, String); - - type IntoIter = as IntoIterator>::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - vec![("token", self.token), ("email", self.email)].into_iter() - } -} - -#[derive(Debug, Deserialize)] -struct AwsData { - key_id: String, - secret_access_key: String, -} - -impl AwsData { - pub fn read_from_env() -> anyhow::Result { - let key_id = common::env::read_env("AWS_ACCESS_KEY_ID")?; - let secret_access_key = common::env::read_env("AWS_SECRET_ACCESS_KEY")?; - Ok(Self { - key_id, - secret_access_key, - }) - } - - pub fn read_from_bao() -> anyhow::Result { - let data = read_bao_data("aws")?; - Ok(data) - } - - pub fn into_env_data(self) -> Vec<(&'static str, String)> { - vec![ - ("AWS_ACCESS_KEY_ID", self.key_id), - ("AWS_SECRET_ACCESS_KEY", self.secret_access_key), - ] - } -} - -impl IntoIterator for AwsData { - type Item = (&'static str, String); - - type IntoIter = as IntoIterator>::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - vec![ - ("key_id", self.key_id), - ("secret_access_key", self.secret_access_key), - ] - .into_iter() - } -} - -#[derive(Debug, Deserialize)] -struct HcloudData { - api_token: String, -} - -impl HcloudData { - pub fn read_from_env() -> anyhow::Result { - let api_token = common::env::read_env("TF_VAR_hcloud_api_token")?; - Ok(Self { api_token }) - } - - pub fn read_from_bao() -> anyhow::Result { - let data = read_bao_data("hcloud")?; - Ok(data) - } - - pub fn into_env_data(self) -> Vec<(&'static str, String)> { - vec![("TF_VAR_hcloud_api_token", self.api_token)] - } -} - -impl IntoIterator for HcloudData { - type Item = (&'static str, String); - - type IntoIter = as IntoIterator>::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - vec![("api_token", self.api_token)].into_iter() - } -} - -#[derive(Debug, Deserialize)] -struct UnifiData { - username: String, - password: String, - url: String, -} - -impl UnifiData { - pub fn read_from_env() -> anyhow::Result { - let username = common::env::read_env("UNIFI_USERNAME")?; - let password = common::env::read_env("UNIFI_PASSWORD")?; - let url = common::env::read_env("UNIFI_API")?; - Ok(Self { - username, - password, - url, - }) - } - - pub fn read_from_bao() -> anyhow::Result { - let data = read_bao_data("unifi")?; - Ok(data) - } - - pub fn into_env_data(self) -> Vec<(&'static str, String)> { - vec![ - ("UNIFI_USERNAME", self.username), - ("UNIFI_PASSWORD", self.password), - ("UNIFI_API", self.url), - ] - } -} - -impl IntoIterator for UnifiData { - type Item = (&'static str, String); - - type IntoIter = as IntoIterator>::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - vec![ - ("username", self.username), - ("password", self.password), - ("url", self.url), - ] - .into_iter() - } -} +entry_definition!( + OpenstackDataConfig, + OpenstackData, + "openstack", + &[ + "TF_VAR_openstack_username", + "TF_VAR_openstack_password", + "TF_VAR_openstack_tenant_name", + "TF_VAR_openstack_auth_url", + "TF_VAR_openstack_endpoint_type", + "TF_VAR_openstack_region" + ] +); +entry_definition!( + CloudflareDataConfig, + CloudflareData, + "cloudflare", + &["TF_VAR_cloudflare_token", "TF_VAR_cloudflare_email"] +); +entry_definition!( + AwsDataConfig, + AwsData, + "aws", + &["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] +); +entry_definition!( + HcloudDataConfig, + HcloudData, + "hcloud", + &["TF_VAR_hcloud_api_token"] +); +entry_definition!( + UnifiDataConfig, + UnifiData, + "unifi", + &["UNIFI_USERNAME", "UNIFI_PASSWORD", "UNIFI_API"] +); +entry_definition!(VaultDataConfig, VaultData, "vault", &["VAULT_TOKEN"]); +entry_definition!( + AuthentikDataConfig, + AuthentikData, + "authentik", + &["AUTHENTIK_TOKEN", "TF_VAR_authentik_username"] +); fn transfer() -> anyhow::Result<()> { - let openstack = OpenstackData::read_from_env()?; - let cloudflare = CloudflareData::read_from_env()?; - let aws = AwsData::read_from_env()?; - let hcloud = HcloudData::read_from_env()?; - let unifi = UnifiData::read_from_env()?; + let openstack = OpenstackData::try_new_from_env()?; + let cloudflare = CloudflareData::try_new_from_env()?; + let aws = AwsData::try_new_from_env()?; + let hcloud = HcloudData::try_new_from_env()?; + let unifi = UnifiData::try_new_from_env()?; + let authentik = AuthentikData::try_new_from_env()?; + let vault = VaultData::try_new_from_env()?; - write_kv_data("openstack", openstack)?; - write_kv_data("cloudflare", cloudflare)?; - write_kv_data("aws", aws)?; - write_kv_data("hcloud", hcloud)?; - write_kv_data("unifi", unifi)?; + write_kv_data(openstack)?; + write_kv_data(cloudflare)?; + write_kv_data(aws)?; + write_kv_data(hcloud)?; + write_kv_data(unifi)?; + write_kv_data(authentik)?; + write_kv_data(vault)?; Ok(()) } -fn write_kv_data( - key: &str, - data: impl IntoIterator, -) -> anyhow::Result<()> { +fn write_kv_data(entry: EnvEntry) -> anyhow::Result<()> { let mut cmd = common::proc::Command::new("bao"); cmd.args(["kv", "put", "-mount=opentofu"]); - cmd.arg(key); - for (key, value) in data { + cmd.arg(T::BAO_KEY); + for (key, value) in entry { cmd.arg(format!("{key}={value}")); } cmd.try_spawn_to_string()?; @@ -359,18 +199,13 @@ fn wrap_program(wrap_program: WrapProgram) -> anyhow::Result<()> { return Err(anyhow::format_err!("No command to execute was specified")); } let unique: BTreeSet<_> = BTreeSet::from_iter(endpoint); - let mut env = Vec::::new(); + let mut env = Vec::<(OsString, OsString)>::new(); for (key, value) in std::env::vars() { - env.push( - CString::new(format!("{key}={value}")) - .with_context(|| format!("Environment variable {key} contained a null byte"))?, - ); + env.push((OsString::from(key), OsString::from(value))); } for env_set in unique { for (key, value) in env_set.try_into_env_data()? { - env.push(CString::new(format!("{key}={value}")).with_context(|| { - format!("Environment variable {key} contained a null byte") - })?); + env.push((OsString::from(key), OsString::from(value))); } } let mut args = Vec::new(); @@ -381,7 +216,65 @@ fn wrap_program(wrap_program: WrapProgram) -> anyhow::Result<()> { } (args, env) }; - nix::unistd::execvpe(&args[0], args.as_slice(), env.as_slice())?; + unsafe { + execvpe(&args[0], args.as_slice(), env.as_slice())?; + } // This will never get executed Ok(()) } + +#[cfg(not(target_os = "macos"))] +/// Safety: No other threads may read or write environment variables when this function is called. +/// The easiest way to ensure this is using a single threaded program. +/// Note: On Linux specifically this safety requirement is not needed +unsafe fn execvpe, SEK: AsRef, SEV: AsRef>( + filename: &CStr, + args: &[SA], + environ: &[(SEK, SEV)], +) -> anyhow::Result { + let environ = environ + .iter() + .map(|(k, v)| { + CString::new(format!( + "{k}={v}", + k = k.as_ref().display(), + v = v.as_ref().display() + )) + .with_context(|| { + format!( + "Environment variable {k} contains null bytes", + k = k.as_ref().display() + ) + }) + }) + .collect::>>()?; + Ok(nix::unistd::execvpe(filename, args, &environ)?) +} + +#[cfg(target_os = "macos")] +/// Safety: No other threads may read or write environment variables when this function is called. +/// The easiest way to ensure this is using a single threaded program. +// Simple "bad" version of execvpe that also works on OSX +unsafe fn execvpe, SEK: AsRef, SEV: AsRef>( + filename: &CStr, + args: &[SA], + environ: &[(SEK, SEV)], +) -> anyhow::Result { + let current_env = std::env::vars_os(); + // Safety: Same as this function + unsafe { nix::env::clearenv()? }; + for (key, val) in environ { + // Safety: Same as this function + unsafe { std::env::set_var(key.as_ref(), val.as_ref()) }; + } + match nix::unistd::execvp(filename, args) { + Err(err) => { + unsafe { nix::env::clearenv()? }; + for (key, val) in current_env { + unsafe { std::env::set_var(key.as_os_str(), val.as_os_str()) }; + } + Err(err.into()) + } + _ => unreachable!("execvp doesn't return on success"), + } +}