feat(server): add remote host metrics

configs/server.nix

@@ -1,5 +1,16 @@
-{config, ...}: {
+{
-  imports = [./common.nix];
+  config,
+  _utils,
+  ...
+}: let
+  secrets = _utils.setupSharedSecrets config {
+    secrets = ["vmAuthToken"];
+  };
+in {
+  imports = [
+    ./common.nix
+    secrets.generate
+  ];
 
   _module.args.nixinate = {
     host = config.networking.hostName;
@@ -27,5 +38,29 @@
       port = 9091;
       enabledCollectors = ["systemd"];
     };
+
+    vmagent = {
+      enable = true;
+      remoteWrite.url = "https://metrics.uku3lig.net/api/v1/write";
+      extraArgs = ["-remoteWrite.bearerToken $VM_AUTH_TOKEN"];
+      prometheusConfig = {
+        global.scrape_interval = "15s";
+
+        scrape_configs = [
+          {
+            job_name = "node";
+            static_configs = [{targets = ["localhost:${builtins.toString config.services.prometheus.exporters.node.port}"];}];
+            relabel_configs = [
+              {
+                target_label = "instance";
+                replacement = config.networking.hostName;
+              }
+            ];
+          }
+        ];
+      };
+    };
   };
+
+  systemd.services.vmagent.serviceConfig.EnvironmentFile = secrets.get "vmAuthToken";
 }

flake.lock

@@ -299,6 +299,28 @@
         "type": "github"
       }
     },
+    "mystia": {
+      "inputs": {
+        "flake-compat": [],
+        "nix-update-soopy": [],
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1722240118,
+        "narHash": "sha256-SkKJH1Qc2+rMQsas9YWtIfgWNv9Be788Mw6VvcubW4Y=",
+        "owner": "soopyc",
+        "repo": "mystia",
+        "rev": "f0020f82a83957ba6924e46907820f1e05fcf66c",
+        "type": "github"
+      },
+      "original": {
+        "owner": "soopyc",
+        "repo": "mystia",
+        "type": "github"
+      }
+    },
     "nixinate": {
       "inputs": {
         "nixpkgs": [
@@ -401,6 +423,7 @@
         "ghostty": "ghostty",
         "home-manager": "home-manager",
         "lanzaboote": "lanzaboote",
+        "mystia": "mystia",
         "nixinate": "nixinate",
         "nixos-wsl": "nixos-wsl",
         "nixpkgs": "nixpkgs",

flake.nix

@@ -77,6 +77,13 @@
       inputs.flake-compat.follows = "";
     };
 
+    mystia = {
+      url = "github:soopyc/mystia";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.nix-update-soopy.follows = "";
+      inputs.flake-compat.follows = "";
+    };
+
     nixinate = {
       url = "github:matthewcroughan/nixinate";
       inputs.nixpkgs.follows = "nixpkgs";

secrets/secrets.nix

@@ -9,7 +9,8 @@ let
 in {
   "shared/userPassword.age".publicKeys = all;
   "shared/tailscaleKey.age".publicKeys = all;
-  "shared/frpToken.age".publicKeys = main ++ [etna vesuvio];
+  "shared/frpToken.age".publicKeys = all;
+  "shared/vmAuthToken.age".publicKeys = all;
 
   "fuji/rootPassword.age".publicKeys = main;
   "fuji-wsl/rootPassword.age".publicKeys = main;

secrets/shared/vmAuthToken.age

@@ -0,0 +1,17 @@
+-----BEGIN AGE ENCRYPTED FILE-----
+YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCaTI0Qm9kL2VSZFlQdlla
+bnJOVElDMGFFM0tDVHVpb0hZcDVKUDFmNndJClMyYytuc01yY0E1MGJncjRuNWRV
+U2JsMzJtdWY2L2EyRjRHMk91em9ITzgKLT4gWDI1NTE5IDhEbGR2ZTFydmcrdE8y
+V0p4V20yMXN0S250cDJFS2lYWnpRYWRwQk9oQjQKc2E4WHQ3OVdmdUE1TjJhU3Vn
+ckJ5aXVNbmJ3WEh4U3VHMFpKVW9vYUIwWQotPiBYMjU1MTkgYXBJcGlOdmlXaHlR
+bVlBazRyeUhMZ0NBNFl5UUM0SmhLNEZGd0RWNndnbwpIRFBsa1ZFMlNReEJIMWsz
+OEFjQUoyR1M0NVpVRzdodlFNK2VRcm55a09BCi0+IFgyNTUxOSB4bDVGTERBaXRG
+ZXo0V0ZkdmtMTlBhaWhvWDQ1UzVoRVZoeWlQYWlpWlhFCkVKNVFSOE0vMHMvNFBj
+eUxSZk10UTZxSzN5OXVHVHpCUFRNYUVkbFlLaTgKLS0tIFVpa05XQmhWeGtEa0Nr
+ZUYwMmg5cExiVndYcVppQlRIS0JNMGliTHQvREkKpwP8aD6RCqL/rKZ3YF4pG245
+2jeBRoLLWP5uYMvBDURL27LvvfI4WdXtvDXAoWU4bpxQU+o6Vixc2MKE0cNeCrhX
+vQt4x5csJXN+jqWqI5JEKasI2p1nSWv3TNS7yAS+K8fCXv+x2pUD2vLR+fcQqpkM
+X/5deZFdMWa4zmGE5sPR4oaZjGu7gLvLEm6JBZrG+pU+Kabwk1HOekjuSelsW2Tg
+cGn8nJ2yNSX7s2a+8tTFxnd+mor+TuKBr+czvjY6mhDLvCgQs79IfK5Id7K95i87
+XC28zxHyHP0t
+-----END AGE ENCRYPTED FILE-----

systems/etna/metrics.nix

@@ -1,8 +1,24 @@
-{config, ...}: let
+{
+  config,
+  mystia,
+  _utils,
+  ...
+}: let
   vmcfg = config.services.victoriametrics;
-  pmcfg = config.services.prometheus;
+  secrets = _utils.setupSharedSecrets config {secrets = ["vmAuthToken"];};
 in {
-  cfTunnels."grafana.uku3lig.net" = "http://localhost:2432";
+  imports = [
+    mystia.nixosModules.vmauth
+    secrets.generate
+  ];
+
+  cfTunnels = {
+    "grafana.uku3lig.net" = "http://localhost:2432";
+    "metrics.uku3lig.net" = {
+      service = "http://localhost:9089";
+      path = "/api/.*/write";
+    };
+  };
 
   services.grafana = {
     enable = true;
@@ -22,16 +38,11 @@ in {
 
   services.vmagent = {
     enable = true;
-    remoteWrite.url = "http://${vmcfg.listenAddress}/api/v1/write";
     prometheusConfig = {
       global.scrape_interval = "15s";
 
+      # node scraping is sent to vm directly via vmauth
       scrape_configs = [
-        {
-          job_name = "node";
-          static_configs = [{targets = ["localhost:${builtins.toString pmcfg.exporters.node.port}"];}];
-        }
-
         {
           job_name = "victoriametrics";
           static_configs = [{targets = ["${builtins.toString vmcfg.listenAddress}"];}];
@@ -44,4 +55,16 @@ in {
       ];
     };
   };
+
+  services.vmauth = {
+    enable = true;
+    listenAddress = "127.0.0.1:9089";
+    environmentFile = secrets.get "vmAuthToken";
+    authConfig.users = [
+      {
+        bearer_token = "%{VM_AUTH_TOKEN}";
+        url_prefix = "http://${vmcfg.listenAddress}";
+      }
+    ];
+  };
 }