fix: victoriametrics (#172)

* chore: update aqua's host key

* fix: victoriametrics

* feat: adjust order of nix cache mirrors

* feat: update mysecrets

* fix: use bind mount - Failed at step STATE_DIRECTORY

hosts/idols-aquamarine/monitoring/module/default.nix

@@ -8,28 +8,23 @@ with lib; let
   cfg = config.services.my-victoriametrics;
   settingsFormat = pkgs.formats.yaml {};
 
-  workingDir = "/var/lib/" + cfg.stateDir;
-  startCommandLine =
-    lib.escapeShellArgs [
+  startCLIList =
+    [
       "${cfg.package}/bin/victoria-metrics"
-      "-storageDataPath=${workingDir}"
+      "-storageDataPath=/var/lib/${cfg.stateDir}"
       "-httpListenAddr=${cfg.listenAddress}"
       "-retentionPeriod=${cfg.retentionPeriod}"
     ]
-    ++ lib.optional (cfg.prometheusConfig != null) "-promscrape.config=${prometheusConfigYml}"
     ++ cfg.extraOptions;
   prometheusConfigYml = checkedConfig (
     settingsFormat.generate "prometheusConfig.yaml" cfg.prometheusConfig
   );
 
   checkedConfig = file:
-    if cfg.checkConfig
-    then
-      pkgs.runCommand "checked-config" {nativeBuildInputs = [cfg.package];} ''
-        ln -s ${file} $out
-        ${startCommandLine} -dryRun
-      ''
-    else file;
+    pkgs.runCommand "checked-config" {nativeBuildInputs = [cfg.package];} ''
+      ln -s ${file} $out
+      ${lib.escapeShellArgs startCLIList} -promscrape.config=${file} -dryRun
+    '';
 in {
   options.services.my-victoriametrics = {
     enable = mkEnableOption "VictoriaMetrics, a time series database.";
@@ -130,15 +125,17 @@ in {
       startLimitBurst = 5;
 
       serviceConfig = {
-        ExecStart = startCommandLine;
+        ExecStart = lib.escapeShellArgs (
+          startCLIList
+          ++ lib.optional (cfg.prometheusConfig != null) ["-promscrape.config=${prometheusConfigYml}"]
+        );
+
         DynamicUser = true;
         User = "victoriametrics";
-        Group = "victoriametrics";
         RestartSec = 1;
         Restart = "on-failure";
         RuntimeDirectory = "victoriametrics";
         RuntimeDirectoryMode = "0700";
-        WorkingDirectory = workingDir;
         StateDirectory = cfg.stateDir;
         StateDirectoryMode = "0700";
 

hosts/idols-aquamarine/monitoring/victoriametrics.nix

@@ -3,13 +3,23 @@
   myvars,
   ...
 }: {
+  # Since victoriametrics use DynamicUser, the user & group do not exists before the service starts.
+  # this group is used as a supplementary Unix group for the service to access our data dir(/data/apps/xxx)
+  users.groups.victoriametrics-data = {};
+
   # Workaround for victoriametrics to store data in another place
   # https://www.freedesktop.org/software/systemd/man/latest/tmpfiles.d.html#Type
   systemd.tmpfiles.rules = [
-    "D /data/apps/victoriametrics 0751 victoriametrics victoriametrics - -"
-    "L+ /var/lib/victoriametrics - - - - /data/apps/victoriametrics"
+    "D /data/apps/victoriametrics 0770 root victoriametrics-data - -"
   ];
 
+  # Symlinks do not work with DynamicUser, so we should use bind mount here.
+  # https://github.com/systemd/systemd/issues/25097#issuecomment-1929074961
+  systemd.services.victoriametrics.serviceConfig = {
+    SupplementaryGroups = ["victoriametrics-data"];
+    BindPaths = ["/data/apps/victoriametrics:/var/lib/victoriametrics:rbind"];
+  };
+
   # https://victoriametrics.io/docs/victoriametrics/latest/configuration/configuration/
   services.my-victoriametrics = {
     enable = true;

modules/base.nix

@@ -90,11 +90,11 @@
     # substituers that will be considered before the official ones(https://cache.nixos.org)
     substituters = [
       # cache mirror located in China
-      # status: https://mirror.sjtu.edu.cn/
-      "https://mirror.sjtu.edu.cn/nix-channels/store"
       # status: https://mirrors.ustc.edu.cn/status/
       "https://mirrors.ustc.edu.cn/nix-channels/store"
       "https://mirrors.tuna.tsinghua.edu.cn/nix-channels/store"
+      # status: https://mirror.sjtu.edu.cn/
+      "https://mirror.sjtu.edu.cn/nix-channels/store"
 
       "https://nix-community.cachix.org"
       # my own cache server

vars/networking.nix

@@ -193,7 +193,7 @@
         publicKey = value.publicKey;
       })
       {
-        aquamarine.publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIbIecyrmrBpjD497lA2adJeTpsubZ3dozEraLGCcgVi root@aquamarine";
+        aquamarine.publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEOXFhFu9Duzp6ZBE288gDZ6VLrNaeWL4kDrFUh9Neic root@aquamarine";
         # ruby.publicKey = "";
         # kana.publicKey = "";
       };