Working version of ilo fan control script

hosts/cap-apollo-n01/configuration.nix

@@ -24,11 +24,5 @@
 
   networking.hostName = "cap-apollo-n01";
 
-  # This value determines the NixOS release from which the default
+
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "25.11"; # Did you read the comment?
 }

hosts/cap-apollo-n02/configuration.nix

@@ -4,31 +4,10 @@
     # Hardware Scan
     ./hardware-configuration.nix
 
-    # Users
+    # Host Groups
-    ../../users/apollo-admin/apollo-admin.nix
+    ../../modules/host-groups/apollo-2000.nix
 
-    # System Configuration
-    ../../modules/system/cpu-intel.nix
-    ../../modules/system/fonts.nix
-    ../../modules/system/home-manager-settings.nix
-    ../../modules/system/internationalization.nix
-    ../../modules/system/networking.nix
-    ../../modules/system/nix-settings.nix
-    ../../modules/system/security.nix
-    ../../modules/system/systemd-boot.nix
-
-    # Application Groups
-    ../../modules/application-groups/system-utilities-cluster.nix
-    ../../modules/application-groups/virtualization.nix
   ];
 
   networking.hostName = "cap-apollo-n02";
-
-  # This value determines the NixOS release from which the default
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "25.11"; # Did you read the comment?
 }

hosts/cap-clust-01/configuration.nix

@@ -12,12 +12,4 @@
   ];
 
   networking.hostName = "cap-clust-01";
-
-  # This value determines the NixOS release from which the default
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "25.11"; # Did you read the comment?
 }

hosts/cap-clust-02/configuration.nix

@@ -12,12 +12,4 @@
   ];
 
   networking.hostName = "cap-clust-02";
-
-  # This value determines the NixOS release from which the default
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "25.11"; # Did you read the comment?
 }

hosts/cap-clust-03/configuration.nix

@@ -12,12 +12,4 @@
   ];
 
   networking.hostName = "cap-clust-03";
-
-  # This value determines the NixOS release from which the default
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "25.11"; # Did you read the comment?
 }

hosts/cap-clust-04/configuration.nix

@@ -10,11 +10,5 @@
 
   networking.hostName = "cap-clust-04";
 
-  # This value determines the NixOS release from which the default
+
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "25.11"; # Did you read the comment?
 }

hosts/cap-clust-05/configuration.nix

@@ -10,11 +10,5 @@
 
   networking.hostName = "cap-clust-05";
 
-  # This value determines the NixOS release from which the default
+
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "25.11"; # Did you read the comment?
 }

hosts/cap-clust-06/configuration.nix

@@ -10,11 +10,5 @@
 
   networking.hostName = "cap-clust-06";
 
-  # This value determines the NixOS release from which the default
+
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "25.11"; # Did you read the comment?
 }

hosts/cap-clust-07/configuration.nix

@@ -10,11 +10,5 @@
 
   networking.hostName = "cap-clust-07";
 
-  # This value determines the NixOS release from which the default
+
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "25.11"; # Did you read the comment?
 }

hosts/cap-clust-08/configuration.nix

@@ -10,11 +10,5 @@
 
   networking.hostName = "cap-clust-08";
 
-  # This value determines the NixOS release from which the default
+
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "25.11"; # Did you read the comment?
 }

hosts/cap-clust-09/configuration.nix

@@ -10,11 +10,5 @@
 
   networking.hostName = "cap-clust-09";
 
-  # This value determines the NixOS release from which the default
+
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "25.11"; # Did you read the comment?
 }

hosts/cap-nr200p/configuration.nix

@@ -31,6 +31,8 @@
     ../../modules/system/security.nix
     ../../modules/system/systemd-boot.nix
 
+    ../../modules/system/ilo-management.nix
+
     # Application Groups
     ../../modules/application-groups/3d-design.nix
     ../../modules/application-groups/android.nix

modules/application-groups/system-utilities-cluster.nix

@@ -18,6 +18,7 @@
     nmap
     nvtopPackages.full
     pciutils
+    screen
     unzip
     usbutils
     util-linux

modules/application-groups/system-utilities.nix

@@ -52,6 +52,7 @@
     rpiboot
     s-tui
     scrcpy
+    screen
     speedcrunch
     streamdeck-ui
     stress

modules/host-groups/apollo-2000.nix

@@ -0,0 +1,109 @@
+{ config, pkgs, ... }:
+
+{
+  imports = [
+    # Users
+    ../../users/apollo-admin/apollo-admin.nix
+
+    # System Configuration
+    ../../modules/system/cpu-intel.nix
+    ../../modules/system/fonts.nix
+    ../../modules/system/home-manager-settings.nix
+    ../../modules/system/ilo-management.nix
+    ../../modules/system/internationalization.nix
+    ../../modules/system/networking.nix
+    ../../modules/system/nix-settings.nix
+    ../../modules/system/security.nix
+    ../../modules/system/systemd-boot.nix
+
+    # Application Groups
+    ../../modules/application-groups/system-utilities-cluster.nix
+    ../../modules/application-groups/virtualization.nix
+  ];
+
+  time.timeZone = "America/Los_Angeles";
+
+  systemd = {
+    services.hpe-ilo-keepalive = {
+      enable = true;
+      after = [ "network.target" ];
+      description = "Maintains ilo ssh session via sending periodic command";
+
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = ''screen -S ilofansession -X stuff "fan info^M"'';
+      };
+
+      path = with pkgs; [
+        bash
+        screen
+        config.programs.ssh.package
+      ];
+    };
+    services.hpe-silent-fans = {
+      enable = true;
+      after = [ "network.target" ];
+      description = "Lowers fan speeds by using ilo over ssh to manually set fan parameters.";
+
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = "${pkgs.writeShellScript "hpe-silent-fans.sh" ''
+          set -e
+
+          SCREEN_NAME=ilofansession
+
+          SSH_USER=ilouser
+          SSH_HOST=cap-apollo-ilo02
+          SSH_KEY=/root/.ssh/ilo_id_rsa
+          SSH_OPTIONS="-o KexAlgorithms=diffie-hellman-group14-sha1,diffie-hellman-group1-sha1 -o PubkeyAcceptedKeyTypes=+ssh-rsa -o HostKeyAlgorithms=ssh-rsa -o StrictHostKeyChecking=no"
+
+          # Create screen session
+          screen -dmS $SCREEN_NAME
+
+          # Make initial iLO connection
+          screen -S $SCREEN_NAME -X stuff "ssh -i $SSH_KEY -t $SSH_USER@$SSH_HOST $SSH_OPTIONS^M"
+
+          sleep 5
+
+          ##### Tune pid for all non-segmented fans
+          for sensor in 1 2 3 4 5 6 7 9 10 11 12 13 14 15 16 17 18 19 20 21 26 28 29 30 31 32 38 40 41; do
+            screen -S $SCREEN_NAME -X stuff "fan pid $sensor lo 1600^M"
+            sleep 0.5
+          done
+
+          ##### Tune pid for segmented fans
+          for sensor in 8 22 23 24 25 27 39; do
+            screen -S $SCREEN_NAME -X stuff "fan a $sensor 0 0 16 41 16 25^M"
+            sleep 0.5
+          done
+
+          ##### Set minimum for fan group
+          screen -S $SCREEN_NAME -X stuff "fan p 0 min 16^M"
+        ''}";
+
+      };
+
+      path = with pkgs; [
+        bash
+        screen
+        config.programs.ssh.package
+      ];
+    };
+
+    timers.hpe-ilo-keepalive = {
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = "*-*-* *:0/5:00";
+        Unit = "hpe-ilo-keepalive.service";
+      };
+    };
+  };
+
+  # This value determines the NixOS release from which the default
+  # settings for stateful data, like file locations and database versions
+  # on your system were taken. It‘s perfectly fine and recommended to leave
+  # this value at the release version of the first install of this system.
+  # Before changing this value read the documentation for this option
+  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
+  system.stateVersion = "25.11"; # Did you read the comment?
+}

modules/system/ilo-management.nix

@@ -0,0 +1,13 @@
+{ config, pkgs, ... }:
+{
+  sops.secrets = {
+    "ssh/ilouser/id_rsa" = {
+      sopsFile = ../../secrets/default.yaml;
+      path = "/root/.ssh/ilo_id_rsa";
+    };
+    "ssh/ilouser/id_rsa_pub" = {
+      sopsFile = ../../secrets/default.yaml;
+      path = "/root/.ssh/ilo_id_rsa.pub";
+    };
+  };
+}

secrets/default.yaml

@@ -1,4 +1,7 @@
-default: ENC[AES256_GCM,data:hblL4UM//g==,iv:pu+XlfdZl8XZFk16iwV5juImHosUfOhZJ54UAzi9iwo=,tag:8h2ybkmNoqUT85L2JfXLrA==,type:str]
+ssh:
+    ilouser:
+        id_rsa: ENC[AES256_GCM,data:BuoM1E1pf9srSfWWV+bIy9H/JAQIMsucJv8T+Sky6HgSsR1WLdOX8pYJSVukuMMisSBG8DJiGy7PgzqeZohrFbkaHpc7gI0S3+803pZQjILzs9kEIrpTzqb+i5UgGPNn7Q9Rk2Etm7UlQAaUSLZg1XfCWV421YeTDFnM42bLy8Lhpb5J/GyP4iYJN5osmlQdjUbzR4wWaYIofWHjTqbNo0yNdMTDPf/bVDw+/34Xx/Xcvc90KtbO/k/gWpOam/Mq/jMoaA0IcZu7p6pu3WWFlilkeBtLefLo8nDEGi13hT2KxrB89npTNWWXf4sejQxkZgp8x1FxA8LH9v6vRunvdBRNTw/JjmtxaF8vKfjd2oxaKztLA8NROYQRpavqv6/NGlyBCU+yncq/E5dl51t7BwDwpSUvYvVEP20qH1JBKtiyuHlLOTOwTJtigOE3EuNxuEViz56Q0tVpQOcPQuSRmwdOg8hMZx/HnaDv52JVxeQHdF8jpzz3E3noB1AbRIAb84XXWVo2QLxFb8W8t5ta5cdOq/Pqte5PNXfT+2pjBIOlfg5mzA9kjH4QOZaO7zBbMj/ZD480gZY+kUMgeXgOtoLSZCPvdBbSjd7Lsd7xYbZSk6m4jOtvbTzpaFX8ZIT4ompC/+G84t3HyxE+UUeBip7mTeOJBCc1WjguKSzdBMmYZUxJAGMmWJ+TQDZSFu2m7bMmLGTgya5isqtcqGN5cGcICxIfOVfuoJ5T6vO3t2ZdUuV2U4+gfCYhtEZbsT/lhgRC0NlMWBhowCRpzhowYLHHGqmUfl5M+JTCAOS1QDmZvaGVc8BC0WWkc+162NGME1D+GRolHF44TfB5IuTQ1cfrv+VvK4DIsEslleAMCxqHPPQPGpCTcCNfe/3Yylkl/Pmogas2P8Q1fyGVTZCsS2G5wugptt8AdfdM+tvmqzlyponw3lNKe+mj1ehunJ8V93ea95cdmVikM6OLykzSrbxsejX8EgIQ0BChAFly38ot7YcmqgfqABCGeiYwOnL/90EuRmYRaMuUihbiRVzGPd5oc0OKDj5xdM2Eq/VRaODCFuhmwK5eLHB3tGab2Bin/8VGZkpWjQsd3H/UYRfPpJqjnhI3/dxD619ouZPfsTfQCVnBaZMYVKzpmeb3geNMeV1h8cDxNsAk85nAnG6RUF7xlqmgF6yUZ/vFtjUq/vCoNHsKvS5sSy73LPXljrVvZyG2K9P6dfTR4n8yYGJCoj95ahCAD4djsr9ylRzj1YTFQ7XdZGxw3GML7rmzuMS/Kk0qMKNYSlS9ru0yIL2baZmntjUKjcl0qigzO74bi3TxyoylmxXA2/80AcRU/fWi0t9g9aTJmuVztPiRQpCLsDzSS/xTjriwThx6v7tgmVwO05SdlBQ5Fk6R1nOsC+4X35l2f6pWJ91nlkfimJM3mY/MZDxWusfgXcS3SBswj59n7zX7AwykcX6/XHVWg91EfB6sRUZyHWsMxwn+i/P5aoIiO43leCD9KX5T0KjRAxa0DG092zM6pRaKEJWeBXTgcFTS5AQdMLVlt5GBHBHw2Slr6SMsUo7A1ycJqP7nw+YdLje86h4edfvbAtowhaBT3WPFoxbtAoBWqkYPgTGs/AiKRsfjecypuCm52bz8VTiw8laRkYFO0nqyYZBm4u2BbVODS6Ji9ZxNpjN/NIDh/5ffhQ611iwL2xgob/yFrjqaRyzPx1DXseMO+Lc0TMEhdSVGG9i8fTJXN8VasDDHxWCQD/ZSyERGCqIDc1+pZBjwABiECsDtQy3DI+oQ3UEPLOFHGS95Kc5+KKij3ptvf61o2J6XKEUY0jqv+puNvmjz7jqsNBY+S524G+06zzcvXsUnWQ3o1lOseFDSl4ZV+a2GiRA+AHj5gSKXTNVUMDsEu7E+4Sg1WVFKLw4KgZ0KubHYQYz56QLojv2TUL3NrK1mUfdn9jeJ01L+CIMTVjz+4UjVI/AZJA370Le01pl0ZMxaFRa0fep1Pa6MikTS7SO8NBtTtbOI2Dt/hBrXpzbicfRU/2pNJ8hEn9in0eXkZ3h0/nHm5J/OrrbVOdvQK9RkF9J0sdgmLtnAxdQjKK7r8Tg17q4vefFRtTEFPz1hV8sOrNXJZc3GRnvINE/7CVYdBQZuq7Bqze9WS+Yqzo2Z7T+cgpLTThBWlL7uGRsNcOdxQOctvoHCDFMV+9+QSH9Cc55Jdbd1T/clludubmyCJLvd5+25haXtRNSivJ/qB/ZbaZq/1PrJw2rl9XuiTczVrG/5Gwpz7a7PVrP8ydIEoTAglMdr9WICKl2cqHo1+k7HOFZluJAugNTV5aRwCvb6roKXsENH6TE68RuiYI8GXD/AzrY86TADCZroeQ+arLW+UuDqQfsnJ6MM02NcSOriAqTbwikGSzNIwkOiuZmwc5Y1pVqe1pyVDf5gN6Q=,iv:IDe6vkBvgAzfxee+/odkLk1TLZRghVEf8hqH2r3+V9I=,tag:OFCA57fQjQxc+CT9DOq+VA==,type:str]
+        id_rsa_pub: ENC[AES256_GCM,data:hYdf7KifFqkHDa8UF1zvhirc2AkrYpL2VrmV2IJjHzzgbrJxshdA629mTXwXwXORbLwNcMpRR7DFLyHCRDu5cWsxluZ04rNU9sJFmL+6pPUonaa4Wc+9z5VysV8UItY/BZ/5mOMQ2ph/wMQ3pkFnBD58vOFOYsmZ7OCir5dQZfm8GdGDX76bOlOI/CAg/LtwFveHK4trY3EUkxbJQ1soZdpm7ML/7bDN52klaS/EEgwk+FJrNAcm8M+N3kMSZoJyJW7c1OG81d4G84tryt2fa7s2HGT79IX6h4wI0GGu+bl87MgmFC48KdvjPdWowfh2QxkphSLduYQ4ss4EQYgJrmkQznQtt67acCFxNrEyE5psSMQPXtgAbwV94xeqtc+QHzmjHAwICTzk7Q4sFF85cDV54ZyoQDYdfubjpJ6mdE65dftN2eYD/5ywdTq09vRpnlh24PTrp7nn4H+Yddakv5KGhQOnj0p6JHNytXEClmiqxANrsYyhKvZe9gS+5w1RIcrsv9dHr5uL7Q==,iv:PQiVjFf2LlOKa6i7V/DcxYU54m/AbJGwTwUmA9asKI4=,tag:y27R0sMuOno0Al9iD3+MsQ==,type:str]
 sops:
     age:
         - recipient: age1xjnkqv32a5nqftw6pqthapnzmgjl4lnqfpxy9utqm56yzm2mvfhqzch648
@@ -109,7 +112,7 @@ sops:
             dW1YMTBIa202Tkp3WC9KRUhTckFCMUEKgUhihP1CN+kNOcbtfsr/gofI0tVzMVwo
             4aQPOxmvp3gyKdvPtUUTxJ3QrZ3laAHcVmsxPjEPnaAjfmGSUZh/YQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-12-13T11:02:46Z"
+    lastmodified: "2025-12-31T05:09:48Z"
-    mac: ENC[AES256_GCM,data:roAByCemPPNz6kkAX1nOL/TU3p2Jv67paQKlouek40FEf5cwVRMmygKDhs1vV8ZO4Ot0xGjXwiq+ylD0aSzbzvdcD/gG+cZ67XpqcW7CQMMtCrQ3Rt+U7q4rxyUeR55VxJdusjwtPp8qPVutKNJlebOUdBgaSKzDzwbnRppDUxk=,iv:PZVwlU3uUO+hHisHaoQAAfcBR2jlB0UHSU7ZFRXYfPo=,tag:0hPLfuSoSLRR1LiOWHFpfQ==,type:str]
+    mac: ENC[AES256_GCM,data:VP4URW/zRZFa4A3Q0gVzs06Zre+GzT3DNcrYxOcktgR1ooyvCjPE6l5t3Jf2LvVanSuBfIQMP7w67OcBar89QqGjn38E6V/U5Lyj7hHF9AtqNd/3l3P91xt+69UBOEqhZI0oASrTA3MKAZVeg6kWtU7YWajPH0PVxOsxMHeD9g4=,iv:LciFXM9JdXwmR56dgO6OskfcGauy8Q5gYIKZH2sES90=,tag:VJbexnwD+N1mGzADfXhp7g==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.11.0