From 76c10ec22ee178d61dfbcb88009c332c63f8bae3 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 14 Jun 2023 12:05:15 +0200 Subject: [PATCH 01/11] Add ncdu to system packages --- common/main.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/common/main.nix b/common/main.nix index d993956..fd5296e 100644 --- a/common/main.nix +++ b/common/main.nix @@ -14,6 +14,7 @@ environment.systemPackages = with pkgs; [ vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree + ncdu ]; systemd.services."serial-getty@ttyS0" = { -- GitLab From 70304d26ff063f8b46789c7c4921d09f52ba4110 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 14 Jun 2023 13:55:19 +0200 Subject: [PATCH 02/11] Remove profiles older than 30 days with gc --- common/main.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/common/main.nix b/common/main.nix index fd5296e..59d5286 100644 --- a/common/main.nix +++ b/common/main.nix @@ -46,6 +46,7 @@ nix.settings.trusted-users = [ "@wheel" ]; nix.gc.automatic = true; nix.gc.dates = "weekly"; + nix.gc.options = "--delete-older-than 30d"; programs.zsh.enable = true; programs.zsh.histSize = 100000; -- GitLab From f787343f29ea41d6f954da258ac0675b96c4445b Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 14 Jun 2023 11:15:00 +0200 Subject: [PATCH 03/11] Rename xeon07 to hut --- common/net.nix | 2 +- common/slurm.nix | 2 +- doc/install.md | 10 +++++----- flake.nix | 6 +++--- {xeon07 => hut}/configuration.nix | 2 +- {xeon07 => hut}/gitlab-runner.nix | 0 {xeon07 => hut}/monitoring.nix | 0 {xeon07 => hut}/nfs.nix | 0 {xeon07 => hut}/secrets.nix | 0 {xeon07 => hut}/secrets/nosv-token.age | Bin {xeon07 => hut}/secrets/ovni-token.age | 0 {xeon07 => hut}/slurm-daemon.nix | 0 12 files changed, 11 insertions(+), 11 deletions(-) rename {xeon07 => hut}/configuration.nix (95%) rename {xeon07 => hut}/gitlab-runner.nix (100%) rename {xeon07 => hut}/monitoring.nix (100%) rename {xeon07 => hut}/nfs.nix (100%) rename {xeon07 => hut}/secrets.nix (100%) rename {xeon07 => hut}/secrets/nosv-token.age (100%) rename {xeon07 => hut}/secrets/ovni-token.age (100%) rename {xeon07 => hut}/slurm-daemon.nix (100%) diff --git a/common/net.nix b/common/net.nix index 6e5d5bf..51f80f4 100644 --- a/common/net.nix +++ b/common/net.nix @@ -73,7 +73,7 @@ 10.0.40.106 xeon06-ipmi0 # Node Entry for node: xeon07 (ID=21) - 10.0.40.7 xeon07 xeon07-eth0 + 10.0.40.7 xeon07 xeon07-eth0 hut 10.0.42.7 xeon07-ib0 10.0.40.107 xeon07-ipmi0 diff --git a/common/slurm.nix b/common/slurm.nix index e4890e3..2da5a02 100644 --- a/common/slurm.nix +++ b/common/slurm.nix @@ -3,7 +3,7 @@ { services.slurm = { client.enable = true; - controlMachine = "xeon07"; + controlMachine = "hut"; clusterName = "owl"; nodeName = [ "xeon[01-02,07] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=xeon" diff --git a/doc/install.md b/doc/install.md index 41e7684..8eae8c7 100644 --- a/doc/install.md +++ b/doc/install.md @@ -32,19 +32,19 @@ sdX ## Prepare nix and nixos-install -Mount the nix store from the xeon07 node in read-only /nix. +Mount the nix store from the hut node in read-only /nix. ``` # mkdir /nix -# mount -o ro xeon07:/nix /nix +# mount -o ro hut:/nix /nix ``` -Get the nix binary and nixos-install tool from xeon07: +Get the nix binary and nixos-install tool from hut: ``` -# ssh xeon07 'readlink -f $(which nix)' +# ssh hut 'readlink -f $(which nix)' /nix/store/0sxbaj71c4c4n43qhdxm31f56gjalksw-nix-2.13.3/bin/nix -# ssh xeon07 'readlink -f $(which nixos-install)' +# ssh hut 'readlink -f $(which nixos-install)' /nix/store/9yq8ps06ysr2pfiwiij39ny56yk3pdcs-nixos-install/bin/nixos-install ``` diff --git a/flake.nix b/flake.nix index e07bff9..d3fc4ae 100644 --- a/flake.nix +++ b/flake.nix @@ -52,7 +52,7 @@ ./xeon02/configuration.nix ]; }; - xeon07 = nixpkgs.lib.nixosSystem { + hut = nixpkgs.lib.nixosSystem { system = "x86_64-linux"; modules = [ ( {options, ...}: { @@ -61,7 +61,7 @@ nix.nixPath = [ "nixpkgs=${nixpkgs}" "bscpkgs=${bscpkgs}" - "nixos-config=${self.outPath}/xeon07/configuration.nix" + "nixos-config=${self.outPath}/hut/configuration.nix" "nixpkgs-overlays=${self.outPath}/overlays-compat" ]; nix.registry.nixpkgs.flake = nixpkgs; @@ -72,7 +72,7 @@ else throw ("Refusing to build from a dirty Git tree!"); }) agenix.nixosModules.default - ./xeon07/configuration.nix + ./hut/configuration.nix ]; }; xeon08 = nixpkgs.lib.nixosSystem { diff --git a/xeon07/configuration.nix b/hut/configuration.nix similarity index 95% rename from xeon07/configuration.nix rename to hut/configuration.nix index f3b6ab4..423b18d 100644 --- a/xeon07/configuration.nix +++ b/hut/configuration.nix @@ -14,7 +14,7 @@ boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN"; networking = { - hostName = "xeon07"; + hostName = "hut"; interfaces.eno1.ipv4.addresses = [ { address = "10.0.40.7"; prefixLength = 24; diff --git a/xeon07/gitlab-runner.nix b/hut/gitlab-runner.nix similarity index 100% rename from xeon07/gitlab-runner.nix rename to hut/gitlab-runner.nix diff --git a/xeon07/monitoring.nix b/hut/monitoring.nix similarity index 100% rename from xeon07/monitoring.nix rename to hut/monitoring.nix diff --git a/xeon07/nfs.nix b/hut/nfs.nix similarity index 100% rename from xeon07/nfs.nix rename to hut/nfs.nix diff --git a/xeon07/secrets.nix b/hut/secrets.nix similarity index 100% rename from xeon07/secrets.nix rename to hut/secrets.nix diff --git a/xeon07/secrets/nosv-token.age b/hut/secrets/nosv-token.age similarity index 100% rename from xeon07/secrets/nosv-token.age rename to hut/secrets/nosv-token.age diff --git a/xeon07/secrets/ovni-token.age b/hut/secrets/ovni-token.age similarity index 100% rename from xeon07/secrets/ovni-token.age rename to hut/secrets/ovni-token.age diff --git a/xeon07/slurm-daemon.nix b/hut/slurm-daemon.nix similarity index 100% rename from xeon07/slurm-daemon.nix rename to hut/slurm-daemon.nix -- GitLab From f4ac9f3186600dc2914e4924b713a1dee4b75ca2 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 14 Jun 2023 17:28:00 +0200 Subject: [PATCH 04/11] Simplify flake and expose host pkgs The configuration of the machines is now moved to m/ --- common/overlays.nix | 9 -- flake.nix | 105 +++--------------- {common => m/common}/boot.nix | 0 {common => m/common}/fs.nix | 0 {common => m/common}/hw.nix | 0 {common => m/common}/main.nix | 19 +++- {common => m/common}/net.nix | 4 +- {common => m/common}/slurm.nix | 3 +- {common => m/common}/ssh.nix | 0 {common => m/common}/users.nix | 0 {hut => m/hut}/configuration.nix | 3 +- {hut => m/hut}/gitlab-runner.nix | 0 {hut => m/hut}/monitoring.nix | 0 {hut => m/hut}/nfs.nix | 0 {hut => m/hut}/secrets.nix | 0 {hut => m/hut}/secrets/nosv-token.age | Bin {hut => m/hut}/secrets/ovni-token.age | 0 {hut => m/hut}/slurm-daemon.nix | 0 {xeon01 => m/owl1}/configuration.nix | 0 {xeon02 => m/owl2}/configuration.nix | 0 {xeon08 => m/xeon08}/configuration.nix | 0 {xeon08 => m/xeon08}/cpufreq.nix | 0 {xeon08 => m/xeon08}/fs.nix | 0 {xeon08 => m/xeon08}/kernel/configs/defconfig | 0 {xeon08 => m/xeon08}/kernel/configs/lockdep | 0 {xeon08 => m/xeon08}/kernel/kernel.nix | 0 {xeon08 => m/xeon08}/kernel/lttng.nix | 0 {xeon08 => m/xeon08}/kernel/perf.nix | 0 {xeon08 => m/xeon08}/slurm.nix | 0 {xeon08 => m/xeon08}/users.nix | 0 overlays-compat/overlays.nix | 8 -- 31 files changed, 39 insertions(+), 112 deletions(-) delete mode 100644 common/overlays.nix rename {common => m/common}/boot.nix (100%) rename {common => m/common}/fs.nix (100%) rename {common => m/common}/hw.nix (100%) rename {common => m/common}/main.nix (80%) rename {common => m/common}/net.nix (96%) rename {common => m/common}/slurm.nix (61%) rename {common => m/common}/ssh.nix (100%) rename {common => m/common}/users.nix (100%) rename {hut => m/hut}/configuration.nix (89%) rename {hut => m/hut}/gitlab-runner.nix (100%) rename {hut => m/hut}/monitoring.nix (100%) rename {hut => m/hut}/nfs.nix (100%) rename {hut => m/hut}/secrets.nix (100%) rename {hut => m/hut}/secrets/nosv-token.age (100%) rename {hut => m/hut}/secrets/ovni-token.age (100%) rename {hut => m/hut}/slurm-daemon.nix (100%) rename {xeon01 => m/owl1}/configuration.nix (100%) rename {xeon02 => m/owl2}/configuration.nix (100%) rename {xeon08 => m/xeon08}/configuration.nix (100%) rename {xeon08 => m/xeon08}/cpufreq.nix (100%) rename {xeon08 => m/xeon08}/fs.nix (100%) rename {xeon08 => m/xeon08}/kernel/configs/defconfig (100%) rename {xeon08 => m/xeon08}/kernel/configs/lockdep (100%) rename {xeon08 => m/xeon08}/kernel/kernel.nix (100%) rename {xeon08 => m/xeon08}/kernel/lttng.nix (100%) rename {xeon08 => m/xeon08}/kernel/perf.nix (100%) rename {xeon08 => m/xeon08}/slurm.nix (100%) rename {xeon08 => m/xeon08}/users.nix (100%) delete mode 100644 overlays-compat/overlays.nix diff --git a/common/overlays.nix b/common/overlays.nix deleted file mode 100644 index fdf0706..0000000 --- a/common/overlays.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ options, ... }: - -{ - nix.nixPath = - # Prepend default nixPath values. - options.nix.nixPath.default ++ - # Append our nixpkgs-overlays. - [ "nixpkgs-overlays=${../overlays-compat}" ]; -} diff --git a/flake.nix b/flake.nix index d3fc4ae..54642d0 100644 --- a/flake.nix +++ b/flake.nix @@ -6,97 +6,22 @@ bscpkgs.url = "git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git"; }; - outputs = { self, nixpkgs, agenix, bscpkgs, ... }: { + outputs = { self, nixpkgs, agenix, bscpkgs, ... }: +let + mkConf = name: nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + specialArgs = { inherit nixpkgs bscpkgs agenix; theFlake = self; }; + modules = [ "${self.outPath}/m/${name}/configuration.nix" ]; + }; +in + { nixosConfigurations = { - xeon01 = nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = [ - ( {options, ...}: { - # Sel the nixos-config path to the one of the current flake - nixpkgs.overlays = [ bscpkgs.bscOverlay ]; - nix.nixPath = [ - "nixpkgs=${nixpkgs}" - "bscpkgs=${bscpkgs}" - "nixos-config=${self.outPath}/xeon01/configuration.nix" - "nixpkgs-overlays=${self.outPath}/overlays-compat" - ]; - nix.registry.nixpkgs.flake = nixpkgs; - nix.registry.bscpkgs.flake = bscpkgs; - system.configurationRevision = - if self ? rev - then self.rev - else throw ("Refusing to build from a dirty Git tree!"); - }) - ./xeon01/configuration.nix - ]; - }; - xeon02 = nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = [ - ( {options, ...}: { - # Sel the nixos-config path to the one of the current flake - nixpkgs.overlays = [ bscpkgs.bscOverlay ]; - nix.nixPath = [ - "nixpkgs=${nixpkgs}" - "bscpkgs=${bscpkgs}" - "nixos-config=${self.outPath}/xeon02/configuration.nix" - "nixpkgs-overlays=${self.outPath}/overlays-compat" - ]; - nix.registry.nixpkgs.flake = nixpkgs; - nix.registry.bscpkgs.flake = bscpkgs; - system.configurationRevision = - if self ? rev - then self.rev - else throw ("Refusing to build from a dirty Git tree!"); - }) - ./xeon02/configuration.nix - ]; - }; - hut = nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = [ - ( {options, ...}: { - # Sel the nixos-config path to the one of the current flake - nixpkgs.overlays = [ bscpkgs.bscOverlay ]; - nix.nixPath = [ - "nixpkgs=${nixpkgs}" - "bscpkgs=${bscpkgs}" - "nixos-config=${self.outPath}/hut/configuration.nix" - "nixpkgs-overlays=${self.outPath}/overlays-compat" - ]; - nix.registry.nixpkgs.flake = nixpkgs; - nix.registry.bscpkgs.flake = bscpkgs; - system.configurationRevision = - if self ? rev - then self.rev - else throw ("Refusing to build from a dirty Git tree!"); - }) - agenix.nixosModules.default - ./hut/configuration.nix - ]; - }; - xeon08 = nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = [ - ( {options, ...}: { - # Sel the nixos-config path to the one of the current flake - nixpkgs.overlays = [ bscpkgs.bscOverlay ]; - nix.nixPath = [ - "nixpkgs=${nixpkgs}" - "bscpkgs=${bscpkgs}" - "nixos-config=${self.outPath}/xeon08/configuration.nix" - "nixpkgs-overlays=${self.outPath}/overlays-compat" - ]; - nix.registry.nixpkgs.flake = nixpkgs; - nix.registry.bscpkgs.flake = bscpkgs; - system.configurationRevision = - if self ? rev - then self.rev - else throw ("Refusing to build from a dirty Git tree!"); - }) - ./xeon08/configuration.nix - ]; - }; + hut = mkConf "hut"; + owl1 = mkConf "owl1"; + owl2 = mkConf "owl2"; + xeon08 = mkConf "xeon08"; }; + + packages.x86_64-linux.hut = self.nixosConfigurations.hut.pkgs; }; } diff --git a/common/boot.nix b/m/common/boot.nix similarity index 100% rename from common/boot.nix rename to m/common/boot.nix diff --git a/common/fs.nix b/m/common/fs.nix similarity index 100% rename from common/fs.nix rename to m/common/fs.nix diff --git a/common/hw.nix b/m/common/hw.nix similarity index 100% rename from common/hw.nix rename to m/common/hw.nix diff --git a/common/main.nix b/m/common/main.nix similarity index 80% rename from common/main.nix rename to m/common/main.nix index 59d5286..ba50376 100644 --- a/common/main.nix +++ b/m/common/main.nix @@ -1,4 +1,4 @@ -{ config, pkgs, ... }: +{ config, pkgs, nixpkgs, bscpkgs, agenix, theFlake, ... }: { imports = [ @@ -11,6 +11,23 @@ ./users.nix ]; + nixpkgs.overlays = [ bscpkgs.bscOverlay ]; + + nix.nixPath = [ + "nixpkgs=${nixpkgs}" + "bscpkgs=${bscpkgs}" + "jungle=${theFlake.outPath}" + ]; + + nix.registry.nixpkgs.flake = nixpkgs; + nix.registry.bscpkgs.flake = bscpkgs; + nix.registry.jungle.flake = theFlake; + + system.configurationRevision = + if theFlake ? rev + then theFlake.rev + else throw ("Refusing to build from a dirty Git tree!"); + environment.systemPackages = with pkgs; [ vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree diff --git a/common/net.nix b/m/common/net.nix similarity index 96% rename from common/net.nix rename to m/common/net.nix index 51f80f4..504246c 100644 --- a/common/net.nix +++ b/m/common/net.nix @@ -43,12 +43,12 @@ 10.0.40.143 oss02-ipmi0 # Node Entry for node: xeon01 (ID=15) - 10.0.40.1 xeon01 xeon01-eth0 + 10.0.40.1 xeon01 xeon01-eth0 owl1 10.0.42.1 xeon01-ib0 10.0.40.101 xeon01-ipmi0 # Node Entry for node: xeon02 (ID=16) - 10.0.40.2 xeon02 xeon02-eth0 + 10.0.40.2 xeon02 xeon02-eth0 owl2 10.0.42.2 xeon02-ib0 10.0.40.102 xeon02-ipmi0 diff --git a/common/slurm.nix b/m/common/slurm.nix similarity index 61% rename from common/slurm.nix rename to m/common/slurm.nix index 2da5a02..c208b88 100644 --- a/common/slurm.nix +++ b/m/common/slurm.nix @@ -6,7 +6,8 @@ controlMachine = "hut"; clusterName = "owl"; nodeName = [ - "xeon[01-02,07] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=xeon" + "owl[1,2] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=owl" + "hut Sockets=2 CoresPerSocket=14 ThreadsPerCore=2" ]; extraConfig = '' MpiDefault=pmix diff --git a/common/ssh.nix b/m/common/ssh.nix similarity index 100% rename from common/ssh.nix rename to m/common/ssh.nix diff --git a/common/users.nix b/m/common/users.nix similarity index 100% rename from common/users.nix rename to m/common/users.nix diff --git a/hut/configuration.nix b/m/hut/configuration.nix similarity index 89% rename from hut/configuration.nix rename to m/hut/configuration.nix index 423b18d..82d4c34 100644 --- a/hut/configuration.nix +++ b/m/hut/configuration.nix @@ -1,4 +1,4 @@ -{ config, pkgs, ... }: +{ config, pkgs, agenix, ... }: { imports = [ @@ -8,6 +8,7 @@ ./monitoring.nix ./nfs.nix ./slurm-daemon.nix + agenix.nixosModules.default ]; # Select the this using the ID to avoid mismatches diff --git a/hut/gitlab-runner.nix b/m/hut/gitlab-runner.nix similarity index 100% rename from hut/gitlab-runner.nix rename to m/hut/gitlab-runner.nix diff --git a/hut/monitoring.nix b/m/hut/monitoring.nix similarity index 100% rename from hut/monitoring.nix rename to m/hut/monitoring.nix diff --git a/hut/nfs.nix b/m/hut/nfs.nix similarity index 100% rename from hut/nfs.nix rename to m/hut/nfs.nix diff --git a/hut/secrets.nix b/m/hut/secrets.nix similarity index 100% rename from hut/secrets.nix rename to m/hut/secrets.nix diff --git a/hut/secrets/nosv-token.age b/m/hut/secrets/nosv-token.age similarity index 100% rename from hut/secrets/nosv-token.age rename to m/hut/secrets/nosv-token.age diff --git a/hut/secrets/ovni-token.age b/m/hut/secrets/ovni-token.age similarity index 100% rename from hut/secrets/ovni-token.age rename to m/hut/secrets/ovni-token.age diff --git a/hut/slurm-daemon.nix b/m/hut/slurm-daemon.nix similarity index 100% rename from hut/slurm-daemon.nix rename to m/hut/slurm-daemon.nix diff --git a/xeon01/configuration.nix b/m/owl1/configuration.nix similarity index 100% rename from xeon01/configuration.nix rename to m/owl1/configuration.nix diff --git a/xeon02/configuration.nix b/m/owl2/configuration.nix similarity index 100% rename from xeon02/configuration.nix rename to m/owl2/configuration.nix diff --git a/xeon08/configuration.nix b/m/xeon08/configuration.nix similarity index 100% rename from xeon08/configuration.nix rename to m/xeon08/configuration.nix diff --git a/xeon08/cpufreq.nix b/m/xeon08/cpufreq.nix similarity index 100% rename from xeon08/cpufreq.nix rename to m/xeon08/cpufreq.nix diff --git a/xeon08/fs.nix b/m/xeon08/fs.nix similarity index 100% rename from xeon08/fs.nix rename to m/xeon08/fs.nix diff --git a/xeon08/kernel/configs/defconfig b/m/xeon08/kernel/configs/defconfig similarity index 100% rename from xeon08/kernel/configs/defconfig rename to m/xeon08/kernel/configs/defconfig diff --git a/xeon08/kernel/configs/lockdep b/m/xeon08/kernel/configs/lockdep similarity index 100% rename from xeon08/kernel/configs/lockdep rename to m/xeon08/kernel/configs/lockdep diff --git a/xeon08/kernel/kernel.nix b/m/xeon08/kernel/kernel.nix similarity index 100% rename from xeon08/kernel/kernel.nix rename to m/xeon08/kernel/kernel.nix diff --git a/xeon08/kernel/lttng.nix b/m/xeon08/kernel/lttng.nix similarity index 100% rename from xeon08/kernel/lttng.nix rename to m/xeon08/kernel/lttng.nix diff --git a/xeon08/kernel/perf.nix b/m/xeon08/kernel/perf.nix similarity index 100% rename from xeon08/kernel/perf.nix rename to m/xeon08/kernel/perf.nix diff --git a/xeon08/slurm.nix b/m/xeon08/slurm.nix similarity index 100% rename from xeon08/slurm.nix rename to m/xeon08/slurm.nix diff --git a/xeon08/users.nix b/m/xeon08/users.nix similarity index 100% rename from xeon08/users.nix rename to m/xeon08/users.nix diff --git a/overlays-compat/overlays.nix b/overlays-compat/overlays.nix deleted file mode 100644 index 8a606c5..0000000 --- a/overlays-compat/overlays.nix +++ /dev/null @@ -1,8 +0,0 @@ -self: super: -with super.lib; -let - # Load the system config and get the `nixpkgs.overlays` option - overlays = (import { }).config.nixpkgs.overlays; -in - # Apply all overlays to the input of the current "main" overlay - foldl' (flip extends) (_: super) overlays self -- GitLab From 2e95281af5b892bf56904e398703bd33967dd6ec Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 16 Jun 2023 11:34:00 +0200 Subject: [PATCH 05/11] Add owl and all partition --- m/hut/slurm-daemon.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/m/hut/slurm-daemon.nix b/m/hut/slurm-daemon.nix index a8dd3b8..e6ab227 100644 --- a/m/hut/slurm-daemon.nix +++ b/m/hut/slurm-daemon.nix @@ -4,7 +4,8 @@ services.slurm = { server.enable = true; partitionName = [ - "xeon Nodes=xeon[01-02,07] Default=YES MaxTime=INFINITE State=UP" + "owl Nodes=owl[1-2] Default=YES MaxTime=INFINITE State=UP" + "all Nodes=owl[1-2],hut Default=NO MaxTime=INFINITE State=UP" ]; }; } -- GitLab From fba0f7b739491a31e7d474c95b06cd6c4c1204cc Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 16 Jun 2023 11:42:39 +0200 Subject: [PATCH 06/11] Change owl hostnames --- m/owl1/configuration.nix | 2 +- m/owl2/configuration.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/m/owl1/configuration.nix b/m/owl1/configuration.nix index 1daecec..a14ab21 100644 --- a/m/owl1/configuration.nix +++ b/m/owl1/configuration.nix @@ -7,7 +7,7 @@ boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53566c"; networking = { - hostName = "xeon01"; + hostName = "owl1"; interfaces.eno1.ipv4.addresses = [ { address = "10.0.40.1"; prefixLength = 24; diff --git a/m/owl2/configuration.nix b/m/owl2/configuration.nix index 1b06afd..8022b36 100644 --- a/m/owl2/configuration.nix +++ b/m/owl2/configuration.nix @@ -12,7 +12,7 @@ #programs.ssh.setXAuthLocation = lib.mkForce true; networking = { - hostName = "xeon02"; + hostName = "owl2"; interfaces.eno1.ipv4.addresses = [ { address = "10.0.40.2"; prefixLength = 24; -- GitLab From 9f5941c2be5e047f26f3e4305ef0d27ca1453efc Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 16 Jun 2023 12:00:54 +0200 Subject: [PATCH 07/11] Set the name of the slurm cluster to jungle --- m/common/slurm.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/m/common/slurm.nix b/m/common/slurm.nix index c208b88..650156f 100644 --- a/m/common/slurm.nix +++ b/m/common/slurm.nix @@ -4,7 +4,7 @@ services.slurm = { client.enable = true; controlMachine = "hut"; - clusterName = "owl"; + clusterName = "jungle"; nodeName = [ "owl[1,2] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=owl" "hut Sockets=2 CoresPerSocket=14 ThreadsPerCore=2" -- GitLab From 94320d9256c349695e7aa3cd8b33cb03bb9cdeea Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 16 Jun 2023 12:01:12 +0200 Subject: [PATCH 08/11] Add ssh host keys --- m/common/ssh.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/m/common/ssh.nix b/m/common/ssh.nix index d3676f7..08c958c 100644 --- a/m/common/ssh.nix +++ b/m/common/ssh.nix @@ -17,6 +17,7 @@ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINa0tvnNgwkc5xOwd6xTtaIdFi5jv0j2FrE7jl5MTLoE ram@mio" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut" ]; rarias.openssh.authorizedKeys.keys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop" @@ -30,6 +31,10 @@ }; programs.ssh.knownHosts = { + "hut".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1"; + "owl1".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv"; + "owl2".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK"; + "gitlab-internal.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3"; "bscpm03.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS"; }; -- GitLab From 4e9409db10baa26736e69b04e0536a3e2c533cfb Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 16 Jun 2023 12:13:07 +0200 Subject: [PATCH 09/11] Update rebuild script for all nodes --- rebuild.sh | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/rebuild.sh b/rebuild.sh index 5df9616..1320c24 100755 --- a/rebuild.sh +++ b/rebuild.sh @@ -1,19 +1,16 @@ -#!/bin/sh -e +#!/bin/sh -ex if [ "$(id -u)" != 0 ]; then echo "Needs root permissions" exit 1 fi -host=$(hostname) - -#conf="$(readlink -f .)/${host}/configuration.nix" -# -#if [ ! -e "$conf" ]; then -# echo "Missing config $conf" -# exit 1 -#fi -# -#NIXOS_CONFIG="${conf}" nixos-rebuild switch +if [ "$(hostname)" != "hut" ]; then + >&2 echo "must run from machine hut, not $(hostname)" + exit 1 +fi +# Update all nodes nixos-rebuild switch --flake . +nixos-rebuild switch --flake .#owl1 --target-host owl1 +nixos-rebuild switch --flake .#owl2 --target-host owl2 -- GitLab From c916157fccc9a8905e97aa70b3fa9028d8521698 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 16 Jun 2023 17:16:05 +0200 Subject: [PATCH 10/11] Rename xeon08 to eudy From Eudyptula, a little penguin. --- flake.nix | 8 ++++---- m/common/net.nix | 2 +- m/{xeon08 => eudy}/configuration.nix | 2 +- m/{xeon08 => eudy}/cpufreq.nix | 0 m/{xeon08 => eudy}/fs.nix | 0 m/{xeon08 => eudy}/kernel/configs/defconfig | 0 m/{xeon08 => eudy}/kernel/configs/lockdep | 0 m/{xeon08 => eudy}/kernel/kernel.nix | 0 m/{xeon08 => eudy}/kernel/lttng.nix | 0 m/{xeon08 => eudy}/kernel/perf.nix | 0 m/{xeon08 => eudy}/slurm.nix | 0 m/{xeon08 => eudy}/users.nix | 0 12 files changed, 6 insertions(+), 6 deletions(-) rename m/{xeon08 => eudy}/configuration.nix (97%) rename m/{xeon08 => eudy}/cpufreq.nix (100%) rename m/{xeon08 => eudy}/fs.nix (100%) rename m/{xeon08 => eudy}/kernel/configs/defconfig (100%) rename m/{xeon08 => eudy}/kernel/configs/lockdep (100%) rename m/{xeon08 => eudy}/kernel/kernel.nix (100%) rename m/{xeon08 => eudy}/kernel/lttng.nix (100%) rename m/{xeon08 => eudy}/kernel/perf.nix (100%) rename m/{xeon08 => eudy}/slurm.nix (100%) rename m/{xeon08 => eudy}/users.nix (100%) diff --git a/flake.nix b/flake.nix index 54642d0..ebc935b 100644 --- a/flake.nix +++ b/flake.nix @@ -16,10 +16,10 @@ let in { nixosConfigurations = { - hut = mkConf "hut"; - owl1 = mkConf "owl1"; - owl2 = mkConf "owl2"; - xeon08 = mkConf "xeon08"; + hut = mkConf "hut"; + owl1 = mkConf "owl1"; + owl2 = mkConf "owl2"; + eudy = mkConf "eudy"; }; packages.x86_64-linux.hut = self.nixosConfigurations.hut.pkgs; diff --git a/m/common/net.nix b/m/common/net.nix index 504246c..cfcd686 100644 --- a/m/common/net.nix +++ b/m/common/net.nix @@ -78,7 +78,7 @@ 10.0.40.107 xeon07-ipmi0 # Node Entry for node: xeon08 (ID=22) - 10.0.40.8 xeon08 xeon08-eth0 + 10.0.40.8 xeon08 xeon08-eth0 eudy 10.0.42.8 xeon08-ib0 10.0.40.108 xeon08-ipmi0 ''; diff --git a/m/xeon08/configuration.nix b/m/eudy/configuration.nix similarity index 97% rename from m/xeon08/configuration.nix rename to m/eudy/configuration.nix index 195117f..c627bf8 100644 --- a/m/xeon08/configuration.nix +++ b/m/eudy/configuration.nix @@ -24,7 +24,7 @@ # set up both ethernet and infiniband ips networking = { - hostName = "xeon08"; + hostName = "eudy"; interfaces.eno1.ipv4.addresses = [ { address = "10.0.40.8"; prefixLength = 24; diff --git a/m/xeon08/cpufreq.nix b/m/eudy/cpufreq.nix similarity index 100% rename from m/xeon08/cpufreq.nix rename to m/eudy/cpufreq.nix diff --git a/m/xeon08/fs.nix b/m/eudy/fs.nix similarity index 100% rename from m/xeon08/fs.nix rename to m/eudy/fs.nix diff --git a/m/xeon08/kernel/configs/defconfig b/m/eudy/kernel/configs/defconfig similarity index 100% rename from m/xeon08/kernel/configs/defconfig rename to m/eudy/kernel/configs/defconfig diff --git a/m/xeon08/kernel/configs/lockdep b/m/eudy/kernel/configs/lockdep similarity index 100% rename from m/xeon08/kernel/configs/lockdep rename to m/eudy/kernel/configs/lockdep diff --git a/m/xeon08/kernel/kernel.nix b/m/eudy/kernel/kernel.nix similarity index 100% rename from m/xeon08/kernel/kernel.nix rename to m/eudy/kernel/kernel.nix diff --git a/m/xeon08/kernel/lttng.nix b/m/eudy/kernel/lttng.nix similarity index 100% rename from m/xeon08/kernel/lttng.nix rename to m/eudy/kernel/lttng.nix diff --git a/m/xeon08/kernel/perf.nix b/m/eudy/kernel/perf.nix similarity index 100% rename from m/xeon08/kernel/perf.nix rename to m/eudy/kernel/perf.nix diff --git a/m/xeon08/slurm.nix b/m/eudy/slurm.nix similarity index 100% rename from m/xeon08/slurm.nix rename to m/eudy/slurm.nix diff --git a/m/xeon08/users.nix b/m/eudy/users.nix similarity index 100% rename from m/xeon08/users.nix rename to m/eudy/users.nix -- GitLab From 316ea18e24e0f58f65d067ac1f931fc53fc9e878 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 16 Jun 2023 17:29:48 +0200 Subject: [PATCH 11/11] Add eudy host key to known hosts --- m/common/ssh.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/m/common/ssh.nix b/m/common/ssh.nix index 08c958c..3e5bbd6 100644 --- a/m/common/ssh.nix +++ b/m/common/ssh.nix @@ -34,6 +34,7 @@ "hut".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1"; "owl1".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv"; "owl2".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK"; + "eudy".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG"; "gitlab-internal.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3"; "bscpm03.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS"; -- GitLab