diff --git a/common/overlays.nix b/common/overlays.nix deleted file mode 100644 index fdf0706e8324de755ba99caac0dd54b9d6439e4d..0000000000000000000000000000000000000000 --- a/common/overlays.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ options, ... }: - -{ - nix.nixPath = - # Prepend default nixPath values. - options.nix.nixPath.default ++ - # Append our nixpkgs-overlays. - [ "nixpkgs-overlays=${../overlays-compat}" ]; -} diff --git a/common/slurm.nix b/common/slurm.nix deleted file mode 100644 index e4890e36a47730d08bcaff24525c8445b9dae55d..0000000000000000000000000000000000000000 --- a/common/slurm.nix +++ /dev/null @@ -1,16 +0,0 @@ -{ ... }: - -{ - services.slurm = { - client.enable = true; - controlMachine = "xeon07"; - clusterName = "owl"; - nodeName = [ - "xeon[01-02,07] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=xeon" - ]; - extraConfig = '' - MpiDefault=pmix - ReturnToService=2 - ''; - }; -} diff --git a/doc/install.md b/doc/install.md index 41e7684ea09f1e5336d4011a4b8e74803b561bce..8eae8c750cb7028ea5bbabbd48c26c6e0b376545 100644 --- a/doc/install.md +++ b/doc/install.md @@ -32,19 +32,19 @@ sdX ## Prepare nix and nixos-install -Mount the nix store from the xeon07 node in read-only /nix. +Mount the nix store from the hut node in read-only /nix. ``` # mkdir /nix -# mount -o ro xeon07:/nix /nix +# mount -o ro hut:/nix /nix ``` -Get the nix binary and nixos-install tool from xeon07: +Get the nix binary and nixos-install tool from hut: ``` -# ssh xeon07 'readlink -f $(which nix)' +# ssh hut 'readlink -f $(which nix)' /nix/store/0sxbaj71c4c4n43qhdxm31f56gjalksw-nix-2.13.3/bin/nix -# ssh xeon07 'readlink -f $(which nixos-install)' +# ssh hut 'readlink -f $(which nixos-install)' /nix/store/9yq8ps06ysr2pfiwiij39ny56yk3pdcs-nixos-install/bin/nixos-install ``` diff --git a/flake.nix b/flake.nix index e07bff97b5250ed118e60d3a81be07bdd207d407..ebc935b33242afe0389dd78eea7d5a4a4a54f0c1 100644 --- a/flake.nix +++ b/flake.nix @@ -6,97 +6,22 @@ bscpkgs.url = "git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git"; }; - outputs = { self, nixpkgs, agenix, bscpkgs, ... }: { + outputs = { self, nixpkgs, agenix, bscpkgs, ... }: +let + mkConf = name: nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + specialArgs = { inherit nixpkgs bscpkgs agenix; theFlake = self; }; + modules = [ "${self.outPath}/m/${name}/configuration.nix" ]; + }; +in + { nixosConfigurations = { - xeon01 = nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = [ - ( {options, ...}: { - # Sel the nixos-config path to the one of the current flake - nixpkgs.overlays = [ bscpkgs.bscOverlay ]; - nix.nixPath = [ - "nixpkgs=${nixpkgs}" - "bscpkgs=${bscpkgs}" - "nixos-config=${self.outPath}/xeon01/configuration.nix" - "nixpkgs-overlays=${self.outPath}/overlays-compat" - ]; - nix.registry.nixpkgs.flake = nixpkgs; - nix.registry.bscpkgs.flake = bscpkgs; - system.configurationRevision = - if self ? rev - then self.rev - else throw ("Refusing to build from a dirty Git tree!"); - }) - ./xeon01/configuration.nix - ]; - }; - xeon02 = nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = [ - ( {options, ...}: { - # Sel the nixos-config path to the one of the current flake - nixpkgs.overlays = [ bscpkgs.bscOverlay ]; - nix.nixPath = [ - "nixpkgs=${nixpkgs}" - "bscpkgs=${bscpkgs}" - "nixos-config=${self.outPath}/xeon02/configuration.nix" - "nixpkgs-overlays=${self.outPath}/overlays-compat" - ]; - nix.registry.nixpkgs.flake = nixpkgs; - nix.registry.bscpkgs.flake = bscpkgs; - system.configurationRevision = - if self ? rev - then self.rev - else throw ("Refusing to build from a dirty Git tree!"); - }) - ./xeon02/configuration.nix - ]; - }; - xeon07 = nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = [ - ( {options, ...}: { - # Sel the nixos-config path to the one of the current flake - nixpkgs.overlays = [ bscpkgs.bscOverlay ]; - nix.nixPath = [ - "nixpkgs=${nixpkgs}" - "bscpkgs=${bscpkgs}" - "nixos-config=${self.outPath}/xeon07/configuration.nix" - "nixpkgs-overlays=${self.outPath}/overlays-compat" - ]; - nix.registry.nixpkgs.flake = nixpkgs; - nix.registry.bscpkgs.flake = bscpkgs; - system.configurationRevision = - if self ? rev - then self.rev - else throw ("Refusing to build from a dirty Git tree!"); - }) - agenix.nixosModules.default - ./xeon07/configuration.nix - ]; - }; - xeon08 = nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = [ - ( {options, ...}: { - # Sel the nixos-config path to the one of the current flake - nixpkgs.overlays = [ bscpkgs.bscOverlay ]; - nix.nixPath = [ - "nixpkgs=${nixpkgs}" - "bscpkgs=${bscpkgs}" - "nixos-config=${self.outPath}/xeon08/configuration.nix" - "nixpkgs-overlays=${self.outPath}/overlays-compat" - ]; - nix.registry.nixpkgs.flake = nixpkgs; - nix.registry.bscpkgs.flake = bscpkgs; - system.configurationRevision = - if self ? rev - then self.rev - else throw ("Refusing to build from a dirty Git tree!"); - }) - ./xeon08/configuration.nix - ]; - }; + hut = mkConf "hut"; + owl1 = mkConf "owl1"; + owl2 = mkConf "owl2"; + eudy = mkConf "eudy"; }; + + packages.x86_64-linux.hut = self.nixosConfigurations.hut.pkgs; }; } diff --git a/common/boot.nix b/m/common/boot.nix similarity index 100% rename from common/boot.nix rename to m/common/boot.nix diff --git a/common/fs.nix b/m/common/fs.nix similarity index 100% rename from common/fs.nix rename to m/common/fs.nix diff --git a/common/hw.nix b/m/common/hw.nix similarity index 100% rename from common/hw.nix rename to m/common/hw.nix diff --git a/common/main.nix b/m/common/main.nix similarity index 77% rename from common/main.nix rename to m/common/main.nix index d993956c695595aa370c84ee2fe40ce8a3a06db7..ba503760a0ad769ede9b80494d9042046f3578df 100644 --- a/common/main.nix +++ b/m/common/main.nix @@ -1,4 +1,4 @@ -{ config, pkgs, ... }: +{ config, pkgs, nixpkgs, bscpkgs, agenix, theFlake, ... }: { imports = [ @@ -11,9 +11,27 @@ ./users.nix ]; + nixpkgs.overlays = [ bscpkgs.bscOverlay ]; + + nix.nixPath = [ + "nixpkgs=${nixpkgs}" + "bscpkgs=${bscpkgs}" + "jungle=${theFlake.outPath}" + ]; + + nix.registry.nixpkgs.flake = nixpkgs; + nix.registry.bscpkgs.flake = bscpkgs; + nix.registry.jungle.flake = theFlake; + + system.configurationRevision = + if theFlake ? rev + then theFlake.rev + else throw ("Refusing to build from a dirty Git tree!"); + environment.systemPackages = with pkgs; [ vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree + ncdu ]; systemd.services."serial-getty@ttyS0" = { @@ -45,6 +63,7 @@ nix.settings.trusted-users = [ "@wheel" ]; nix.gc.automatic = true; nix.gc.dates = "weekly"; + nix.gc.options = "--delete-older-than 30d"; programs.zsh.enable = true; programs.zsh.histSize = 100000; diff --git a/common/net.nix b/m/common/net.nix similarity index 92% rename from common/net.nix rename to m/common/net.nix index 6e5d5bfccefd9532df97322a3863899b9bd12243..cfcd686edda544c4b895e03d1312aa113cc9aa06 100644 --- a/common/net.nix +++ b/m/common/net.nix @@ -43,12 +43,12 @@ 10.0.40.143 oss02-ipmi0 # Node Entry for node: xeon01 (ID=15) - 10.0.40.1 xeon01 xeon01-eth0 + 10.0.40.1 xeon01 xeon01-eth0 owl1 10.0.42.1 xeon01-ib0 10.0.40.101 xeon01-ipmi0 # Node Entry for node: xeon02 (ID=16) - 10.0.40.2 xeon02 xeon02-eth0 + 10.0.40.2 xeon02 xeon02-eth0 owl2 10.0.42.2 xeon02-ib0 10.0.40.102 xeon02-ipmi0 @@ -73,12 +73,12 @@ 10.0.40.106 xeon06-ipmi0 # Node Entry for node: xeon07 (ID=21) - 10.0.40.7 xeon07 xeon07-eth0 + 10.0.40.7 xeon07 xeon07-eth0 hut 10.0.42.7 xeon07-ib0 10.0.40.107 xeon07-ipmi0 # Node Entry for node: xeon08 (ID=22) - 10.0.40.8 xeon08 xeon08-eth0 + 10.0.40.8 xeon08 xeon08-eth0 eudy 10.0.42.8 xeon08-ib0 10.0.40.108 xeon08-ipmi0 ''; diff --git a/m/common/slurm.nix b/m/common/slurm.nix new file mode 100644 index 0000000000000000000000000000000000000000..650156fa2abc962357c009768c020b7bb97ddfed --- /dev/null +++ b/m/common/slurm.nix @@ -0,0 +1,17 @@ +{ ... }: + +{ + services.slurm = { + client.enable = true; + controlMachine = "hut"; + clusterName = "jungle"; + nodeName = [ + "owl[1,2] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=owl" + "hut Sockets=2 CoresPerSocket=14 ThreadsPerCore=2" + ]; + extraConfig = '' + MpiDefault=pmix + ReturnToService=2 + ''; + }; +} diff --git a/common/ssh.nix b/m/common/ssh.nix similarity index 75% rename from common/ssh.nix rename to m/common/ssh.nix index d3676f79dbba5af07353c652d19b9072ce65e18d..3e5bbd670dc0d222ea90aa5b9ea27dacd37794b4 100644 --- a/common/ssh.nix +++ b/m/common/ssh.nix @@ -17,6 +17,7 @@ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINa0tvnNgwkc5xOwd6xTtaIdFi5jv0j2FrE7jl5MTLoE ram@mio" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut" ]; rarias.openssh.authorizedKeys.keys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop" @@ -30,6 +31,11 @@ }; programs.ssh.knownHosts = { + "hut".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1"; + "owl1".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv"; + "owl2".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK"; + "eudy".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG"; + "gitlab-internal.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3"; "bscpm03.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS"; }; diff --git a/common/users.nix b/m/common/users.nix similarity index 100% rename from common/users.nix rename to m/common/users.nix diff --git a/xeon08/configuration.nix b/m/eudy/configuration.nix similarity index 97% rename from xeon08/configuration.nix rename to m/eudy/configuration.nix index 195117f8afc6a8ad987cc23ed8dfa632f7524f4e..c627bf8ff3200ea97af630e9a671e85fa349133f 100644 --- a/xeon08/configuration.nix +++ b/m/eudy/configuration.nix @@ -24,7 +24,7 @@ # set up both ethernet and infiniband ips networking = { - hostName = "xeon08"; + hostName = "eudy"; interfaces.eno1.ipv4.addresses = [ { address = "10.0.40.8"; prefixLength = 24; diff --git a/xeon08/cpufreq.nix b/m/eudy/cpufreq.nix similarity index 100% rename from xeon08/cpufreq.nix rename to m/eudy/cpufreq.nix diff --git a/xeon08/fs.nix b/m/eudy/fs.nix similarity index 100% rename from xeon08/fs.nix rename to m/eudy/fs.nix diff --git a/xeon08/kernel/configs/defconfig b/m/eudy/kernel/configs/defconfig similarity index 100% rename from xeon08/kernel/configs/defconfig rename to m/eudy/kernel/configs/defconfig diff --git a/xeon08/kernel/configs/lockdep b/m/eudy/kernel/configs/lockdep similarity index 100% rename from xeon08/kernel/configs/lockdep rename to m/eudy/kernel/configs/lockdep diff --git a/xeon08/kernel/kernel.nix b/m/eudy/kernel/kernel.nix similarity index 100% rename from xeon08/kernel/kernel.nix rename to m/eudy/kernel/kernel.nix diff --git a/xeon08/kernel/lttng.nix b/m/eudy/kernel/lttng.nix similarity index 100% rename from xeon08/kernel/lttng.nix rename to m/eudy/kernel/lttng.nix diff --git a/xeon08/kernel/perf.nix b/m/eudy/kernel/perf.nix similarity index 100% rename from xeon08/kernel/perf.nix rename to m/eudy/kernel/perf.nix diff --git a/xeon08/slurm.nix b/m/eudy/slurm.nix similarity index 100% rename from xeon08/slurm.nix rename to m/eudy/slurm.nix diff --git a/xeon08/users.nix b/m/eudy/users.nix similarity index 100% rename from xeon08/users.nix rename to m/eudy/users.nix diff --git a/xeon07/configuration.nix b/m/hut/configuration.nix similarity index 85% rename from xeon07/configuration.nix rename to m/hut/configuration.nix index f3b6ab4bfdca4bfe7e27793d8f204c969e8b5588..82d4c346959da5714dc2cd4df1c1ffdea62214c0 100644 --- a/xeon07/configuration.nix +++ b/m/hut/configuration.nix @@ -1,4 +1,4 @@ -{ config, pkgs, ... }: +{ config, pkgs, agenix, ... }: { imports = [ @@ -8,13 +8,14 @@ ./monitoring.nix ./nfs.nix ./slurm-daemon.nix + agenix.nixosModules.default ]; # Select the this using the ID to avoid mismatches boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN"; networking = { - hostName = "xeon07"; + hostName = "hut"; interfaces.eno1.ipv4.addresses = [ { address = "10.0.40.7"; prefixLength = 24; diff --git a/xeon07/gitlab-runner.nix b/m/hut/gitlab-runner.nix similarity index 100% rename from xeon07/gitlab-runner.nix rename to m/hut/gitlab-runner.nix diff --git a/xeon07/monitoring.nix b/m/hut/monitoring.nix similarity index 100% rename from xeon07/monitoring.nix rename to m/hut/monitoring.nix diff --git a/xeon07/nfs.nix b/m/hut/nfs.nix similarity index 100% rename from xeon07/nfs.nix rename to m/hut/nfs.nix diff --git a/xeon07/secrets.nix b/m/hut/secrets.nix similarity index 100% rename from xeon07/secrets.nix rename to m/hut/secrets.nix diff --git a/xeon07/secrets/nosv-token.age b/m/hut/secrets/nosv-token.age similarity index 100% rename from xeon07/secrets/nosv-token.age rename to m/hut/secrets/nosv-token.age diff --git a/xeon07/secrets/ovni-token.age b/m/hut/secrets/ovni-token.age similarity index 100% rename from xeon07/secrets/ovni-token.age rename to m/hut/secrets/ovni-token.age diff --git a/m/hut/slurm-daemon.nix b/m/hut/slurm-daemon.nix new file mode 100644 index 0000000000000000000000000000000000000000..e6ab227898f52bf5b976688f1479d586da26f0e0 --- /dev/null +++ b/m/hut/slurm-daemon.nix @@ -0,0 +1,11 @@ +{ ... }: + +{ + services.slurm = { + server.enable = true; + partitionName = [ + "owl Nodes=owl[1-2] Default=YES MaxTime=INFINITE State=UP" + "all Nodes=owl[1-2],hut Default=NO MaxTime=INFINITE State=UP" + ]; + }; +} diff --git a/xeon01/configuration.nix b/m/owl1/configuration.nix similarity index 94% rename from xeon01/configuration.nix rename to m/owl1/configuration.nix index 1daecec2be784fa4de213fa40779e57a9434e4c8..a14ab21802beb73c5376c5ad7c89917b05ea9dff 100644 --- a/xeon01/configuration.nix +++ b/m/owl1/configuration.nix @@ -7,7 +7,7 @@ boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53566c"; networking = { - hostName = "xeon01"; + hostName = "owl1"; interfaces.eno1.ipv4.addresses = [ { address = "10.0.40.1"; prefixLength = 24; diff --git a/xeon02/configuration.nix b/m/owl2/configuration.nix similarity index 95% rename from xeon02/configuration.nix rename to m/owl2/configuration.nix index 1b06afd201f69a35707295976d6429d0b4aa1e8a..8022b36c8eb86f01d427a4cea0986dbc4a9684ad 100644 --- a/xeon02/configuration.nix +++ b/m/owl2/configuration.nix @@ -12,7 +12,7 @@ #programs.ssh.setXAuthLocation = lib.mkForce true; networking = { - hostName = "xeon02"; + hostName = "owl2"; interfaces.eno1.ipv4.addresses = [ { address = "10.0.40.2"; prefixLength = 24; diff --git a/overlays-compat/overlays.nix b/overlays-compat/overlays.nix deleted file mode 100644 index 8a606c5c0e612f7932362444bd866e4b38b2aed4..0000000000000000000000000000000000000000 --- a/overlays-compat/overlays.nix +++ /dev/null @@ -1,8 +0,0 @@ -self: super: -with super.lib; -let - # Load the system config and get the `nixpkgs.overlays` option - overlays = (import { }).config.nixpkgs.overlays; -in - # Apply all overlays to the input of the current "main" overlay - foldl' (flip extends) (_: super) overlays self diff --git a/rebuild.sh b/rebuild.sh index 5df9616e4cb644bfec73bf3c3b2f3793b77590fa..1320c24294896d301587f944c07d8594eb4db553 100755 --- a/rebuild.sh +++ b/rebuild.sh @@ -1,19 +1,16 @@ -#!/bin/sh -e +#!/bin/sh -ex if [ "$(id -u)" != 0 ]; then echo "Needs root permissions" exit 1 fi -host=$(hostname) - -#conf="$(readlink -f .)/${host}/configuration.nix" -# -#if [ ! -e "$conf" ]; then -# echo "Missing config $conf" -# exit 1 -#fi -# -#NIXOS_CONFIG="${conf}" nixos-rebuild switch +if [ "$(hostname)" != "hut" ]; then + >&2 echo "must run from machine hut, not $(hostname)" + exit 1 +fi +# Update all nodes nixos-rebuild switch --flake . +nixos-rebuild switch --flake .#owl1 --target-host owl1 +nixos-rebuild switch --flake .#owl2 --target-host owl2 diff --git a/xeon07/slurm-daemon.nix b/xeon07/slurm-daemon.nix deleted file mode 100644 index a8dd3b83d209a20e0235ff29aeab230506ee80d5..0000000000000000000000000000000000000000 --- a/xeon07/slurm-daemon.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ ... }: - -{ - services.slurm = { - server.enable = true; - partitionName = [ - "xeon Nodes=xeon[01-02,07] Default=YES MaxTime=INFINITE State=UP" - ]; - }; -}