diff --git a/common/boot.nix b/common/boot.nix index cf85951c56e8f656c161ece9dba130f27e90b6ff..ca18e0e3e47db36dcb8bb9d04f6fdea928a7df10 100644 --- a/common/boot.nix +++ b/common/boot.nix @@ -1,8 +1,8 @@ -{ lib, ... }: +{ lib, pkgs, ... }: { # Use the GRUB 2 boot loader. - boot.loader.grub.enable = true; + boot.loader.grub.enable = lib.mkForce true; boot.loader.grub.version = 2; # Enable GRUB2 serial console @@ -18,14 +18,16 @@ "console=ttyS0,115200" ]; - boot.kernelPatches = lib.singleton { - name = "osnoise-tracer"; - patch = null; - extraStructuredConfig = with lib.kernel; { - OSNOISE_TRACER = yes; - HWLAT_TRACER = yes; - }; - }; + boot.kernelPackages = pkgs.linuxPackages_latest; + + #boot.kernelPatches = lib.singleton { + # name = "osnoise-tracer"; + # patch = null; + # extraStructuredConfig = with lib.kernel; { + # OSNOISE_TRACER = yes; + # HWLAT_TRACER = yes; + # }; + #}; boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "ehci_pci" "nvme" "usbhid" "sd_mod" ]; boot.initrd.kernelModules = [ ]; diff --git a/common/main.nix b/common/main.nix index ef16e0c68b832419587bb061d394870163f76e21..c7fc6ce10d52a4040db3c614c834e2c89686518f 100644 --- a/common/main.nix +++ b/common/main.nix @@ -6,7 +6,7 @@ ./fs.nix ./hw.nix ./net.nix - ./overlays.nix + ./slurm.nix ./ssh.nix ./users.nix ]; @@ -22,6 +22,16 @@ serviceConfig.Restart = "always"; }; + # Increase limits + security.pam.loginLimits = [ + { + domain = "*"; + type = "-"; + item = "memlock"; + value = "1048576"; # 1 GiB of mem locked + } + ]; + time.timeZone = "Europe/Madrid"; i18n.defaultLocale = "en_DK.UTF-8"; @@ -39,10 +49,14 @@ programs.zsh.enable = true; programs.zsh.histSize = 100000; + programs.bash.promptInit = '' + PS1="\h\\$ " + ''; + # Copy the NixOS configuration file and link it from the resulting system # (/run/current-system/configuration.nix). This is useful in case you # accidentally delete configuration.nix. - system.copySystemConfiguration = true; + #system.copySystemConfiguration = true; # This value determines the NixOS release from which the default # settings for stateful data, like file locations and database versions diff --git a/common/overlays.nix b/common/overlays.nix index 23fd4c67aca8bd569c5c26b503ccfd1ffdd0f29f..fdf0706e8324de755ba99caac0dd54b9d6439e4d 100644 --- a/common/overlays.nix +++ b/common/overlays.nix @@ -1,25 +1,9 @@ { options, ... }: -let - - bscpkgsSrc = builtins.fetchTarball "https://pm.bsc.es/gitlab/rarias/bscpkgs/-/archive/master/bscpkgs-master.tar.gz"; - bscpkgs = import "${bscpkgsSrc}/overlay.nix"; - - xeon07Overlay = (self: super: { - slurm = super.bsc.slurm-16-05-8-1; - }); - -in - { nix.nixPath = # Prepend default nixPath values. options.nix.nixPath.default ++ # Append our nixpkgs-overlays. - [ "nixpkgs-overlays=/config/overlays-compat/" ] - ; - - nixpkgs.overlays = [ - bscpkgs xeon07Overlay - ]; + [ "nixpkgs-overlays=${../overlays-compat}" ]; } diff --git a/common/slurm.nix b/common/slurm.nix new file mode 100644 index 0000000000000000000000000000000000000000..c1b09d68b535bfd62f761e49753869fcf496a2b0 --- /dev/null +++ b/common/slurm.nix @@ -0,0 +1,15 @@ +{ ... }: + +{ + services.slurm = { + client.enable = true; + controlMachine = "xeon07"; + clusterName = "owl"; + nodeName = [ + "xeon[01-02,07] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=xeon" + ]; + extraConfig = '' + MpiDefault=pmix + ''; + }; +} diff --git a/doc/install.md b/doc/install.md new file mode 100644 index 0000000000000000000000000000000000000000..05a60222e491248c55596e96a0c5dd2a1ce9ecdd --- /dev/null +++ b/doc/install.md @@ -0,0 +1,62 @@ +# Installing NixOS in a new node + +This article shows the steps to install NixOS in a node following the +configuration of the repo. + +## Prepare the disk + +Create a main partition and label it `nixos` following [the manual][1]. + +[1]: https://nixos.org/manual/nixos/stable/index.html#sec-installation-manual-partitioning. + +``` +# disk=/dev/sdX +# parted $disk -- mklabel msdos +# parted $disk -- mkpart primary 1MB 100% +# parted $disk -- set 1 boot on +``` + +Then create an etx4 filesystem, labeled `nixos` where the system will be +installed. **Ensure that no other partition has the same label.** + +``` +# mkfs.ext4 -L nixos "${disk}1" +# mount ${disk}1 /mnt +# lsblk -f $disk +NAME FSTYPE LABEL UUID MOUNTPOINT +sdX +`-sdX1 ext4 nixos 10d73b75-809c-4fa3-b99d-4fab2f0d0d8e /mnt +``` + +## Prepare nix and nixos-install + +Mount the nix store from the xeon07 node in read-only /nix. + +``` +# mkdir /nix +# mount -o ro xeon07:/nix /nix +``` + +Get the nix binary and nixos-install tool from xeon07: + +``` +# ssh xeon07 'readlink -f $(which nix)' +/nix/store/0sxbaj71c4c4n43qhdxm31f56gjalksw-nix-2.13.3/bin/nix +# ssh xeon07 'readlink -f $(which nixos-install)' +/nix/store/9yq8ps06ysr2pfiwiij39ny56yk3pdcs-nixos-install/bin/nixos-install +``` + +And add them to the PATH: + +``` +# export PATH=$PATH:/nix/store/0sxbaj71c4c4n43qhdxm31f56gjalksw-nix-2.13.3/bin +# export PATH=$PATH:/nix/store/9yq8ps06ysr2pfiwiij39ny56yk3pdcs-nixos-install/bin/ +# nix --version +nix (Nix) 2.13.3 +``` + +## Build the nixos kexec image + +``` +# nix build .#nixosConfigurations.xeon02.config.system.build.kexecTree -v +``` diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000000000000000000000000000000000..a8e8e792ea9363da1b9aaa0aa9142164918f9bce --- /dev/null +++ b/flake.lock @@ -0,0 +1,87 @@ +{ + "nodes": { + "agenix": { + "inputs": { + "darwin": "darwin", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1682101079, + "narHash": "sha256-MdAhtjrLKnk2uiqun1FWABbKpLH090oeqCSiWemtuck=", + "owner": "ryantm", + "repo": "agenix", + "rev": "2994d002dcff5353ca1ac48ec584c7f6589fe447", + "type": "github" + }, + "original": { + "owner": "ryantm", + "repo": "agenix", + "type": "github" + } + }, + "bscpkgs": { + "locked": { + "lastModified": 1682521628, + "narHash": "sha256-uRIDCuJNt3rdikWiRcM3VPsQSk0vpQB1JO3Wx24psJo=", + "ref": "refs/heads/master", + "rev": "c775ee4d6f76aded05b08ae13924c302f18f9b2c", + "revCount": 807, + "type": "git", + "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git" + }, + "original": { + "type": "git", + "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git" + } + }, + "darwin": { + "inputs": { + "nixpkgs": [ + "agenix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1673295039, + "narHash": "sha256-AsdYgE8/GPwcelGgrntlijMg4t3hLFJFCRF3tL5WVjA=", + "owner": "lnl7", + "repo": "nix-darwin", + "rev": "87b9d090ad39b25b2400029c64825fc2a8868943", + "type": "github" + }, + "original": { + "owner": "lnl7", + "ref": "master", + "repo": "nix-darwin", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1682526928, + "narHash": "sha256-2cKh4O6t1rQ8Ok+v16URynmb0rV7oZPEbXkU0owNLQs=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "d6b863fd9b7bb962e6f9fdf292419a775e772891", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "agenix": "agenix", + "bscpkgs": "bscpkgs", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000000000000000000000000000000000..7c4b67d220e40bba8b16080def4aff85aa9ee8fc --- /dev/null +++ b/flake.nix @@ -0,0 +1,80 @@ +{ + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + agenix.url = "github:ryantm/agenix"; + agenix.inputs.nixpkgs.follows = "nixpkgs"; + bscpkgs.url = "git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git"; + }; + + outputs = { self, nixpkgs, agenix, bscpkgs, ... }: { + nixosConfigurations = { + xeon01 = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + ( {options, ...}: { + # Sel the nixos-config path to the one of the current flake + nixpkgs.overlays = [ bscpkgs.bscOverlay ]; + nix.nixPath = [ + "nixpkgs=${nixpkgs}" + "bscpkgs=${bscpkgs}" + "nixos-config=${self.outPath}/xeon01/configuration.nix" + "nixpkgs-overlays=${self.outPath}/overlays-compat" + ]; + nix.registry.nixpkgs.flake = nixpkgs; + nix.registry.bscpkgs.flake = bscpkgs; + system.configurationRevision = + if self ? rev + then self.rev + else throw ("Refusing to build from a dirty Git tree!"); + }) + ./xeon01/configuration.nix + ]; + }; + xeon02 = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + ( {options, ...}: { + # Sel the nixos-config path to the one of the current flake + nixpkgs.overlays = [ bscpkgs.bscOverlay ]; + nix.nixPath = [ + "nixpkgs=${nixpkgs}" + "bscpkgs=${bscpkgs}" + "nixos-config=${self.outPath}/xeon02/configuration.nix" + "nixpkgs-overlays=${self.outPath}/overlays-compat" + ]; + nix.registry.nixpkgs.flake = nixpkgs; + nix.registry.bscpkgs.flake = bscpkgs; + system.configurationRevision = + if self ? rev + then self.rev + else throw ("Refusing to build from a dirty Git tree!"); + }) + ./xeon02/configuration.nix + ]; + }; + xeon07 = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + ( {options, ...}: { + # Sel the nixos-config path to the one of the current flake + nixpkgs.overlays = [ bscpkgs.bscOverlay ]; + nix.nixPath = [ + "nixpkgs=${nixpkgs}" + "bscpkgs=${bscpkgs}" + "nixos-config=${self.outPath}/xeon07/configuration.nix" + "nixpkgs-overlays=${self.outPath}/overlays-compat" + ]; + nix.registry.nixpkgs.flake = nixpkgs; + nix.registry.bscpkgs.flake = bscpkgs; + system.configurationRevision = + if self ? rev + then self.rev + else throw ("Refusing to build from a dirty Git tree!"); + }) + agenix.nixosModules.default + ./xeon07/configuration.nix + ]; + }; + }; + }; +} diff --git a/rebuild.sh b/rebuild.sh index 592ed384153e50113d0677b6ffa48b395fa9a727..5df9616e4cb644bfec73bf3c3b2f3793b77590fa 100755 --- a/rebuild.sh +++ b/rebuild.sh @@ -6,11 +6,14 @@ if [ "$(id -u)" != 0 ]; then fi host=$(hostname) -conf="$(readlink -f .)/${host}/configuration.nix" -if [ ! -e "$conf" ]; then - echo "Missing config $conf" - exit 1 -fi +#conf="$(readlink -f .)/${host}/configuration.nix" +# +#if [ ! -e "$conf" ]; then +# echo "Missing config $conf" +# exit 1 +#fi +# +#NIXOS_CONFIG="${conf}" nixos-rebuild switch -NIXOS_CONFIG="${conf}" nixos-rebuild switch +nixos-rebuild switch --flake . diff --git a/xeon01/configuration.nix b/xeon01/configuration.nix index 3bf8bba1d2bf356efd5487c7b1af941842923161..1daecec2be784fa4de213fa40779e57a9434e4c8 100644 --- a/xeon01/configuration.nix +++ b/xeon01/configuration.nix @@ -1,9 +1,7 @@ { config, pkgs, ... }: { - imports = [ - ../common/main.nix - ]; + imports = [ ../common/main.nix ]; # Select the this using the ID to avoid mismatches boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53566c"; diff --git a/xeon02/configuration.nix b/xeon02/configuration.nix new file mode 100644 index 0000000000000000000000000000000000000000..1b06afd201f69a35707295976d6429d0b4aa1e8a --- /dev/null +++ b/xeon02/configuration.nix @@ -0,0 +1,25 @@ +{ config, pkgs, modulesPath, lib, ... }: + +{ + imports = [ + #(modulesPath + "/installer/netboot/netboot-minimal.nix") + ../common/main.nix + ]; + + # Select the this using the ID to avoid mismatches + boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d535629"; + #programs.ssh.forwardX11 = false; + #programs.ssh.setXAuthLocation = lib.mkForce true; + + networking = { + hostName = "xeon02"; + interfaces.eno1.ipv4.addresses = [ { + address = "10.0.40.2"; + prefixLength = 24; + } ]; + interfaces.ibp129s0.ipv4.addresses = [ { + address = "10.0.42.2"; + prefixLength = 24; + } ]; + }; +} diff --git a/xeon07/configuration.nix b/xeon07/configuration.nix index df6350837290169d05fa8cf224dcd66afc04987d..f3b6ab4bfdca4bfe7e27793d8f204c969e8b5588 100644 --- a/xeon07/configuration.nix +++ b/xeon07/configuration.nix @@ -7,9 +7,7 @@ ./gitlab-runner.nix ./monitoring.nix ./nfs.nix - ./slurm.nix - - + ./slurm-daemon.nix ]; # Select the this using the ID to avoid mismatches @@ -26,8 +24,4 @@ prefixLength = 24; } ]; }; - - environment.systemPackages = with pkgs; [ - (pkgs.callPackage {}) - ]; } diff --git a/xeon07/slurm-daemon.nix b/xeon07/slurm-daemon.nix new file mode 100644 index 0000000000000000000000000000000000000000..a8dd3b83d209a20e0235ff29aeab230506ee80d5 --- /dev/null +++ b/xeon07/slurm-daemon.nix @@ -0,0 +1,10 @@ +{ ... }: + +{ + services.slurm = { + server.enable = true; + partitionName = [ + "xeon Nodes=xeon[01-02,07] Default=YES MaxTime=INFINITE State=UP" + ]; + }; +} diff --git a/xeon07/slurm.nix b/xeon07/slurm.nix deleted file mode 100644 index 379681d8edf0d8dd8348a113219c28f796914708..0000000000000000000000000000000000000000 --- a/xeon07/slurm.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ ... }: - -{ - services.slurm = { - client.enable = true; - controlMachine = "ssfhead"; - clusterName = "owl"; - nodeName = [ - "xeon[01-08] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=xeon" - ]; - }; -}