From 4c309dea2fa708ce8c369bf01ad608bbfc379579 Mon Sep 17 00:00:00 2001 From: Aleix Roca Nonell Date: Thu, 29 Jun 2023 09:14:39 +0200 Subject: [PATCH 1/5] eudy: disable all cpu mitigations --- m/eudy/kernel/kernel.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/m/eudy/kernel/kernel.nix b/m/eudy/kernel/kernel.nix index 10bfead..17a580b 100644 --- a/m/eudy/kernel/kernel.nix +++ b/m/eudy/kernel/kernel.nix @@ -65,4 +65,9 @@ in { ./perf.nix ]; boot.kernelPackages = lib.mkForce kernel; + + # disable all cpu mitigations + boot.kernelParams = [ + "mitigations=off" + ]; } -- GitLab From a0b9d10b1463d28f6bc3a64032357aeb0d4778df Mon Sep 17 00:00:00 2001 From: Aleix Roca Nonell Date: Fri, 30 Jun 2023 12:49:44 +0200 Subject: [PATCH 2/5] eudy: Enable memory overcommit --- m/eudy/kernel/kernel.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/m/eudy/kernel/kernel.nix b/m/eudy/kernel/kernel.nix index 17a580b..3078fad 100644 --- a/m/eudy/kernel/kernel.nix +++ b/m/eudy/kernel/kernel.nix @@ -70,4 +70,8 @@ in { boot.kernelParams = [ "mitigations=off" ]; + + # enable memory overcommit, needed to build a taglibc system using nix after + # increasing the openblas memory footprint + boot.kernel.sysctl."vm.overcommit_memory" = 1; } -- GitLab From 40b1a8f0df9c340e2dd04c9d7a610555a13a312b Mon Sep 17 00:00:00 2001 From: Aleix Roca Nonell Date: Wed, 12 Jul 2023 13:22:42 +0200 Subject: [PATCH 3/5] eudy: Add fcsv3 and intermediate versions for testing --- m/eudy/kernel/kernel.nix | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/m/eudy/kernel/kernel.nix b/m/eudy/kernel/kernel.nix index 3078fad..016b8cb 100644 --- a/m/eudy/kernel/kernel.nix +++ b/m/eudy/kernel/kernel.nix @@ -21,14 +21,14 @@ let # configfile = if lockdep then ./configs/lockdep else ./configs/defconfig; #}; - kernel = nixos-fcsv2; + kernel = nixos-fcsv3; - nixos-fcs-kernel = {gitCommit, lockStat ? false, preempt ? false}: pkgs.linuxPackagesFor (pkgs.buildLinux rec { + nixos-fcs-kernel = {gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec { version = "6.2.8"; src = builtins.fetchGit { url = "git@bscpm03.bsc.es:ompss-kernel/linux.git"; rev = gitCommit; - ref = "fcs"; + ref = branch; }; structuredExtraConfig = with lib.kernel; { # add general custom kernel options here @@ -44,6 +44,17 @@ let nixos-fcsv1 = nixos-fcs-kernel {gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";}; nixos-fcsv2 = nixos-fcs-kernel {gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";}; + nixos-fcsv3 = nixos-fcs-kernel {gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";}; + + # always use fcs_sched_setaffinity + #nixos-debug = nixos-fcs-kernel {gitCommit = "7d0bf285fca92badc8df3c9907a9ab30db4418aa";}; + # remove need_check_cgroup + #nixos-debug = nixos-fcs-kernel {gitCommit = "4cc4efaab5e4a0bfa3089e935215b981c1922919";}; + # merge again fcs_wake and fcs_wait + #nixos-debug = nixos-fcs-kernel {gitCommit = "40c6f72f4ae54b0b636b193ac0648fb5730c810d";}; + # start from scratch, this is the working version with split fcs_wake and fcs_wait + nixos-debug = nixos-fcs-kernel {gitCommit = "c9a39d6a4ca83845b4e71fcc268fb0a76aff1bdf"; branch = "fcs-test"; }; + nixos-fcsv1-lockstat = nixos-fcs-kernel { gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be"; lockStat = true; @@ -52,8 +63,12 @@ let gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1"; lockStat = true; }; - nixos-fcsv2-lockstat-preempt = nixos-fcs-kernel { - gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1"; + nixos-fcsv3-lockstat = nixos-fcs-kernel { + gitCommit = "6c17394890704c3345ac1a521bb547164b36b154"; + lockStat = true; + }; + nixos-fcsv3-lockstat-preempt = nixos-fcs-kernel { + gitCommit = "6c17394890704c3345ac1a521bb547164b36b154"; lockStat = true; preempt = true; }; -- GitLab From 66fb848ba8567c2fee342ccbde8301fe36e27b8c Mon Sep 17 00:00:00 2001 From: Aleix Roca Nonell Date: Fri, 21 Jul 2023 10:34:19 +0200 Subject: [PATCH 4/5] Add koro node --- flake.nix | 1 + m/common/net.nix | 2 +- m/common/ssh.nix | 1 + m/koro/configuration.nix | 37 +++++++++++++++++++++++ m/koro/kernel.nix | 64 ++++++++++++++++++++++++++++++++++++++++ m/koro/users.nix | 17 +++++++++++ 6 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 m/koro/configuration.nix create mode 100644 m/koro/kernel.nix create mode 100644 m/koro/users.nix diff --git a/flake.nix b/flake.nix index ebc935b..5f67afc 100644 --- a/flake.nix +++ b/flake.nix @@ -20,6 +20,7 @@ in owl1 = mkConf "owl1"; owl2 = mkConf "owl2"; eudy = mkConf "eudy"; + koro = mkConf "koro"; }; packages.x86_64-linux.hut = self.nixosConfigurations.hut.pkgs; diff --git a/m/common/net.nix b/m/common/net.nix index 4970f40..b2c09ca 100644 --- a/m/common/net.nix +++ b/m/common/net.nix @@ -63,7 +63,7 @@ 10.0.40.104 xeon04-ipmi0 # Node Entry for node: xeon05 (ID=19) - 10.0.40.5 xeon05 xeon05-eth0 + 10.0.40.5 koro xeon05 xeon05-eth0 10.0.42.5 xeon05-ib0 10.0.40.105 xeon05-ipmi0 diff --git a/m/common/ssh.nix b/m/common/ssh.nix index 6dcc9f3..2d805bf 100644 --- a/m/common/ssh.nix +++ b/m/common/ssh.nix @@ -16,6 +16,7 @@ "owl1".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv"; "owl2".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK"; "eudy".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG"; + "koro".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67"; "gitlab-internal.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3"; "bscpm03.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS"; diff --git a/m/koro/configuration.nix b/m/koro/configuration.nix new file mode 100644 index 0000000..221a871 --- /dev/null +++ b/m/koro/configuration.nix @@ -0,0 +1,37 @@ +{ config, pkgs, lib, modulesPath, ... }: + +{ + imports = [ + ../common/main.nix + #(modulesPath + "/installer/netboot/netboot-minimal.nix") + + ../eudy/cpufreq.nix + ../eudy/users.nix + ../eudy/slurm.nix + ./users.nix + ./kernel.nix + ]; + + # Select this using the ID to avoid mismatches + boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d5376d2"; + + # disable automatic garbage collector + nix.gc.automatic = lib.mkForce false; + + # members of the tracing group can use the lttng-provided kernel events + # without root permissions + users.groups.tracing.members = [ "arocanon" ]; + + # set up both ethernet and infiniband ips + networking = { + hostName = "koro"; + interfaces.eno1.ipv4.addresses = [ { + address = "10.0.40.5"; + prefixLength = 24; + } ]; + interfaces.ibp5s0.ipv4.addresses = [ { + address = "10.0.42.5"; + prefixLength = 24; + } ]; + }; +} diff --git a/m/koro/kernel.nix b/m/koro/kernel.nix new file mode 100644 index 0000000..016ac52 --- /dev/null +++ b/m/koro/kernel.nix @@ -0,0 +1,64 @@ +{ pkgs, lib, ... }: + +let + kernel = nixos-fcsv4; + + nixos-fcs-kernel = {gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec { + version = "6.2.8"; + src = builtins.fetchGit { + url = "git@bscpm03.bsc.es:ompss-kernel/linux.git"; + rev = gitCommit; + ref = branch; + }; + structuredExtraConfig = with lib.kernel; { + # add general custom kernel options here + } // lib.optionalAttrs lockStat { + LOCK_STAT = yes; + } // lib.optionalAttrs preempt { + PREEMPT = lib.mkForce yes; + PREEMPT_VOLUNTARY = lib.mkForce no; + }; + kernelPatches = []; + extraMeta.branch = lib.versions.majorMinor version; + }); + + nixos-fcsv1 = nixos-fcs-kernel {gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";}; + nixos-fcsv2 = nixos-fcs-kernel {gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";}; + nixos-fcsv3 = nixos-fcs-kernel {gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";}; + nixos-fcsv4 = nixos-fcs-kernel {gitCommit = "c94c3d946f33ac3e5782a02ee002cc1164c0cb4f";}; + + nixos-fcsv1-lockstat = nixos-fcs-kernel { + gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be"; + lockStat = true; + }; + nixos-fcsv2-lockstat = nixos-fcs-kernel { + gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1"; + lockStat = true; + }; + nixos-fcsv3-lockstat = nixos-fcs-kernel { + gitCommit = "6c17394890704c3345ac1a521bb547164b36b154"; + lockStat = true; + }; + nixos-fcsv3-lockstat-preempt = nixos-fcs-kernel { + gitCommit = "6c17394890704c3345ac1a521bb547164b36b154"; + lockStat = true; + preempt = true; + }; + latest = pkgs.linuxPackages_latest; + +in { + imports = [ + ../eudy/kernel/lttng.nix + ../eudy/kernel/perf.nix + ]; + boot.kernelPackages = lib.mkForce kernel; + + # disable all cpu mitigations + boot.kernelParams = [ + "mitigations=off" + ]; + + # enable memory overcommit, needed to build a taglibc system using nix after + # increasing the openblas memory footprint + boot.kernel.sysctl."vm.overcommit_memory" = lib.mkForce 1; +} diff --git a/m/koro/users.nix b/m/koro/users.nix new file mode 100644 index 0000000..4a4f794 --- /dev/null +++ b/m/koro/users.nix @@ -0,0 +1,17 @@ +{ ... }: + +{ + users.users = { + vlopez = { + uid = 4334; + isNormalUser = true; + home = "/home/Computational/vlopez"; + description = "Victor López"; + group = "Computational"; + hashedPassword = "$6$0ZBkgIYE/renVqtt$1uWlJsb0FEezRVNoETTzZMx4X2SvWiOsKvi0ppWCRqI66S6TqMBXBdP4fcQyvRRBt0e4Z7opZIvvITBsEtO0f0"; + openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch" + ]; + }; + }; +} -- GitLab From 79adbe76a8f7026f15233ca573b6572a11bfb1b1 Mon Sep 17 00:00:00 2001 From: Aleix Roca Nonell Date: Fri, 21 Jul 2023 10:34:37 +0200 Subject: [PATCH 5/5] koro: Add vlopez user --- m/koro/configuration.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/m/koro/configuration.nix b/m/koro/configuration.nix index 221a871..6623549 100644 --- a/m/koro/configuration.nix +++ b/m/koro/configuration.nix @@ -20,7 +20,7 @@ # members of the tracing group can use the lttng-provided kernel events # without root permissions - users.groups.tracing.members = [ "arocanon" ]; + users.groups.tracing.members = [ "arocanon" "vlopez" ]; # set up both ethernet and infiniband ips networking = { -- GitLab