From a4c254fcd6da928907d801e7ce5b0e405335ec84 Mon Sep 17 00:00:00 2001 From: Aleix Roca Nonell Date: Wed, 7 Jun 2023 19:52:24 +0200 Subject: [PATCH 1/5] xeon08: Add lttng module and tools --- xeon08/kernel/kernel.nix | 3 +++ xeon08/kernel/lttng.nix | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 xeon08/kernel/lttng.nix diff --git a/xeon08/kernel/kernel.nix b/xeon08/kernel/kernel.nix index 5aca93c..ae3b91d 100644 --- a/xeon08/kernel/kernel.nix +++ b/xeon08/kernel/kernel.nix @@ -46,5 +46,8 @@ let latest = pkgs.linuxPackages_latest; in { + imports = [ + ./lttng.nix + ]; boot.kernelPackages = lib.mkForce kernel; } diff --git a/xeon08/kernel/lttng.nix b/xeon08/kernel/lttng.nix new file mode 100644 index 0000000..b9d6e4d --- /dev/null +++ b/xeon08/kernel/lttng.nix @@ -0,0 +1,36 @@ +{ config, pkgs, lib, ... }: + +let + + # the lttng btrfs probe crashes at compile time because of an undefined + # function. This disables the btrfs tracepoints to avoid the issue. + lttng-modules-fixed = config.boot.kernelPackages.lttng-modules.overrideAttrs (finalAttrs: previousAttrs: { + patchPhase = (lib.optionalString (previousAttrs ? patchPhase) previousAttrs.patchPhase) + '' + substituteInPlace src/probes/Kbuild \ + --replace " obj-\$(CONFIG_LTTNG) += lttng-probe-btrfs.o" " #obj-\$(CONFIG_LTTNG) += lttng-probe-btrfs.o" + ''; + }); +in { + + # add the lttng tools and modules to the system environment + boot.extraModulePackages = [ lttng-modules-fixed ]; + environment.systemPackages = with pkgs; [ + lttng-tools lttng-ust babeltrace + ]; + + # start the lttng root daemon to manage kernel events + systemd.services.lttng-sessiond = { + wantedBy = [ "multi-user.target" ]; + description = "LTTng session daemon for the root user"; + serviceConfig = { + User = "root"; + ExecStart = '' + ${pkgs.lttng-tools}/bin/lttng-sessiond + ''; + }; + }; + + # members of the tracing group can use the lttng-provided kernel events + # without root permissions + users.groups.tracing.members = [ "arocanon" ]; +} -- GitLab From 8769f3d4182b9d13300e920a39750645bb8014fd Mon Sep 17 00:00:00 2001 From: Aleix Roca Nonell Date: Fri, 9 Jun 2023 08:04:30 +0200 Subject: [PATCH 2/5] xeon08: Enable lttng lockdep tracepoints --- xeon08/configuration.nix | 4 ++++ xeon08/kernel/lttng.nix | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/xeon08/configuration.nix b/xeon08/configuration.nix index 7899173..195117f 100644 --- a/xeon08/configuration.nix +++ b/xeon08/configuration.nix @@ -18,6 +18,10 @@ # disable automatic garbage collector nix.gc.automatic = lib.mkForce false; + # members of the tracing group can use the lttng-provided kernel events + # without root permissions + users.groups.tracing.members = [ "arocanon" ]; + # set up both ethernet and infiniband ips networking = { hostName = "xeon08"; diff --git a/xeon08/kernel/lttng.nix b/xeon08/kernel/lttng.nix index b9d6e4d..eb45911 100644 --- a/xeon08/kernel/lttng.nix +++ b/xeon08/kernel/lttng.nix @@ -2,12 +2,23 @@ let - # the lttng btrfs probe crashes at compile time because of an undefined + # The lttng btrfs probe crashes at compile time because of an undefined # function. This disables the btrfs tracepoints to avoid the issue. + + # Also enable lockdep tracepoints, this is disabled by default because it + # does not work well on architectures other than x86_64 (i think that arm) as + # I was told on the mailing list. lttng-modules-fixed = config.boot.kernelPackages.lttng-modules.overrideAttrs (finalAttrs: previousAttrs: { patchPhase = (lib.optionalString (previousAttrs ? patchPhase) previousAttrs.patchPhase) + '' + # disable btrfs substituteInPlace src/probes/Kbuild \ --replace " obj-\$(CONFIG_LTTNG) += lttng-probe-btrfs.o" " #obj-\$(CONFIG_LTTNG) += lttng-probe-btrfs.o" + + # enable lockdep tracepoints + substituteInPlace src/probes/Kbuild \ + --replace "#ifneq (\$(CONFIG_LOCKDEP),)" "ifneq (\$(CONFIG_LOCKDEP),)" \ + --replace "# obj-\$(CONFIG_LTTNG) += lttng-probe-lock.o" " obj-\$(CONFIG_LTTNG) += lttng-probe-lock.o" \ + --replace "#endif # CONFIG_LOCKDEP" "endif # CONFIG_LOCKDEP" ''; }); in { @@ -29,8 +40,4 @@ in { ''; }; }; - - # members of the tracing group can use the lttng-provided kernel events - # without root permissions - users.groups.tracing.members = [ "arocanon" ]; } -- GitLab From 1552eeca128b9be7c544ee1ad4a9385dd68ead31 Mon Sep 17 00:00:00 2001 From: Aleix Roca Nonell Date: Fri, 9 Jun 2023 10:58:11 +0200 Subject: [PATCH 3/5] xeon08: Add perf --- xeon08/kernel/kernel.nix | 1 + xeon08/kernel/perf.nix | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 xeon08/kernel/perf.nix diff --git a/xeon08/kernel/kernel.nix b/xeon08/kernel/kernel.nix index ae3b91d..45ecce6 100644 --- a/xeon08/kernel/kernel.nix +++ b/xeon08/kernel/kernel.nix @@ -48,6 +48,7 @@ let in { imports = [ ./lttng.nix + ./perf.nix ]; boot.kernelPackages = lib.mkForce kernel; } diff --git a/xeon08/kernel/perf.nix b/xeon08/kernel/perf.nix new file mode 100644 index 0000000..51340df --- /dev/null +++ b/xeon08/kernel/perf.nix @@ -0,0 +1,22 @@ +{ config, pkgs, lib, ... }: + +{ + # add the perf tool + environment.systemPackages = with pkgs; [ + config.boot.kernelPackages.perf + ]; + + # allow non-root users to read tracing data from the kernel + boot.kernel.sysctl."kernel.perf_event_paranoid" = -2; + boot.kernel.sysctl."kernel.kptr_restrict" = 0; + + # specify additionl options to the tracefs directory to allow members of the + # tracing group to access tracefs. + fileSystems."/sys/kernel/tracing" = { + options = [ + "mode=755" + "gid=tracing" + ]; + }; +} + -- GitLab From c1f138a9c186a4fe726d20dbf753741bbdca5a9c Mon Sep 17 00:00:00 2001 From: Aleix Roca Nonell Date: Mon, 12 Jun 2023 17:16:01 +0200 Subject: [PATCH 4/5] xeon08: Add config for kernel non-voluntary preemption --- xeon08/kernel/kernel.nix | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/xeon08/kernel/kernel.nix b/xeon08/kernel/kernel.nix index 45ecce6..10bfead 100644 --- a/xeon08/kernel/kernel.nix +++ b/xeon08/kernel/kernel.nix @@ -23,7 +23,7 @@ let kernel = nixos-fcsv2; - nixos-fcs-kernel = {gitCommit, lockStat ? false}: pkgs.linuxPackagesFor (pkgs.buildLinux rec { + nixos-fcs-kernel = {gitCommit, lockStat ? false, preempt ? false}: pkgs.linuxPackagesFor (pkgs.buildLinux rec { version = "6.2.8"; src = builtins.fetchGit { url = "git@bscpm03.bsc.es:ompss-kernel/linux.git"; @@ -31,9 +31,12 @@ let ref = "fcs"; }; structuredExtraConfig = with lib.kernel; { - # add cutom kernel options here + # add general custom kernel options here } // lib.optionalAttrs lockStat { LOCK_STAT = yes; + } // lib.optionalAttrs preempt { + PREEMPT = lib.mkForce yes; + PREEMPT_VOLUNTARY = lib.mkForce no; }; kernelPatches = []; extraMeta.branch = lib.versions.majorMinor version; @@ -41,8 +44,19 @@ let nixos-fcsv1 = nixos-fcs-kernel {gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";}; nixos-fcsv2 = nixos-fcs-kernel {gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";}; - nixos-fcsv1-lockstat = nixos-fcs-kernel {gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be"; lockStat = true;}; - nixos-fcsv2-lockstat = nixos-fcs-kernel {gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1"; lockStat = true;}; + nixos-fcsv1-lockstat = nixos-fcs-kernel { + gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be"; + lockStat = true; + }; + nixos-fcsv2-lockstat = nixos-fcs-kernel { + gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1"; + lockStat = true; + }; + nixos-fcsv2-lockstat-preempt = nixos-fcs-kernel { + gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1"; + lockStat = true; + preempt = true; + }; latest = pkgs.linuxPackages_latest; in { -- GitLab From 011e8c2bf8fec5fb52dda08601d211563660f0f2 Mon Sep 17 00:00:00 2001 From: Aleix Roca Nonell Date: Wed, 14 Jun 2023 16:16:46 +0200 Subject: [PATCH 5/5] Move arocanon user from xeon08 to common --- common/users.nix | 28 ++++++++++++++++++++-------- xeon08/users.nix | 12 ------------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/common/users.nix b/common/users.nix index 03a5eb1..be34db7 100644 --- a/common/users.nix +++ b/common/users.nix @@ -3,14 +3,26 @@ { users = { mutableUsers = false; - users.rarias = { - uid = 1880; - isNormalUser = true; - home = "/home/Computational/rarias"; - description = "Rodrigo Arias"; - group = "Computational"; - extraGroups = [ "wheel" ]; - hashedPassword = "$6$u06tkCy13enReBsb$xiI.twRvvTfH4jdS3s68NZ7U9PSbGKs5.LXU/UgoawSwNWhZo2hRAjNL5qG0/lAckzcho2LjD0r3NfVPvthY6/"; + users = { + rarias = { + uid = 1880; + isNormalUser = true; + home = "/home/Computational/rarias"; + description = "Rodrigo Arias"; + group = "Computational"; + extraGroups = [ "wheel" ]; + hashedPassword = "$6$u06tkCy13enReBsb$xiI.twRvvTfH4jdS3s68NZ7U9PSbGKs5.LXU/UgoawSwNWhZo2hRAjNL5qG0/lAckzcho2LjD0r3NfVPvthY6/"; + }; + + arocanon = { + uid = 1042; + isNormalUser = true; + home = "/home/Computational/arocanon"; + description = "Aleix Roca"; + group = "Computational"; + extraGroups = [ "wheel" ]; + hashedPassword = "$6$hliZiW4tULC/tH7p$pqZarwJkNZ7vS0G5llWQKx08UFG9DxDYgad7jplMD8WkZh5k58i4dfPoWtnEShfjTO6JHiIin05ny5lmSXzGM/"; + }; }; groups = { diff --git a/xeon08/users.nix b/xeon08/users.nix index 4a01344..a1cfab4 100644 --- a/xeon08/users.nix +++ b/xeon08/users.nix @@ -1,18 +1,6 @@ { ... }: { - users = { - users.arocanon = { - uid = 1042; - isNormalUser = true; - home = "/home/Computational/arocanon"; - description = "Aleix Roca"; - group = "Computational"; - extraGroups = [ "wheel" ]; - hashedPassword = "$6$hliZiW4tULC/tH7p$pqZarwJkNZ7vS0G5llWQKx08UFG9DxDYgad7jplMD8WkZh5k58i4dfPoWtnEShfjTO6JHiIin05ny5lmSXzGM/"; - }; - }; - security.sudo.extraRules= [{ users = [ "arocanon" ]; commands = [{ -- GitLab