From b9aff1dba54ea30656a242a24c97b04fdb29d71c Mon Sep 17 00:00:00 2001
From: Rodrigo Arias <rodrigo.arias@bsc.es>
Date: Wed, 26 Apr 2023 13:28:04 +0200
Subject: [PATCH 01/18] Remove xeon07 overlay to load upstream slurm

---
 common/overlays.nix | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/common/overlays.nix b/common/overlays.nix
index 23fd4c6..2fb80d7 100644
--- a/common/overlays.nix
+++ b/common/overlays.nix
@@ -5,10 +5,6 @@ let
   bscpkgsSrc = builtins.fetchTarball "https://pm.bsc.es/gitlab/rarias/bscpkgs/-/archive/master/bscpkgs-master.tar.gz";
   bscpkgs = import "${bscpkgsSrc}/overlay.nix";
 
-  xeon07Overlay = (self: super: {
-    slurm = super.bsc.slurm-16-05-8-1;
-  });
-
 in
 
 {
@@ -20,6 +16,6 @@ in
   ;
 
   nixpkgs.overlays = [
-    bscpkgs xeon07Overlay
+    bscpkgs
   ];
 }
-- 
GitLab


From de2af7981011b1cb98ba26d92a252c7ec2c78eaa Mon Sep 17 00:00:00 2001
From: Rodrigo Arias <rodrigo.arias@bsc.es>
Date: Wed, 26 Apr 2023 13:29:28 +0200
Subject: [PATCH 02/18] Use xeon07 as control machine

---
 xeon07/slurm.nix | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/xeon07/slurm.nix b/xeon07/slurm.nix
index 379681d..48a77d7 100644
--- a/xeon07/slurm.nix
+++ b/xeon07/slurm.nix
@@ -3,10 +3,15 @@
 {
   services.slurm = {
     client.enable = true;
-    controlMachine = "ssfhead";
+    server.enable = true;
+    controlMachine = "xeon07";
     clusterName = "owl";
     nodeName = [
-      "xeon[01-08] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=xeon"
+      "xeon[01-02,07] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=xeon"
+    ];
+
+    partitionName = [
+      "xeon Nodes=xeon[01-02,07] Default=YES MaxTime=INFINITE State=UP"
     ];
   };
 }
-- 
GitLab


From 95bbeeb646408677685718e6a183df494760c038 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias <rodrigo.arias@bsc.es>
Date: Wed, 26 Apr 2023 13:35:06 +0200
Subject: [PATCH 03/18] Enable slurm in xeon01

---
 xeon01/configuration.nix |  1 +
 xeon01/slurm.nix         | 12 ++++++++++++
 2 files changed, 13 insertions(+)
 create mode 100644 xeon01/slurm.nix

diff --git a/xeon01/configuration.nix b/xeon01/configuration.nix
index 3bf8bba..9f651d8 100644
--- a/xeon01/configuration.nix
+++ b/xeon01/configuration.nix
@@ -3,6 +3,7 @@
 {
   imports = [
     ../common/main.nix
+    ./slurm.nix
   ];
 
   # Select the this using the ID to avoid mismatches
diff --git a/xeon01/slurm.nix b/xeon01/slurm.nix
new file mode 100644
index 0000000..815c94a
--- /dev/null
+++ b/xeon01/slurm.nix
@@ -0,0 +1,12 @@
+{ ... }:
+
+{
+  services.slurm = {
+    client.enable = true;
+    controlMachine = "xeon07";
+    clusterName = "owl";
+    nodeName = [
+      "xeon[01-02,07] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=xeon"
+    ];
+  };
+}
-- 
GitLab


From 543983e9f3d24cadc26bee3e958d92306c960c55 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Wed, 26 Apr 2023 14:26:39 +0200
Subject: [PATCH 04/18] Test flakes

---
 flake.nix | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 flake.nix

diff --git a/flake.nix b/flake.nix
new file mode 100644
index 0000000..0a90a63
--- /dev/null
+++ b/flake.nix
@@ -0,0 +1,16 @@
+{
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/22.11";
+
+  outputs = { nixpkgs, ... }: {
+    nixosConfigurations = {
+      xeon01 = nixpkgs.lib.nixosSystem {
+        system = "x86_64-linux";
+        modules = [ ./xeon01/configuration.nix ];
+      };
+      xeon07 = nixpkgs.lib.nixosSystem {
+        system = "x86_64-linux";
+        modules = [ ./xeon07/configuration.nix ];
+      };
+    };
+  };
+}
-- 
GitLab


From 44fcb97ec7ff597780da57bb358adf7ab674f5aa Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Wed, 26 Apr 2023 17:36:36 +0200
Subject: [PATCH 05/18] Lock flakes and add inputs

---
 common/main.nix          |  4 +-
 common/overlays.nix      | 14 +------
 flake.lock               | 87 ++++++++++++++++++++++++++++++++++++++++
 flake.nix                | 50 +++++++++++++++++++++--
 rebuild.sh               | 15 ++++---
 xeon01/slurm.nix         |  2 +-
 xeon07/configuration.nix |  6 ---
 7 files changed, 146 insertions(+), 32 deletions(-)
 create mode 100644 flake.lock

diff --git a/common/main.nix b/common/main.nix
index ef16e0c..ad54622 100644
--- a/common/main.nix
+++ b/common/main.nix
@@ -6,7 +6,6 @@
     ./fs.nix
     ./hw.nix
     ./net.nix
-    ./overlays.nix
     ./ssh.nix
     ./users.nix
   ];
@@ -38,11 +37,12 @@
 
   programs.zsh.enable = true;
   programs.zsh.histSize = 100000;
+  users.defaultUserShell = pkgs.zsh;
 
   # Copy the NixOS configuration file and link it from the resulting system
   # (/run/current-system/configuration.nix). This is useful in case you
   # accidentally delete configuration.nix.
-  system.copySystemConfiguration = true;
+  #system.copySystemConfiguration = true;
 
   # This value determines the NixOS release from which the default
   # settings for stateful data, like file locations and database versions
diff --git a/common/overlays.nix b/common/overlays.nix
index 2fb80d7..fdf0706 100644
--- a/common/overlays.nix
+++ b/common/overlays.nix
@@ -1,21 +1,9 @@
 { options, ... }:
 
-let
-
-  bscpkgsSrc = builtins.fetchTarball "https://pm.bsc.es/gitlab/rarias/bscpkgs/-/archive/master/bscpkgs-master.tar.gz";
-  bscpkgs = import "${bscpkgsSrc}/overlay.nix";
-
-in
-
 {
   nix.nixPath =
     # Prepend default nixPath values.
     options.nix.nixPath.default ++
     # Append our nixpkgs-overlays.
-    [ "nixpkgs-overlays=/config/overlays-compat/" ]
-  ;
-
-  nixpkgs.overlays = [
-    bscpkgs
-  ];
+    [ "nixpkgs-overlays=${../overlays-compat}" ];
 }
diff --git a/flake.lock b/flake.lock
new file mode 100644
index 0000000..c8d3383
--- /dev/null
+++ b/flake.lock
@@ -0,0 +1,87 @@
+{
+  "nodes": {
+    "agenix": {
+      "inputs": {
+        "darwin": "darwin",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1682101079,
+        "narHash": "sha256-MdAhtjrLKnk2uiqun1FWABbKpLH090oeqCSiWemtuck=",
+        "owner": "ryantm",
+        "repo": "agenix",
+        "rev": "2994d002dcff5353ca1ac48ec584c7f6589fe447",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ryantm",
+        "repo": "agenix",
+        "type": "github"
+      }
+    },
+    "bscpkgs": {
+      "locked": {
+        "lastModified": 1682521628,
+        "narHash": "sha256-uRIDCuJNt3rdikWiRcM3VPsQSk0vpQB1JO3Wx24psJo=",
+        "ref": "refs/heads/master",
+        "rev": "c775ee4d6f76aded05b08ae13924c302f18f9b2c",
+        "revCount": 807,
+        "type": "git",
+        "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
+      }
+    },
+    "darwin": {
+      "inputs": {
+        "nixpkgs": [
+          "agenix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1673295039,
+        "narHash": "sha256-AsdYgE8/GPwcelGgrntlijMg4t3hLFJFCRF3tL5WVjA=",
+        "owner": "lnl7",
+        "repo": "nix-darwin",
+        "rev": "87b9d090ad39b25b2400029c64825fc2a8868943",
+        "type": "github"
+      },
+      "original": {
+        "owner": "lnl7",
+        "ref": "master",
+        "repo": "nix-darwin",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1669833724,
+        "narHash": "sha256-mlqo1r+TZUOuypWdrZHluxWL+E5WzXlUXNZ9Y0WLDFU=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "9a6aabc4740790ef3bbb246b86d029ccf6759658",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "22.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "agenix": "agenix",
+        "bscpkgs": "bscpkgs",
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
diff --git a/flake.nix b/flake.nix
index 0a90a63..70cdb8b 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,15 +1,57 @@
 {
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/22.11";
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/22.11";
+    agenix.url = "github:ryantm/agenix";
+    agenix.inputs.nixpkgs.follows = "nixpkgs";
+    bscpkgs.url = "git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git";
+  };
 
-  outputs = { nixpkgs, ... }: {
+  outputs = { self, nixpkgs, agenix, bscpkgs, ... }: {
     nixosConfigurations = {
       xeon01 = nixpkgs.lib.nixosSystem {
         system = "x86_64-linux";
-        modules = [ ./xeon01/configuration.nix ];
+        modules = [
+          ( {options, ...}: {
+            # Sel the nixos-config path to the one of the current flake
+            nixpkgs.overlays = [ bscpkgs.bscOverlay ];
+            nix.nixPath = [
+                "nixpkgs=${nixpkgs}"
+                "bscpkgs=${bscpkgs}"
+                "nixos-config=${self.outPath}/xeon01/configuration.nix"
+                "nixpkgs-overlays=${self.outPath}/overlays-compat"
+            ];
+            nix.registry.nixpkgs.flake = nixpkgs;
+            nix.registry.bscpkgs.flake = bscpkgs;
+            system.configurationRevision =
+              if self ? rev
+              then self.rev
+              else throw ("Refusing to build from a dirty Git tree!");
+          })
+          ./xeon01/configuration.nix
+        ];
       };
       xeon07 = nixpkgs.lib.nixosSystem {
         system = "x86_64-linux";
-        modules = [ ./xeon07/configuration.nix ];
+        modules = [
+          ( {options, ...}: {
+            # Sel the nixos-config path to the one of the current flake
+            nixpkgs.overlays = [ bscpkgs.bscOverlay ];
+            nix.nixPath = [
+                "nixpkgs=${nixpkgs}"
+                "bscpkgs=${bscpkgs}"
+                "nixos-config=${self.outPath}/xeon07/configuration.nix"
+                "nixpkgs-overlays=${self.outPath}/overlays-compat"
+            ];
+            nix.registry.nixpkgs.flake = nixpkgs;
+            nix.registry.bscpkgs.flake = bscpkgs;
+            system.configurationRevision =
+              if self ? rev
+              then self.rev
+              else throw ("Refusing to build from a dirty Git tree!");
+          })
+          agenix.nixosModules.default
+          ./xeon07/configuration.nix
+        ];
       };
     };
   };
diff --git a/rebuild.sh b/rebuild.sh
index 592ed38..5df9616 100755
--- a/rebuild.sh
+++ b/rebuild.sh
@@ -6,11 +6,14 @@ if [ "$(id -u)" != 0 ]; then
 fi
 
 host=$(hostname)
-conf="$(readlink -f .)/${host}/configuration.nix"
 
-if [ ! -e "$conf" ]; then
- echo "Missing config $conf"
- exit 1
-fi
+#conf="$(readlink -f .)/${host}/configuration.nix"
+#
+#if [ ! -e "$conf" ]; then
+# echo "Missing config $conf"
+# exit 1
+#fi
+#
+#NIXOS_CONFIG="${conf}" nixos-rebuild switch
 
-NIXOS_CONFIG="${conf}" nixos-rebuild switch
+nixos-rebuild switch --flake .
diff --git a/xeon01/slurm.nix b/xeon01/slurm.nix
index 815c94a..672bfc5 100644
--- a/xeon01/slurm.nix
+++ b/xeon01/slurm.nix
@@ -6,7 +6,7 @@
     controlMachine = "xeon07";
     clusterName = "owl";
     nodeName = [
-      "xeon[01-02,07] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=xeon"
+      "xeon[01-02,07]  Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=xeon"
     ];
   };
 }
diff --git a/xeon07/configuration.nix b/xeon07/configuration.nix
index df63508..4cb68cc 100644
--- a/xeon07/configuration.nix
+++ b/xeon07/configuration.nix
@@ -8,8 +8,6 @@
     ./monitoring.nix
     ./nfs.nix
     ./slurm.nix
-
-    <agenix/modules/age.nix>
   ];
 
   # Select the this using the ID to avoid mismatches
@@ -26,8 +24,4 @@
       prefixLength = 24;
     } ];
   };
-
-  environment.systemPackages = with pkgs; [
-    (pkgs.callPackage <agenix/pkgs/agenix.nix> {})
-  ];
 }
-- 
GitLab


From 141d77e2b61ebe912175f82bcf4ea505f143a6fb Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 27 Apr 2023 16:27:04 +0200
Subject: [PATCH 06/18] Refacto slurm configuration into compute/control

---
 common/main.nix              |  1 +
 {xeon01 => common}/slurm.nix |  0
 xeon01/configuration.nix     |  5 +----
 xeon07/configuration.nix     |  2 +-
 xeon07/slurm-daemon.nix      | 10 ++++++++++
 xeon07/slurm.nix             | 17 -----------------
 6 files changed, 13 insertions(+), 22 deletions(-)
 rename {xeon01 => common}/slurm.nix (100%)
 create mode 100644 xeon07/slurm-daemon.nix
 delete mode 100644 xeon07/slurm.nix

diff --git a/common/main.nix b/common/main.nix
index ad54622..8a76649 100644
--- a/common/main.nix
+++ b/common/main.nix
@@ -6,6 +6,7 @@
     ./fs.nix
     ./hw.nix
     ./net.nix
+    ./slurm.nix
     ./ssh.nix
     ./users.nix
   ];
diff --git a/xeon01/slurm.nix b/common/slurm.nix
similarity index 100%
rename from xeon01/slurm.nix
rename to common/slurm.nix
diff --git a/xeon01/configuration.nix b/xeon01/configuration.nix
index 9f651d8..1daecec 100644
--- a/xeon01/configuration.nix
+++ b/xeon01/configuration.nix
@@ -1,10 +1,7 @@
 { config, pkgs, ... }:
 
 {
-  imports = [
-    ../common/main.nix
-    ./slurm.nix
-  ];
+  imports = [ ../common/main.nix ];
 
   # Select the this using the ID to avoid mismatches
   boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53566c";
diff --git a/xeon07/configuration.nix b/xeon07/configuration.nix
index 4cb68cc..f3b6ab4 100644
--- a/xeon07/configuration.nix
+++ b/xeon07/configuration.nix
@@ -7,7 +7,7 @@
     ./gitlab-runner.nix
     ./monitoring.nix
     ./nfs.nix
-    ./slurm.nix
+    ./slurm-daemon.nix
   ];
 
   # Select the this using the ID to avoid mismatches
diff --git a/xeon07/slurm-daemon.nix b/xeon07/slurm-daemon.nix
new file mode 100644
index 0000000..a8dd3b8
--- /dev/null
+++ b/xeon07/slurm-daemon.nix
@@ -0,0 +1,10 @@
+{ ... }:
+
+{
+  services.slurm = {
+    server.enable = true;
+    partitionName = [
+      "xeon Nodes=xeon[01-02,07] Default=YES MaxTime=INFINITE State=UP"
+    ];
+  };
+}
diff --git a/xeon07/slurm.nix b/xeon07/slurm.nix
deleted file mode 100644
index 48a77d7..0000000
--- a/xeon07/slurm.nix
+++ /dev/null
@@ -1,17 +0,0 @@
-{ ... }:
-
-{
-  services.slurm = {
-    client.enable = true;
-    server.enable = true;
-    controlMachine = "xeon07";
-    clusterName = "owl";
-    nodeName = [
-      "xeon[01-02,07] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=xeon"
-    ];
-
-    partitionName = [
-      "xeon Nodes=xeon[01-02,07] Default=YES MaxTime=INFINITE State=UP"
-    ];
-  };
-}
-- 
GitLab


From 919f211536fe11c4c4c47a832ae14d174afb4387 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 27 Apr 2023 16:28:12 +0200
Subject: [PATCH 07/18] Add xeon02 configuration

---
 flake.nix                | 22 ++++++++++++++++++++++
 xeon02/configuration.nix | 20 ++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 xeon02/configuration.nix

diff --git a/flake.nix b/flake.nix
index 70cdb8b..a4c037d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -30,6 +30,28 @@
           ./xeon01/configuration.nix
         ];
       };
+      xeon02 = nixpkgs.lib.nixosSystem {
+        system = "x86_64-linux";
+        modules = [
+          ( {options, ...}: {
+            # Sel the nixos-config path to the one of the current flake
+            nixpkgs.overlays = [ bscpkgs.bscOverlay ];
+            nix.nixPath = [
+                "nixpkgs=${nixpkgs}"
+                "bscpkgs=${bscpkgs}"
+                "nixos-config=${self.outPath}/xeon02/configuration.nix"
+                "nixpkgs-overlays=${self.outPath}/overlays-compat"
+            ];
+            nix.registry.nixpkgs.flake = nixpkgs;
+            nix.registry.bscpkgs.flake = bscpkgs;
+            system.configurationRevision =
+              if self ? rev
+              then self.rev
+              else throw ("Refusing to build from a dirty Git tree!");
+          })
+          ./xeon02/configuration.nix
+        ];
+      };
       xeon07 = nixpkgs.lib.nixosSystem {
         system = "x86_64-linux";
         modules = [
diff --git a/xeon02/configuration.nix b/xeon02/configuration.nix
new file mode 100644
index 0000000..5d097dd
--- /dev/null
+++ b/xeon02/configuration.nix
@@ -0,0 +1,20 @@
+{ config, pkgs, ... }:
+
+{
+  imports = [ ../common/main.nix ];
+
+  # Select the this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d535629";
+
+  networking = {
+    hostName = "xeon02";
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "10.0.40.2";
+      prefixLength = 24;
+    } ];
+    interfaces.ibp5s0.ipv4.addresses = [ {
+      address = "10.0.42.2";
+      prefixLength = 24;
+    } ];
+  };
+}
-- 
GitLab


From c3ea8864bbefd1cad39954d250e4de81ef6d1742 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 27 Apr 2023 16:36:15 +0200
Subject: [PATCH 08/18] Add minimal netboot module to build kexec image

---
 common/boot.nix          | 2 +-
 xeon02/configuration.nix | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/common/boot.nix b/common/boot.nix
index cf85951..1ff695f 100644
--- a/common/boot.nix
+++ b/common/boot.nix
@@ -2,7 +2,7 @@
 
 {
   # Use the GRUB 2 boot loader.
-  boot.loader.grub.enable = true;
+  boot.loader.grub.enable = lib.mkForce true;
   boot.loader.grub.version = 2;
 
   # Enable GRUB2 serial console
diff --git a/xeon02/configuration.nix b/xeon02/configuration.nix
index 5d097dd..83d4114 100644
--- a/xeon02/configuration.nix
+++ b/xeon02/configuration.nix
@@ -1,10 +1,12 @@
-{ config, pkgs, ... }:
+{ config, pkgs, modulesPath, lib, ... }:
 
 {
-  imports = [ ../common/main.nix ];
+  imports = [ ../common/main.nix (modulesPath + "/installer/netboot/netboot-minimal.nix") ];
 
   # Select the this using the ID to avoid mismatches
   boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d535629";
+  #programs.ssh.forwardX11 = false;
+  programs.ssh.setXAuthLocation = lib.mkForce true;
 
   networking = {
     hostName = "xeon02";
-- 
GitLab


From 613a76ac294a6e12df3b36beec19cf1c5760effe Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 27 Apr 2023 16:36:48 +0200
Subject: [PATCH 09/18] Add steps in install documentation

---
 doc/install.md           | 62 ++++++++++++++++++++++++++++++++++++++++
 xeon02/configuration.nix |  7 +++--
 2 files changed, 67 insertions(+), 2 deletions(-)
 create mode 100644 doc/install.md

diff --git a/doc/install.md b/doc/install.md
new file mode 100644
index 0000000..05a6022
--- /dev/null
+++ b/doc/install.md
@@ -0,0 +1,62 @@
+# Installing NixOS in a new node
+
+This article shows the steps to install NixOS in a node following the
+configuration of the repo.
+
+## Prepare the disk
+
+Create a main partition and label it `nixos` following [the manual][1].
+
+[1]: https://nixos.org/manual/nixos/stable/index.html#sec-installation-manual-partitioning.
+
+```
+# disk=/dev/sdX
+# parted $disk -- mklabel msdos
+# parted $disk -- mkpart primary 1MB 100%
+# parted $disk -- set 1 boot on
+```
+
+Then create an etx4 filesystem, labeled `nixos` where the system will be
+installed. **Ensure that no other partition has the same label.**
+
+```
+# mkfs.ext4 -L nixos "${disk}1"
+# mount ${disk}1 /mnt
+# lsblk -f $disk
+NAME   FSTYPE LABEL UUID                                 MOUNTPOINT
+sdX
+`-sdX1 ext4   nixos 10d73b75-809c-4fa3-b99d-4fab2f0d0d8e /mnt
+```
+
+## Prepare nix and nixos-install
+
+Mount the nix store from the xeon07 node in read-only /nix.
+
+```
+# mkdir /nix
+# mount -o ro xeon07:/nix /nix
+```
+
+Get the nix binary and nixos-install tool from xeon07:
+
+```
+# ssh xeon07 'readlink -f $(which nix)'
+/nix/store/0sxbaj71c4c4n43qhdxm31f56gjalksw-nix-2.13.3/bin/nix
+# ssh xeon07 'readlink -f $(which nixos-install)'
+/nix/store/9yq8ps06ysr2pfiwiij39ny56yk3pdcs-nixos-install/bin/nixos-install
+```
+
+And add them to the PATH:
+
+```
+# export PATH=$PATH:/nix/store/0sxbaj71c4c4n43qhdxm31f56gjalksw-nix-2.13.3/bin
+# export PATH=$PATH:/nix/store/9yq8ps06ysr2pfiwiij39ny56yk3pdcs-nixos-install/bin/
+# nix --version
+nix (Nix) 2.13.3
+```
+
+## Build the nixos kexec image
+
+```
+# nix build .#nixosConfigurations.xeon02.config.system.build.kexecTree -v
+```
diff --git a/xeon02/configuration.nix b/xeon02/configuration.nix
index 83d4114..06d88ff 100644
--- a/xeon02/configuration.nix
+++ b/xeon02/configuration.nix
@@ -1,12 +1,15 @@
 { config, pkgs, modulesPath, lib, ... }:
 
 {
-  imports = [ ../common/main.nix (modulesPath + "/installer/netboot/netboot-minimal.nix") ];
+  imports = [
+    #(modulesPath + "/installer/netboot/netboot-minimal.nix")
+    ../common/main.nix
+  ];
 
   # Select the this using the ID to avoid mismatches
   boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d535629";
   #programs.ssh.forwardX11 = false;
-  programs.ssh.setXAuthLocation = lib.mkForce true;
+  #programs.ssh.setXAuthLocation = lib.mkForce true;
 
   networking = {
     hostName = "xeon02";
-- 
GitLab


From 74639d3ecec8f9de225e8b17972bcb742b6abbf6 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 27 Apr 2023 18:29:32 +0200
Subject: [PATCH 10/18] Update ib interface name in xeon02

It seems to be plugged in another PCI port
---
 xeon02/configuration.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xeon02/configuration.nix b/xeon02/configuration.nix
index 06d88ff..1b06afd 100644
--- a/xeon02/configuration.nix
+++ b/xeon02/configuration.nix
@@ -17,7 +17,7 @@
       address = "10.0.40.2";
       prefixLength = 24;
     } ];
-    interfaces.ibp5s0.ipv4.addresses = [ {
+    interfaces.ibp129s0.ipv4.addresses = [ {
       address = "10.0.42.2";
       prefixLength = 24;
     } ];
-- 
GitLab


From 399103a9b4edfd8ac6488c4c0aaa7d58ee543c48 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Fri, 28 Apr 2023 11:13:46 +0200
Subject: [PATCH 11/18] Update nixpkgs

---
 flake.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flake.lock b/flake.lock
index c8d3383..3a78c3b 100644
--- a/flake.lock
+++ b/flake.lock
@@ -61,10 +61,10 @@
     "nixpkgs": {
       "locked": {
         "lastModified": 1669833724,
-        "narHash": "sha256-mlqo1r+TZUOuypWdrZHluxWL+E5WzXlUXNZ9Y0WLDFU=",
+        "narHash": "sha256-/HEZNyGbnQecrgJnfE8d0WC5c1xuPSD2LUpB6YXlg4c=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9a6aabc4740790ef3bbb246b86d029ccf6759658",
+        "rev": "4d2b37a84fad1091b9de401eb450aae66f1a741e",
         "type": "github"
       },
       "original": {
-- 
GitLab


From aa1dd14b624d8badb3e3d6c6746d81965cef299a Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Fri, 28 Apr 2023 11:18:37 +0200
Subject: [PATCH 12/18] Update nixpkgs to nixos-unstable

---
 flake.lock | 8 ++++----
 flake.nix  | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/flake.lock b/flake.lock
index 3a78c3b..a8e8e79 100644
--- a/flake.lock
+++ b/flake.lock
@@ -60,16 +60,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1669833724,
-        "narHash": "sha256-/HEZNyGbnQecrgJnfE8d0WC5c1xuPSD2LUpB6YXlg4c=",
+        "lastModified": 1682526928,
+        "narHash": "sha256-2cKh4O6t1rQ8Ok+v16URynmb0rV7oZPEbXkU0owNLQs=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "4d2b37a84fad1091b9de401eb450aae66f1a741e",
+        "rev": "d6b863fd9b7bb962e6f9fdf292419a775e772891",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "22.11",
+        "ref": "nixos-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index a4c037d..7c4b67d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,6 +1,6 @@
 {
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/22.11";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
     agenix.url = "github:ryantm/agenix";
     agenix.inputs.nixpkgs.follows = "nixpkgs";
     bscpkgs.url = "git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git";
-- 
GitLab


From c1bbbd779393a195b1bbca9c86cce07757ec68a9 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Fri, 28 Apr 2023 11:19:47 +0200
Subject: [PATCH 13/18] Disable osnoise and hwlat tracer for now

Reuse nix cache to avoid rebuilding the kernel.
---
 common/boot.nix | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/common/boot.nix b/common/boot.nix
index 1ff695f..a893e44 100644
--- a/common/boot.nix
+++ b/common/boot.nix
@@ -18,14 +18,14 @@
     "console=ttyS0,115200"
   ];
 
-  boot.kernelPatches = lib.singleton {
-    name = "osnoise-tracer";
-    patch = null;
-    extraStructuredConfig = with lib.kernel; {
-      OSNOISE_TRACER = yes;
-      HWLAT_TRACER = yes;
-    };
-  };
+  #boot.kernelPatches = lib.singleton {
+  #  name = "osnoise-tracer";
+  #  patch = null;
+  #  extraStructuredConfig = with lib.kernel; {
+  #    OSNOISE_TRACER = yes;
+  #    HWLAT_TRACER = yes;
+  #  };
+  #};
 
   boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "ehci_pci" "nvme" "usbhid" "sd_mod" ];
   boot.initrd.kernelModules = [ ];
-- 
GitLab


From 2815f5bcfde32813fa324be885dbe38a6cca31d1 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Fri, 28 Apr 2023 11:50:43 +0200
Subject: [PATCH 14/18] Use the latest kernel

---
 common/boot.nix | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/common/boot.nix b/common/boot.nix
index a893e44..ca18e0e 100644
--- a/common/boot.nix
+++ b/common/boot.nix
@@ -1,4 +1,4 @@
-{ lib, ... }:
+{ lib, pkgs, ... }:
 
 {
   # Use the GRUB 2 boot loader.
@@ -18,6 +18,8 @@
     "console=ttyS0,115200"
   ];
 
+  boot.kernelPackages = pkgs.linuxPackages_latest;
+
   #boot.kernelPatches = lib.singleton {
   #  name = "osnoise-tracer";
   #  patch = null;
-- 
GitLab


From 818edccb340a9d78e5054f7aaec3a2642aedac90 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Fri, 28 Apr 2023 12:34:51 +0200
Subject: [PATCH 15/18] Increase locked memory to 1 GiB

---
 common/main.nix | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/common/main.nix b/common/main.nix
index 8a76649..b0147f9 100644
--- a/common/main.nix
+++ b/common/main.nix
@@ -22,6 +22,16 @@
     serviceConfig.Restart = "always";
   };
 
+  # Increase limits
+  security.pam.loginLimits = [
+    {
+      domain = "*";
+      type = "-";
+      item = "memlock";
+      value = "1048576"; # 1 GiB of mem locked
+    }
+  ];
+
   time.timeZone = "Europe/Madrid";
   i18n.defaultLocale = "en_DK.UTF-8";
 
-- 
GitLab


From 0d5609ecc267eae62a7368632f5ea94c7c8994a5 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Fri, 28 Apr 2023 17:07:48 +0200
Subject: [PATCH 16/18] Use pmix by default in slurm

---
 common/slurm.nix | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/common/slurm.nix b/common/slurm.nix
index 672bfc5..c1b09d6 100644
--- a/common/slurm.nix
+++ b/common/slurm.nix
@@ -8,5 +8,8 @@
     nodeName = [
       "xeon[01-02,07]  Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=xeon"
     ];
+    extraConfig = ''
+      MpiDefault=pmix
+    '';
   };
 }
-- 
GitLab


From ca3a7d98f5a41c1ff49ec5257e2d17fe8d15e61a Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Fri, 28 Apr 2023 17:59:19 +0200
Subject: [PATCH 17/18] Roolback to bash as default shell

Zsh doesn't behave properly, it needs further configuration.
---
 common/main.nix | 1 -
 1 file changed, 1 deletion(-)

diff --git a/common/main.nix b/common/main.nix
index b0147f9..0b76d22 100644
--- a/common/main.nix
+++ b/common/main.nix
@@ -48,7 +48,6 @@
 
   programs.zsh.enable = true;
   programs.zsh.histSize = 100000;
-  users.defaultUserShell = pkgs.zsh;
 
   # Copy the NixOS configuration file and link it from the resulting system
   # (/run/current-system/configuration.nix). This is useful in case you
-- 
GitLab


From f47734b524858c6acfa2627b46f94af013762cd4 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Fri, 28 Apr 2023 18:12:10 +0200
Subject: [PATCH 18/18] Simplify bash prompt

---
 common/main.nix | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/common/main.nix b/common/main.nix
index 0b76d22..c7fc6ce 100644
--- a/common/main.nix
+++ b/common/main.nix
@@ -49,6 +49,10 @@
   programs.zsh.enable = true;
   programs.zsh.histSize = 100000;
 
+  programs.bash.promptInit = ''
+    PS1="\h\\$ "
+  '';
+
   # Copy the NixOS configuration file and link it from the resulting system
   # (/run/current-system/configuration.nix). This is useful in case you
   # accidentally delete configuration.nix.
-- 
GitLab