From a7e66e2246a8bb8490b356ed7d1770a9cfd9647a Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 28 Jul 2023 19:49:48 +0200 Subject: [PATCH 1/9] Add bay node --- flake.nix | 1 + m/bay/configuration.nix | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 m/bay/configuration.nix diff --git a/flake.nix b/flake.nix index 5f67afc..6ce0689 100644 --- a/flake.nix +++ b/flake.nix @@ -21,6 +21,7 @@ in owl2 = mkConf "owl2"; eudy = mkConf "eudy"; koro = mkConf "koro"; + bay = mkConf "bay"; }; packages.x86_64-linux.hut = self.nixosConfigurations.hut.pkgs; diff --git a/m/bay/configuration.nix b/m/bay/configuration.nix new file mode 100644 index 0000000..04965fd --- /dev/null +++ b/m/bay/configuration.nix @@ -0,0 +1,26 @@ +{ config, pkgs, modulesPath, ... }: + +{ + imports = [ + (modulesPath + "/installer/netboot/netboot-minimal.nix") + ../common/main.nix + ]; + + services.openssh.settings.X11Forwarding = false; + nixpkgs.config.allowBroken = true; + + # Select the this using the ID to avoid mismatches + boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53562d"; + + networking = { + hostName = "bay"; + interfaces.eno1.ipv4.addresses = [ { + address = "10.0.40.40"; + prefixLength = 24; + } ]; + interfaces.ibp5s0.ipv4.addresses = [ { + address = "10.0.42.40"; + prefixLength = 24; + } ]; + }; +} -- GitLab From b1d5185cca1decea79e102624c286d8d8d315115 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 28 Jul 2023 20:31:44 +0200 Subject: [PATCH 2/9] Remove netboot and fixes --- m/bay/configuration.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/m/bay/configuration.nix b/m/bay/configuration.nix index 04965fd..b42abf1 100644 --- a/m/bay/configuration.nix +++ b/m/bay/configuration.nix @@ -1,14 +1,10 @@ -{ config, pkgs, modulesPath, ... }: +{ config, pkgs, ... }: { imports = [ - (modulesPath + "/installer/netboot/netboot-minimal.nix") ../common/main.nix ]; - services.openssh.settings.X11Forwarding = false; - nixpkgs.config.allowBroken = true; - # Select the this using the ID to avoid mismatches boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53562d"; -- GitLab From b677b827d4715686264abd126f8ca8e2906cb6e7 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 22 Aug 2023 15:56:09 +0200 Subject: [PATCH 3/9] Add the bay host name --- m/common/net.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/m/common/net.nix b/m/common/net.nix index 9d6a28a..354fa5d 100644 --- a/m/common/net.nix +++ b/m/common/net.nix @@ -31,7 +31,7 @@ 84.88.53.236 ssfhead.bsc.es ssfhead # Node Entry for node: mds01 (ID=72) - 10.0.40.40 mds01 mds01-eth0 + 10.0.40.40 bay mds01 mds01-eth0 10.0.42.40 mds01-ib0 10.0.40.141 mds01-ipmi0 -- GitLab From cd5853cf53714bc6c1a99591d4f13ce59ad2d8bf Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 22 Aug 2023 15:57:25 +0200 Subject: [PATCH 4/9] Add ceph config in bay --- m/bay/configuration.nix | 68 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/m/bay/configuration.nix b/m/bay/configuration.nix index b42abf1..847e162 100644 --- a/m/bay/configuration.nix +++ b/m/bay/configuration.nix @@ -1,4 +1,4 @@ -{ config, pkgs, ... }: +{ config, pkgs, lib, ... }: { imports = [ @@ -8,6 +8,14 @@ # Select the this using the ID to avoid mismatches boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53562d"; + environment.systemPackages = with pkgs; [ + ceph + ]; + + services.slurm = { + client.enable = lib.mkForce false; + }; + networking = { hostName = "bay"; interfaces.eno1.ipv4.addresses = [ { @@ -19,4 +27,62 @@ prefixLength = 24; } ]; }; + + services.ceph = { + enable = true; + global = { + fsid = "9c8d06e0-485f-4aaf-b16b-06d6daf1232b"; + monHost = "10.0.40.40"; + monInitialMembers = "bay"; + clusterNetwork = "10.0.40.40/24"; # Use Ethernet only + }; + mds = { + enable = true; + daemons = [ "mds0" "mds1" ]; + extraConfig = { + "host" = "bay"; + }; + }; + mgr = { + enable = true; + daemons = [ "bay" ]; + }; + mon = { + enable = true; + daemons = [ "bay" ]; + }; + osd = { + enable = true; + # One daemon per NVME disk + daemons = [ "0" "1" "2" "3" ]; + extraConfig = { + "osd crush chooseleaf type" = "0"; + "osd journal size" = "10000"; + "osd pool default min size" = "2"; + "osd pool default pg num" = "200"; + "osd pool default pgp num" = "200"; + "osd pool default size" = "3"; + }; + }; + }; + + # Missing service for volumes, see: + # https://www.reddit.com/r/ceph/comments/14otjyo/comment/jrd69vt/ + systemd.services.ceph-volume = { + enable = true; + description = "Ceph Volume activation"; + unitConfig = { + Type = "oneshot"; + After = "local-fs.target"; + Wants = "local-fs.target"; + }; + path = [ pkgs.ceph pkgs.util-linux pkgs.lvm2 pkgs.cryptsetup ]; + serviceConfig = { + KillMode = "none"; + Environment = "CEPH_VOLUME_TIMEOUT=10000"; + ExecStart = "/bin/sh -c 'timeout $CEPH_VOLUME_TIMEOUT ${pkgs.ceph}/bin/ceph-volume lvm activate --all --no-systemd'"; + TimeoutSec = "0"; + }; + wantedBy = [ "multi-user.target" ]; + }; } -- GitLab From 8257c245b10581d698e43f8dd7d66b2a1d666238 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 22 Aug 2023 15:57:49 +0200 Subject: [PATCH 5/9] Mount the ceph filesystem in hut --- m/hut/ceph.nix | 14 ++++++++++++++ m/hut/configuration.nix | 1 + 2 files changed, 15 insertions(+) create mode 100644 m/hut/ceph.nix diff --git a/m/hut/ceph.nix b/m/hut/ceph.nix new file mode 100644 index 0000000..fab5493 --- /dev/null +++ b/m/hut/ceph.nix @@ -0,0 +1,14 @@ +{ pkgs, ... }: + +{ + environment.systemPackages = [ pkgs.ceph-client ]; + + # We need the ceph module loaded as the mount.ceph binary fails to run the + # modprobe command. + boot.kernelModules = [ "ceph" ]; + + fileSystems."/ceph" = { + fsType = "ceph"; + device = "animal@9c8d06e0-485f-4aaf-b16b-06d6daf1232b.cephfs=/"; + }; +} diff --git a/m/hut/configuration.nix b/m/hut/configuration.nix index 82d4c34..8e677d0 100644 --- a/m/hut/configuration.nix +++ b/m/hut/configuration.nix @@ -8,6 +8,7 @@ ./monitoring.nix ./nfs.nix ./slurm-daemon.nix + ./ceph.nix agenix.nixosModules.default ]; -- GitLab From dfffc0bdce57ae792d43417aaa7b8cd14d6e1f18 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 22 Aug 2023 16:33:55 +0200 Subject: [PATCH 6/9] Add ceph metrics to prometheus --- m/common/net.nix | 2 +- m/hut/monitoring.nix | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/m/common/net.nix b/m/common/net.nix index 354fa5d..de2b28d 100644 --- a/m/common/net.nix +++ b/m/common/net.nix @@ -12,7 +12,7 @@ nameservers = ["8.8.8.8"]; proxy = { default = "http://localhost:23080/"; - noProxy = "127.0.0.1,localhost,internal.domain"; + noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40"; # Don't set all_proxy as go complains and breaks the gitlab runner, see: # https://github.com/golang/go/issues/16715 allProxy = null; diff --git a/m/hut/monitoring.nix b/m/hut/monitoring.nix index a00fb9b..4bc3d24 100644 --- a/m/hut/monitoring.nix +++ b/m/hut/monitoring.nix @@ -67,6 +67,7 @@ "127.0.0.1:9323" "127.0.0.1:9252" "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}" + "10.0.40.40:9283" # Ceph ]; }]; } -- GitLab From e0852ee89b3685f2a31c1b14d4e037caf4abd505 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 23 Aug 2023 17:15:26 +0200 Subject: [PATCH 7/9] Add rarias key for secrets --- m/hut/secrets.nix | 11 ++++++----- m/hut/secrets/nosv-token.age | Bin 501 -> 541 bytes m/hut/secrets/ovni-token.age | Bin 538 -> 610 bytes 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/m/hut/secrets.nix b/m/hut/secrets.nix index 7c5aea7..2c4d823 100644 --- a/m/hut/secrets.nix +++ b/m/hut/secrets.nix @@ -1,9 +1,10 @@ let - root = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb"; - system = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1"; - systems = [ root system ]; + rarias = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1oZTPtlEXdGt0Ak+upeCIiBdaDQtcmuWoTUCVuSVIR rarias@hut"; + root = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb"; + hut = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1"; + default = [ rarias root hut ]; in { - "secrets/ovni-token.age".publicKeys = systems; - "secrets/nosv-token.age".publicKeys = systems; + "secrets/ovni-token.age".publicKeys = default; + "secrets/nosv-token.age".publicKeys = default; } diff --git a/m/hut/secrets/nosv-token.age b/m/hut/secrets/nosv-token.age index b26d482ab5e0522be3058a61ea400bb213e6c327..4b495a944de69fccbfdde7394b7f63f80e53e06e 100644 GIT binary patch delta 514 zcmey$JeOsHYQ3{#xVuT9f}5ead10DsdVyzTvU`xXUr4Y~L8`t{ZjfcLnL&1TXq9(p zVt}`Waan{TSF(RtRd#8%bEu_-YeZ?1t9MzBzkZsJXIgPlMoF=^Q*ohXfV;ViX{NR> zm#&>cadC!jYKoDmsiCEUZ?KzjO1eUZe^{=$p+R_RaecCRdQ?DAv5T*1iDQ~}fpK|y zg>$Z1aAaw@VSZ6iYL*38NpYTUWPU)BvtfF9UU+syk$I`JxnX%xN@|6FgkfcBSw*mq zkBhmnr%yh*bsmvMl|ktWp*iNpnMv+mg`wu9MS(t1Mfw(%K8D7Dd3k~LX1TeRA(fVu z6_pkFp@Bx8T$zSdkr}y>&gQ21RqmChh2bU{B|*uBi5W@G$?lbgreUc8J|;#%5#gn- zAnQ`Kob-Qh%CmFoTYYEC*C)ChdEYTBdQwYo)tjPAyJbrj_{;jG#&_hKYAIXGXC7s~ydvl2 s#;@M2JJzn-Xg=w<+8^O>4KF`l)!Fi}4UuNZKc_@AZ0ImnTuK)l5 delta 454 zcmV;%0XhDi1oZ=uC0(Olm}RQZi&&SaxSZab;_3Yj9*w zVNp|2R%KdBc`|rw3N>s{M^8~mGHy#VH)c(2VrEoHGGS^#QAm1Ia%VwnQCV(6Sb9ok zPfloAk?|LQa4$7Xa(6c|RZ(noaauz|c3Nt3bV5`!WlC2yL2gSmbT(KvLvBtpHfU7} zWH5SbOgU0kIap3iR90g_WKT&&MpR{JOH@*HQ8a9NNib6@Qgw1KZ#H8JEj}P+PGe+K zM>8#Fa%Ew2WgsPWX(Cy9F+>Vlcxq!>Omk;2FE>ShXGvu%LS`{obW?9~Ggx6_T54`( zHE&8eH)uIwMrwFkcSq)gR`ossI20 diff --git a/m/hut/secrets/ovni-token.age b/m/hut/secrets/ovni-token.age index 8241b6ddcd826a5515d2446465ffd67fc36e4453..bb850ef38e40d426b50aba25ae6993fe521b4d74 100644 GIT binary patch literal 610 zcmZ9|J&%)M003Z(#-VYz!JG~k!(be);aX^+&_ol;M@vgR`j&Dn-d%)3;cbByD6}-W zIC)EVKcdq`CkJnGa4Cd{`c=McTFR8xQaNdE38CG*;;8UMzkhPmY%F-ftd?`^@9yn?YRC uXU6e`&z|t~*Q0K?-dSzA>lNkV>yzcdi#PUzt3QpeFK?_{*OxH8vilFpr_yf# delta 492 zcmV1eyepC0cErGDcZf zFg9p!PGnIlW;ii83QkLIby0FuQ9@X1R6%lBM>a1tOI9^wWM?lqN>ytwY%_LNWLa@A zW>`-}k?|LQF}2aZ55(LvMLQSz2UuXEs@7N^4CD zLPcd%XhdsjW?@S?Q#D~YF-T*1VOn-MdT(M&K~7dpMQLqhbz?6{Pc<|OEj}P7WkMx= zc`avhWnpt=Aa6S^SXm%hX>xiXeOPE*L`MokYfMsqLu_$ZQfotUH*Pp@HC0wQD{Ev! zX=^fRR7WplNilOaHa0^`V`q3`K~QgKZE|@^dQVGaXHragdP8O~3PV#+O;u)cX?SgQ zQFL`vXlZA3ZCN%@Z8AX$EiEk|M@4l`O>S3OGb=(iQBGKPH*+^)W=dpAGGa(g3Z}9%hr=$WE#uR!(iq0SM5BO`dtABf)FrCiQXDX(-?y;-wz1~0 zeAZCEMw{_?o)~im9*bX4u&;PvG{h`{dNVcrOqk07kz{gey)vu;GNA6KLl& ienC*p!i(f(GiFraMZ;_=`8awm`}fs&GMyL$Fse7heYeK| -- GitLab From b3ef53de515183cd40c779f1bed67f3f7eb2e089 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 23 Aug 2023 17:18:17 +0200 Subject: [PATCH 8/9] Store ceph secret key in age This allows a node to mount the ceph FS without any extra ceph configuration in /etc/ceph. --- m/hut/ceph.nix | 10 ++++++++-- m/hut/secrets.nix | 1 + m/hut/secrets/ceph-user.age | 11 +++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 m/hut/secrets/ceph-user.age diff --git a/m/hut/ceph.nix b/m/hut/ceph.nix index fab5493..722466f 100644 --- a/m/hut/ceph.nix +++ b/m/hut/ceph.nix @@ -1,4 +1,4 @@ -{ pkgs, ... }: +{ config, pkgs, ... }: { environment.systemPackages = [ pkgs.ceph-client ]; @@ -7,8 +7,14 @@ # modprobe command. boot.kernelModules = [ "ceph" ]; + age.secrets."secrets/ceph-user".file = ./secrets/ceph-user.age; + fileSystems."/ceph" = { fsType = "ceph"; - device = "animal@9c8d06e0-485f-4aaf-b16b-06d6daf1232b.cephfs=/"; + device = "user@9c8d06e0-485f-4aaf-b16b-06d6daf1232b.cephfs=/"; + options = [ + "mon_addr=10.0.40.40" + "secretfile=${config.age.secrets."secrets/ceph-user".path}" + ]; }; } diff --git a/m/hut/secrets.nix b/m/hut/secrets.nix index 2c4d823..d22add6 100644 --- a/m/hut/secrets.nix +++ b/m/hut/secrets.nix @@ -7,4 +7,5 @@ in { "secrets/ovni-token.age".publicKeys = default; "secrets/nosv-token.age".publicKeys = default; + "secrets/ceph-user.age".publicKeys = default; } diff --git a/m/hut/secrets/ceph-user.age b/m/hut/secrets/ceph-user.age new file mode 100644 index 0000000..735afca --- /dev/null +++ b/m/hut/secrets/ceph-user.age @@ -0,0 +1,11 @@ +age-encryption.org/v1 +-> ssh-ed25519 CAWG4Q 35Ak+Mep9k5KnDLF1ywDbMD4l4mRFg6D0et19tqXxAw +Wgr+CX4rzrPmUszSidtLAVSvgD80F2dqtd92hGZIFwo +-> ssh-ed25519 MSF3dg OVFvpkAyWTowtxsafstX31H/hJpNZmnOCbvqMIN0+AQ +VxjRcQmp+BadEh2y0PB96EeizIl3tTQpVu0CWHmsc1s +-> ssh-ed25519 HY2yRg MJSQIpre9m0XnojgXuKQ/+hVBZNrZNGZqplwhqicpjI +CLkE52iqpoqSnbzisNjQgxTfNqKeaRl5ntcw1d+ZDyQ +-> m$8`De%~-grease '85p}`by +52zMpprONcawWDDtzHdWNwFoYXErPUnVjhSONbUBpDlqAmJmD1LcAnsU +--- 0vZOPyXQIMMGTwgFfvm8Sn8O7vjrsjGUEy5m/BASCyc +|)*_DUS`r sN[֌^e+A 1G.#mW 5 ( \ No newline at end of file -- GitLab From 077eece6b9e7362007c4b12c1131e4b2a79857c7 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 23 Aug 2023 17:42:50 +0200 Subject: [PATCH 9/9] Add agenix to PATH in hut --- m/hut/configuration.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/m/hut/configuration.nix b/m/hut/configuration.nix index 8e677d0..14c7881 100644 --- a/m/hut/configuration.nix +++ b/m/hut/configuration.nix @@ -12,6 +12,10 @@ agenix.nixosModules.default ]; + environment.systemPackages = [ + agenix.packages.x86_64-linux.default + ]; + # Select the this using the ID to avoid mismatches boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN"; -- GitLab