diff --git a/flake.nix b/flake.nix index 5f67afc936daea75b2c25d823e16d204473df462..6ce0689bf78390d1e9c8ddc2fdc57369078dc3ed 100644 --- a/flake.nix +++ b/flake.nix @@ -21,6 +21,7 @@ in owl2 = mkConf "owl2"; eudy = mkConf "eudy"; koro = mkConf "koro"; + bay = mkConf "bay"; }; packages.x86_64-linux.hut = self.nixosConfigurations.hut.pkgs; diff --git a/m/bay/configuration.nix b/m/bay/configuration.nix new file mode 100644 index 0000000000000000000000000000000000000000..847e1621f17872a0dd0d375dccf3b943ccfa17ba --- /dev/null +++ b/m/bay/configuration.nix @@ -0,0 +1,88 @@ +{ config, pkgs, lib, ... }: + +{ + imports = [ + ../common/main.nix + ]; + + # Select the this using the ID to avoid mismatches + boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53562d"; + + environment.systemPackages = with pkgs; [ + ceph + ]; + + services.slurm = { + client.enable = lib.mkForce false; + }; + + networking = { + hostName = "bay"; + interfaces.eno1.ipv4.addresses = [ { + address = "10.0.40.40"; + prefixLength = 24; + } ]; + interfaces.ibp5s0.ipv4.addresses = [ { + address = "10.0.42.40"; + prefixLength = 24; + } ]; + }; + + services.ceph = { + enable = true; + global = { + fsid = "9c8d06e0-485f-4aaf-b16b-06d6daf1232b"; + monHost = "10.0.40.40"; + monInitialMembers = "bay"; + clusterNetwork = "10.0.40.40/24"; # Use Ethernet only + }; + mds = { + enable = true; + daemons = [ "mds0" "mds1" ]; + extraConfig = { + "host" = "bay"; + }; + }; + mgr = { + enable = true; + daemons = [ "bay" ]; + }; + mon = { + enable = true; + daemons = [ "bay" ]; + }; + osd = { + enable = true; + # One daemon per NVME disk + daemons = [ "0" "1" "2" "3" ]; + extraConfig = { + "osd crush chooseleaf type" = "0"; + "osd journal size" = "10000"; + "osd pool default min size" = "2"; + "osd pool default pg num" = "200"; + "osd pool default pgp num" = "200"; + "osd pool default size" = "3"; + }; + }; + }; + + # Missing service for volumes, see: + # https://www.reddit.com/r/ceph/comments/14otjyo/comment/jrd69vt/ + systemd.services.ceph-volume = { + enable = true; + description = "Ceph Volume activation"; + unitConfig = { + Type = "oneshot"; + After = "local-fs.target"; + Wants = "local-fs.target"; + }; + path = [ pkgs.ceph pkgs.util-linux pkgs.lvm2 pkgs.cryptsetup ]; + serviceConfig = { + KillMode = "none"; + Environment = "CEPH_VOLUME_TIMEOUT=10000"; + ExecStart = "/bin/sh -c 'timeout $CEPH_VOLUME_TIMEOUT ${pkgs.ceph}/bin/ceph-volume lvm activate --all --no-systemd'"; + TimeoutSec = "0"; + }; + wantedBy = [ "multi-user.target" ]; + }; +} diff --git a/m/common/net.nix b/m/common/net.nix index 9d6a28a26296a05748b9ebcc4ac96859820ece66..de2b28dce2ea35f47fe130f47b09cab2755eb90a 100644 --- a/m/common/net.nix +++ b/m/common/net.nix @@ -12,7 +12,7 @@ nameservers = ["8.8.8.8"]; proxy = { default = "http://localhost:23080/"; - noProxy = "127.0.0.1,localhost,internal.domain"; + noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40"; # Don't set all_proxy as go complains and breaks the gitlab runner, see: # https://github.com/golang/go/issues/16715 allProxy = null; @@ -31,7 +31,7 @@ 84.88.53.236 ssfhead.bsc.es ssfhead # Node Entry for node: mds01 (ID=72) - 10.0.40.40 mds01 mds01-eth0 + 10.0.40.40 bay mds01 mds01-eth0 10.0.42.40 mds01-ib0 10.0.40.141 mds01-ipmi0 diff --git a/m/hut/ceph.nix b/m/hut/ceph.nix new file mode 100644 index 0000000000000000000000000000000000000000..722466f4e0ce2bbe98009febbad5d2f66aac7d99 --- /dev/null +++ b/m/hut/ceph.nix @@ -0,0 +1,20 @@ +{ config, pkgs, ... }: + +{ + environment.systemPackages = [ pkgs.ceph-client ]; + + # We need the ceph module loaded as the mount.ceph binary fails to run the + # modprobe command. + boot.kernelModules = [ "ceph" ]; + + age.secrets."secrets/ceph-user".file = ./secrets/ceph-user.age; + + fileSystems."/ceph" = { + fsType = "ceph"; + device = "user@9c8d06e0-485f-4aaf-b16b-06d6daf1232b.cephfs=/"; + options = [ + "mon_addr=10.0.40.40" + "secretfile=${config.age.secrets."secrets/ceph-user".path}" + ]; + }; +} diff --git a/m/hut/configuration.nix b/m/hut/configuration.nix index 82d4c346959da5714dc2cd4df1c1ffdea62214c0..14c7881050ba0ae48e873bed044f9f5c680cfa90 100644 --- a/m/hut/configuration.nix +++ b/m/hut/configuration.nix @@ -8,9 +8,14 @@ ./monitoring.nix ./nfs.nix ./slurm-daemon.nix + ./ceph.nix agenix.nixosModules.default ]; + environment.systemPackages = [ + agenix.packages.x86_64-linux.default + ]; + # Select the this using the ID to avoid mismatches boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN"; diff --git a/m/hut/monitoring.nix b/m/hut/monitoring.nix index a00fb9b12264f9115ac210e124e50824552d1202..4bc3d246ab70bacb4407c2fdd0c46fdbe04ff5d7 100644 --- a/m/hut/monitoring.nix +++ b/m/hut/monitoring.nix @@ -67,6 +67,7 @@ "127.0.0.1:9323" "127.0.0.1:9252" "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}" + "10.0.40.40:9283" # Ceph ]; }]; } diff --git a/m/hut/secrets.nix b/m/hut/secrets.nix index 7c5aea70a3b89a7c5c0bfa1a615c58720b3dff2d..d22add65e72ca20f981fdd48d78705dbbfce5122 100644 --- a/m/hut/secrets.nix +++ b/m/hut/secrets.nix @@ -1,9 +1,11 @@ let - root = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb"; - system = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1"; - systems = [ root system ]; + rarias = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1oZTPtlEXdGt0Ak+upeCIiBdaDQtcmuWoTUCVuSVIR rarias@hut"; + root = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb"; + hut = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1"; + default = [ rarias root hut ]; in { - "secrets/ovni-token.age".publicKeys = systems; - "secrets/nosv-token.age".publicKeys = systems; + "secrets/ovni-token.age".publicKeys = default; + "secrets/nosv-token.age".publicKeys = default; + "secrets/ceph-user.age".publicKeys = default; } diff --git a/m/hut/secrets/ceph-user.age b/m/hut/secrets/ceph-user.age new file mode 100644 index 0000000000000000000000000000000000000000..735afca6652d99ab5888fa8c23b06127003d3fd9 --- /dev/null +++ b/m/hut/secrets/ceph-user.age @@ -0,0 +1,11 @@ +age-encryption.org/v1 +-> ssh-ed25519 CAWG4Q 35Ak+Mep9k5KnDLF1ywDbMD4l4mRFg6D0et19tqXxAw +Wgr+CX4rzrPmUszSidtLAVSvgD80F2dqtd92hGZIFwo +-> ssh-ed25519 MSF3dg OVFvpkAyWTowtxsafstX31H/hJpNZmnOCbvqMIN0+AQ +VxjRcQmp+BadEh2y0PB96EeizIl3tTQpVu0CWHmsc1s +-> ssh-ed25519 HY2yRg MJSQIpre9m0XnojgXuKQ/+hVBZNrZNGZqplwhqicpjI +CLkE52iqpoqSnbzisNjQgxTfNqKeaRl5ntcw1d+ZDyQ +-> m$8`De%~-grease '85p}`by +52zMpprONcawWDDtzHdWNwFoYXErPUnVjhSONbUBpDlqAmJmD1LcAnsU +--- 0vZOPyXQIMMGTwgFfvm8Sn8O7vjrsjGUEy5m/BASCyc +|)*_DUS`r sN[֌^e+A 1G.#mW 5 ( \ No newline at end of file diff --git a/m/hut/secrets/nosv-token.age b/m/hut/secrets/nosv-token.age index b26d482ab5e0522be3058a61ea400bb213e6c327..4b495a944de69fccbfdde7394b7f63f80e53e06e 100644 Binary files a/m/hut/secrets/nosv-token.age and b/m/hut/secrets/nosv-token.age differ diff --git a/m/hut/secrets/ovni-token.age b/m/hut/secrets/ovni-token.age index 8241b6ddcd826a5515d2446465ffd67fc36e4453..bb850ef38e40d426b50aba25ae6993fe521b4d74 100644 Binary files a/m/hut/secrets/ovni-token.age and b/m/hut/secrets/ovni-token.age differ