From b06399dc70471dcac07632d9d59eb3e995fc408f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 24 Aug 2023 12:29:44 +0200 Subject: [PATCH 01/57] Add section to enable serial console --- doc/install.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/doc/install.md b/doc/install.md index 8eae8c7..d5e279d 100644 --- a/doc/install.md +++ b/doc/install.md @@ -3,6 +3,34 @@ This article shows the steps to install NixOS in a node following the configuration of the repo. +## Enable the serial console + +By default, the nodes have the serial console disabled in the GRUB and also boot +without the serial enabled. + +To enable the serial console in the GRUB, set in /etc/default/grub the following +lines: + +``` +GRUB_TERMINAL="console serial" +GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1" +``` + +To boot Linux with the serial enabled, so you can see the boot log and login via +serial set: + +``` +GRUB_CMDLINE_LINUX="console=ttyS0,115200n8 console=tty0" +``` + +Then update the grub config: + +``` +# grub2-mkconfig -o /boot/grub2/grub.cfg +``` + +And reboot. + ## Prepare the disk Create a main partition and label it `nixos` following [the manual][1]. -- GitLab From 58c1cc1f7cf47a77bf0aebd00a9c5fe173690ea8 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 24 Aug 2023 12:30:46 +0200 Subject: [PATCH 02/57] Add lake2 bootstrap config --- flake.nix | 1 + m/lake2/configuration.nix | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 m/lake2/configuration.nix diff --git a/flake.nix b/flake.nix index 6ce0689..7bd69c0 100644 --- a/flake.nix +++ b/flake.nix @@ -22,6 +22,7 @@ in eudy = mkConf "eudy"; koro = mkConf "koro"; bay = mkConf "bay"; + lake2 = mkConf "lake2"; }; packages.x86_64-linux.hut = self.nixosConfigurations.hut.pkgs; diff --git a/m/lake2/configuration.nix b/m/lake2/configuration.nix new file mode 100644 index 0000000..37bd22a --- /dev/null +++ b/m/lake2/configuration.nix @@ -0,0 +1,32 @@ +{ config, pkgs, lib, modulesPath, ... }: + +{ + imports = [ + ../common/main.nix + (modulesPath + "/installer/netboot/netboot-minimal.nix") + ]; + + # For now we install NixOS in the first nvme disk (nvme0n1), as this node only + # has one SSD already used for SUSE. + boot.loader.grub.device = "/dev/disk/by-id/nvme-SNVMe_INTEL_SSDPE2MD02CVFT5281004L2P0KGN"; + + environment.systemPackages = with pkgs; [ + ceph + ]; + + services.slurm = { + client.enable = lib.mkForce false; + }; + + networking = { + hostName = "lake2"; + interfaces.eno1.ipv4.addresses = [ { + address = "10.0.40.42"; + prefixLength = 24; + } ]; + interfaces.ibp5s0.ipv4.addresses = [ { + address = "10.0.42.42"; + prefixLength = 24; + } ]; + }; +} -- GitLab From a19347161f91d47db2b1c5d9ae158aeeba3b8cd4 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 24 Aug 2023 13:54:22 +0200 Subject: [PATCH 03/57] Prepare lake2 config after bootstrap The disk ID is different under NixOS. --- m/lake2/configuration.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/m/lake2/configuration.nix b/m/lake2/configuration.nix index 37bd22a..1b89226 100644 --- a/m/lake2/configuration.nix +++ b/m/lake2/configuration.nix @@ -3,12 +3,11 @@ { imports = [ ../common/main.nix - (modulesPath + "/installer/netboot/netboot-minimal.nix") ]; # For now we install NixOS in the first nvme disk (nvme0n1), as this node only # has one SSD already used for SUSE. - boot.loader.grub.device = "/dev/disk/by-id/nvme-SNVMe_INTEL_SSDPE2MD02CVFT5281004L2P0KGN"; + boot.loader.grub.device = "/dev/disk/by-id/nvme-INTEL_SSDPE2MD020T4_CVFT5281004L2P0KGN"; environment.systemPackages = with pkgs; [ ceph -- GitLab From 95fa67ede1d165b7b5decb109210bb4fe90d0839 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 24 Aug 2023 15:27:37 +0200 Subject: [PATCH 04/57] Specify the disk by path --- m/lake2/configuration.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/m/lake2/configuration.nix b/m/lake2/configuration.nix index 1b89226..99fffa3 100644 --- a/m/lake2/configuration.nix +++ b/m/lake2/configuration.nix @@ -7,7 +7,7 @@ # For now we install NixOS in the first nvme disk (nvme0n1), as this node only # has one SSD already used for SUSE. - boot.loader.grub.device = "/dev/disk/by-id/nvme-INTEL_SSDPE2MD020T4_CVFT5281004L2P0KGN"; + boot.loader.grub.device = "/dev/disk/by-path/pci-0000:83:00.0-nvme-1"; environment.systemPackages = with pkgs; [ ceph -- GitLab From 029d9cb1db37f0c5ceb9a7378f72649a11e2cc5a Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 24 Aug 2023 19:08:23 +0200 Subject: [PATCH 05/57] Enable netboot again for PXE --- m/lake2/configuration.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/m/lake2/configuration.nix b/m/lake2/configuration.nix index 99fffa3..618acfc 100644 --- a/m/lake2/configuration.nix +++ b/m/lake2/configuration.nix @@ -3,6 +3,7 @@ { imports = [ ../common/main.nix + (modulesPath + "/installer/netboot/netboot-minimal.nix") ]; # For now we install NixOS in the first nvme disk (nvme0n1), as this node only -- GitLab From 815888fb07fe984fd3aa517f57a303ee81af733d Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 25 Aug 2023 12:03:30 +0200 Subject: [PATCH 06/57] Add PXE helper --- m/hut/configuration.nix | 1 + m/hut/pxe.nix | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 m/hut/pxe.nix diff --git a/m/hut/configuration.nix b/m/hut/configuration.nix index 14c7881..1356780 100644 --- a/m/hut/configuration.nix +++ b/m/hut/configuration.nix @@ -9,6 +9,7 @@ ./nfs.nix ./slurm-daemon.nix ./ceph.nix + ./pxe.nix agenix.nixosModules.default ]; diff --git a/m/hut/pxe.nix b/m/hut/pxe.nix new file mode 100644 index 0000000..65e6fb4 --- /dev/null +++ b/m/hut/pxe.nix @@ -0,0 +1,26 @@ +{ theFlake, pkgs, ... }: + +# This module describes a script that can launch the pixiecore daemon to serve a +# NixOS image via PXE to a node to directly boot from there, without requiring a +# working disk. + +let + # The host config must have the netboot-minimal.nix module too + host = theFlake.nixosConfigurations.lake2; + sys = host.config.system; + build = sys.build; + kernel = "${build.kernel}/bzImage"; + initrd = "${build.netbootRamdisk}/initrd"; + init = "${build.toplevel}/init"; + + script = pkgs.writeShellScriptBin "pixiecore-helper" '' + #!/usr/bin/env bash -x + + ${pkgs.pixiecore}/bin/pixiecore \ + boot ${kernel} ${initrd} --cmdline "init=${init} loglevel=4" \ + --debug --dhcp-no-bind --port 64172 --status-port 64172 "$@" + ''; +in +{ + environment.systemPackages = [ script ]; +} -- GitLab From 3416416864df3efee754efcf4c872affdaff2d30 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 25 Aug 2023 13:21:00 +0200 Subject: [PATCH 07/57] Disable pixiecore in hut for now --- m/hut/configuration.nix | 2 +- m/hut/pxe.nix | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/m/hut/configuration.nix b/m/hut/configuration.nix index 1356780..b4ba36f 100644 --- a/m/hut/configuration.nix +++ b/m/hut/configuration.nix @@ -9,7 +9,7 @@ ./nfs.nix ./slurm-daemon.nix ./ceph.nix - ./pxe.nix + #./pxe.nix agenix.nixosModules.default ]; diff --git a/m/hut/pxe.nix b/m/hut/pxe.nix index 65e6fb4..e3a74e2 100644 --- a/m/hut/pxe.nix +++ b/m/hut/pxe.nix @@ -22,5 +22,14 @@ let ''; in { + ## We need a DHCP server to provide the IP + #services.dnsmasq = { + # enable = true; + # settings = { + # domain-needed = true; + # dhcp-range = [ "192.168.0.2,192.168.0.254" ]; + # }; + #}; + environment.systemPackages = [ script ]; } -- GitLab From 8f1f6f92a8521dfa15ba1c93ce867e73d43fe609 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 25 Aug 2023 13:39:01 +0200 Subject: [PATCH 08/57] Remove netboot module --- m/lake2/configuration.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/m/lake2/configuration.nix b/m/lake2/configuration.nix index 618acfc..99fffa3 100644 --- a/m/lake2/configuration.nix +++ b/m/lake2/configuration.nix @@ -3,7 +3,6 @@ { imports = [ ../common/main.nix - (modulesPath + "/installer/netboot/netboot-minimal.nix") ]; # For now we install NixOS in the first nvme disk (nvme0n1), as this node only -- GitLab From a2075cfd655a106b7fc5c15f18acc7bdef1bc0c1 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 25 Aug 2023 13:40:10 +0200 Subject: [PATCH 09/57] Use the sda for lake2 --- m/lake2/configuration.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/m/lake2/configuration.nix b/m/lake2/configuration.nix index 99fffa3..255a53a 100644 --- a/m/lake2/configuration.nix +++ b/m/lake2/configuration.nix @@ -5,9 +5,7 @@ ../common/main.nix ]; - # For now we install NixOS in the first nvme disk (nvme0n1), as this node only - # has one SSD already used for SUSE. - boot.loader.grub.device = "/dev/disk/by-path/pci-0000:83:00.0-nvme-1"; + boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53563a"; environment.systemPackages = with pkgs; [ ceph -- GitLab From f1ce815edd00b37fe53a8455a3cce1b6e418742e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 25 Aug 2023 14:44:35 +0200 Subject: [PATCH 10/57] Add the lake2 hostname to the hosts --- m/common/net.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/m/common/net.nix b/m/common/net.nix index de2b28d..09765c4 100644 --- a/m/common/net.nix +++ b/m/common/net.nix @@ -41,7 +41,7 @@ 10.0.40.142 oss01-ipmi0 # Node Entry for node: oss02 (ID=74) - 10.0.40.42 oss02 oss02-eth0 + 10.0.40.42 lake2 oss02 oss02-eth0 10.0.42.42 oss02-ib0 10.0.40.143 oss02-ipmi0 -- GitLab From f5d6bf627bbb3357218f4f9e3cf51e88e669dafe Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 25 Aug 2023 14:44:53 +0200 Subject: [PATCH 11/57] Enable ceph osd daemons in lake2 --- m/lake2/configuration.nix | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/m/lake2/configuration.nix b/m/lake2/configuration.nix index 255a53a..c8225db 100644 --- a/m/lake2/configuration.nix +++ b/m/lake2/configuration.nix @@ -15,6 +15,29 @@ client.enable = lib.mkForce false; }; + services.ceph = { + enable = true; + global = { + fsid = "9c8d06e0-485f-4aaf-b16b-06d6daf1232b"; + monHost = "10.0.40.40"; + monInitialMembers = "bay"; + clusterNetwork = "10.0.40.40/24"; # Use Ethernet only + }; + osd = { + enable = true; + # One daemon per NVME disk + daemons = [ "4" "5" "6" "7" ]; + extraConfig = { + "osd crush chooseleaf type" = "0"; + "osd journal size" = "10000"; + "osd pool default min size" = "2"; + "osd pool default pg num" = "200"; + "osd pool default pgp num" = "200"; + "osd pool default size" = "3"; + }; + }; + }; + networking = { hostName = "lake2"; interfaces.eno1.ipv4.addresses = [ { -- GitLab From cb3a7b19f757aefbdb906601a237b5a53c2abfa7 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 25 Aug 2023 18:12:00 +0200 Subject: [PATCH 12/57] Move pkgs overlay to overlay.nix --- m/common/main.nix | 2 +- pkgs/{mpi.nix => overlay.nix} | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) rename pkgs/{mpi.nix => overlay.nix} (91%) diff --git a/m/common/main.nix b/m/common/main.nix index 7054495..5d77ddc 100644 --- a/m/common/main.nix +++ b/m/common/main.nix @@ -14,7 +14,7 @@ nixpkgs.overlays = [ bscpkgs.bscOverlay - (import ../../pkgs/mpi.nix) + (import ../../pkgs/overlay.nix) ]; nix.nixPath = [ diff --git a/pkgs/mpi.nix b/pkgs/overlay.nix similarity index 91% rename from pkgs/mpi.nix rename to pkgs/overlay.nix index cdaeadd..b86d2d9 100644 --- a/pkgs/mpi.nix +++ b/pkgs/overlay.nix @@ -1,7 +1,10 @@ final: prev: { bsc = prev.bsc.extend (bscFinal: bscPrev: { + # Set MPICH as default mpi = bscFinal.mpich; + + # Configure the network for MPICH mpich = with final; prev.mpich.overrideAttrs (old: { buildInput = old.buildInputs ++ [ libfabric -- GitLab From ac4fa9abd4f5572337518cd44adf64ab8ce95f1d Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 25 Aug 2023 18:12:46 +0200 Subject: [PATCH 13/57] Update ceph to 18.2.0 in overlay --- pkgs/ceph.nix | 405 +++++++++++++++++++++++++++++++++++++++++++++++ pkgs/overlay.nix | 7 + 2 files changed, 412 insertions(+) create mode 100644 pkgs/ceph.nix diff --git a/pkgs/ceph.nix b/pkgs/ceph.nix new file mode 100644 index 0000000..5247862 --- /dev/null +++ b/pkgs/ceph.nix @@ -0,0 +1,405 @@ +{ lib +, stdenv +, runCommand +, fetchurl +, fetchFromGitHub +, fetchPypi + +# Build time +, cmake +, ensureNewerSourcesHook +, fmt +, git +, makeWrapper +, nasm +, pkg-config +, which + +# Tests +, nixosTests + +# Runtime dependencies +, arrow-cpp +, babeltrace +, boost179 +, bzip2 +, cryptsetup +, cunit +, doxygen +, gperf +, graphviz +, gtest +, icu +, libcap +, libcap_ng +, libnl +, libxml2 +, lttng-ust +, lua +, lz4 +, oath-toolkit +, openldap +, python310 +, rdkafka +, rocksdb +, snappy +, sqlite +, utf8proc +, zlib +, zstd + +# Optional Dependencies +, curl ? null +, expat ? null +, fuse ? null +, libatomic_ops ? null +, libedit ? null +, libs3 ? null +, yasm ? null + +# Mallocs +, gperftools ? null +, jemalloc ? null + +# Crypto Dependencies +, cryptopp ? null +, nspr ? null +, nss ? null + +# Linux Only Dependencies +, linuxHeaders +, util-linux +, libuuid +, udev +, keyutils +, rdma-core +, rabbitmq-c +, libaio ? null +, libxfs ? null +, liburing ? null +, zfs ? null +, ... +}: + +# We must have one crypto library +assert cryptopp != null || (nss != null && nspr != null); + +let + shouldUsePkg = pkg: if pkg != null && pkg.meta.available then pkg else null; + + optYasm = shouldUsePkg yasm; + optExpat = shouldUsePkg expat; + optCurl = shouldUsePkg curl; + optFuse = shouldUsePkg fuse; + optLibedit = shouldUsePkg libedit; + optLibatomic_ops = shouldUsePkg libatomic_ops; + optLibs3 = shouldUsePkg libs3; + + optJemalloc = shouldUsePkg jemalloc; + optGperftools = shouldUsePkg gperftools; + + optCryptopp = shouldUsePkg cryptopp; + optNss = shouldUsePkg nss; + optNspr = shouldUsePkg nspr; + + optLibaio = shouldUsePkg libaio; + optLibxfs = shouldUsePkg libxfs; + optZfs = shouldUsePkg zfs; + + # Downgrade rocksdb, 7.10 breaks ceph + rocksdb' = rocksdb.overrideAttrs { + version = "7.9.2"; + src = fetchFromGitHub { + owner = "facebook"; + repo = "rocksdb"; + rev = "refs/tags/v7.9.2"; + hash = "sha256-5P7IqJ14EZzDkbjaBvbix04ceGGdlWBuVFH/5dpD5VM="; + }; + }; + + hasRadosgw = optExpat != null && optCurl != null && optLibedit != null; + + # Malloc implementation (can be jemalloc, tcmalloc or null) + malloc = if optJemalloc != null then optJemalloc else optGperftools; + + # We prefer nss over cryptopp + cryptoStr = if optNss != null && optNspr != null then "nss" else + if optCryptopp != null then "cryptopp" else "none"; + + cryptoLibsMap = { + nss = [ optNss optNspr ]; + cryptopp = [ optCryptopp ]; + none = [ ]; + }; + + getMeta = description: with lib; { + homepage = "https://ceph.io/en/"; + inherit description; + license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ]; + maintainers = with maintainers; [ adev ak johanot krav ]; + platforms = [ "x86_64-linux" "aarch64-linux" ]; + }; + + ceph-common = with python.pkgs; buildPythonPackage { + pname = "ceph-common"; + inherit src version; + + sourceRoot = "ceph-${version}/src/python-common"; + + propagatedBuildInputs = [ + pyyaml + ]; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + disabledTests = [ + # requires network access + "test_valid_addr" + ]; + + meta = getMeta "Ceph common module for code shared by manager modules"; + }; + + # Watch out for python <> boost compatibility + python = python310.override { + packageOverrides = self: super: { + sqlalchemy = super.sqlalchemy.overridePythonAttrs rec { + version = "1.4.46"; + src = fetchPypi { + pname = "SQLAlchemy"; + inherit version; + hash = "sha256-aRO4JH2KKS74MVFipRkx4rQM6RaB8bbxj2lwRSAMSjA="; + }; + disabledTestPaths = [ + "test/aaa_profiling" + "test/ext/mypy" + ]; + }; + }; + }; + + boost = boost179.override { + enablePython = true; + inherit python; + }; + + # TODO: split this off in build and runtime environment + ceph-python-env = python.withPackages (ps: with ps; [ + ceph-common + + # build time + cython + + # debian/control + bcrypt + cherrypy + influxdb + jinja2 + kubernetes + natsort + numpy + pecan + prettytable + pyjwt + pyopenssl + python-dateutil + pyyaml + requests + routes + scikit-learn + scipy + setuptools + sphinx + virtualenv + werkzeug + + # src/pybind/mgr/requirements-required.txt + cryptography + jsonpatch + + # src/tools/cephfs/shell/setup.py + cmd2 + colorama + ]); + inherit (ceph-python-env.python) sitePackages; + + version = "18.2.0"; + src = fetchurl { + url = "https://download.ceph.com/tarballs/ceph-${version}.tar.gz"; + hash = "sha256:0k9nl6xi5brva51rr14m7ig27mmmd7vrpchcmqc40q3c2khn6ns9"; + }; +in rec { + ceph = stdenv.mkDerivation { + pname = "ceph"; + inherit src version; + + nativeBuildInputs = [ + cmake + fmt + git + makeWrapper + nasm + pkg-config + python + python.pkgs.python # for the toPythonPath function + python.pkgs.wrapPython + which + (ensureNewerSourcesHook { year = "1980"; }) + # for building docs/man-pages presumably + doxygen + graphviz + ]; + + enableParallelBuilding = true; + + buildInputs = cryptoLibsMap.${cryptoStr} ++ [ + arrow-cpp + babeltrace + boost + bzip2 + ceph-python-env + cryptsetup + cunit + gperf + gtest + icu + libcap + libnl + libxml2 + lttng-ust + lua + lz4 + malloc + oath-toolkit + openldap + optLibatomic_ops + optLibs3 + optYasm + rdkafka + rocksdb' + snappy + sqlite + utf8proc + zlib + zstd + ] ++ lib.optionals stdenv.isLinux [ + keyutils + libcap_ng + liburing + libuuid + linuxHeaders + optLibaio + optLibxfs + optZfs + rabbitmq-c + rdma-core + udev + util-linux + ] ++ lib.optionals hasRadosgw [ + optCurl + optExpat + optFuse + optLibedit + ]; + + pythonPath = [ ceph-python-env "${placeholder "out"}/${ceph-python-env.sitePackages}" ]; + + preConfigure ='' + substituteInPlace src/common/module.c --replace "/sbin/modinfo" "modinfo" + substituteInPlace src/common/module.c --replace "/sbin/modprobe" "modprobe" + substituteInPlace src/common/module.c --replace "/bin/grep" "grep" + + # install target needs to be in PYTHONPATH for "*.pth support" check to succeed + # set PYTHONPATH, so the build system doesn't silently skip installing ceph-volume and others + export PYTHONPATH=${ceph-python-env}/${sitePackages}:$lib/${sitePackages}:$out/${sitePackages} + patchShebangs src/ + ''; + + cmakeFlags = [ + "-DCMAKE_INSTALL_DATADIR=${placeholder "lib"}/lib" + + "-DWITH_CEPHFS_SHELL:BOOL=ON" + "-DWITH_SYSTEMD:BOOL=OFF" + # `WITH_JAEGER` requires `thrift` as a depenedncy (fine), but the build fails with: + # CMake Error at src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-Release.cmake:49 (message): + # Command failed: 2 + # + # 'make' 'opentelemetry_trace' 'opentelemetry_exporter_jaeger_trace' + # + # See also + # + # /build/ceph-18.2.0/build/src/opentelemetry-cpp/src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-*.log + # and that file contains: + # /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc: In member function 'virtual void opentelemetry::v1::exporter::jaeger::TUDPTransport::close()': + # /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc:71:7: error: '::close' has not been declared; did you mean 'pclose'? + # 71 | ::THRIFT_CLOSESOCKET(socket_); + # | ^~~~~~~~~~~~~~~~~~ + # Looks like `close()` is somehow not included. + # But the relevant code is already removed in `open-telemetry` 1.10: https://github.com/open-telemetry/opentelemetry-cpp/pull/2031 + # So it's proably not worth trying to fix that for this Ceph version, + # and instead just disable Ceph's Jaeger support. + "-DWITH_JAEGER:BOOL=OFF" + "-DWITH_TESTS:BOOL=OFF" + + # Use our own libraries, where possible + "-DWITH_SYSTEM_ARROW:BOOL=ON" # Only used if other options enable Arrow support. + "-DWITH_SYSTEM_BOOST:BOOL=ON" + "-DWITH_SYSTEM_GTEST:BOOL=ON" + "-DWITH_SYSTEM_ROCKSDB:BOOL=ON" + "-DWITH_SYSTEM_UTF8PROC:BOOL=ON" + "-DWITH_SYSTEM_ZSTD:BOOL=ON" + + # TODO breaks with sandbox, tries to download stuff with npm + "-DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF" + # WITH_XFS has been set default ON from Ceph 16, keeping it optional in nixpkgs for now + ''-DWITH_XFS=${if optLibxfs != null then "ON" else "OFF"}'' + ] ++ lib.optional stdenv.isLinux "-DWITH_SYSTEM_LIBURING=ON"; + + postFixup = '' + wrapPythonPrograms + wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "$(toPythonPath ${placeholder "out"}):$(toPythonPath ${ceph-python-env})" + + # Test that ceph-volume exists since the build system has a tendency to + # silently drop it with misconfigurations. + test -f $out/bin/ceph-volume + ''; + + outputs = [ "out" "lib" "dev" "doc" "man" ]; + + doCheck = false; # uses pip to install things from the internet + + # Takes 7+h to build with 2 cores. + requiredSystemFeatures = [ "big-parallel" ]; + + meta = getMeta "Distributed storage system"; + + passthru = { + inherit version; + tests = { + inherit (nixosTests) + ceph-multi-node + ceph-single-node + ceph-single-node-bluestore; + }; + }; + }; + + ceph-client = runCommand "ceph-client-${version}" { + meta = getMeta "Tools needed to mount Ceph's RADOS Block Devices/Cephfs"; + } '' + mkdir -p $out/{bin,etc,${sitePackages},share/bash-completion/completions} + cp -r ${ceph}/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $out/bin + cp -r ${ceph}/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $out/bin + cp -r ${ceph}/bin/rbd-replay* $out/bin + cp -r ${ceph}/sbin/mount.ceph $out/bin + cp -r ${ceph}/sbin/mount.fuse.ceph $out/bin + ln -s bin $out/sbin + cp -r ${ceph}/${sitePackages}/* $out/${sitePackages} + cp -r ${ceph}/etc/bash_completion.d $out/share/bash-completion/completions + # wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths + substituteInPlace $out/bin/ceph --replace ${ceph} $out + substituteInPlace $out/bin/.ceph-wrapped --replace ${ceph} $out + ''; +} diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix index b86d2d9..03208c5 100644 --- a/pkgs/overlay.nix +++ b/pkgs/overlay.nix @@ -25,4 +25,11 @@ final: prev: ]; }); }); + + # Update ceph to 18.2.0 until it lands in nixpkgs, see: + # https://github.com/NixOS/nixpkgs/pull/247849 + inherit (prev.callPackage ./ceph.nix { + lua = prev.lua5_4; + fmt = prev.fmt_8; + }) ceph ceph-client; } -- GitLab From 599613d139816749ef4da231c2b42f7ce7b6be4e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 28 Aug 2023 17:58:08 +0200 Subject: [PATCH 14/57] Switch ceph logs to journal --- m/bay/configuration.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/m/bay/configuration.nix b/m/bay/configuration.nix index 847e162..41d0a56 100644 --- a/m/bay/configuration.nix +++ b/m/bay/configuration.nix @@ -36,6 +36,14 @@ monInitialMembers = "bay"; clusterNetwork = "10.0.40.40/24"; # Use Ethernet only }; + extraConfig = { + # Only log to stderr so it appears in the journal + "log_file" = "/dev/null"; + "mon_cluster_log_file" = "/dev/null"; + "log_to_stderr" = "true"; + "err_to_stderr" = "true"; + "log_to_file" = "false"; + }; mds = { enable = true; daemons = [ "mds0" "mds1" ]; -- GitLab From d2a80c8c180e9957155ff875ba79c3df49808971 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 28 Aug 2023 17:58:21 +0200 Subject: [PATCH 15/57] Add ceph tools in hut too --- m/hut/ceph.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/m/hut/ceph.nix b/m/hut/ceph.nix index 722466f..59920e8 100644 --- a/m/hut/ceph.nix +++ b/m/hut/ceph.nix @@ -1,7 +1,7 @@ { config, pkgs, ... }: { - environment.systemPackages = [ pkgs.ceph-client ]; + environment.systemPackages = [ pkgs.ceph pkgs.ceph-client ]; # We need the ceph module loaded as the mount.ceph binary fails to run the # modprobe command. -- GitLab From fad9df61e1ba251aa9104f6fa402f6e56556cd70 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 29 Aug 2023 11:27:50 +0200 Subject: [PATCH 16/57] Add fio tool --- m/hut/ceph.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/m/hut/ceph.nix b/m/hut/ceph.nix index 59920e8..3bebe11 100644 --- a/m/hut/ceph.nix +++ b/m/hut/ceph.nix @@ -1,7 +1,11 @@ { config, pkgs, ... }: { - environment.systemPackages = [ pkgs.ceph pkgs.ceph-client ]; + environment.systemPackages = with pkgs; [ + ceph + ceph-client + fio # For benchmarks + ]; # We need the ceph module loaded as the mount.ceph binary fails to run the # modprobe command. -- GitLab From 5bd1d673334b96e63ac2a9a5d01ef6ef43e36f5e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 29 Aug 2023 11:53:32 +0200 Subject: [PATCH 17/57] Add monitoring in the bay node --- m/bay/configuration.nix | 1 + m/bay/monitoring.nix | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 m/bay/monitoring.nix diff --git a/m/bay/configuration.nix b/m/bay/configuration.nix index 41d0a56..6aff18e 100644 --- a/m/bay/configuration.nix +++ b/m/bay/configuration.nix @@ -3,6 +3,7 @@ { imports = [ ../common/main.nix + ./monitoring.nix ]; # Select the this using the ID to avoid mismatches diff --git a/m/bay/monitoring.nix b/m/bay/monitoring.nix new file mode 100644 index 0000000..0ef9209 --- /dev/null +++ b/m/bay/monitoring.nix @@ -0,0 +1,25 @@ +{ config, lib, ... }: + +{ + # We need access to the devices to monitor the disk space + systemd.services.prometheus-node-exporter.serviceConfig.PrivateDevices = lib.mkForce false; + systemd.services.prometheus-node-exporter.serviceConfig.ProtectHome = lib.mkForce "read-only"; + + # Required to allow the smartctl exporter to read the nvme0 character device, + # see the commit message on: + # https://github.com/NixOS/nixpkgs/commit/12c26aca1fd55ab99f831bedc865a626eee39f80 + services.udev.extraRules = '' + SUBSYSTEM=="nvme", KERNEL=="nvme[0-9]*", GROUP="disk" + ''; + + services.prometheus = { + exporters = { + node = { + enable = true; + enabledCollectors = [ "systemd" ]; + port = 9002; + }; + smartctl.enable = true; + }; + }; +} -- GitLab From 70321ce23777217b058471e3763c657a5dbf5a06 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 29 Aug 2023 11:58:00 +0200 Subject: [PATCH 18/57] Scrape metrics from bay --- m/hut/monitoring.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/m/hut/monitoring.nix b/m/hut/monitoring.nix index 4bc3d24..65e68cc 100644 --- a/m/hut/monitoring.nix +++ b/m/hut/monitoring.nix @@ -67,7 +67,15 @@ "127.0.0.1:9323" "127.0.0.1:9252" "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}" + ]; + }]; + } + { + job_name = "bay"; + static_configs = [{ + targets = [ "10.0.40.40:9283" # Ceph + "10.0.40.40:9002" # Node exporter ]; }]; } -- GitLab From beb0d5940e283ed615c67e430cc8832f5c37db5f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 29 Aug 2023 12:29:41 +0200 Subject: [PATCH 19/57] Also enable monitoring in lake2 --- m/bay/configuration.nix | 2 +- m/{bay => common}/monitoring.nix | 0 m/lake2/configuration.nix | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) rename m/{bay => common}/monitoring.nix (100%) diff --git a/m/bay/configuration.nix b/m/bay/configuration.nix index 6aff18e..5e2b342 100644 --- a/m/bay/configuration.nix +++ b/m/bay/configuration.nix @@ -3,7 +3,7 @@ { imports = [ ../common/main.nix - ./monitoring.nix + ../common/monitoring.nix ]; # Select the this using the ID to avoid mismatches diff --git a/m/bay/monitoring.nix b/m/common/monitoring.nix similarity index 100% rename from m/bay/monitoring.nix rename to m/common/monitoring.nix diff --git a/m/lake2/configuration.nix b/m/lake2/configuration.nix index c8225db..4b05147 100644 --- a/m/lake2/configuration.nix +++ b/m/lake2/configuration.nix @@ -3,6 +3,7 @@ { imports = [ ../common/main.nix + ../common/monitoring.nix ]; boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53563a"; -- GitLab From 0f0a861896dea197941442a099bfc7b17f0873f6 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 29 Aug 2023 12:33:26 +0200 Subject: [PATCH 20/57] Scrape lake2 too --- m/hut/monitoring.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/m/hut/monitoring.nix b/m/hut/monitoring.nix index 65e68cc..8a8646a 100644 --- a/m/hut/monitoring.nix +++ b/m/hut/monitoring.nix @@ -71,11 +71,12 @@ }]; } { - job_name = "bay"; + job_name = "ceph"; static_configs = [{ targets = [ - "10.0.40.40:9283" # Ceph + "10.0.40.40:9283" # Ceph statistics "10.0.40.40:9002" # Node exporter + "10.0.40.42:9002" # Node exporter ]; }]; } -- GitLab From 7d09108c9ff77acaa4f3a08c09ce5c28449676ea Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 29 Aug 2023 18:47:25 +0200 Subject: [PATCH 21/57] Enable all osd on boot in lake2 --- m/lake2/configuration.nix | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/m/lake2/configuration.nix b/m/lake2/configuration.nix index 4b05147..58bfef8 100644 --- a/m/lake2/configuration.nix +++ b/m/lake2/configuration.nix @@ -50,4 +50,24 @@ prefixLength = 24; } ]; }; + + # Missing service for volumes, see: + # https://www.reddit.com/r/ceph/comments/14otjyo/comment/jrd69vt/ + systemd.services.ceph-volume = { + enable = true; + description = "Ceph Volume activation"; + unitConfig = { + Type = "oneshot"; + After = "local-fs.target"; + Wants = "local-fs.target"; + }; + path = [ pkgs.ceph pkgs.util-linux pkgs.lvm2 pkgs.cryptsetup ]; + serviceConfig = { + KillMode = "none"; + Environment = "CEPH_VOLUME_TIMEOUT=10000"; + ExecStart = "/bin/sh -c 'timeout $CEPH_VOLUME_TIMEOUT ${pkgs.ceph}/bin/ceph-volume lvm activate --all --no-systemd'"; + TimeoutSec = "0"; + }; + wantedBy = [ "multi-user.target" ]; + }; } -- GitLab From 3c99c2a662eb67e2c4c0ed67de718e40d742744c Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 29 Aug 2023 22:26:12 +0200 Subject: [PATCH 22/57] Enable watchdog --- m/common/main.nix | 1 + m/common/watchdog.nix | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 m/common/watchdog.nix diff --git a/m/common/main.nix b/m/common/main.nix index 5d77ddc..f9141aa 100644 --- a/m/common/main.nix +++ b/m/common/main.nix @@ -10,6 +10,7 @@ ./slurm.nix ./ssh.nix ./users.nix + ./watchdog.nix ]; nixpkgs.overlays = [ diff --git a/m/common/watchdog.nix b/m/common/watchdog.nix new file mode 100644 index 0000000..d4d297d --- /dev/null +++ b/m/common/watchdog.nix @@ -0,0 +1,9 @@ +{ ... }: + +{ + # The boards have a BMC watchdog controlled by IPMI + boot.kernelModules = [ "ipmi_watchdog" ]; + + # Enable systemd watchdog with 30 s interval + systemd.watchdog.runtimeTime = "30s"; +} -- GitLab From 9d487845f6a9a3533d9d4e25b2b21891f56ae72e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 31 Aug 2023 17:22:36 +0200 Subject: [PATCH 23/57] Enable binary emulation for other architectures --- m/hut/configuration.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/m/hut/configuration.nix b/m/hut/configuration.nix index b4ba36f..1be9dc3 100644 --- a/m/hut/configuration.nix +++ b/m/hut/configuration.nix @@ -17,6 +17,8 @@ agenix.packages.x86_64-linux.default ]; + boot.binfmt.emulatedSystems = [ "aarch64-linux" "powerpc64le-linux" "riscv64-linux" ]; + # Select the this using the ID to avoid mismatches boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN"; -- GitLab From f9c77b433ac00fd7d267f875f7f2a70b77cbdf0e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Sat, 2 Sep 2023 23:37:11 +0200 Subject: [PATCH 24/57] Store nixos config in /etc/nixos/config.rev --- m/common/main.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/m/common/main.nix b/m/common/main.nix index f9141aa..25a209c 100644 --- a/m/common/main.nix +++ b/m/common/main.nix @@ -33,6 +33,9 @@ then theFlake.rev else throw ("Refusing to build from a dirty Git tree!"); + # Save the commit of the config in /etc/nixos/config.rev + environment.etc."nixos/config.rev".text = system.configurationRevision; + environment.systemPackages = with pkgs; [ vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree -- GitLab From 13a70411aa8332f7351d8b70d7014e140bd895e8 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Sat, 2 Sep 2023 23:37:59 +0200 Subject: [PATCH 25/57] Configure bscpkgs.nixpkgs to follow nixpkgs --- flake.lock | 13 +++++++++---- flake.nix | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 1a3da19..44b83b7 100644 --- a/flake.lock +++ b/flake.lock @@ -23,12 +23,17 @@ } }, "bscpkgs": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, "locked": { - "lastModified": 1690560045, - "narHash": "sha256-39ZP+FIzlWoN3c43hReBYpStg4RLYw/z7TdxCQmOvTM=", + "lastModified": 1693479395, + "narHash": "sha256-/C0ZEafHZlhD0m145P5crGPcD7Ck9GGSTbiqbiAlgUo=", "ref": "refs/heads/master", - "rev": "b4a20d7c3af854b39682484adfd1c7979319f439", - "revCount": 841, + "rev": "18d64c352c10f9ce74aabddeba5a5db02b74ec27", + "revCount": 845, "type": "git", "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git" }, diff --git a/flake.nix b/flake.nix index 7bd69c0..10be75a 100644 --- a/flake.nix +++ b/flake.nix @@ -4,6 +4,7 @@ agenix.url = "github:ryantm/agenix"; agenix.inputs.nixpkgs.follows = "nixpkgs"; bscpkgs.url = "git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git"; + bscpkgs.inputs.nixpkgs.follows = "nixpkgs"; }; outputs = { self, nixpkgs, agenix, bscpkgs, ... }: -- GitLab From 6848b58e39839812d601eb1c519c33125d257730 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Sat, 2 Sep 2023 23:49:41 +0200 Subject: [PATCH 26/57] Keep a log over time with the config commits --- m/common/main.nix | 14 ++++++-------- m/common/rev.nix | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 m/common/rev.nix diff --git a/m/common/main.nix b/m/common/main.nix index 25a209c..3792135 100644 --- a/m/common/main.nix +++ b/m/common/main.nix @@ -11,6 +11,7 @@ ./ssh.nix ./users.nix ./watchdog.nix + ./rev.nix ]; nixpkgs.overlays = [ @@ -18,6 +19,11 @@ (import ../../pkgs/overlay.nix) ]; + system.configurationRevision = + if theFlake ? rev + then theFlake.rev + else throw ("Refusing to build from a dirty Git tree!"); + nix.nixPath = [ "nixpkgs=${nixpkgs}" "bscpkgs=${bscpkgs}" @@ -28,14 +34,6 @@ nix.registry.bscpkgs.flake = bscpkgs; nix.registry.jungle.flake = theFlake; - system.configurationRevision = - if theFlake ? rev - then theFlake.rev - else throw ("Refusing to build from a dirty Git tree!"); - - # Save the commit of the config in /etc/nixos/config.rev - environment.etc."nixos/config.rev".text = system.configurationRevision; - environment.systemPackages = with pkgs; [ vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree diff --git a/m/common/rev.nix b/m/common/rev.nix new file mode 100644 index 0000000..80d019b --- /dev/null +++ b/m/common/rev.nix @@ -0,0 +1,18 @@ +{ theFlake, ... }: + +let + rev = if theFlake ? rev then theFlake.rev + else throw ("Refusing to build from a dirty Git tree!"); +in { + # Save the commit of the config in /etc/configrev + environment.etc.configrev.text = rev + "\n"; + + # Keep a log with the config over time + system.activationScripts.configRevLog.text = '' + BOOTED=$(cat /run/booted-system/etc/configrev 2>/dev/null || echo unknown) + CURRENT=$(cat /run/current-system/etc/configrev 2>/dev/null || echo unknown) + NEXT=${rev} + DATENOW=$(date --iso-8601=seconds) + echo "$DATENOW booted=$BOOTED current=$CURRENT next=$NEXT" >> /var/configrev.log + ''; +} -- GitLab From 3418e57907622bbfb35a3d93939324d97e4fd924 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Sun, 3 Sep 2023 11:51:53 +0200 Subject: [PATCH 27/57] Enable zsh and fix key bindings --- m/common/main.nix | 4 +-- m/common/zsh.nix | 89 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 m/common/zsh.nix diff --git a/m/common/main.nix b/m/common/main.nix index 3792135..f664ef8 100644 --- a/m/common/main.nix +++ b/m/common/main.nix @@ -12,6 +12,7 @@ ./users.nix ./watchdog.nix ./rev.nix + ./zsh.nix ]; nixpkgs.overlays = [ @@ -73,9 +74,6 @@ nix.gc.dates = "weekly"; nix.gc.options = "--delete-older-than 30d"; - programs.zsh.enable = true; - programs.zsh.histSize = 100000; - programs.bash.promptInit = '' PS1="\h\\$ " ''; diff --git a/m/common/zsh.nix b/m/common/zsh.nix new file mode 100644 index 0000000..c437a49 --- /dev/null +++ b/m/common/zsh.nix @@ -0,0 +1,89 @@ +{ pkgs, ... }: + +{ + environment.systemPackages = with pkgs; [ + direnv + zsh-completions + nix-zsh-completions + ]; + + programs.zsh = { + enable = true; + histSize = 1000000; + + shellInit = '' + # Disable new user prompt + if [ ! -e ~/.zshrc ]; then + touch ~/.zshrc + fi + ''; + + promptInit = '' + # Note that to manually override this in ~/.zshrc you should run `prompt off` + # before setting your PS1 and etc. Otherwise this will likely to interact with + # your ~/.zshrc configuration in unexpected ways as the default prompt sets + # a lot of different prompt variables. + autoload -U promptinit && promptinit && prompt default && setopt prompt_sp + ''; + + # Taken from Ulli Kehrle config: + # https://git.hrnz.li/Ulli/nixos/src/commit/2e203b8d8d671f4e3ced0f1744a51d5c6ee19846/profiles/shell.nix#L199-L205 + interactiveShellInit = '' + source "${pkgs.zsh-history-substring-search}/share/zsh-history-substring-search/zsh-history-substring-search.zsh" + + # dircolors doesn't support alacritty: + # https://lists.gnu.org/archive/html/bug-coreutils/2019-05/msg00029.html + export LS_COLORS='rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=00:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.avif=01;35:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.webp=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:*~=00;90:*#=00;90:*.bak=00;90:*.old=00;90:*.orig=00;90:*.part=00;90:*.rej=00;90:*.swp=00;90:*.tmp=00;90:*.dpkg-dist=00;90:*.dpkg-old=00;90:*.ucf-dist=00;90:*.ucf-new=00;90:*.ucf-old=00;90:*.rpmnew=00;90:*.rpmorig=00;90:*.rpmsave=00;90:'; + + # From Arch Linux and GRML + bindkey "^R" history-incremental-pattern-search-backward + bindkey "^S" history-incremental-pattern-search-forward + + # Auto rehash for new binaries + zstyle ':completion:*' rehash true + # show a nice menu with the matches + zstyle ':completion:*' menu yes select + + bindkey '^[OA' history-substring-search-up # Up + bindkey '^[[A' history-substring-search-up # Up + + bindkey '^[OB' history-substring-search-down # Down + bindkey '^[[B' history-substring-search-down # Down + + bindkey '\e[1~' beginning-of-line # Home + bindkey '\e[7~' beginning-of-line # Home + bindkey '\e[H' beginning-of-line # Home + bindkey '\eOH' beginning-of-line # Home + + bindkey '\e[4~' end-of-line # End + bindkey '\e[8~' end-of-line # End + bindkey '\e[F ' end-of-line # End + bindkey '\eOF' end-of-line # End + + bindkey '^?' backward-delete-char # Backspace + bindkey '\e[3~' delete-char # Del + # bindkey '\e[3;5~' delete-char # sometimes Del, sometimes C-Del + bindkey '\e[2~' overwrite-mode # Ins + + bindkey '^H' backward-kill-word # C-Backspace + + bindkey '5~' kill-word # C-Del + bindkey '^[[3;5~' kill-word # C-Del + bindkey '^[[3^' kill-word # C-Del + + bindkey "^[[1;5H" backward-kill-line # C-Home + bindkey "^[[7^" backward-kill-line # C-Home + + bindkey "^[[1;5F" kill-line # C-End + bindkey "^[[8^" kill-line # C-End + + bindkey '^[[1;5C' forward-word # C-Right + bindkey '^[0c' forward-word # C-Right + bindkey '^[[5C' forward-word # C-Right + + bindkey '^[[1;5D' backward-word # C-Left + bindkey '^[0d' backward-word # C-Left + bindkey '^[[5D' backward-word # C-Left + ''; + }; +} -- GitLab From 80c98041b5cbac7bd0c85ba8ecea51c0126b3215 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Sun, 3 Sep 2023 16:46:27 +0200 Subject: [PATCH 28/57] Set zsh shell for rarias --- m/common/users.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/m/common/users.nix b/m/common/users.nix index 3399ed3..d333c20 100644 --- a/m/common/users.nix +++ b/m/common/users.nix @@ -1,4 +1,4 @@ -{ ... }: +{ pkgs, ... }: { users = { @@ -26,6 +26,7 @@ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINa0tvnNgwkc5xOwd6xTtaIdFi5jv0j2FrE7jl5MTLoE ram@mio" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGYcXIxe0poOEGLpk8NjiRozls7fMRX0N3j3Ar94U+Gl rarias@hal" ]; + shell = pkgs.zsh; }; arocanon = { -- GitLab From 9344daa31cf9881cef3b2b9a066488d1b6cff490 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Sun, 3 Sep 2023 16:57:53 +0200 Subject: [PATCH 29/57] Set zsh inc_append_history option --- m/common/zsh.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/m/common/zsh.nix b/m/common/zsh.nix index c437a49..666e34b 100644 --- a/m/common/zsh.nix +++ b/m/common/zsh.nix @@ -31,6 +31,9 @@ interactiveShellInit = '' source "${pkgs.zsh-history-substring-search}/share/zsh-history-substring-search/zsh-history-substring-search.zsh" + # Save history immediately, but only load it when the shell starts + setopt inc_append_history + # dircolors doesn't support alacritty: # https://lists.gnu.org/archive/html/bug-coreutils/2019-05/msg00029.html export LS_COLORS='rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=00:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.avif=01;35:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.webp=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:*~=00;90:*#=00;90:*.bak=00;90:*.old=00;90:*.orig=00;90:*.part=00;90:*.rej=00;90:*.swp=00;90:*.tmp=00;90:*.dpkg-dist=00;90:*.dpkg-old=00;90:*.ucf-dist=00;90:*.ucf-new=00;90:*.ucf-old=00;90:*.rpmnew=00;90:*.rpmorig=00;90:*.rpmsave=00;90:'; -- GitLab From 2d16709648ae41a4337dec916775f163e088b6b3 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 4 Sep 2023 16:00:01 +0200 Subject: [PATCH 30/57] Add anavarro user --- m/common/users.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/m/common/users.nix b/m/common/users.nix index d333c20..8451196 100644 --- a/m/common/users.nix +++ b/m/common/users.nix @@ -54,6 +54,18 @@ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFYfXg37mauGeurqsLpedgA2XQ9d4Nm0ZGo/hI1f7wwH rpenacob@bsc" ]; }; + + anavarro = { + uid = 1037; + isNormalUser = true; + home = "/home/Computational/anavarro"; + description = "Antoni Navarro"; + group = "Computational"; + hashedPassword = "$6$QdNDsuLehoZTYZlb$CDhCouYDPrhoiB7/seu7RF.Gqg4zMQz0n5sA4U1KDgHaZOxy2as9pbIGeF8tOHJKRoZajk5GiaZv0rZMn7Oq31"; + openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWjRSlKgzBPZQhIeEtk6Lvws2XNcYwHcwPv4osSgst5 anavarro@ssfhead" + ]; + }; }; groups = { -- GitLab From 2bb366b9ac5639ad9e4d43c887411ba95c65ae81 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 4 Sep 2023 21:36:31 +0200 Subject: [PATCH 31/57] Reorganize secrets and ssh keys The agenix tools needs to read the secrets from a standalone file, but we also need the same information for the SSH keys. --- keys.nix | 29 +++++++++++++++++++++++++++++ m/common/ssh.nix | 14 ++++++-------- m/hut/ceph.nix | 4 ++-- m/hut/gitlab-runner.nix | 10 +++++----- m/hut/secrets/ceph-user.age | 11 ----------- m/hut/secrets/nosv-token.age | Bin 541 -> 0 bytes m/hut/secrets/ovni-token.age | Bin 610 -> 0 bytes secrets/ceph-user.age | 21 +++++++++++++++++++++ secrets/nosv-token.age | 11 +++++++++++ secrets/ovni-token.age | Bin 0 -> 553 bytes secrets/secrets.nix | 13 +++++++++++++ 11 files changed, 87 insertions(+), 26 deletions(-) create mode 100644 keys.nix delete mode 100644 m/hut/secrets/ceph-user.age delete mode 100644 m/hut/secrets/nosv-token.age delete mode 100644 m/hut/secrets/ovni-token.age create mode 100644 secrets/ceph-user.age create mode 100644 secrets/nosv-token.age create mode 100644 secrets/ovni-token.age create mode 100644 secrets/secrets.nix diff --git a/keys.nix b/keys.nix new file mode 100644 index 0000000..681fcbc --- /dev/null +++ b/keys.nix @@ -0,0 +1,29 @@ +# As agenix needs to parse the secrets from a standalone .nix file, we describe +# here all the public keys +rec { + hosts = { + hut = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1 hut"; + owl1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv owl1"; + owl2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK owl2"; + eudy = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG eudy"; + koro = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro"; + bay = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay"; + lake2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2"; + }; + + hostGroup = with hosts; rec { + compute = [ owl1 owl2 ]; + playground = [ eudy koro ]; + storage = [ bay lake2 ]; + monitor = [ hut ]; + + system = storage ++ monitor; + safe = system ++ compute; + all = safe ++ playground; + }; + + admins = { + rarias = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1oZTPtlEXdGt0Ak+upeCIiBdaDQtcmuWoTUCVuSVIR rarias@hut"; + root = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut"; + }; +} diff --git a/m/common/ssh.nix b/m/common/ssh.nix index 2d805bf..b8cb5c1 100644 --- a/m/common/ssh.nix +++ b/m/common/ssh.nix @@ -1,5 +1,9 @@ -{ ... }: +{ lib, ... }: +let + keys = import ../../keys.nix; + hostsKeys = lib.mapAttrs (name: value: { publicKey = value; }) keys.hosts; +in { # Enable the OpenSSH daemon. services.openssh.enable = true; @@ -11,13 +15,7 @@ ProxyCommand nc -X connect -x localhost:23080 %h %p ''; - programs.ssh.knownHosts = { - "hut".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1"; - "owl1".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv"; - "owl2".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK"; - "eudy".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG"; - "koro".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67"; - + programs.ssh.knownHosts = hostsKeys // { "gitlab-internal.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3"; "bscpm03.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS"; }; diff --git a/m/hut/ceph.nix b/m/hut/ceph.nix index 3bebe11..ebbb885 100644 --- a/m/hut/ceph.nix +++ b/m/hut/ceph.nix @@ -11,14 +11,14 @@ # modprobe command. boot.kernelModules = [ "ceph" ]; - age.secrets."secrets/ceph-user".file = ./secrets/ceph-user.age; + age.secrets.cephUser.file = ../../secrets/ceph-user.age; fileSystems."/ceph" = { fsType = "ceph"; device = "user@9c8d06e0-485f-4aaf-b16b-06d6daf1232b.cephfs=/"; options = [ "mon_addr=10.0.40.40" - "secretfile=${config.age.secrets."secrets/ceph-user".path}" + "secretfile=${config.age.secrets.cephUser.path}" ]; }; } diff --git a/m/hut/gitlab-runner.nix b/m/hut/gitlab-runner.nix index 6255005..d640de9 100644 --- a/m/hut/gitlab-runner.nix +++ b/m/hut/gitlab-runner.nix @@ -1,15 +1,15 @@ { pkgs, lib, config, ... }: { - age.secrets."secrets/ovni-token".file = ./secrets/ovni-token.age; - age.secrets."secrets/nosv-token".file = ./secrets/nosv-token.age; + age.secrets.ovniToken.file = ../../secrets/ovni-token.age; + age.secrets.nosvToken.file = ../../secrets/nosv-token.age; services.gitlab-runner = { enable = true; settings.concurrent = 5; services = { ovni-shell = { - registrationConfigFile = config.age.secrets."secrets/ovni-token".path; + registrationConfigFile = config.age.secrets.ovniToken.path; executor = "shell"; tagList = [ "nix" "xeon" ]; environmentVariables = { @@ -17,7 +17,7 @@ }; }; ovni-docker = { - registrationConfigFile = config.age.secrets."secrets/ovni-token".path; + registrationConfigFile = config.age.secrets.ovniToken.path; dockerImage = "debian:stable"; tagList = [ "docker" "xeon" ]; registrationFlags = [ "--docker-network-mode host" ]; @@ -27,7 +27,7 @@ }; }; nosv-docker = { - registrationConfigFile = config.age.secrets."secrets/nosv-token".path; + registrationConfigFile = config.age.secrets.nosvToken.path; dockerImage = "debian:stable"; tagList = [ "docker" "xeon" ]; registrationFlags = [ diff --git a/m/hut/secrets/ceph-user.age b/m/hut/secrets/ceph-user.age deleted file mode 100644 index 735afca..0000000 --- a/m/hut/secrets/ceph-user.age +++ /dev/null @@ -1,11 +0,0 @@ -age-encryption.org/v1 --> ssh-ed25519 CAWG4Q 35Ak+Mep9k5KnDLF1ywDbMD4l4mRFg6D0et19tqXxAw -Wgr+CX4rzrPmUszSidtLAVSvgD80F2dqtd92hGZIFwo --> ssh-ed25519 MSF3dg OVFvpkAyWTowtxsafstX31H/hJpNZmnOCbvqMIN0+AQ -VxjRcQmp+BadEh2y0PB96EeizIl3tTQpVu0CWHmsc1s --> ssh-ed25519 HY2yRg MJSQIpre9m0XnojgXuKQ/+hVBZNrZNGZqplwhqicpjI -CLkE52iqpoqSnbzisNjQgxTfNqKeaRl5ntcw1d+ZDyQ --> m$8`De%~-grease '85p}`by -52zMpprONcawWDDtzHdWNwFoYXErPUnVjhSONbUBpDlqAmJmD1LcAnsU ---- 0vZOPyXQIMMGTwgFfvm8Sn8O7vjrsjGUEy5m/BASCyc -|)*_DUS`r sN[֌^e+A 1G.#mW 5 ( \ No newline at end of file diff --git a/m/hut/secrets/nosv-token.age b/m/hut/secrets/nosv-token.age deleted file mode 100644 index 4b495a944de69fccbfdde7394b7f63f80e53e06e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 541 zcmZ9`O>5I&003Y?JlKO5&ts8^V!N6pY5Fz@l{U@NH0jc&P20?ar0tuJrb*f){YWl) z*kOXg*m3jV(Tl?73W6S$T~tu<=wTOOpopLoiU;+&U+_Gt4wKO7c;O&0UFW*%>F5}X zC9{a{8%fy3NfM(FI@e@VRb&+dV5l+rfKxiG#IMzB_yD4~Em5`fieHkfq#LmzIc%nkKCd{`c=McTFR8xQaNdE38CG*;;8UMzkhPmY%F-ftd?`^@9yn?YRC uXU6e`&z|t~*Q0K?-dSzA>lNkV>yzcdi#PUzt3QpeFK?_{*OxH8vilFpr_yf# diff --git a/secrets/ceph-user.age b/secrets/ceph-user.age new file mode 100644 index 0000000..f23e2ff --- /dev/null +++ b/secrets/ceph-user.age @@ -0,0 +1,21 @@ +age-encryption.org/v1 +-> ssh-ed25519 AY8zKw J00a6ZOhkupkhLU5WQ0kD05HEF4KKsSs2hwjHKbnnHU +J14VoNOCqLpScVO7OLXbqTcLI4tcVUHt5cqY/XQmbGs +-> ssh-ed25519 sgAamA k8R/bSUdvVmlBI6yHPi5NBQPBGM36lPJwsir8DFGgxE +4ZKC3gYvic6AVrNGgNjwztbUzhxP8ViX5O3wFo9wlrk +-> ssh-ed25519 HY2yRg 966xf2fTnA6Wq0uYXbXZQOManqITJcCbQS9LZCGEOh4 +Qg5echQSrzqeDqvaMx+5fqi8XyTjAeCsY/UFJX6YnDs +-> ssh-ed25519 tcumPQ e0U2okrGIoUpLfPYjIRx1V92rE3hZW13nJef+l3kBQg +LejAUKBl+tPhwocCF00ZHTzFISnwX8og8GvemiMIcyo +-> ssh-ed25519 JJ1LWg QkzTsPq9Gdh+FNz/a4bDb9LQOreFyxeTC51UNd1fsj0 +ayrlKenETfQzH1Z9drVEWqszQebicGVJve0/pCnxAE8 +-> ssh-ed25519 CAWG4Q lJLW9+dxvyoD4hYzeXeE/4rzJ6HIeEQOB1+fbhV3xw0 +T2RrVCtTuQvya9HiJB7txk3QGrntpsMX9Tt1cyXoW5E +-> ssh-ed25519 MSF3dg JOZkFb2CfqWKvZIz7lYxXWgv8iEVDkQF8hInDMZvknc +MHDWxjUw4dNiC1h4MrU9uKKcI3rwkxABm0+5FYMZkok +-> ~8m;7f-grease +lDIullfC98RhpTZ4Mk87Td+VtPmwPdgz+iIilpKugUkmV5r4Uqd7yE+5ArA6ekr/ +G/X4EA +--- Cz4sv9ZunBcVdZCozdTh1zlg1zIASjk2MjYeYfcN9eA +N $[HQ +d'7Ͳ)x9yEM7^[M+&$8tMв \ No newline at end of file diff --git a/secrets/nosv-token.age b/secrets/nosv-token.age new file mode 100644 index 0000000..31a354b --- /dev/null +++ b/secrets/nosv-token.age @@ -0,0 +1,11 @@ +age-encryption.org/v1 +-> ssh-ed25519 HY2yRg hrdS7Dl/j+u3XVfM79ZJpZSlre9TcD7DTQ+EEAT6kEE +avUO96P1h7w2BYWgrQ7GpUgdaCV9AZL7eOTTcF9gfro +-> ssh-ed25519 CAWG4Q A5raRY1CAgFYZgoQ92GMyNejYNdHx/7Y6uTS+EjLPWA +FRFqT2Jz7qRcybaxkQTKHGl797LVXoHpYG4RZSrX/70 +-> ssh-ed25519 MSF3dg D+R80Bg7W9AuiOMAqtGFZQl994dRBIegYRLmmTaeZ3o +BHvZsugRiuZ91b4jk91h30o3eF3hadSnVCwxXge95T8 +-> BT/El`a-grease W{nq|Vm )bld 2Nl}4 N$#JGB4t +oLG+0S1aGfO/ohCfgGmhDhwwLi4H +--- 2I5C+FvBG/K1ZHh7C5QD39feTSLoFGwcTeZAmeILNsI +Wo d;C._(u G#vgɝyYl9ϵ.0x޽N./tBbK:Q\T_txm_Jޞ- \ No newline at end of file diff --git a/secrets/ovni-token.age b/secrets/ovni-token.age new file mode 100644 index 0000000000000000000000000000000000000000..4378c388dd465e6d39681bbca5354136454855f8 GIT binary patch literal 553 zcmZ9HO>5Ht06;}VFiaHjvWo=4Q%IAv`B;Sw+9XZVHf_^%$r42DN18TClcq`2q$-Fg zUWCC;A}V+gL7XSSgCg_eA}A=EF!kW@Ac#2k2L=w0`vH&lUeh#S!)=d}Vc7TFls7Vw z2m{~+FbI0E(Mb~omIZk&ozzWG*F9CSg6_bM1W4%GOU@Es#E^7FX<@cepxS+|f%H1E zH$Z{VfHY#v>Mb7E*k(JT(E{i9GpK^N95V${Wei9WI84Y<>H#f&JlB%qnTJmsrO{AIh;~Oi-Pc}6?t*XnsC`9ujYVG zl+@8FLW-hW_ZXq3MxxJ9iIY^y6rrhFAi48Sl9s)T?dS~{l)tlQ(sQ73LqT$^-B7YU*maoltTlga_p@3ny=E1UU^m1B76 zJAHfh=by_TpKLs~-ah?s_dtM(()zR2m#ORbFV1fL4z#Olb@|QCb}-jyF` zPpp0+FU-6*U+|BX?|h>lY@h!$f9PcV{MGSy$kFBXouMLc-+Xv(_sslE_V8MBcj5Kc NU+43!ujyYm)PDq_#q9t9 literal 0 HcmV?d00001 diff --git a/secrets/secrets.nix b/secrets/secrets.nix new file mode 100644 index 0000000..34fb177 --- /dev/null +++ b/secrets/secrets.nix @@ -0,0 +1,13 @@ +let + keys = import ../keys.nix; + adminsKeys = builtins.attrValues keys.admins; + hut = [ keys.hosts.hut ] ++ adminsKeys; + # Only expose ceph keys to safe nodes and admins + ceph = keys.hostGroup.safe ++ adminsKeys; +in +{ + "ovni-token.age".publicKeys = hut; + "nosv-token.age".publicKeys = hut; + + "ceph-user.age".publicKeys = ceph; +} -- GitLab From 3b6be8a2fc1b52a090799feb1a7ffb65fef79352 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 4 Sep 2023 21:59:04 +0200 Subject: [PATCH 32/57] Move the ceph client config to an external module --- m/hut/configuration.nix | 2 +- m/{hut => module}/ceph.nix | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) rename m/{hut => module}/ceph.nix (93%) diff --git a/m/hut/configuration.nix b/m/hut/configuration.nix index 1be9dc3..c3e72ab 100644 --- a/m/hut/configuration.nix +++ b/m/hut/configuration.nix @@ -4,11 +4,11 @@ imports = [ ../common/main.nix + ../module/ceph.nix ./gitlab-runner.nix ./monitoring.nix ./nfs.nix ./slurm-daemon.nix - ./ceph.nix #./pxe.nix agenix.nixosModules.default ]; diff --git a/m/hut/ceph.nix b/m/module/ceph.nix similarity index 93% rename from m/hut/ceph.nix rename to m/module/ceph.nix index ebbb885..ff3fd22 100644 --- a/m/hut/ceph.nix +++ b/m/module/ceph.nix @@ -1,5 +1,6 @@ { config, pkgs, ... }: +# Mounts the /ceph filesystem at boot { environment.systemPackages = with pkgs; [ ceph -- GitLab From 98374bd303b987e56abf9c0562fe55265f7efee6 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 4 Sep 2023 21:59:56 +0200 Subject: [PATCH 33/57] Clean owl2 configuration --- m/owl2/configuration.nix | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/m/owl2/configuration.nix b/m/owl2/configuration.nix index 8022b36..ed1aab4 100644 --- a/m/owl2/configuration.nix +++ b/m/owl2/configuration.nix @@ -1,15 +1,12 @@ -{ config, pkgs, modulesPath, lib, ... }: +{ config, pkgs, ... }: { imports = [ - #(modulesPath + "/installer/netboot/netboot-minimal.nix") ../common/main.nix ]; # Select the this using the ID to avoid mismatches boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d535629"; - #programs.ssh.forwardX11 = false; - #programs.ssh.setXAuthLocation = lib.mkForce true; networking = { hostName = "owl2"; -- GitLab From a69a71d1b03f235c34332182501407db7606b2e1 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 4 Sep 2023 22:00:17 +0200 Subject: [PATCH 34/57] Warn about the owl2 omnipath device --- m/owl2/configuration.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/m/owl2/configuration.nix b/m/owl2/configuration.nix index ed1aab4..8da8f7d 100644 --- a/m/owl2/configuration.nix +++ b/m/owl2/configuration.nix @@ -14,6 +14,7 @@ address = "10.0.40.2"; prefixLength = 24; } ]; + # Watch out! The OmniPath device is not in the same place here: interfaces.ibp129s0.ipv4.addresses = [ { address = "10.0.42.2"; prefixLength = 24; -- GitLab From 9b5bfbb7a34d19c0bd445cb93c57400409bb5994 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 4 Sep 2023 22:00:36 +0200 Subject: [PATCH 35/57] Mount /ceph in owl1 and owl2 --- m/owl1/configuration.nix | 5 ++++- m/owl2/configuration.nix | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/m/owl1/configuration.nix b/m/owl1/configuration.nix index a14ab21..8255898 100644 --- a/m/owl1/configuration.nix +++ b/m/owl1/configuration.nix @@ -1,7 +1,10 @@ { config, pkgs, ... }: { - imports = [ ../common/main.nix ]; + imports = [ + ../common/main.nix + ../module/ceph.nix + ]; # Select the this using the ID to avoid mismatches boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53566c"; diff --git a/m/owl2/configuration.nix b/m/owl2/configuration.nix index 8da8f7d..0af3c40 100644 --- a/m/owl2/configuration.nix +++ b/m/owl2/configuration.nix @@ -3,6 +3,7 @@ { imports = [ ../common/main.nix + ../module/ceph.nix ]; # Select the this using the ID to avoid mismatches -- GitLab From ab55aac5ff540db4104aae26e15adac74deb5e31 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 4 Sep 2023 22:04:32 +0200 Subject: [PATCH 36/57] Remove old secrets --- m/hut/secrets.nix | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 m/hut/secrets.nix diff --git a/m/hut/secrets.nix b/m/hut/secrets.nix deleted file mode 100644 index d22add6..0000000 --- a/m/hut/secrets.nix +++ /dev/null @@ -1,11 +0,0 @@ -let - rarias = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1oZTPtlEXdGt0Ak+upeCIiBdaDQtcmuWoTUCVuSVIR rarias@hut"; - root = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb"; - hut = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1"; - default = [ rarias root hut ]; -in -{ - "secrets/ovni-token.age".publicKeys = default; - "secrets/nosv-token.age".publicKeys = default; - "secrets/ceph-user.age".publicKeys = default; -} -- GitLab From 77d43b6da9228e2be8e054a3ccb4ba1bbed33ebf Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 4 Sep 2023 22:06:20 +0200 Subject: [PATCH 37/57] Add agenix module to ceph --- m/module/ceph.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/m/module/ceph.nix b/m/module/ceph.nix index ff3fd22..ac7b3d3 100644 --- a/m/module/ceph.nix +++ b/m/module/ceph.nix @@ -1,7 +1,9 @@ -{ config, pkgs, ... }: +{ config, pkgs, agenix, ... }: # Mounts the /ceph filesystem at boot { + imports = [ agenix.nixosModules.default ]; + environment.systemPackages = with pkgs; [ ceph ceph-client -- GitLab From 02f40a8217f3d0e129d8230c6887cad769f57628 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 4 Sep 2023 22:09:40 +0200 Subject: [PATCH 38/57] Add agenix to all nodes --- m/common/agenix.nix | 9 +++++++++ m/common/main.nix | 1 + m/hut/configuration.nix | 7 +------ m/module/ceph.nix | 4 +--- 4 files changed, 12 insertions(+), 9 deletions(-) create mode 100644 m/common/agenix.nix diff --git a/m/common/agenix.nix b/m/common/agenix.nix new file mode 100644 index 0000000..3d5bb65 --- /dev/null +++ b/m/common/agenix.nix @@ -0,0 +1,9 @@ +{ agenix, ... }: + +{ + imports = [ agenix.nixosModules.default ]; + + environment.systemPackages = [ + agenix.packages.x86_64-linux.default + ]; +} diff --git a/m/common/main.nix b/m/common/main.nix index f664ef8..a5f8e45 100644 --- a/m/common/main.nix +++ b/m/common/main.nix @@ -2,6 +2,7 @@ { imports = [ + ./agenix.nix ./boot.nix ./fs.nix ./hw.nix diff --git a/m/hut/configuration.nix b/m/hut/configuration.nix index c3e72ab..fc4d2ab 100644 --- a/m/hut/configuration.nix +++ b/m/hut/configuration.nix @@ -1,4 +1,4 @@ -{ config, pkgs, agenix, ... }: +{ config, pkgs, ... }: { imports = [ @@ -10,11 +10,6 @@ ./nfs.nix ./slurm-daemon.nix #./pxe.nix - agenix.nixosModules.default - ]; - - environment.systemPackages = [ - agenix.packages.x86_64-linux.default ]; boot.binfmt.emulatedSystems = [ "aarch64-linux" "powerpc64le-linux" "riscv64-linux" ]; diff --git a/m/module/ceph.nix b/m/module/ceph.nix index ac7b3d3..ff3fd22 100644 --- a/m/module/ceph.nix +++ b/m/module/ceph.nix @@ -1,9 +1,7 @@ -{ config, pkgs, agenix, ... }: +{ config, pkgs, ... }: # Mounts the /ceph filesystem at boot { - imports = [ agenix.nixosModules.default ]; - environment.systemPackages = with pkgs; [ ceph ceph-client -- GitLab From 64e856e8b92e24009f107093293968f0322bb6c6 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 5 Sep 2023 15:03:26 +0200 Subject: [PATCH 39/57] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'bscpkgs': 'git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git?ref=refs/heads/master&rev=18d64c352c10f9ce74aabddeba5a5db02b74ec27' (2023-08-31) → 'git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git?ref=refs/heads/master&rev=ee24b910a1cb95bd222e253da43238e843816f2f' (2023-09-01) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/d680ded26da5cf104dd2735a51e88d2d8f487b4d' (2023-08-19) → 'github:NixOS/nixpkgs/e56990880811a451abd32515698c712788be5720' (2023-09-02) --- flake.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 44b83b7..97c2222 100644 --- a/flake.lock +++ b/flake.lock @@ -29,11 +29,11 @@ ] }, "locked": { - "lastModified": 1693479395, - "narHash": "sha256-/C0ZEafHZlhD0m145P5crGPcD7Ck9GGSTbiqbiAlgUo=", + "lastModified": 1693579892, + "narHash": "sha256-dcgnHtutDrKmWW8mcr1jb4JoM7kbH9Q8QSW6aA6i2dI=", "ref": "refs/heads/master", - "rev": "18d64c352c10f9ce74aabddeba5a5db02b74ec27", - "revCount": 845, + "rev": "ee24b910a1cb95bd222e253da43238e843816f2f", + "revCount": 848, "type": "git", "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git" }, @@ -87,11 +87,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1692447944, - "narHash": "sha256-fkJGNjEmTPvqBs215EQU4r9ivecV5Qge5cF/QDLVn3U=", + "lastModified": 1693663421, + "narHash": "sha256-ImMIlWE/idjcZAfxKK8sQA7A1Gi/O58u5/CJA+mxvl8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d680ded26da5cf104dd2735a51e88d2d8f487b4d", + "rev": "e56990880811a451abd32515698c712788be5720", "type": "github" }, "original": { -- GitLab From c33909f32fcb2049b06497e36f52c4de0da3c377 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 29 Aug 2023 11:12:30 +0200 Subject: [PATCH 40/57] Update email contact to jungle mail list --- web/content/access/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/content/access/index.md b/web/content/access/index.md index e639b24..4787544 100644 --- a/web/content/access/index.md +++ b/web/content/access/index.md @@ -17,6 +17,6 @@ Then, to request access to the machines we will need some information about you: 1. The salted hash of your login password, generated with `mkpasswd -m sha-512` 1. An SSH public key of type Ed25519 (can be generated with `ssh-keygen -t ed25519`) -You can send us both an email at and - with the details, or directly open a merge request in -the [jungle repository](https://pm.bsc.es/gitlab/rarias/jungle/). +Send an email to with the details, or directly open a +merge request in the [jungle +repository](https://pm.bsc.es/gitlab/rarias/jungle/). -- GitLab From dca274d0201caa2b258ae220fd320341dc963663 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 5 Sep 2023 16:24:27 +0200 Subject: [PATCH 41/57] Unlock ovni gitlab runners --- m/hut/gitlab-runner.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/m/hut/gitlab-runner.nix b/m/hut/gitlab-runner.nix index d640de9..d9fedb5 100644 --- a/m/hut/gitlab-runner.nix +++ b/m/hut/gitlab-runner.nix @@ -12,6 +12,10 @@ registrationConfigFile = config.age.secrets.ovniToken.path; executor = "shell"; tagList = [ "nix" "xeon" ]; + registrationFlags = [ + # Using space doesn't work, and causes it to misread the next flag + "--locked='false'" + ]; environmentVariables = { SHELL = "${pkgs.bash}/bin/bash"; }; @@ -20,7 +24,10 @@ registrationConfigFile = config.age.secrets.ovniToken.path; dockerImage = "debian:stable"; tagList = [ "docker" "xeon" ]; - registrationFlags = [ "--docker-network-mode host" ]; + registrationFlags = [ + "--locked='false'" + "--docker-network-mode host" + ]; environmentVariables = { https_proxy = "http://localhost:23080"; http_proxy = "http://localhost:23080"; -- GitLab From 3e347e673c477786eaadadc450abe2ac6d490866 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 7 Sep 2023 11:13:45 +0200 Subject: [PATCH 42/57] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'bscpkgs': 'git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git?ref=refs/heads/master&rev=ee24b910a1cb95bd222e253da43238e843816f2f' (2023-09-01) → 'git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git?ref=refs/heads/master&rev=6122fef92701701e1a0622550ac0fc5c2beb5906' (2023-09-07) --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 97c2222..294fa56 100644 --- a/flake.lock +++ b/flake.lock @@ -29,11 +29,11 @@ ] }, "locked": { - "lastModified": 1693579892, - "narHash": "sha256-dcgnHtutDrKmWW8mcr1jb4JoM7kbH9Q8QSW6aA6i2dI=", + "lastModified": 1694077645, + "narHash": "sha256-72bvRBhq8Q8V6ibsR9lyBE92V2EC6C6Ek3J5cOM79So=", "ref": "refs/heads/master", - "rev": "ee24b910a1cb95bd222e253da43238e843816f2f", - "revCount": 848, + "rev": "6122fef92701701e1a0622550ac0fc5c2beb5906", + "revCount": 860, "type": "git", "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git" }, -- GitLab From 6db5772ac4cba7c6dda162fa7ed162f7fe519dc0 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 8 Sep 2023 13:21:37 +0200 Subject: [PATCH 43/57] Add IB and IPMI node host names --- m/common/net.nix | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/m/common/net.nix b/m/common/net.nix index 09765c4..e542e5f 100644 --- a/m/common/net.nix +++ b/m/common/net.nix @@ -32,8 +32,8 @@ # Node Entry for node: mds01 (ID=72) 10.0.40.40 bay mds01 mds01-eth0 - 10.0.42.40 mds01-ib0 - 10.0.40.141 mds01-ipmi0 + 10.0.42.40 bay-ib mds01-ib0 + 10.0.40.141 bay-ipmi mds01-ipmi0 # Node Entry for node: oss01 (ID=73) 10.0.40.41 oss01 oss01-eth0 @@ -42,18 +42,18 @@ # Node Entry for node: oss02 (ID=74) 10.0.40.42 lake2 oss02 oss02-eth0 - 10.0.42.42 oss02-ib0 - 10.0.40.143 oss02-ipmi0 + 10.0.42.42 lake2-ib oss02-ib0 + 10.0.40.143 lake2-ipmi oss02-ipmi0 # Node Entry for node: xeon01 (ID=15) 10.0.40.1 owl1 xeon01 xeon01-eth0 - 10.0.42.1 xeon01-ib0 - 10.0.40.101 xeon01-ipmi0 + 10.0.42.1 owl1-ib xeon01-ib0 + 10.0.40.101 owl1-ipmi xeon01-ipmi0 # Node Entry for node: xeon02 (ID=16) 10.0.40.2 owl2 xeon02 xeon02-eth0 - 10.0.42.2 xeon02-ib0 - 10.0.40.102 xeon02-ipmi0 + 10.0.42.2 owl2-ib xeon02-ib0 + 10.0.40.102 owl2-ipmi xeon02-ipmi0 # Node Entry for node: xeon03 (ID=17) 10.0.40.3 xeon03 xeon03-eth0 @@ -67,8 +67,8 @@ # Node Entry for node: xeon05 (ID=19) 10.0.40.5 koro xeon05 xeon05-eth0 - 10.0.42.5 xeon05-ib0 - 10.0.40.105 xeon05-ipmi0 + 10.0.42.5 koro-ib xeon05-ib0 + 10.0.40.105 koro-ipmi xeon05-ipmi0 # Node Entry for node: xeon06 (ID=20) 10.0.40.6 xeon06 xeon06-eth0 @@ -77,13 +77,13 @@ # Node Entry for node: xeon07 (ID=21) 10.0.40.7 hut xeon07 xeon07-eth0 - 10.0.42.7 xeon07-ib0 - 10.0.40.107 xeon07-ipmi0 + 10.0.42.7 hut-ib xeon07-ib0 + 10.0.40.107 hut-ipmi xeon07-ipmi0 # Node Entry for node: xeon08 (ID=22) 10.0.40.8 eudy xeon08 xeon08-eth0 - 10.0.42.8 xeon08-ib0 - 10.0.40.108 xeon08-ipmi0 + 10.0.42.8 eudy-ib xeon08-ib0 + 10.0.40.108 eudy-ipmi xeon08-ipmi0 ''; }; } -- GitLab From 77cb3c494ec530c3cf8f33376d2c620863eef553 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 8 Sep 2023 13:31:23 +0200 Subject: [PATCH 44/57] Poweroff idle slurm nodes after 1 hour --- m/common/slurm.nix | 47 ++++++++++++++++++++++++++++++++++++++++-- m/hut/slurm-daemon.nix | 4 ---- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/m/common/slurm.nix b/m/common/slurm.nix index b02a914..988e1d8 100644 --- a/m/common/slurm.nix +++ b/m/common/slurm.nix @@ -1,6 +1,33 @@ -{ lib, ... }: +{ pkgs, lib, ... }: -{ +let + suspendProgram = pkgs.writeScript "suspend.sh" '' + #!/usr/bin/env bash + exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log + set -x + export "PATH=/run/current-system/sw/bin:$PATH" + echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log + hosts=$(scontrol show hostnames $1) + for host in $hosts; do + echo Shutting down host: $host + ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power off + done + ''; + + resumeProgram = pkgs.writeScript "resume.sh" '' + #!/usr/bin/env bash + exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log + set -x + export "PATH=/run/current-system/sw/bin:$PATH" + echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log + hosts=$(scontrol show hostnames $1) + for host in $hosts; do + echo Starting host: $host + ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power on + done + ''; + +in { systemd.services.slurmd.serviceConfig = { # Kill all processes in the control group on stop/restart. This will kill # all the jobs running, so ensure that we only upgrade when the nodes are @@ -9,6 +36,7 @@ # https://bugs.schedmd.com/show_bug.cgi?id=2095#c24 KillMode = lib.mkForce "control-group"; }; + services.slurm = { client.enable = true; controlMachine = "hut"; @@ -18,6 +46,11 @@ "hut Sockets=2 CoresPerSocket=14 ThreadsPerCore=2" ]; + partitionName = [ + "owl Nodes=owl[1-2] Default=YES MaxTime=INFINITE State=UP" + "all Nodes=owl[1-2],hut Default=NO MaxTime=INFINITE State=UP" + ]; + # See slurm.conf(5) for more details about these options. extraConfig = '' # Use PMIx for MPI by default. It works okay with MPICH and OpenMPI, but @@ -37,6 +70,16 @@ # Enable task/affinity to allow the jobs to run in a specified subset of # the resources. Use the task/cgroup plugin to enable process containment. TaskPlugin=task/affinity,task/cgroup + + # Power off unused nodes until they are requested + SuspendProgram=${suspendProgram} + SuspendTimeout=60 + ResumeProgram=${resumeProgram} + ResumeTimeout=300 + SuspendExcNodes=hut + + # Turn the nodes off after 1 hour of inactivity + SuspendTime=3600 ''; }; } diff --git a/m/hut/slurm-daemon.nix b/m/hut/slurm-daemon.nix index e6ab227..e7fab8b 100644 --- a/m/hut/slurm-daemon.nix +++ b/m/hut/slurm-daemon.nix @@ -3,9 +3,5 @@ { services.slurm = { server.enable = true; - partitionName = [ - "owl Nodes=owl[1-2] Default=YES MaxTime=INFINITE State=UP" - "all Nodes=owl[1-2],hut Default=NO MaxTime=INFINITE State=UP" - ]; }; } -- GitLab From 033a1fe97b609a0dce06f54442d64f9651fc398c Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 8 Sep 2023 17:20:32 +0200 Subject: [PATCH 45/57] Block ssfhead from reaching our slurm daemon --- m/common/net.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/m/common/net.nix b/m/common/net.nix index e542e5f..2057143 100644 --- a/m/common/net.nix +++ b/m/common/net.nix @@ -23,7 +23,14 @@ allowedTCPPorts = [ 22 ]; # FIXME: For slurmd as it requests the compute nodes to connect to us - allowedTCPPortRanges = [ { from=1024; to=65535; } ]; + #allowedTCPPortRanges = [ { from=1024; to=65535; } ]; + + extraCommands = '' + # Prevent ssfhead from contacting our slurmd daemon + iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-log-refuse + # But accept traffic to slurm ports from any other node in the subnet + iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817:6819 -j nixos-fw-accept + ''; }; extraHosts = '' -- GitLab From 7050c505b5dcd0113a9d88db1fe75b5c5b70f473 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 8 Sep 2023 17:51:37 +0200 Subject: [PATCH 46/57] Allow only some ports for srun --- m/common/net.nix | 2 ++ m/common/slurm.nix | 3 +++ 2 files changed, 5 insertions(+) diff --git a/m/common/net.nix b/m/common/net.nix index 2057143..57f15be 100644 --- a/m/common/net.nix +++ b/m/common/net.nix @@ -30,6 +30,8 @@ iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-log-refuse # But accept traffic to slurm ports from any other node in the subnet iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817:6819 -j nixos-fw-accept + # We also need to open the srun port range + iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 60000:61000 -j nixos-fw-accept ''; }; diff --git a/m/common/slurm.nix b/m/common/slurm.nix index 988e1d8..5404d0c 100644 --- a/m/common/slurm.nix +++ b/m/common/slurm.nix @@ -80,6 +80,9 @@ in { # Turn the nodes off after 1 hour of inactivity SuspendTime=3600 + + # Reduce port range so we can allow only this range in the firewall + SrunPortRange=60000-61000 ''; }; } -- GitLab From 7ddd1977f3a69066cc287966054b35746307019e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 8 Sep 2023 18:13:04 +0200 Subject: [PATCH 47/57] Make exporters listen in localhost only --- m/hut/monitoring.nix | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/m/hut/monitoring.nix b/m/hut/monitoring.nix index 8a8646a..b455c1d 100644 --- a/m/hut/monitoring.nix +++ b/m/hut/monitoring.nix @@ -19,6 +19,7 @@ enable = true; port = 9001; retentionTime = "1y"; + listenAddress = "127.0.0.1"; }; systemd.services.prometheus-ipmi-exporter.serviceConfig.DynamicUser = lib.mkForce false; @@ -48,13 +49,18 @@ user = "root"; configFile = ./ipmi.yml; #extraFlags = [ "--log.level=debug" ]; + listenAddress = "127.0.0.1"; }; node = { enable = true; enabledCollectors = [ "systemd" ]; port = 9002; + listenAddress = "127.0.0.1"; + }; + smartctl = { + enable = true; + listenAddress = "127.0.0.1"; }; - smartctl.enable = true; }; scrapeConfigs = [ -- GitLab From ec9be9bb6227431d822bd146093b352a59fc1db2 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 8 Sep 2023 18:22:48 +0200 Subject: [PATCH 48/57] Remove unused large port hole in firewall --- m/common/net.nix | 4 ---- 1 file changed, 4 deletions(-) diff --git a/m/common/net.nix b/m/common/net.nix index 57f15be..d90f8ba 100644 --- a/m/common/net.nix +++ b/m/common/net.nix @@ -21,10 +21,6 @@ firewall = { enable = true; allowedTCPPorts = [ 22 ]; - - # FIXME: For slurmd as it requests the compute nodes to connect to us - #allowedTCPPortRanges = [ { from=1024; to=65535; } ]; - extraCommands = '' # Prevent ssfhead from contacting our slurmd daemon iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-log-refuse -- GitLab From 19a451db777efe1c7abbd73a4fc74a8eb2978055 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 8 Sep 2023 19:01:57 +0200 Subject: [PATCH 49/57] Add encrypted munge key with agenix --- m/common/slurm.nix | 13 ++++++++++++- secrets/munge-key.age | Bin 0 -> 2007 bytes secrets/secrets.nix | 5 +++-- 3 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 secrets/munge-key.age diff --git a/m/common/slurm.nix b/m/common/slurm.nix index 5404d0c..22ffae6 100644 --- a/m/common/slurm.nix +++ b/m/common/slurm.nix @@ -1,4 +1,4 @@ -{ pkgs, lib, ... }: +{ config, pkgs, lib, ... }: let suspendProgram = pkgs.writeScript "suspend.sh" '' @@ -85,4 +85,15 @@ in { SrunPortRange=60000-61000 ''; }; + + age.secrets.mungeKey = { + file = ../../secrets/munge-key.age; + owner = "munge"; + group = "munge"; + }; + + services.munge = { + enable = true; + password = config.age.secrets.mungeKey.path; + }; } diff --git a/secrets/munge-key.age b/secrets/munge-key.age new file mode 100644 index 0000000000000000000000000000000000000000..ead42c816eb6d3ba2973e4dcb0a62c55e6f5ac8b GIT binary patch literal 2007 zcmZY8`CpBR0>^Pni%ycsD6Z5o*-yvWH{;cK&c1KY*`5>XEYEq)zVC^`Wv)z?q2Xml zC3h|{O=ud0o1LMCVr)ZPTM=nO!^OB>ulpN+!TX2L_iI34EbKPK_1{#3qaZv>AiSj5E*c`SS>^ufciW<#Oghn8`xNZ&00{h9nkTWFl zkYsc|9YwO+nHUEKv{0xL37!H&T%|f9lrvpohFpXTf;Jn-v#Q80jvDw-3m|~O1)u?+ z5aQuOkld)0%dKGth9xjFs6MS%>i|>;p3gJ-DMYT)rDl;)Y<`^UjIrDVHibeG(tLO| zksydk06tD*khsJUzD{q(+5`D>O!pLWUc$Gwgbyj0XgmHiGi~;4mMrL;`Y2W(2J^(rGZ2 zkJzo)Xk!U4xB1#<*hJlD7$Z;`gdWVk@HhHLGz$JCr5R`sW z7rcR9iLJ{0GQQ|eSNioS%fQCBeJuXtftqcu@a6b6?C}fp3mYekC!ZhFoCuD%#$7*Y z#wpS9VZRsrc;(x4PDJx`-jDZG#iM_%iSG68%HCA`?gnSgjT*tNX-%81W@ia!Tq*7S zT+bfytb6%)FCUMcNVFY2q)SlZ2X8fepC*}RkyWl4xjUfS?U>M+h`hr6_rd(_e=_rK z7Y(wbxW)&Z+LS=|+b{izi+2@X=1(ifo=f<3WqRhT(rU$0?6`Y#Q&zHGd*%~|#R7Vz zRB&%*ZR_al^?uCy`eo%E_+xX@a&jumep|id=as9U?H~oO-4c;kzT1EHbap}2g9DeI ztW~8%tP6q6(v$=Rf7^+18#ee~wv}hR<@6uqwA9s2X_qDMyZ=nIwY^m3lRXUXYV+M) zcf0H0p0eRZ+YJ>zvo59Z?sqrar)Hazx^+@;`@7htt{W%&X85?3=M+!Yc&pB=S~}$X zmq*YLoAIe3md#kCF>J4Sp>!D!)I)EBU7aaN>Xsk=%3PGcYWhjrrXT*fx_I~kZR?{8 zx&8A!?THnA3tx?&y18}w`@d*j98SJQSbKKE;EN3Z=+-O7v0p!*gHvac0ndg>wI8CIRWwef!y2ImJ+FR*cutStO%*k2xZv&=+gUR1!PVA@{=i=`D<=U|o%F(xR8#xtF&+x~+ zrB+r_+ts`g?PEF)RcdcOzw?xMYT-_x=~Z1OTynf%RO|Ib4JWdZdDpWd$*$N}yR`73 z4VKk9I;saJC<%k^d$+fudU5!9V=vd`EH7BK;P&df%TdCGEBQjqPboMzAm$zb`>HYv8BB s{p$z+e{uH`#IZ<`Ht<{if!xgv8cxFh0FV+i@Bjb+ literal 0 HcmV?d00001 diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 34fb177..9dce058 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -3,11 +3,12 @@ let adminsKeys = builtins.attrValues keys.admins; hut = [ keys.hosts.hut ] ++ adminsKeys; # Only expose ceph keys to safe nodes and admins - ceph = keys.hostGroup.safe ++ adminsKeys; + safe = keys.hostGroup.safe ++ adminsKeys; in { "ovni-token.age".publicKeys = hut; "nosv-token.age".publicKeys = hut; - "ceph-user.age".publicKeys = ceph; + "ceph-user.age".publicKeys = safe; + "munge-key.age".publicKeys = safe; } -- GitLab From 75b0f48715b88fe06c159eb33193517e9663738b Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 12 Sep 2023 12:19:43 +0200 Subject: [PATCH 50/57] Serve the nix store from hut --- m/hut/configuration.nix | 1 + m/hut/nix-serve.nix | 16 ++++++++++++++++ secrets/nix-serve.age | 12 ++++++++++++ secrets/secrets.nix | 1 + 4 files changed, 30 insertions(+) create mode 100644 m/hut/nix-serve.nix create mode 100644 secrets/nix-serve.age diff --git a/m/hut/configuration.nix b/m/hut/configuration.nix index fc4d2ab..9187812 100644 --- a/m/hut/configuration.nix +++ b/m/hut/configuration.nix @@ -9,6 +9,7 @@ ./monitoring.nix ./nfs.nix ./slurm-daemon.nix + ./nix-serve.nix #./pxe.nix ]; diff --git a/m/hut/nix-serve.nix b/m/hut/nix-serve.nix new file mode 100644 index 0000000..35ccd72 --- /dev/null +++ b/m/hut/nix-serve.nix @@ -0,0 +1,16 @@ +{ config, ... }: + +{ + age.secrets.nixServe.file = ../../secrets/nix-serve.age; + + services.nix-serve = { + enable = true; + # Only listen locally, as we serve it via ssh + bindAddress = "127.0.0.1"; + port = 5000; + + secretKeyFile = config.age.secrets.nixServe.path; + # Public key: + # jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0= + }; +} diff --git a/secrets/nix-serve.age b/secrets/nix-serve.age new file mode 100644 index 0000000..f59c323 --- /dev/null +++ b/secrets/nix-serve.age @@ -0,0 +1,12 @@ +age-encryption.org/v1 +-> ssh-ed25519 HY2yRg d144D+VvxhYgKtH//uD2qNuVnYX6bh74YqkyM3ZjBwU +0IeVmFAf4U8Sm0d01O6ZwJ1V2jl/mSMl4wF0MP5LrIg +-> ssh-ed25519 CAWG4Q H4nKxue/Cj/3KUF5A+/ygHMjjArwgx3SIWwXcqFtyUo +4k5NJkLUrueLYiPkr2LAwQLWmuaOIsDmV/86ravpleU +-> ssh-ed25519 MSF3dg HpgUAFHLPs4w0cdJHqTwf8lySkTeV9O9NnBf49ClDHs +foPIUUgAYe1YSDy6+aMfjN7xv9xud9fDmhRlIztHoEo +-> vLkF\<-grease +3GRT+W8gYSpjl/a6Ix9+g9UJnTpl1ZH/oucfR801vfE8y77DV2Jxz/XJwzxYxKG5 +YEhiTGMNbXw/V7E5aVSz6Bdc +--- GtiHKCZdHByq9j0BSLd544PhbEwTN138E8TFdxipeiA +G$SRATh]n8,Hs=p'+j9):)Y8I8:olZ3PMF;rY$yLٜΜUs16ǾLb \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 9dce058..95b43ac 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -8,6 +8,7 @@ in { "ovni-token.age".publicKeys = hut; "nosv-token.age".publicKeys = hut; + "nix-serve.age".publicKeys = hut; "ceph-user.age".publicKeys = safe; "munge-key.age".publicKeys = safe; -- GitLab From 10ca572aec0f15cb177493291b924de59826ab23 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 12 Sep 2023 16:39:45 +0200 Subject: [PATCH 51/57] Enable fstrim service --- m/common/fs.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/m/common/fs.nix b/m/common/fs.nix index 60fccef..10339d0 100644 --- a/m/common/fs.nix +++ b/m/common/fs.nix @@ -6,6 +6,9 @@ fsType = "ext4"; }; + # Trim unused blocks weekly + services.fstrim.enable = true; + swapDevices = [ { device = "/dev/disk/by-label/swap"; } ]; -- GitLab From 8d449ba20c23cc959b8ab1c99a903b2e4a7adba5 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 12 Sep 2023 18:13:38 +0200 Subject: [PATCH 52/57] Add update post to website --- web/content/posts/2023-09-12/_index.md | 69 +++++++++++++++++++++++++ web/content/posts/2023-09-12/power.png | Bin 0 -> 58985 bytes 2 files changed, 69 insertions(+) create mode 100644 web/content/posts/2023-09-12/_index.md create mode 100644 web/content/posts/2023-09-12/power.png diff --git a/web/content/posts/2023-09-12/_index.md b/web/content/posts/2023-09-12/_index.md new file mode 100644 index 0000000..eff289a --- /dev/null +++ b/web/content/posts/2023-09-12/_index.md @@ -0,0 +1,69 @@ +--- +title: "Update 2023-09-12" +author: "Rodrigo Arias Mallo" +date: 2023-09-12 +--- + +This is a summary of notable changes introduced in the jungle cluster in the +last months. + +### New Ceph filesystem available + +We have installed the latest [Ceph filesystem][1] (18.2.0) which stores three +redundant copies of the data so a failure in one disk doesn't cause data loss. +It is mounted in /ceph and available for use in the owl1, owl2 and hut nodes. + +[1]: https://en.wikipedia.org/wiki/Ceph_(software) + +The throughput is limited by the 1 Gigabit Ethernet speed, but should be +reasonably fast for most workloads. Here is a test with dd which reaches the +network limit: + +```txt +hut% dd if=/dev/urandom of=/ceph/rarias/urandom bs=1M count=1024 +1024+0 records in +1024+0 records out +1073741824 bytes (1,1 GB, 1,0 GiB) copied, 8,98544 s, 119 MB/s +``` + +### SLURM power save + +The SLURM daemon has been configured to power down the nodes after one hour of +idling. When a new job is allocated to a node that is powered off, it is +automatically turned on and as soon as it becomes available it will execute the +job. Here is an example with two nodes that boot and execute a simple job that +shows the date. + +```txt +hut% date; srun -N 2 date +2023-09-12T17:36:09 CEST +2023-09-12T17:38:26 CEST +2023-09-12T17:38:18 CEST +``` + +You can expect a similar delay (around 2-3 min) while the nodes are starting. +Notice that while the nodes are kept on, the delay is not noticeable: + +```txt +hut% date; srun -N 2 date +2023-09-12T17:40:04 CEST +2023-09-12T17:40:04 CEST +2023-09-12T17:40:04 CEST +``` + +### Power and temperature monitoring + +In the cluster, we monitor the temperature and the power draw of all nodes. This +allows us to understand which machines are not being used and turn them off to +save energy that otherwise would be wasted. Here is an example where some nodes +are powered off to save energy: + +![power](./power.png) + +We also configured the nodes to work at low CPU frequencies, so the temperature +is kept low to increase the lifespan of the node components. Towards these +goals, we have configured two alerts that trigger when the CPUs of a node +exceeds the limit temperature of 80 °C or when the power draw exceeds 350 W. + +By keeping the power consumption and temperatures controlled, we can safely +incorporate more machines that will only be used on demand. diff --git a/web/content/posts/2023-09-12/power.png b/web/content/posts/2023-09-12/power.png new file mode 100644 index 0000000000000000000000000000000000000000..4987d48cb6d3b865e3fa29c609b7ed18db7e580d GIT binary patch literal 58985 zcmd?RWn7hAw?3+bN=bvT2np#B=~fz)ZlqgSbhk)%hjb`ii*BSlrMtVkIrq}%dH4JH z-+RCR^W}UwpWw&3*SzN(d5v+6Nq~%$C<-Dz;-g28P{hTAnR*#Ds{MVnv}(bj<^AXF z^wo8y!48Ha-sI#pbyE!v4nLXi98n{OA8zX$?zwFS0*;T@dIt~qT|cL$5?8;534TQO zF6=#?fL8$XlwfoE~)b5(&p%5jtT$g`cGjPwHih=jN$s;KMQp@n1@UbwaUys z{bL*2tZ*q!jye<69xs(q_@+u|+h#fcd_)RP1swmAfrs|beH}k5nJlSCaC-d5+dp%n zYeRIT>JAJ3>kz+>)Jyh5xAmA}75?8v0Ts-;(C2nv&b^j0-G4hc^gDE6kAhHo6T@-; zDZU@i;xQlNfY!^A1_S>gtk38rJTMaNzFlwlz-?4D^Zd0BPcB|CVHIjvrG%^z z5`AShQ1H&xg;<@8(}v!O^}Gxy(}D(y)%=#RvkDm8>~8kfZuF7B)Z z37;6v6o6zr+kJI&s!9KlaN3K<-*p9TtKC~ErYvH5u(2 z2>jPm#Tr~BhhlzPzs^FdqNL#8a%<-q!tdC_@$VboU6f&Zr*zPEd4lvzgZ)mz}}+bUP>}f1*bo zMB+c+^zfMnK5#KEbZ;nYO#0X%Ln>%pG9nJH|2(;_#H63A?`lUHi(28e447OZ9<9** z8Xa#Wn~VpMmJLx^{|{dv@cb+XA@S}giO5`wzQ1e$MI%1fXcyT+R1D{kC8bk+Eh^R4 z*(qv%JU2VL2(eASRW+W-F`ugq$vNF?LPO3#CSt>AA-U@dXbo%^VaK2mc;Wh7f`xxZd$fjw{JO zDvaCVVEJ^b?D%x$xW)r3D5Rz=EAzER6C#zsrG9@BUKC~UyZ%}fbXui8?coCZ-Pw>- zZrhGh;gfJ~Whtlj_ep%WqZynMG5O@}vdK>*#kA?#eM3H$Sof5A6<-n&vq!1--5M>hMh zZf7X=2QYX6f@H>h|K&T<2C4ME`X;~evjFcE<10PpTTUAsd$1Aa@fQ+EM>i`(UdEmAI#AEi2dL} zTpgq&YTC${&a|LQ*f1?As*_la3%9SL=TFq(fdV(ohn=J4Z3YKk6$nB93QORTvKnwN@qa z`5ymEH&!7|Yi$Q@+k)@CyhGU!FQ*zU(_uL5Z{b2XQN6A&-g!COseq}ni2v~D*BC(D zZ`~Ka`|<@%GbIF@R#{eTvc|U7k~sJ0x92$5N~cHR0`2$LH^dxn=Se*DIbtw^{?S&L z6hD0*cmuKNKPX4Y^&ZxkxUx|M;|07ThiRK_>lR|fK)-VD!sOUH6j*qofHAP+M}4~w z@p3_@lmc^kUitG}1--;H5W{MG@h+%(oz+iZZ@$3+HK2>S(QM~IJe|a+yJ^Ys3O^I=#aKF@0dxjcQ`-}=v>EyM*E z9T?)zkkl?B6tQCe?*<7x__-a56Q^OfjqQi_puj@oqegq(lBd_yg>@{AlU~Z6ckmk@gr##14ugh zTO`(vIF_bw^gsLip@X1xZz-sUtO`b#D!WrrTHs2vUn|8cpDjl1~C`)L8&kV6Gv3n%~F z3oFu~Dp+XJ+qJa0kp{n8FJj%$NtF|yL6hA=grJ|3tV0jjY{+7AW~Y7%sMiKq5pR(4 zuVq#7Zc43L6es7w7NRx-{zvh^0m&RUHkrO|bQ~uE8|^5Y9D0`KlW@}8GZ`0?ncH%1 z=)#>~BNE)nW0V4F<`xa>=3$TBpkrfslk)pqNm@!BjRgdf*p=)vGs8*Dx;*QgC$rDy zIh{YO*FboTAn6fDz~eB+>Fc+pZ_W+YjPptwrshM}lh9X~2ET5I91aZYy%`Ky;|xLx z2?iejE!h|dBdw^s&tCtzJ|Y#0iK3j$>45Va>+=AQKT^x#Y$^zLQ{_yc z0b5j7{V7;IrvY6-vK8QQxFLc5D+7bw-kxCv9hb5O!v8@~z~i5BOa>Tf^|X{ivk+$z z_x6kj@R*2BlAO%p4hVQDPG(h702y^T{;jG1??tLi&jutVe;qQ>mHmdBt_WVxlk=Sr zj-GX&(l`dQYy(02rqGc0sc%nKnCb}r;qaVE+jBklib z6HsRZ+|O{Sh6Vz$Jdz{ec89_22t{lwFr4$fEHpL3qEUGSEobW33S}}JWv_1LTrPB` zPrs^E8uOhKb9h5ADRjn;+da)pCrrJI01UDu0Pi)GcJR4bEQ*WO-DEhpQNuL|0DqYR zHKkn%ub81lqjIrUrv6kQhrQ*2vOxQA$7&RcD4F@JqoI>AVaTK6ME>YMEs}W<_zP1C z4TQ;K2_xxtgV<>sO~zY0N3$@Gr}>i2fp2%F0>x^jEuzW&WSt%nmobEyGHi&tgMi2N z@|m0S*)uRVQ2LNdB|R%tyMF=R`&en-*~4neq_<7JJ6|TP*^7})Vme{*BN{;uV%8}I z)Rk)Y^P4TYt;R?k=B~d2kQJEOoE+-S9Os(h7>o)Ut)W^dw zb-~DXYm_hkRa>|4*0G}H=@0W@0^aB1Q55eA6UJmyigXBP>tF@GeUtbK)1S!3Ht^*k zv`@Fy5|aZA0~K54qQ}$a|2dq>zhV=DwPH1)GCL>cbr(=h?D-rGFM-YIdwl9(Rcd8? zAk`L(CttL-)*DBql;^tG-4T^NI8^3R^~3Ht!5if6$X7Sp`hp5M!ju#(SM?4}o-bHk zpS>{@3*$mb&_4X!cv&dQ7>vhBH{?M^Zt3psPOf{~B`tJ&@Dw)Kk`5|QtbI)e!ORI7@s z(1iJJtzvlc1R=8LLtX>f^?VuEpwry;Jht-`cUD~;1;2)(?#fn~k>k-*Vi3&j-TH{t)GXO4YKkGNHubN+LTPPRib9vK%Gmri5J6Sl4@UvO5c;!3|&>PJG!$9oa~CiFKX^meHjKoBhA z&iyb+0*!_OYTJCTew%{L27?K^aqtv#mbeuOsVpJC6NZi9J@sR?eTale&b7@koa4>q z;>Ppj;d*hhEsG_=2ZoQi_|uj5#Cb}&aHzFkSY75}z%$iSi2mVgi%Vq26A+}HSO#io zHX%;uU5@GU1MkRJfqHbw;P~{C$=6=k1k6v%c*;Xn_u^z7=Wf$?{|M1ZPW#3K0Y?e} zD8t%7+3FX+B1#s+g=G{`8`|!%sVa1*qo^IL(Vq!)m=ALIByZ< zxDT>yH*6F!2&*`$?RU}`BC5o3D+RLL`Eu;nYbbc=ZxeOt_krdv#UK;@+ufkfMSRz@)iA&f91LrMwQvxKD(svQRSQr*W3~~S7df^fH`-|)z zRJ3b>F+0#4th8P{QUibnXb%4V0VR{@Utd&3fRMC^Jf~tNRi_oIw4?n_P&_k zB%cUKj)k~!eY2>)k|RHmWrdP|V*;MihHp0(2U57X8R3&J26V}=;7Ai2V)#tz&F6!u z{_)mC9{@~5s|8E5!JxlJa$-%8vODRj+;5!m@WZ^DNNpF%QTGR*D@iy4@TMi&y1nNN zidkq+l(%O4TL5q)LEoY4)hlEULOVm zFYf_qI;<{3=qfPaRDYf;m=(_0qmCCe@35TEpo zp#B@PH317{Kw%hwtAN9eG?|0_a%@9PRIZNDaX=}93_8cn!~RS7n5{J43t>JV=(F*s zb_=mQpm6rT)i;g^?_byphyl*;&U>%f$l27HN|#rGZOu(o=}m-~IUbxn&jg=lPSg{Q zN$ZzY$;~>>)g#FFVAD=O(Q#p))^{6rD2}G8co7ux7Oqb~GbBrXj;FM+?&tLbNkSAz zL@@zN2LibqRp)VZA$HK5Z@HxTsi2>qH^~$6?7)E4c>%306bF<|l=NeC+lG}-a^1unu+dD}C19bxf5K*H z<=`vQd%{Ws1-Lb;L~0E$P3y8aXEWQ|HUs5JT}y}US_iwLN*|r8&xGPt;B2$9%AaCF zrKa#ddkH1TwG_QuicPK9{ELdLp5BMGTyzYOd+Obgt|;nJqi(lJWn<)P_q?m>#qyk-t~o=io|vnA+x=fFe#cpf0@w+EeM%5L&o&9g z)yEHkX_Fk*8;|TYXMAmse(KG^qtcm_D6Ra|F9PZvac+EU=u!V!@325MYFy5f`a4?o z9}jKAtCp=*bk}e{r%+BMOpgi~!;M|(o)Hy8BSVi()$lh!nb`S%BRqV@d{2(ymFZwo z?D1Zco+XI?L97hL6LmXtgsk!S_jm0-FNMBl*Qjcago=1i^e4XlXO;CUV&T*OU5@>- zUnLK5>^7VCpZQJuf`+<92I)1z`4venFRyg_eX!qnn~$h;pC+U%-3hU|(85PU*Zi&y zvF3$F`JP)vE{9*%A{XQ%d(i3^W@!^t1i>|B^|-r60oW#|x$!q2OQx2IA4< z7>*+I>zk>Zn>S`K;JYti9n{b3wXThADm%lCv`L!kD14Dlb{aUS5EERt#AyLaKA`Mn z1khF^AlKCwzq0~V>AWv*7VjRLDY1(v#BE;Yk1t!+A)C!fFYvxg-J6pkkypIDyw+wG z&fJ#!_O$fXy;9iL*Nm_(eGAsl$F)u;xxS1_g}uP)A7koF*3EE$f=WiOLS`GtSoewv;_pGxOwn+%gcur%mMYib>$(#aF+F@YD`D*F=`Mq=CwO&@h+9(^aVm>3^(_ znf2eT&WZ!Vcw82)oVlgI9`<3!{b-4^)?yp3cu+I`qf5}o zY)P96ARRv&HQrs4wGLFkD(QleL$3XaK6yJ zuuQ)`b|yp5hLf5S6gE$6z(Ol~K0oVeqPml2dwqv6uaM2dH==p7=3@X=RhVS~MtNbKo9b^7-tvpg<-8jZtoH z=jn%GQT@^?{I^cfSl5c1KMqA6g*_UqCu~s9$bsOef)Bez$T_E~PmXaqh+g=1Mn_x7 z-i_r+vvuqf2d+Zoo5#8pE3v2Akg}))6Q;I5xn}hJ{1pty{;z@o@5{QQTl_2nLcbfv z18pm>ioEuYwB&|xdIaH=>)nOZwg$4U2s%o9w+Z$C8>R70Ek=ciS(h$donja!$A^Ao zEYcD8vde21x#&+0=eN^wJwH(fJ@9aZo*pr$38$p8^2x4$`W0Bw|Chkp{jY`$QMiVy z_46R<{>~U9+>(N3=P}*P*4o%;y-a+*E7xeVK|UTIeMnTMWTTYxhe0eiAqtoL$G@DA z)fHl*0d>N>Sw&`xUrvahf7JaX6)Y4P6;c)^)x5Hcq3-NVgtSxSn_t#+AZ+}qJZ>y3 znz#PE3T0_+2-R0x4L6#Ziq0@hIs!dgs@vy;(AE`E5TDb-i*YS=pCIXh z-k~blHLaTSY+*!|+f=7x90P7EnMsJ}kio#?biz1!RUy z^Q}sSA>n-zCsJLdc{J${D1)d`@{BoK@=IA1vs%U8XP`TuVc(B`cy=aSBQZ97+?kd0 zu`UbDZ8NS8o~y_v#8d zh)K?*LnGTGF6RrhN+%;QH%Jj5)RnL7(6;m>G4KB?Nv9BHlpM zAwB1;O|myrA5qeNAtBJh{ra}F6giwmVv z&(VIpd;R=HFiWodN;Xda;rZT2ELsgB?4FnyN-9YZdt3fQv1}<=n1H+bf;K&)^~E)V$R448#;5o2l{II_Pv%{n zfil5wi-}kM6#eUN;S(8Xo*g}G&$EMeL-Vbb`|E);rl{Kr5RoZTr@O-_?SWVw)?tb& zW#+s*u?*3s4?uGQgLDRsJ@lbttv|7fO1++CAc;pH`-RQn;S$;D))*ODHc6`}iPf@@ z@z`;v_N3WVEg|hhL3?m+REf(VpA-H2B;Nv!rpNLW!R?iV+z{P0lPi}@q#%?``o-!a zXkLdbH?TV_AMU|&Tx_y*D1QyTG)Dl~LRwn`lYs#KBf&rzPqY2T+7QYEA|1;brvGe? zXh>L~J1GtKOj=BLe;j&z!@>AK;^WoWYy#EB{t(&?Y4~eJ*ZW(%ucaGp-^(bOtd3bY z>NqmxZz?@G5FVcMBPLIMm%q`T42TLES<7GC^qhOa=}Z7H&0o}5^D}u6lD%9U-ug!? zP|K$|{p?8By}#_1cUoTB-6b!!fxmRmVG19qHN*iuzNeR%$S6-ii(p`2u(V;`=e|q@o2ml~{oG_j;XGT_mj?KpdULlN18kQaGaJn+ z&I|f?s)ZZO<~T0-7@!tNon&3;3wCGk(`*jgRVwETLjRavToTz9?_>Ryq50%vi5MzI ztwte;+#0MjTMCbxZ^~R9ye^kz*K6|kvAg+ED6;7|8W8-+Vs_^XkiTrJ-%CB;)104a z0k&cf|J6KM%@=(}a@0AY{CDdl@N2to!#LO;Lb3}Es{mD+Kg2%^lu+3ra!(`9s1Xs2 zbLBPQud^`AEUCh^+TZ$gO2vWJt@r&|nFqvNIj{wrxAtNE)WBH+p1`F9hAU5_YozrlxvpWE+7DunX}2MF8lbrgN%?x{Dod!aTrkA9c5 zVwZUHKKf9FAi_wU(WTSY!LQB**$!1Td^)eqw;IuIW1p8A1}4xFrdTeQG*7vZ&W#Od zWJR>x^S!9EKWHz<^U%4{^+*i~kk9IFdK#Tz2U`B!dZ`T@UZEK3_6`abs7Ng`40vPS zqJI@~J%UZdQR3p_jaz6W$Q*>iJAqfuSVm>wQpi}~URb|x4w@nJbRoT!t#4AF@8~tP z+N+_wPtQ@flL&j8Q2oh^{$|Y&_ZTz^pi|P-T6PUpzx+WSFNA1Z;*^!uyUcird*X=i zDpT`n;2%3|5A3OGyP#5l=-4ZyOzoS-g?6<8oeS_Fzu>`=rwEOrM9CgZ60VcdV7e&g zkKp~NwABjHpF+Ilbb_hxZd&!vY?lwa_vaV8%NhaX$fj8 zP?=4xZY+d-bq!ohn^PyucKu3mW`w^%4@ihUuTqHe*8#t%pWrgYk}eHrc8bN&yyB`f zBWo9Yi^uJb5leRw{s2}f4h?3*mSNCn{G#dhbTKfSQp2-{>ICesPMD6($jG=#E#oaE zd!1%@mRA~BRmFZ+Ky`h#Ptv-T@UO-cZBv2!X-*J0psDqs{kFX$@1*zebWV?;!R3Vp zqJ!u;`2svF;)ORZ!GZzFfrg_{bcMeSVgu;sKj~W?0E~SzEa_{?$p!NAH_IxswhIVYgV^Ue4=zH+xZ5%;IAcy~wG! zYevj_<3Z?Yx5Mc!bb}FaksycBsNQ`NKDkL-r)qEbSW`Y{I@19)L%!MklI>H`kA33$ z8o$Tf&XGxnt*7`3jBT})#Q1{G*8&!|PhX)gQl^GY+OAp9C3GpjJ4h~ zQDT0Jg1SXdVU%$iGH-Yj)vuSMUB`+-#i$Xd@6WCm_TNYXMqclUN#VPXLzX(K#+?;t~4sqap|HjT!* zwxWxv1`o7DW>9!8Ka&EeOQ{!+b{wG2ei3o0_&A`RN4s@*4qOxd84}>r?4M}E9dxR> zh1aFiE_#%e&P9|*=XkAMPX(h@JF}a|rm+1E2;TP279I&oJUp|`uvMABh$&O(^nGtS zOl5?$>L14(Ke<`EWQ6sQMc4L8WTlocYL_W8yqe;|u5VS> zvPsV2m`HajH(aC78xoS58F@JU*on1t9DM>xr&2{JNR`UtIc#0{6IQ_iFu6fpsYo6n zBz4zm>K#2CmZqr3S#+OEbMxWK9CpS+WLe*v!I2{($9ErTz%+y+4lIcP%ikq|^q6YelM?4D9futk<(ERA(gSlq*LSg&?KP+-25iZb@N|{V1 z`V^=KmE2VOIk@l+>KB&pck1Mj{Yi@?+Jn%weDlrXPeidKG!*XDFHMu2ED>{;vzpsG zjnHT4i4u$~2tR;X)YS-qs_)0sgRthbSxYCo98PZ1z^dWB0uzhE>Z96_JDG74{))XR zfIF>7g6O*ay{p_`yW*#^Xx$ieKkdJUgBlqcRm&$mu^LF=%w$;xU}&xO7g60;MSM+P zATe3u{uBf+JmHuyS2mNqa#FrOmoTDY^k_Uib*(>g_qrU$Ts+TDMC-nG@1Nk70YA$= zuFOwN7ddx6JUa<{jU&S*&yvR=>@4FyXR-I`iQkzSV0G~Rzk;69O|ID>&<@OB3&!60 ztDzerjS_@n@aVoV7gHk>M&#BdL{vPVD3QC@!~`xd(w$7LGmnurf|uvx@-D~OBJqa; zE_nuzzD5!;B1&vd{{zwXJ}4O@89++V=|-(kbXezjeosBXcl*@qeJ}u;q^a2b{OOe5 zXw(#I&)VCOscoR_gNbAd`iBMpZ3YtlCvU(4X_BysC?;hTJRaMw%q2`(435C2q&C+Z z3KY-x#C+~6eb*Ks zF*4P(%)SLCH`nZ(;PcnNWJCD;dSh0u=5k%8UKn&bx#4h+snS7{ydi{9YH4F8grg;)InURCEA z90(D4swUm3VRQV*;(8Qv zclDd>uVj*?$c6AKv-#h&4Ze+Ny)E-1u5CcQio%UeH}{#Gd^emoz{}$kJ*w4Z?OpT< z`-40dYuQ*?HXgxv74}}?Q;uIr>Z~L^fe$+P@Vj#VJ(_UT{mXgT@)IQ>=^`8~8k&i& zPc!s7nn3ynWK+elfR<;9%v*d!jnEh9N$lrl1E(rtB|bZYw}R`tk(Rp}U`HM@l+Le@ zH(IW4>b})b*HwGiA2P4LOqrxdaPRr485mN5%VU{pZAES`eT+Q>V48QK`kcd?b?fKM zUtnD-GaAl5`~7lVA~uDePoaS9cl+x1kOdznK#n|~9LXdyC`vVDfPQ>HP|JCIOvRYmYSfMddlw7EA5 zq^o$2w0)&F<>Y1>Ew1(M8oVlY&WrD{AaH*>5wa&kXC{7mn<%(4JoB{rc$L&QF4?>8 zz+8v)%0r=F4XK#F@g8x%%(j6>b$8V`syM1aaw+`25b&4J0FeOzf&ax{#7WpOlmYW`*1w z6ke8M$o}XtD)2^~Cx&d_%F%r)ky<`INid`m4Y-w>=NF0i{wCdNfBbl(P?`PR_YY%B zH&^fNEyHYrn6hSBY^O$irRcfjl#X!Ib@#4zG46N}ov{vZcQ5#&$z{}gt?jKV>q1Tm zegHnMclW<*UkCKVVU>9(B16{g_$BhCgFmJ3Vco$GAZ- zvR8R&_PWK+Gj( zOqp}_1y5AI1DV9_rW0on#lXeFYOz8!B-Fr3GW4BxfI#+r(FG(`{w@95&DQEqJWq@bnI|udwV3M#q$Uce1mk z!1aq+NB0=ET(SrXbF~@RsTyCPfAzih)YTrZEKIr;l^U^&Vi|vooaAyS$0CXnbftM)ERHB}PRi2+g1AoG-85d7XaeLr;L z4q6Kx@os`gmwROG${9kHJG|`ni`zeYH`TsTmQoDpxV+?^cnz?xwqO0IyXT285b-ed zVB$t6MQYVYp40RV(I>}DPWN)!Hcm@tru900v#;aPK7593W7G@T zxfa6fshez$yF<7vg+$B(O`toB7Sq;|R+Ff-&pK7_o>~h_P|O^oSA)B!BcWi8p%z}NEzEL#5IQY61RRcx<)C9 zUsIj?`+t!Q!x?mJU|6bKMx3Oi5EqMmZQ~j%F(PiCevw2fO7H3?wpUxic(JS=c`a-+ zlXYL9=CFhO^|Qs@n&Jx?=V&YtfAOb4fQNYNosYc)uEV*4eT@Di&_kX=KzAi`P!xRj zt!;K` zQp8txtO(N>VmmUN>F8lyDILlHnocA2sc!fn@R|m}(_Gl4M?O5jPb6JtTNmIeNzoUl zgL)wxLYi14U|o@rkl~mts;{-5wpF_4oxgNI$;uShH>O%pYEiW&cccEdEf)4bZIP0y zemjeH$1;koOat-aEl`F}gUeS>c@Q(Nad5X1r15$%WvS&^6U}xtnt^s{q}>1q+g3iD zYhv+jh?I}JphO_ws|>Gz9l7PzuCZ?TChQ$AT4_d|fg_;VDB;`oa4C2+{CQVnB79k->U7VEIV%p_^-s6MCXCt->S`ns@8Gy zoNu=Qf;o}mk|F3>bjhGk$(-5!n<<`!MOJz><~JPw~* zp-8y%VPjRhwh$sz`TW5ORy+B-d7ATa5DT>7d%7NtcLSx$`@7yZZ0JKNx*=m#98l$B zEP<8FhSqXO!-p$^X%gwcTQ-m+_Zv-f8R2KM_0&jmQvSZbjE|RS8Z=nk+&?C%X*QDY zpCf;vxo}1EcmXJF$nU9jvs;Gd?qqHgN6;n;`C8+Olu=&#-2!thp9o8R{BOUn>+9Wp zAKMhYVi~!2QQVI@pe)aT`!cMd+eJXEL~~wJg*`q9RXDa|o?R$Z;Y!a{(;I%VXJj9L zMR~Awd)2yf_fnnsK&s5k3NxbvtEav{t5(FVmX0{}Kg1*S+f-EYh9{d&uB2n%Bj+xE z6E2SQvQ2ET>?#skG8Lp)oN+RvO<21}0Gf+!)n2W#9ClXf9#+_ze5tFK%Wx(O{TgD# z{PNz?Y?a|&jyEtGhi;`jrKk}qhZmBBizid4HYSW?W+9=QT}0>hZS@tQ&Cm@)_sI(# zn1N3*NzVLD_r2nGI~FK6*4ocD-5tQ_*J^Zzui7*ZoM<>YR#SIbr*?zccAI!6{4=(+ z1_+whWP-QB95;t7a_y2{N)LwBl~Y5Z*rP==;|fBN4a}%g5f(k5xuk5=NbSk}8U)U$ z4XD`nK{GNBby;HQwAfjMgkiVTNKwT7to7O8B{vC_4&c)xR3DHOA!RjeQ$kb-kO|~U zf5cM7*ZQm9yY*1RuTNo4c8bd319IJqs~RLLs?nm3j?W_b(XV9g-|sv}gayVJ zJNs^fw^`W%VoaTpA$QBK-50-HAh#Flfu>t(gI-mrF9jBr$VCs1@n~njqaLCTuBj~S z0U`2Y0^%FfG9#RsK`D%%&<)RlsTTvBgP4t2x#~&mS~`oZtt-ckJ1Na~NUBgWmFBA{ zsad^kyd)nJ|BxLUy61Mq#QUCo-Y+fV=HZm$fHo1v6+$K_9ZYdw!50w4=L@R;Ot1pK z0XTs(#b_E09B`+UKv`8-s)wN+u61762jR;@p(|u3cq!?Wj|gN?7bs|(@E`a!*iGe7 zUJguW!}FT_Itc9mLrZ=bYr{848jj3Qyufz zr>^4C9Is&z<}k&zo-8TwRvxpE2S@{7lOV~hvH9d_pHC!WeT8<7}Qg3 zJxiF_=z1o3K>CvyPRAW8Rkh#rJR|6GDW%)To{M=3VgX!@9L>l;WeGw@OZ>%mxD1h+wRq;sFB zqqGc3pJM@+SW_EFT&uk!UtCphi-|r_GlxoLg;$xc+V0;gvOl%h6Yf4W2=|p@;~8LR zb`ZCTsICs7wB2U$E_HE9`uHVoRS$iT{+)MwM}Ui{eiKTg1;q`A^TkVh^|>zqd8(s! z=0yHw*+ex(aK@K`&3P&6Kon@cQH<8qU!6X2Pjw?JACQj5l8rWI7$&ThVv0w*U;wIf z0de-T<(gL9bgQ>e4YJ5}Ve~^;D>DBcU;t_RSMqF`EF&e z<{B_KK0!kdao*W{Yi4u!nr8DXQq$Rx?@GL0G|BiLWPuJ(;}Ur=w0T2J##QmdJ_g`edK(4dGV70TweqLl+{uC+h^<}R?m)bAb{4KgGr>#+? z4-a=kQsXVW9t!Ujm(yu(4ZdnzgZ#*8is?PHRZbIqQS$vR}$*ZORn&iAmS8%n~GD97= zX~)u;*>FLIJo9yJdO%2~!*f!8VHiv&Hg8O)b)hNChklJ&H>Z#0DX>E6UFU@5IW;17 zDaRS)w$h*Nuj+aj*fcdon;#b~!Xf0nGZ1xKL)DFX_C)EK2Kcn?eJut@*2t)g5!>3~ z3j&HHT%jL$jXS=TVjr%=g<>5<;|=6M@~Z)bTs#8>ke(_l6JNTtRZ1-REHtZX8b$F6-? z5&^yLIbFrdMotriGBcZKYs`WX?g8xwM3B4_xs9)rtPX*WI6rxxFV`@B_|38gRr4qN z0d~g>QP%(~RID$L)`;jSGPvuy@%Wu@WPIUq+(Bx+uX|ESG8{R5mVdY=68HIOoC`XW zACn5}k&GJ?K)x2xcJ7CV&aZ5~qyBl3K5;X9iiE=S4k}45J`%51je7s`c zVCQMy#zz2`6%uc)_O3&q*NY1@YLLIaF%0L=S1V(90^A>*t@&caZO`Hl_$e9Dq1nw^ zHtt&0c4f4Q*3f!!A~U$Zd%1$^g*Z5t1J__&p*)2HeND;MT zNN)|VEp|f5AyM)|IS~?xTXF{o$)l&)n28(2DZ1i!6~FAbg|^Hg1F+-W9b``azwB5X zd^?_rtKlUP##VTHu;`JP~lt2bYRGhC5P z*P*Of5(m0k#Vyn1bQO3mG{$sXw_{X%vcV{|KK7NA;CRu(XKKt2-I>s26^M1kZNsn+ zPz50&&rZo}F4VkLZCeov3JdjdN2!Xao@NF+XaN)rvPQF$uY9?9ikw;Hp69N*U~$}O zxQ5%eCLyeA#x#F_1r(~K6f~ulh?|*joWr?87J162JHGZ**OKCh9y>RRGi71-BoR^g zpSnD`U#9w^*(@Jm<_~x$;cD1m3^#qjVbxtB5w64tfDu(ZbqFVtCUK8ylkgR+I+qpb z?7+gQ^4?~}8+*y(axr;~v>%bCCS#$P^YhwHkGrO03^z_k{reU$7@3~26(pbCp1mPK zxu=7L*B2ptDST%KMl_dwu?ozG{@7`#%KMfanv~DmIkIzAxk0cu%npoork*dxOkG2- zoSgu3xHAs>X_~EulMn;m*}1tj*4)3ZgfuB)E5p7q7z}uCB(1x=`pMGXN+kQ-43?AX zmizXWFY|r&d9x$66oW)u@t4e}_Ij(ER$>V(2qt3%p~to;JB#y#Xr!+cd$5e>%dK); z-EUWOd5(aKMUl#|Fs}T&@7$S#LLNQfG)&O%V)Cllw&S%uJ$MC);J__vs3t9bZr3$I zKTAF5YzOO+93NayjkwgGJ;8Clh}IFOd`}Ni9VZ+oeR{CxV}Tj8KEkQYyyyl%liGY7d7#7YLJOe`|=2U=1=O904(C}JT7heYfL9neA^Wx^67u6YX z3qev#+=8p;eplyYh%d?6?BEM`v4xI%;ZS=AW1^buNKf9Riw$MqcLs5Bx_nx-gK<-k zzCvl0rUg^A?WT^M;Eq_B9THR1R20Egs*CR1?g;-Q&haMQR>k4jki2i5(_RB}G>Pb% zReZr^femNwO#rPmXG^P7oHAIgx=W}7ITAf^7yC*o{Q9Zcf8CK;rY7hPilHHPEInG? zVm{ryH^}7tuOW}&%KvuA1K|nkCX!Xg8^hI-w+|uV9VaIzWaulRFk-hyF&7hF*I%ia zDo410iI;P=GBpU3r55U{RDxC@$`P7f&y7pIbL$MdpQ#f z)j=Xt<+}Y63WCp~B_RWy8(2h>2uE9qmc&}b;Obg@viFiA1;$u-+d&kN0ASF=Ool?^ zyNom&1ApCH9kUfugI$(Y2TN3hIgAx-;cKpwL*ig6W(2Q`3!x%%Cu%vR!{e==l+-K& z*GfKYU)eQ_t&W$_gK74N3qP$k-;=Z0L{4&ew@?R(ljw&nQ@LDB{fsFB@wN^{V!{aG zKGl%kDCAGK_DzV~(Ok_R;arG|nmrtESe;d`KX&%<(~rAvFW1VH^XhDNak1JzqITqK z@=vvKcqg#&EwAyz&_oJN7g5I#lX0SQq=pm=w}bKvqZ#&M(One?OeZEEXn8e5 zOy|L6>VF?g;W~njrL1=di=QyP_{d3}OC%eRrCw+x=EtXRY+q9*M*~qp*Vz4iGVl@v~z2efH9W|@2nwFH!MlJD68bT@@ zyxT-l@4e9Y-EWJF_~|cHd|ve`Ee>{LF7B;(0sojnQe z44$qH-Sc|*$x17YoC#(i<$SRxtV;uLMIP%5MrU@0W=h^jQK@Edbc}TS;voXCQ^~lg z5}rn;au?`EhC5!qMYwK}R)t$gpAX5oH~G~A)@01DG);WZu%FjJg3 zy8wFb5p1}mP0p3IUCFQcgHSZk*&K|V8z%Z1dWwxYci8Mtwu~F0Vy-fQkR;XTkLQV! z>d-ICr38f=>|Tnil+Fs?SL-xzy_r^cTVs2L=zK=_nKmWG7M>=vuPlO7eiPk6$bSy_ z=PW~td3-sl4SBmQGc$?qN?3Q!7vW2`{W3tGs8V2OOwzbj6`NgaKAaE6t0(k%)$sn# z%+~!ilePZ6Z&B-&?pXWU8;Q_i;o>;O-k394G8bNqRh=Ld^yJNf-o$9Iv=bT}NuSz~ zp-Pg!_xj6yfK8cW4EmyAB`=?Ki0Uu;wPnH8*~hEuLPwZVIfdXIFz+(-W)L@(K6CGr=`A2;eN{;(>_VrMaV_C- zj4Qb(2fMGNkAQzlvGEA;J9Qw?4e;Sscsn_xHifUf5kjxyJ66HE;D7JFy=3JNA|sNT z=k*09JTn{vOX=815wqZDw$3!mrbnFSI?VY9`*AC)Am>W-Yy^^MWnrDVt(88jV?@)B zv|Q|Cj=5yMQG*b!*<{B*Mf5QW?<$nqgq^&9U3Fe6Q>HsZq|pRGae&ww{)V%@kKwQDz~?2m`}rW6V&9i*%VOc@CmXgX=mR9%6Ee&Hp7NM1MRq08n~Z9 zt*e0PBuQl~OU5Qtr??TPBNy}4AD8T%wsi+EpF~0}{M|&bd%r+VVEUta6h`HHB1-r? z{URT^tzP(#f&ULmJ2*KSUxVyUqcXtU+uz|tdlHl&{?wSCDO>lR2_uxL* zH_18g*0)t#Tl-_G=I?V`cVE&S+Qk*befmKSJ5B^FiZY)g^X8VGk9LI$(S*;(-G8Sz zAan*DBCEG3T?yCBCYOX>*iTj!`re!zZ^M*{RQ7Xj7^Z{^)o> zXh9J{%)&q-LINtQkvB1xMEh|*^mY8^K(G1Zm-)dwtHtqynbNyPBUNhOUX|l@AJ0!` z?Q3LwyY~+H>L1Ct*CxStDmx75mp@

8-APofT;aO*d{*v`l0c>n@is-CTmIZIbn{ z4DT2pC@DWfAxG@0UZf=GvfQKo%2RKVIF(h2S7M>Jgl9hLAxOzi3pq=GSfcpLk6k_v z0?mq%!N};hTKBXTGO_T>bf9e_xyt>4sRXA6sEGn~|KP~5yxCesz|0JXFq$)<)9moV zki!EWogf>^Pw#PvNlj0Lp4e5kL#-gBHC98W6h3c!t({jT0M%f-K(GI2NmtsA)}Pjw6KxRYg@G=<1+Q|PwxXDn z6ovu^LgEV?;fItl^gxQ9ScZNc-^}aONL%DN!(FgHl0c0r{ZXU$2|nC@>FFEq zC&BAmw)%M-gs-t&lemt}7ncOrbV3^ZyCIs~P|YNRGBY|j;q#NP-)d54dAMchX<;W{ zc*7Tm!`f4t#8Dk|=!04er(ZY8>1~Tc=q9?9nRNtRrTw9<9Be4>I106(ci(c)twE7B zXnzE$`!@*d0gYIlB{s=RiMkLr=BXwyCO~32&Dpdi@|K>vJ0H;e6SiGmwF9>0$i#ed z+k&dA_;zWDYh*ek_ZV~DL3k>Oy3UPicQ4{N)yE16p_rM#=BBjYxBC1Jb?uV1_Zy3j8~ z)!Xb(Suxp--gsD%v~`4-cs~|+EKPCKP*iW0!spN{q3&g+k`)jHX`N0F_P1*{?F>qN zPYv7L`y=&&PXfPwgQxFF0I_@hR76Wzqg6-7y-MC=cey}kCCssl{mC)t_|rqkHIqd& ztrK@=2?E)P@7c|#$Ta~8$bT`cA4c$)B^cQ*PAT83%RXRH`4J|dL={))hHYZ>ViWJ< zm|1s&e<%WfM-x*@`LKnO>mPmzeAb_wWkuK6fN&^3M7oX6LqB`%b4Ig>(3&l^p<${z zl!Yu_4sX*Gu>5Bk(Hjmb9shfidab3PkiuYEqhR3qUmWzy;zPVG)z7(Y6r>VhZ^968 zO>)uCj^jmIHS)3$$rrO~GB7;j_+lb_I)pByAC@0|nLx1Zyo^OyFa&Ctcrd)rc@yRU zBHmBlXl0d1E)6d0*^ZG5 zBvyHnv%58T8fu00oUL@Hd$p48KSYQ*Pk}q)Ky!)hcR`j31S@W3Cw5H~mjI`}?C|W_ z4w4y#23Sk?Q}_>c8I8(UEuej82M&FoLiZJsY>^`o!8}!W2=I1^vTcsqcoY8FQKcyM zwnL&(Uq(B7YYLll^LVNx(f=hUfc-TGd{CYm(|2uyG(2^m~sY9jgd@ z{0nUS(;)uGDtcQBy;r|;*LT2oei5tl?#9Vc6WP!1E+wJDpj{zX)!Wq-**Jfjd_eF- z0i!i%-qxO%;7?29m-py)3?GB=G}XGGi<2pafT%iGGNZCM+o|vU%yS3N9

ZeHTL~ zexplSY$d0g5R1vt$Kz##bS?`dTM3@&eib>K!QzuR1uY4noMVfY-$SyPB z`hC!obzsEt4E@OCdv9ZUsO8S8z+D+W{8EWIxsOwl!i1ZcR7yGK-}?m~S@UP0`tcSD zyET*>kD4@hSCjF`d8q2(|8Tcz1lt4o`d@oc=@3;6!|pESW()}PgnD^yAJniB{@lO$ zGEA41_fFE`a(o1!F$rY@(IXnY?8TBL7O|KpDs6dj2h-dkld|k$s-InIpv;(Ppeptt znmEH-VU94`qe&>aA!fq|X?S`PyQUY8?a(wrFNafO;2skZW}*lw3Qa;Ff0qnr!q~wj zBE6ojTnS0Zcg@IClkpz5QrG5=|Kah@z*k9iRt&Dh@i9c!A5VLKgJdfu$1otHg3#jA zD?dgGFTAzfAd7Xw06!QBv)OPl3QrXH9VilCevO|!f_c*H^*f(m^P!PvVWImpU&DSU8U9U z;o^hErEnySi{!rdt4Pw^Yp9EJ}HzvVyWG74#HM~L?orp?8g5T(7zklS%tFJ(1PYA{8z!e?- z%!||MW}hD+&4=<->tX zuf>$WMNEI(gUGhhmm>$lU%IOr*`^*5PwzCM<5P&aO;{pEX%B0#X5 z33UXL^;y<@rCVSYB!!;)!Y}**YThQT z02T>;vb%DZQX1MuSMgh&K9;j$QY2?e+2kA^PLp0%}L zr(swiU!&?w8GmUm0Heb+3=6jAHy9ZnSGKhahc@p%d@tyaa(0fbhlkFSd)#*paBbtv&nsJPN?=A zEo;{}tUu9G4ot^>fzR;~8R&`2LUcuSt&F{06C?W`l;9yIchq!qdEUn>$6y2AU8%N$fvF+ModcnB#R?r{$EF?gq^-VqMfCx6%?V0CwFY1sNho+d)EgU#?djj<_l=6|D@*~|%f={F9; z7f|W*Br^sCMQTMP@y-q?^65SuKr}utDuBe6b|4?tiw1g4#hG&Hj~~@2c~$=;;Ts;h z0NUphZ_R#rLA6#c`cVz&X*GYv7!>4lh*QCT@APS*t$2ww2NgSW1UYwOLd7V;)$$!w zY8pm=D7E*yj*Ci(Gut;U*_XQes7^I z*3BB{N`FB=a)nHWB`O!f1l2r~&J(L~!bC!oeNp9T3E_lAEX~o;8~L3w>S5bFvM)D} z2PZMM6Xewx;pbhVm@f|^sPbghuJDx-7cp&G;)Z9?#;ZMW=){*x$vA>y0<&n(VF^hs zX#@ohEnAG-Ki^d5o&=l-ou^f^FEMbmeWx3sZ(OsLd#uwF`WtL29J$G_{?o6n$4hkS zZ<_Ks?oL#q5aGVh2U$`OL)AxYd1eSzSWaW!%-J%rU=;WnB%w4uIkCx=?v*k2b(q5?$L8s&@;nKDt*EYpIcp)q^5Sosv7n_mpT? zl(-KPPg&ys?F+H2SN_Ex91K;SJ9SH7y=Y^_-L!hwbqu7k4r_0(d3e09GwqY=N_|OX zH72G~-P-(JY(@zIN%i<-KM02y`M^fs?63x-9!_=PGNf@Q16`1<2rZOZsoEii zK%E<1>A*rpMr+7`sWu{+fpB0t1mvoH5UglbLXdyep7`AUheuO|S)B@+pbARB)|AR*c~blycG7g zk~C){Zfvbtd4k1?3snN>cw?QF{Eb8*cN3>ju^Ck1o;aThwO|nKedeGwsA?VP-S|^% z!%oieE25!d<@cis-b{eg){AOdPmOhTa;k%I;EF_qj7lE%D`1ns0&kAqWmOvBQmDvW zlW*=9WIixAQEgm?O?XC|Gn9S2Y|+)3D~v=gaQpMx`BGf6A{6`XLwdSDSRmI*a_fKg+Q?pC{PEH%}HcW)htQFI=rH?wU9!E$X z3CX3@|6%ik-}*|NQfPS~KQbADd=ZQEH&h##$LED-@wo5#ZecJHPQp9CSumb7$8UGd zYc4L+U1s~>ua1LBdKCvDl3BG%T<^~B$*e_ZhE#KUgVjD$=+wMM6UXo0%FLC1mHWCF zS(?^B?CQD7$bG#LxA@5Vuf2#lLPXU0TdcAEt)F6)Vwe^}RA|RWUv*MJr2o}C=Y2jR z|Mn@OC!nj7*SzoRzrHeUA#PAdjgy(z)@hu>*_<*zZ@Abvb@6n_gs;|&($Q8zgBs~i zmU{-N(@(URhoC8*-Mpj#7(Z264H(zoxY|D@tW3$wSH3)hO;Gh_!jT84XeEP3Y8D%R!)Q74r zzn_}sVjX-`e^F0xPi+siwTbAxoQ_jq%6#rER_ErSW7Jm{>Dm{?iX-!d*)(FbD^n?P zOBdS>moe(XrL#J0dUnT0mRpcYG;>EG10H=Z&1(Pl)Wqv+(_G0PL>HaEjQtZz9}Y3N z)D`qS*i?(PeT~fc5arxxT!)SxUuHE8^mgaprmR!#%Q2F|{(F?${rCqD(7nqk_BwPKxq`y=~wb{WFBHI&#~jloH@pOwOMk$8!P#B4i;YT@{6 zPr5~HmR1|FlD{c2A1lar)q*n&!x1P_u4F1MQ4WzI>|dk$DDGoRvH|V?qS>QKz1hRe zozNvB_0D*W_kXkiSmqDzy3FK;y}Z%r5|=atT=r;^#HCQ=t`q>H&cGc%HnFL#ekM2WnN;Z; z-wW$z*N_u`IMJ7Qk4e~#FKI%hFg-`4*d`K}_9-{h;#@G^haL0y%n3L}EVWNqi+5 zsyo1-E{+#tGuQG)SCzY`_^@j?@er7_pd9E~efK5dbEHJ-zRSgw{40v8gngnBPa}nX zyQa&BQ5@K!KUKp$A&0QUVUOU^mlTL-opCuOdI-5Ga~&i zo!ItI`-!&4gNHAb%guqlB1yDiQ#xp|fw^Y+cE3sJ!rf*4{|71uqHZWa8t0>Y;8qM) zEhOve9*x%A1~9u+xpQ&jT$FrlGs*n+en*qE`vx2*W|+F{mJw9l6g><+_B&)LeRvn% zycI9tx#sYQdTc>etrGTxMIKn>(1=6$T38qVn$v##CgMo13mk6Rx#oTL1rz$KyTgkj zFpII4GK-*=!i4^fq@(t{UGwg(u5;Ib!BSCqE6U%I7J-Ne0JXZ`t*v?}39qa3+Kr8@ zX1!*oJs^+u0w%dAbCWVHmh?J#@gI%6%wC=6MLHFxiTsM?pSU?`v_-bo?{cSVS{mn_ zKVDie2iOGJsGNR2YWgx4fUK0OB<%_JiGgxgjMZdS`i5F2U_i(0{CafqkX8`_SA5=z zU)9lYYo!)tW*=WnrBA)~^MJp0Y|mXM+y@xn_pp!Y7X5iUzv>|PE4ClirLfY*%O36f zBADQF=Z+GU5}hAC4m8YXJ|oAZ7oCMlocj+*=#Ljs3n3E5gDn$5>0c0=6)dt5RYT`^ zt-Zf(Dnq4ic4A>z7a_NOhzvSTG7T{bNtZrIzaD+maXy8yq$@PR0URufJl;8R*p;%# zo7YPk{mP1Cxl^XBMGrbFGY(Exv;+N0)bzI_Yib-TS#l%2uBz(zu%Kdf@eYtG{JY2! zM@4sB5KAq=_&_>z1z;RyvV+CL$$L&qg~NeHs|5Pkf+JC+0aOD#lM}MhUOlaU%AnIU z4-bopmy0n(Y!(BqmSn&X1q)ZYkZE+&`n^)^zC9mB?teWFIXXauJghmE>i^7>{cId2 zD;3{8^yYLt`lI3S8?mSZ2%ltHB9bZ@id_b*4or?@@W zVcPVYJ6whn>KzA6jTh8g@=|Tl7mPpm7@KJ^*fB8B@ex8dV!ZUokTK{p^TB;Yyd48^ zMdM0151;@(aAk=SvN`@j;5%ZnV5eihr(6-T9Tov%_(~uAs z94U6kw{Pz891)|&$SAF~N*>efA)E_7-NyOj;NwsQ#q<_MC17~w1b_Vf7=#-{)vJ05 zx~DYL!_Y+xH3Ke8-EYk!A-u< z4_V?#OK9tXQ!4bZf&S`nn!MQfwAn)NJxAPbR2%s{@4I>ISDFQ@{YypaTg?0UUNIeN zR;0_zd#Tdanbcyu3Ol1pv`c-Q-}>lSwsC%?`NlvE&v@{)c_^LC6s)hzCvu27Gt;WN zy@FCr&0RK`GSy}{US{027zFh|l`(ZFQTA(?3T(UikiND1!W{KtfD}LsD^2gjJZ&OYdUv5a5xs_>r7XQ;x(2ltk5Tr>1M(gh>&6Dp22qn?2!=18MA-hI zOf+vu)6B?gbYhc3=NaP#)lA-UK&Bd38U~*nO$U>ua3rLcu(Mr_}_Jo!qxE zMlt94WC53dcGyWF9(N%wm{`F%t*SNy%Mf7A~g52N1V@-@kp

|`O+?yvE905WnYIq;|&n|I=r#C5vyq)aVCh5 z1Bo=V-QPcMop})81k@o@%W8Hfa_^lulw>e{`qB2{Lv-pIWN(GjX!I3*AyvxqHhs9c zak4S=M|^|oGX8_d9j%w=rJPq-zJf67gPb*uBW@cMd!@eTCiw!m28OX%6$Ep~=Iv>vt78I>!1jVxGi_)17scbjxP{HjW zRb|C)WYi@zmrm^JoybPa9eP!SUA;WRA9tqTI;9&JU9U%TI&hxgcODmE%j+S z@?+e=zWNV+MPA_b0J0pi3Vn+zL?hcvw=yip)^r()$?5^eN89UMwqhq<^19eDwz~q2 zsxkRY;4R1UG#I=p`+_SfC?|e<5)qwh{rZgT)a9SBFjIkDDf;I3x#wPP{clic)^cix z#}`;mOCO=yw&hne416TNt8pHqx1Gux*iN{79Q$3VkR0Rd+q{F%pToxQIwQQNMt{%n z051GrS~EYtqkRx`<)`ZDc;r2;h_(TK17=4uixy0=0VGEJFwdF9dw|&pdbHV|Wt5Dd9-U)U8A-C3+ zM$`O@ZL29M`uX9|*pe7*2h)*+i7MFWUN?OiYhX#-!hUgZ{v#pM4sTDXZLcA0mK&c_ zN#Zu^YVvs$W{AS!(xP-pVQiCpB1iWV77~Cru)-qE?LeM1Oq6Q<>AFB4X3TM;ZzHku z&2eVhG*A#kAJobwv&jyI0FLCwoG30jpVoYl$Ap9>4e0jaSd-rke?oh~GpS=?E7&!N z%*FV50uSh!w!_s<*)qsk1g`Bk+hgqOnakK_icnG2t+!OXk}^W5@X!tk*7pL@>9&62 ztjmaOWn}om$USvGvDQK!2k$K_<5DX0*mC7%nM*Sw)_*oAX}Rdzp)YL6S^21U^`d9~ zDm#gcT;dM2Co8DSf3#7IQLqDXVQT>$|Bva!XC*0L>2kp&hhvBAH6`cotTac(03znz z=-9D~nP=f|4_+KvH;ZCp2WW$;rf>4y61H6)Rh-yDrWJr`EihI9A4b@kvu_=FP`t~hV z1DRwEgRyM#V{t%Jy+rFDQS({_{!w>UYrOD)z`nMC{VM?-8QC_2;zrBa7d4RDnB93X z5%MSfCm)1Y`go>1#_GD{_yyp@Vy}bqC_NN?B>0i~yOt((+U?tQf%;m%o;_%EMp+4m}{~7qLGGSPKVGSiEC1HR?xO(8;&**F(`- z#y2@9P=w!NP&~gZbtJYKt_oIgnnQE4J@y%zjAMHIUk_Kq-@_iu_l^MbLYIgt#>p&r zOHRWrVf}Xrz<|mIH?F#fytp8r?-*FwwbZdZ>8@yrl!x#+Q>X0j7JO7mGaMtE;LC$} z?=1hQ!wN6R$+SbnuNDD`0cV|6xeABXgaga^fP)-p3s#2gbdJm0n@5{fu?ODF1`K7Ts_F|S>;yh`l?tugMhUM{w`~sNH+e?90lU$X zSkcpp2qEWnC+*s{%`V4Xq@nQV$!)TyS*{-JL2k2&@r6;p5}OPS(5uX=r!Z)zxtNX5 zQ{VKj@bJL29058y*R0g5e<0iUnKY+aOrw;4;qUH6nJwM_Lq><=Vxf-HM zNkZ1CvzaU)EWvb=EM!FWdy)Ep;JfP1e^SoA#daX zRgi>h5gNbvu|uhfU-@~@;HFIz8Z+$Y6d<30Gd1%lMCoix%z)6A$`ovp0g>e)WYo9f z-1StE$z@(Mq_& z`2Y6oo_AOMC~7+Slk2!0n?~NFC6wtwr^fwtzB;ugTofg|N~7LYVO4fxs;Up}Eq3|T z9|t~TiukV9+x6ujFjAn@eTZ;hQ@szhlfVX;(-#CAu<=A)9^yEh9?b^=9Lpy+&rlz@h`~1&e}EzIU!>-HlUh$UUX}e zbPa8fXEPZ?gK)`A&J-L&OIa)aP(3I&!Cr!t((_z5?PMK((!{L_Z%Kl%=q2;J(2$*7 z%63eYoTRXp&v$Ah@yfw)JP=}(D@V2+K)H)Yt*~S1Bie5<2N)*nD?N$4)?J3ThTE(@ zX%bg|m`&DKH@U&V{d-xRq zM(C~H(Pu1&rSv5`ak%w1ziDWmPPBQ1Pzf=pyuqWHsAL%}=evDgNaE9QSkFy3hG#Z1 zVal-u+Wua;BK@K*0pL$^e5xyq+sYkG5PUnd#_hWl+8MADcO?2spSAKQ)4c@2l(4!+ z%698{qpv4Sz#f?2G^JnUe^n)oTlS~+g5C@owK2j!w6SQeO>uQRw%%CHaeTl-q=CKl}0?y(zP! zsDTUhpqP)VfIzju`DpAG{FP(_kR4qD#Yj=YW81p5#;B1vc``tcoUSe%#HFPUr_l2A zZ~|zD3lDctZiT-D7Dn535Up@Ds+d@ay8o$>(Bg@i|c zC0r~fS*_hIxa{s*ntc37hu{8DFGb$epQq^{@C!v*rZZ=KlDsoo{0zv(oV;O0xIS&+W$#hMSd{0^6{a3*k4U{ zlGc3KY60Ff`FJJmGPb~w2|q7K>_(T)vl%s5G`IUh;V8 zaAcEv*V@qe^B${a*wE*|P`XYjn06uHmp;~tRs+!PpXoH{W8{bDW3&Ycr8v)2>Ep{g zDIBsk*y63IAnYJlYfWZHz&!|kW=Gpt_VKOK=#&FrF!h;i2|YCQ>mBwFk2A%EQniJ4E#WySn?+=}Mqu!3Lk>vF}ooEbtxpW^ouH88WnfHi@pkJ=SvyC7A*Yd=Kg^{g& zr{uN`n&#_u@UvR2H|J*H0T*R8+rlIr|4FBvf3|clTQLKWi`Spxisb0hvbnVuk6N80 zMSFdUEyr^FMa2a7?4jCpWl`;vJ?Q$R}h_ns{Ck z1!ivzE<-_79+MPp62pjSt8<&Y6Aa&@k48Nn0!waYeCnurm=)TT>ch`i8Rya;*pqHD zH-MyuG15#i0WIhw^EB1f?W@c=e^$rp-!ny<$J{PO6fA&xjVV}Z;f(ya-b4FPim$bK zvicsGbK1sZ>D3;T7~DK@?e0w!EJ;ySRnnoB zR&f+RLw>q6OeQ*+xtvh4#e&Icnq2X_*&t54jUW{Q<Z<)>ElX?VPKc75WLSv`%7s6D_$Mgy)IF=4qq41JJ#@>F#ncugVICDtnC1`G|&1HfKH5U(v`{* z9$nKthQbyD2zsUrsojHmgdD5o1-4=bGx$W@)>q|}d^8&#cb)y7ra}zm%=4NSXU{qR zSl6>%g!b>*+SV1XyB+SY6K6%Udk%GnWKL_dq2HHFI>p+%bz*&@KFbJjN1RV7C&jm& ze_xFu;7#qbB=N$8pq;gCscp**kRKNZgF)k5RL&(t!z;Z0hW8&jRXg$YY+=^e)FsD~ zZA&uc6+tI-Aw|xJpmpY2vcE|I(>p?Up|j0tmSPztaP`Hw)BX_DhwGf@{@QtOwd25P zo9-~}c-{OlnaSvzYsE`nSVBpi@gxSo5X|ogh*^8SxCpGXS|IfVjahS87I@T|;@!R& z2L=V@$qqMK%@=dgYthDtzK4)ea$=3@H}yZL_p86fmKSkIyq*7sT+P^RZoU4~FxN%S zT5vtl!Ff1#milVZ3BwunglC8JAqcY7Ks{Z1FeAuGo16$U9Iy$`#^t|!+HbB;Aim>& z5*G7AG(693X#V|p*ZVR-TX+3%(@BG5-c_D#!vc_znO+1CWB&ed1a(8_c_j^C_;|G^(Mp`44MD%!j;2z(PF_Ah+N@Tg7h%@zL?#C86hgMtZJ5 zmDTe2oqRly2HmOMKhjmg|H(_PEA^%SW+wVD-1mE1TSde1r?(`6k!_w%W|#Xpxv~X; zO=X(4v)}sT-a{?ct=DHMEZq~e*764J9?AA>FA|@BWBPFIMo185aJUOY2bV`|iic1( zwE8}xnoiHz-QVm}@p_GjBMfgX$Jzkq;&3lHNYm^<$jA*TI-XV&ptJ$^p?4~V@(Oi@5{gSV4 z?bLAYaG!lEm6e;Y)=$^Lmq=8k)QHua4`Os}QT8G!SWfrCIKE!jxV#O2?d6HiHu-EZ z`Blwjk6s4fU`l<(r)|(W8e>65e$x+91dPF42rao7mHWlbJEsOp8rYOEyV+gywB-m_ zx61SryORVvW58PoBLuZbj+w>%=>st^#@Sv){nDA*Mcv8QuVV>w?9l3T-z2%kixDR_ zu}6rtA8cE3KPc`_!*Zyw|LGkQ;W z9sn|WVcb}*$L*=;qAGEHVrn!HKQ*u!VscluxyJlgn&g>1bNyo>B#pP{*I#~b6@fP* zltrY*PfvLrkFbPup0M5QmqUF#ukyua^_so6olFgTkD(3RtDTTnI^57B{Pluem=9U) zU7!Q%_T78>!F@fBgNua^d-)d2O1$ouw0pi_v=0b7((ZTFR`Mn~CDA1BekA5gB5b#d zzDHU*p8idG*hlfD<)<2N3iDkzao?yh1@GR6z3j|j?a7+j$^yIcx)DA$Y=a1Y`{760%ZvFH}rG@poM(^^Tm1P9Czi7vjo7hb?~$ zy9ySU(@)IL@J^7lEchQS0HR?&Z^O!-i}BGNuwuoBx-s#KqJVZjkgREHS|0C=7Sad@ zc&fW3In**46_M>xEa>--l){3Ytx;w<{JMR8(e2Z%5l6=-8Wbe2`xqgDA5nzk)m!J??alZepX8To0xKdawu*|0(n2k-C9mcC{VTJF z4cRl^6Xp;RQBe)03~QqMv5@46ofZbpSYjb;Jv>qGaVk5d-&9^D|v44QZ3HDFAF{Z&piBN02hbCBsA!JDQ%k2i3UqWerf z$VcJ=j)|mbRe2biq6GF^z7q)#r2w zjy%9F!fm6I*=Np}9|45jnMi>-Q7J)quTDJCj_b}^VSQ;pV|I*R|DL%*;AWxUO1Fzs zH|NfoC9tRC_YQ1>d02NN=gV)_`ZHd}vvUz$vq<;)-wRO+0+Jn$Y3;w z-~mEqa$0sc*MZ`t#6h5B=S}f?6pzu9FpFNTb&2Myv{SSjHd#ThaSu z5$8&;0H{BBl1znIuY#>+n*qJX=W%BDX-=%%)rt@?e6FwW(z&5LV>=yEuoLu6&R6XY zB#NYvjxk%T0h>G@1!hL|%BRTs&vV@wKC@bPHTS$g#C*UG=)QxkGMc4IDpoj6K@oo6 zBO(yz>=!~4q*3)48Wj8478X3+5wn9oZV+A2D%gu*H`3;*Tbu2jAnmM-rt>zGhI5a1UJDyLVE2ME_db`=jb(PKmS0cHNxi?aBaLA~+aY&-^WA0#YQy_g) zV14(4@2V8SpZAqM)y6q7zuZUI%eetl-bf6&9gXPvaJrG}H367c0S*H0FYlzHi9_~A z(@o#50(iyDYU}&9m^EPV0f{a?IzKum_fUI%R3pIxjM z+j2U&Lag10Y=)}RVqtM`{)3Al{>8zFHFrMzZijcYSJ z?bGAZ4FP6TKdu_z7ip`ulEIG$ZY3J>@PPB>hipv<-4X$f_G=8@ue6?OtLQDf7yHXh z9La=bc52ECxL>!c_!)c%hVp>Yg68KlXpH!}COe6l~F(W#ImteOJj-lag zt9?|6Ei&^5hcHbnjl8gr2Jm~noxJv+nDHqh>q5$lL#9vLIMOtU(66S}+|_kIA<-B% zLd>wQ6de^in@X|9kiV=*WhL@;fCJy@BzQ;(gg&RsZ(W|mQt)iLR2;k5+9B#4>PzDc zB#pqTCN*}T5k3S+E?6bcXCtR;=m{DlWK@&#=B z5dK5eR@UjYvl)Oa7C|JFxgMC5wPs*)H^2iG1u_%(Wr^OQSyhkRsP+5+DcSW=ev{cw zU}Z*pxCw{m0GTmF>7=FMLvOQzRR`r(Z)mDEuZKSipW-nl&#!QM$9%Jb*eJP=0#4II z4>J&jz?Gx#jDCv~RZc836yN(|8rxYMhb8(Y(H%G3o(cl`w>H|_HxTqB>(iJgA1Qa> zt6^U<7-uFEkzvt*D9*>M3#pQVq>>G|(@W-6dPa(t=G__N9{&8T25HV_F6}W@OASo0 z(5jW`*0zH0pD3hps$j+QzoHN)rdb(q!5!+B5x9X?RD1I&`RK)2IgC$FR^RLEx9bl( z@zg@xhDK4dzM&Rba`*gl95+yvZ0q}MQFqE@K(zc=Ive2;j;!ggj^i zh}d6Vy4sq9KRgunjx5u@zfP13Ns_^rLkB26vU3x`)wS>4C%eL!*#^Fu^Ezz!z0`Oz zCq=&OqtHhBps0k7cZmg6JIdo6S=j^tYk6?8(<5R-aZ}}1h^P)5Ch#-T#|Dmu8PKJg z+I$#xj^fc12z<*HUfF*2n=~K$K5*tM=w85d%ZyS? z#IM19YM)oDxkl}pNKxP(-v%41E4DJ*_ciTiQK18GQz~b8SU%OlU1eoS>?B8GloPt_MdJs*^g}h}45B!Ml3;OkRZ9Rp+EUTv- zxs(B!1m&iqW^P?oaQS1N-iHO4)y63{qlU{M4_@ z!@NlJh5lA$St_d9a>&D9s0k*G!(5aaw|kcbzl}}&GQDFNbN|(8U4pJI+kbyRk$3fo ze^qb!yB0)Pes+mBupC7Q(*TgCdhvBi)d6LuzS>`wg}c(41645@j@t-RB(w~Ni1@qN z+T%sj+6nZ6W{Wmm^n-^l7zN=9j@7Z5$|d$A6-QZiu{DkaFjuYaRq?RF0T1N=?5Sw-vUDddEI2c$jgyA0j5CWy>FEN~G5wi_2|!2PU<1PvDAE z|G2%GY$Q97_#NPk-fUOU13sgR9GbBUr4zRZU<0U&TvuEWlbew{9v`|6SqwxwA`83X zJ#t&Lt$UA1)e%v}j`c8o2nnwG10suiau~Tau`;UW*lm^IkBkiF&QH23yUBwC&3@1| zAj)@oIl|b`ar|~~i7=m_V0>JKe4p2Tlf%@_gBt53dhgt;fe7y(MzPjPH{%$=INd5~ zUR=TQDv8~Yt&)F*H$$JPk)<*RgEA#M%3^`?X*7)w_eq>sep6#w&!=FKuJnHnG!#&# zSS)D2-J*44{tauL+MO!}Bn$^sFJjmZJ7bsD2?*!j@A+7%mNP9SDUY5LNNtLlHTef* zYd%u>eRIC#@ZX#(WTmxDS3Vbi^0*Y>WX%4Z_{_V5{?z4DHZ)9bTih^=r`{*v`bya* zKBh_(my2&};CEHR45qLwHc(6H)Nm|;R;(@wm=-C|OEzl%9_D2^;M`b2QU}X}(Pt=! zshR>+mRxYHtzZF4%I%z&7;D3K5{9S!8Z8<5wf{t?Hsd8oW4zE4dABv1MU6Z`D0Xczz_+MH=Eg-4~6Agkx zDj%_bUiFSpff)A_E>k<(3=nLjL+$(4h!pmGJe`{L?kIGxzHt1z%203WPxZE(qx90?a~LYiorrd>`RUK;E-wPG(Dv&@4G^^y15bCVYXu7 z%0BeZw(7<2^A9qkxR@UYWUqZ5yLtG!n~)NlWk>i#8xuRzK|h^@s}?Jm`u`mW?I#vB z>?RDDrDM~_(Woh=k4>xYjjHc%=Mw|6H01QEvui#W#DyzrFsMxbL7Tfn8IP*J5_ic- zG9Wek9F?bL+RDMrl~w!T^4@?`pAA}b7UIgSr7qjoeU&aQUP*_i`yKp-kmY;aEED2X z39hmD-(`49_!M-%T9>9)gq8A>xm5H|+sc!na9?|Fs{>8)8WdoNr4#?EU~^A3r<%{P zZr?m<*N@|c38YO77wydYt=|*>g;Lf1Nh==n)dfhwq*+c=XKy(i`{_eH^=$dRU~JKs zms)5lg=lklHhbUWM&Ko&DGjuNYxl6y;n_$nGd=G!j(oRl&$ z`Vrsn?h`DDn8tkHc3H?a(lBxWL+^CN+;2Qm&bFC&lx@wB2pfsY^DE^>iaTX}bQAQ& zWpcGBln%U>ubb}QJZS&8X#k*T-5;R@Bf`1vMtDjpppRZIHLxbMr*4jHOC@*OQF82o zSf|JLUjMBV!!RVR&Au~UTvX~ULzUL7H2U!{5@f-yy+rTbyKkbICZM#Xh`}k#a65Sg zOSgmNIX1tAHTc$lU5Lvt6>Y0;cPVa@U;8yUuAX8k_Nkx+932ZR@A)F(qd^{HAja^O$G^eGBi*AoY5bLD^x7Lww>`7R`$ z&)9K%F>aPA2w6WMQb3V#z@G2GckRD(i7uDDlB%CWp!6$_9xTIAVQsK(oOEbJE0Q?1 zF1;88<(4UC#aW4be*f?jpbv^w;hwU4^Gyb1Sypc7W!OR(ZP(d?_Y{pM-L2x>o_+D; z3kA4M1-!!9l1{XnFG4Q~8tM6<311&blIq36(qY-^ zglf`Z`35^twFkH$=a}<5UBmRZt=`S7ocK^Z#T)M4&K7@4akGz#>a@!c>qVDm(~$lI zNV*e*T}z(FPD+ZsJ2pEFy^e#?8XulBkG%D?ay>=N38-`>HUP?jnu1R2QTdyR4g*4uzt6;zrbn3Q-zC!Po6~ z+!xj?i`+xm^WDf>18)O%I|oQwf1czI7m$(^NV53Jvl%FyOWans7|4~25K*y)equ-&*`Kpg5Bj@plGoGH1Ea{EFnT z)BehHx2o54WZqVyd|t?DOxHBqLbuKZa2_pXZOtMNI;F}yHrhPlO6cG&$#D%QH+hhx zTo`SUZgySsZ8gpJ=g1-@sTtr+&me5a+O zds9zWGEFg#re@SHOskZ{wzt}lBQ!#IB_3lx2EC5aG=4LA?74V}C z3=ICYI8R33VzDm;|}B#F%gC>@Y($)2B-cW0P1g#Sf2xWI!qcOJ#*GF^kat-7W5 zi^BS7d64<&d&b22I`&*xk_qvcXh%KseNw!oFW`oYLRX~xI^L1;4$Ubt3oE12IFO+N zv{{KPUH7wMk4j>PbLc@C8;Q-6hkMvrO;NVaw!tw>S4o;ntp zbnW-&YpEQ;dv?WORpCTjzeaiX2PHuW`o>1E0}IX{OQJv-uy=dF+_yy=K48DT9Y?S_ zbzJq_+Ci<2a6E}^8A+ib9~4XmG(>=6^*f5a&-z*zXFRPhowd?~VNY}}x<;puZewEJ z#XWsM(C_WAI8wa#s_>@gAC%3Fn#fF=X)~^tAeJ5a1*lh&w_BtCH=F3#RM)q2*29Df z{OwjszgN;P8XGO{dSTHIaUq1xn&`{3R6bLC)I3j3tZe;G&S^@W!LT%rshiIcG=B>e=7oihH zz+QeFLwhWIx5dXWfUG2TW<_E<)(kYSuwx`_WiFK*##rYQ>WK>*Dcw>KG7{fF4W zq+HC>h1BNXa?1?IRbz8niCL5R9#T$zw>UwCROT}3`CTyA=bvGli*2VHnpR6#Ny!fIg+Vc^61$yQV7i*7 zi;Q59lep~zdV{w==`Deb2<02HJoGu1zbGa1A2B2YVu-AaoH=elc~^zbpV|(ep~K+= zeIY#kj5!JO0V;A-IpsMhYJfpu)E~n06I2b66Ixh7Q$sstx&M?o$RK#^{ zN0)R1D2oyo+Z-AJ!kb!d90E||Co2EG%%dDa#9Q*`Ots!A`A$MD4w+$V8knFpPBJ!& zz9%ukv{~;!2B`q%z2VL~FpfJ)r(GP-Z`R$=F;Pg-j~1lC4FvW|xQ7n54EM2%%uQJD z@Fr}9-38n$ey!sKnFP*rLq<4|^$^M`eo7n7{MKs`-a?hA@aJ?0I!%S3{Vj=61)3BX z;G&s-`CZ@ro9H6WhJA9Y+^6;PlRwoHMlV)itg7d7F_&iW^V89cB&EQ7ByQ3M^qk5A zZ8mdk&ZcK0gi`8^KfXF`f5ZBfv0|SYKI_iaR3^U)CoC*+eqIbeO#y@EpZvz2>EsQi z`YuOUR4HglGdu;$PKERHk?QF9p&fT#I>B#v~&=&oz`;1#(meIVKoVJY_i z$hnmvdzFRk>uhKak;j~F@sAu`P9E^%(lt_U^tV%EjsEFTT|K1ZIe-K>CrrgKkfRCvYhd? zSD7*3;SHSmHlmKJe3g@BNGVV60{s~5Pvi!130)#$dQ1il50igZ+koPX;-zz!YFV%p z8@77#_;|s_8Dp7&{neKGTW@KQH7PpU-|Rf%|Ja!xU4i$kwkAVA2e9bL)~iks&jr#X zT?Xi#>{>G_^6Sx@Q<9nL)^$j9BnSLr>r+am+qJwr}8J)H`QSanjO7BVCuyn4VWA+OUrsN_T(Aa#v<-HyP#65Yo<^k za}utfx~&;u$}5~gX}kQG-Ek$ z*LHZG)Hok`l)yRj(chqmb1r@l)?*JS4QHfULnY`~-x!-bg+OM<(LISvCg%PQ#jqH}I*xlRB|fk3 z!Q-+Sa5D2n((1NUZTwz;*|}MxfIIES{qOxwj{&Ra=vlh389ti)bddH>-i08TZ21`0feH0Vwm6t|U`3%*`hqzxL{%v4}z z6I&bBey4W^hTc(Z(}L-)%Bfu7ikn&8R7{tD&7Q71h4B`T z9Tv2HEaPhU@e(C-bRlsRY=(R(?>fw%4w_-~p?d&Ka7?dO92Ba^w||b}8}l4bwbKVb zfgh6v7KrUYaRTFa{x}ZSEjD-lWeb%Zv zd%8M)bcBXSgu~7f{u`X#Aim~UzDc_%XHtx5MHY->)WQHD{je&ce@Pot3pJLTTV;dJ zJmsWn4t2!c8tO;SpT)hvu1W~$fpm(C5ol(j25P0&9Hnanz8Px{2FHm*yeCAACEo7J zwT3mM`Sg=_6rU{jzdAn_bTg`V{pG3;>DV zJc2<4Aq1bPy<{;c8NOP?Q11Pzd(pAGdd?>tC-ggIo#g?_!>B3NnZS_Hoa7(fJy8S_ zS#{}S@RrUK67JKM(4H5Sg#D7b1=zn6+0zhHa>w zp3@sh{`#+8nm2y@8=4U$Ey?x>pf}}hGy=+!rZ;}1+gWi7Nl+&Ga(~OTuKrYg(?H@V z?mpquDDN}0@w#eade}j8rNa7zYe!##8V8(0I--+Pcnf#5H z46^JFURKe6KC+On_x3cKEJN{p(8DiMEF*rb7fF9z@3PNaXQ3WoYlK4SZ0~sNmpGo6 zp*)CfY@zJ^U$+5Zh!Rz-ekUc-P|q18ig91??c03vTs6_p#rOa6=3^Y71BMS&5Bpx| zt@iix2PfCe7lJbC=WL7S&L8*|t+=KV@YVxg9277YJ05_aNOFGX@BtB@oT^2aGqR!A zZcGWXzGjwB{TKA&lf5$!&_L}w$lrW;94!%BS2USwde(Jkv}H|VlDb(OUB_3gbALBg zdWSK5t&-bo(D~Z{TIT(Y=U+)-ESXC7>smA=LqlJ35 zzTX)pW5!2*eu2?u2e%X+2Tn9ux*XBh%;siG?YD}{w>71ZG|5ss8Xg`DrlK+|IHBT< zAW=0i0eAiL145N?kN=$CzB;4wbmAqRuV{*kJ#uSbeJ*B28IF>?>&IbjuHV2#bz&0! zBiX{l?3)Kg_%dAz^7A|qC9OJp)(`ov%7g{;I8qV7=}&?pdr3lDN%PJ$)P3sLuNbXU zVA&Xq#+kNS;6KuAIJqT58O?KOp9dAy?)_TtdA-q-rgeASAFpRRQ$tdw%_#xG^dkJU z0u4@zvRC3XIdb3VkH`2OfFO3odob_yA-T@-cRA1JwowCoc~$gffAjeUSP2Z8Up@i> zm3rgoyVtf!uW)Ro#XLX$(fYpWutPT(kK*#b|dwsgN@YIt)3o9)_`8~8dJ8*pLCjsZc&Df3bT$h+ zhK}7V7&P;a?G4c)bjSC8I@Gpx)=P zaQ|T}Pb@&VGbs^uFaj))&A+;}=U;P?&J@}vjD4rgexb|axo?bZ-qF>qe^d7aj3Gt@^#B(mNFcH^>Tk;N^VM5<8*|$zr6PqEn{m287`5+}r zZgh>lGXK!H3#rJwS+LBrNBM%%4=tl==Wkr|q>nQW7 zG0~SDEbzlabO(j;;4S()SaeB;>NdQ_zf`a`Hp27mA--Aq+|lY|-hnhW_;gRYcRU04 z%yeEvn`xM@PSWdHpHSX|XPt4{Abq*q7#-Vzh^Zoi z5nu+e-yLa#mR9ZS$$WggTteKha;>5$#RIBAcopFBJei-_+-?>1gXyHx+k%(Fh~dJ? z@~KGo>`e@AZCZDQr6>|4Yp54T~@>Ocg1(okk;nrq;?q}ixDu(!yy0#U0)kVqK! zqfhSxg|WKb9{qKAF(->mqoao1t(;16`W0<#IPP|%1$z7lU6I+{l?T(y6`l_KLtOy* zHK%ny$*tTofl*IW-^+sntWz%vL$pGvNJh^1bG}`Or>(Q|of(UHM{o9fBUqaD0>h87 zC1HdmU5zYVRS46sIiO$850Uo!$gmavTdwX};m1wtpdP=6F`+M#=Q?VCW zwbh1Us$||<$5Z`B!i=grt1;(NwxR71C@|LxbTslVTiD%uD3x88XjAjybee~?B00ep znm5Nhe<-C>#DFZzp?_;^5E5j!{aF|IT#D&Z2?hGdp#K8kXD&oXP0O&R?{TD`F&d<6 z^Dpo1C*xgTbX2!}cAl_g_poZN#NNL3 z1y#ty!nd|Y@N<&(_Ke@A%>*+Qb=q3YObTTrCE*@R=<;M9`6=_MK+qKg*Et)PC`0*o zL;0M8Gx_l&*YkzPgP^I^;Ulw1IuFup|IYN%aYRplbgP6R7Hu8h*p{i^OsT~lA){f{ zMXMp!v-4vO{RDWM>pOGRC%l@7&B2|i$}@88>k7pLAky!NYX4-Zr2b^7{?LE)e_le& zRCH314icmcgT`+sBF*Qfl5m~sK#N15Xb5B=@Qhryw6ht#<5;`ZiVjVk&XmyeRhjmcTk@#O(&{C{7)dzop$ic9$>K>S=>O z3-n>y(Exn6YHz0RO!;~51#>SHL?I(l!pg`q8 zXXwz@E6M^ugAqdhz17zkf0~k3A?h+6D-qGMeH$w ze5(w$mmiD^m0Al}`R(E14=hTz{;IAHApi08|tlAs;`jVJeGRf=*$zHS}H*=X#a za~jLvdUOY|`7K{FmsH~98UP`x^LL;L3MVfaHXwsnIMRk?%2rAy+ak?Z1~VMlviF0f zewonaTn!rTV_&E+3bc#W;o2Nr>oIMwGb zg4VpMb4a`j%&I7iu{aN9x+&}z-x!D)ACk{YH2;(1;VCRt@e)q*h6Lq!n>F+dyw(p_JAVYU`Mts6?el?c$n-^QeD%rPp{tJTlXFlif>R;T%LY983ntm)X>b2?p@Ymn+`!I zH(R}-&#~(SszlV9g$#_W98eDPz9}js4GR})8d0ZdcmWih=m=UV_MhTYc`^5H;cSYH z^{fwpcuzpSZ{X{>Zx$EZp03@U- zyCYcH+?|)ZF16QrIaM)p5lrhqT48JAPxdEsk#!sp@sf0f4cm%NUnhAu zkO0SIng-Z1s{s2DwNG7e&u;FTH7U+iC}?vrwYMX;CCRp(_Wt?jAj{O6^a>57Toc;v zSfeWOvQxM5SgDX21zf+Z!N`@~qD_HX&z+$;!eo-2Gv%f_gnEsLN5eW$xV$_^D<9=$ zYDmtkspH4w0m4}X2?ea^ruvQ3q*%UZ5Nk5;mOIb{^luMMPx0^J&%ZykZ(N?jX*QGj zoQ+B6<8?A5BAhNi#N>>+=KEj?-2iFgE}fwll((Z?7>~j=3$4L)ncIA}tP@EJZ{0Wl zHhbU894TB>_C~?I(S#au%2}uA#IV4Ib#5i;V}@m=zj8Xxu+8K%eS+y|6xUw=p+uvm z^BPWH!&lNngX6`+n;Pi&0~)JKFs?&JHmlyZ^l(7R)U_62(-SyG*8ZCFG7=L;2*s8PiA%3bWoe zR8imLYy73|0#tJg3TSOhgLdxYl>qXn(u7fh?q&oa3Fu?0N#E8B%A?_GdBsM*`Ky*} zntCo9yZ}gg?N>`nK?10%#t)zwjh1PO)Y!D+;@X*zrvSDyxjOQ>{TT0ssV8eH_214h zm;iBC>H$`qInBzx%t7Uu&s33olb%@7v=N}xQeqU)wsbnx86QZ@a?Jc(y7Ky|+!1Nx zi|c~Gl(b;(Z+^?mSx*alAsf#{vQEfKpQu#Ak z4Jgca^776{11`C{00oPG6}R)tL+YsValhPq_F$B}t6QjscRlj+Th3hnx69uJcdf$D zl2-X@&KszQh8qIyou%t`?P?~TiMDPRNuZr8b$LI3m|01NUe?uV+@uIGLE_u+)`?5? z_TV^yR8G;MTnzpp5`Z!lkYJWaL5e@#OSQK-qW7fWlE`9Aam+|c!OCmLXF_`*V<4eeFP3!CkSY@N%mtc$0ozhzW@-rRl9BU1aF z$!D6^-5r-1c^Kz{LU6= zB}2G}S{hz|n}9wBYl^5w8ao|HW7ffQU+7g0C(mm35(=)z)_TS-uq7_ zSRQK8;mr6__+j8nJI(36pV5Y7J|D9y&d}jpw1grg=!fE9&ukI_8TAk3#_q!#Z`IKN zBGxrOVq}#tY$Y$kJ`}S%m*w%El{Im9;zE=rwuNsqd7e8e7GJc5TNzz&2WGUIthMeG zM8xa8;&D92~t}?pn zoG;4t^>GNT@%JQ2_N3dY4+egEY*(wmP<@%!1;^)2lAMRxx0JulEEq1_dEh*~8Oz7V zLiq|iz7CIjW>M7a*z6Fgw<3Jken;RjcTpf$;8>OGq0K3%sEO(qtCX4ioi%#CQSdBg zNC6=pUx(KiMsMvTmtANt`)i?cuOTCI5UtL-lKmUe6|O1D3j|jcJl!i+gk|nPb4xW6 ztXB3LXp$dJ!seRhYxQ_e%UFq4IP}iY>hLt4R*Rm(unNB@# z4SOTraT6?&$CW~yiiCjn#Pl-GuRE8mJD&;J_A8O!zY8$28G<-h-sp`AFQRRUJ}}W_ zq9En&E4m~k%3j$^BADD(7lVO?%_er%f_1%&fspczYUp1vG7P*}_qO(`h&`bR_oPa! zh6`m%E6&_^1G@Xg+{=@yxS;01=(<;bKt_VwVlUJx2qPwXi!C6vHqCk#t4K#lN-Mm# zCcRccoJu*5F3Z7~4Htm#645ZSwBgKL}{qPy7@{?c4E>l^|Wd7VI%7FQaB0Gl( zz61O1zQpxXqI@*lR;|y2xI!;Y`W}c;|2c@O_N|N3Ae6+StkS$1JB3WepVFQI0Gl;} zLTeAg_G8gnM{H3Z=wWKZ^M>Y$p%G8iMWSs1J@!mV2q0S_iE5&ym_L$tU=S!0n8^<6 zZKSk{Use~W?t4R8$@XvGqi&jW5@9W8TbFK^r;((oEbOz2b>%A(6sZ-)K@N}Z-*UYn zEF(|KcCK1pc6JSYdc+23#L+i)U=ml+?ZLzPX{23uSR+c9VIstz- zeT+MJOjw!hRJ0-uQuY7&j+~wy9+CsN80&UktoKA6o3_l0U;4!SJru~8MX)L4fsTv? zdLA8p;4_aQJd$B@?|@2sziCt<{~9cLk0{6=w-Nv?=!z%tAx6u?g#=+~`6py&5^+=R z3MaHQ_cY?}DS++a7rDoBYmK@3dm0aRH^$u|ghNML3eh(h($zb}Mb<{rmg#3B6I0M{HXoY*YAArkW zU||Q~8IWW^bOVntDwY>0#gHw*EWCO-;dQ(JuXn5l!6)L~702UWN|6k6u zi6C&{zC?MGDXI7JtndM%bk|)IZ__i>pRi*jBG^NMSt=fs=PWt@6hjWWWy5??Di3-N#&4O;#`$-QL znp3Ts!H}&^Iz&JR6E`6F^3wYDis;|uk$;)H6r}T^K$ODq17$xj{C;vV@}jdV+E4b! zV415B;YC9D!lO^*g+ccbTk$_DJ>uJ1x#q#*R&r zu}#REh)<#a84Gz)jY0V@T~04_axWC=Uw2SQf|GEHXsR)Y^1?!OIaAQk$={hEuIc$L zIM!4?%Wt%rbKfnhXC*1aELU6j{z7LS&BF{%Skn9rL8`LU&iuoh81j-s17feB%Q4j#U3tN0hXP$pI)Wy?Iu`RU`(NUTG@$*h zY?(hRcSkM1Gnt zIf1EfHvnH5 z(36)gum8C)U?dWY?OKiR!&hOkXLL7qBeJki=K(S&Z{0;a;A=Hx%GbT(p9N|3K~s%? zw4^1_fP|eEQinM`x3&?r{Fq!iL7$I0h4)3n{YVX>kCc7k!%_0T02tLnvmt>xNK*WR z=EeB1YOQf3;Gdq}JX#*79u}BpF_y*d zRXZoVD9ilwrU56+Y1@M1Pik%*5u8si@dihd)SLkiu${_2p#=rx07Ce#JYcE!S6A1d zM&(($l4_+nbR=i)Z=n6k7iV3b3+wzUe;XUq@k}az1gsz0Tnm*h`;H|y9wb-COaAt| zQPZYY{|kal^$6#FYkfXtVLo4ju-O+E>DjeEoWlIB_S@8#;*YPH2k;|>@H*nFZa*9S zM3`vffc1d&$Y&fCK)ljZJ(NpuatDxE9CK82>o+$(ZcUX{+pfOKL%BA?_i4htn$YR{ z>Od^Eb(qq&F4TIiI=6{Pqhu6ai`NNV9rOiVlJF%H_{OeMO{Mr(7m&~Q;Ys+&b0y$( zO&i35({hj4B|;RCtv22@1{scj{i|w|s={(|nbqcu5uIB7`{2F`nI%}{`02*EU6s*X zmM4*S+;~X%RKN4f!DAZ~EUZ9uoW*FFl$m0QEWR(~g@C9jc@oGn{5(G_{|%+Tw6HgP z^6XE>$Qya)=Unr7lxauiOsmY}7<6Z4F2VVKaRE|z`Qyd8PP_e#fNI`yxwjrt4n@qG z8h*8pj1y%>Qr43{_TU^dsz`Ia6IKDn-A@~6oz z`nn7R1ZSIy0$7wYo6T;UNC(`E$!#xMWj7Cc{={st=&$3Y{XM&zGpB+Y3_n|EhY?4s z?)^3Z!Qgiu&DilK<7?)Pl&A;p_FL4Ue0MXOrL^{;(D~c19}yV|v%oYg)0seHP14wb zTfB1)Y9TQ7t-|HJvj@}n_P3GKzm6yBa5JsW)6KSiWzVrIRHXsIkGNrlamh=t{wJ9R ztmPNM?=Q~0UNv7Ak+ZubMImr-NtOPA2v6amw;5uJHQ$2DwK~?);|NkE*h6r*C~byQ zmXSU_W21D3N|U}{)~qp~hu)UP?`%HC$l~>RsF7Bw$^BBI(2eNBCq7E(ce<4E8gkPM z8Bp?A1qM!HEj|wf5ER0*tCXSe#>C-O|Li(So8WV*a$P%S-MB z`5CI7X?o^e3;oVLvx0~Wo@MoT__DCDkTV?nZ!FL^^r3&4N9XoM*$U;HMcy;8b37j7Iq2s)kIlu_ z!L9~0bBI!)++F4|wN9V-rX-s-L&IRJwd6t(m;KWonoqx^6+ykdenaex53YwD^aQ8c z*{bn~5+lvj+(x$JR^@y4*u$_!KkU4{>*>kOo~SnXM9r$1JDXZ(ZZ=yD&cfigC6g|K zg2eR3_ObB#VfXm$_mkX$x0;QIP{ z-9!ISR+ozm^GqSk*n{k-$-T-wc& zLrJTf%^o69c*ML*o4+RB{;U0 z*VuQy;T+?O?FB2N)f6i1j)j^<$835~^m9mcACp|rT%?YPAEPX{11091 zUYEF15_ODUJ5=7#W(f{OVTw-{%QMraYS4H|i4(C5jAw3*6C7x!rKWq?W7on?^OYC* z|4x{kNQ|m|@z@D{nG9ZBVO0Yb@orTJW_j8;`jB9+)_-~jOw{Ter^(k+Xf=6k)8%R3=5b$6J^twC0E0$r2unD4SJ1V%F5RRlYQ zznZLADWf~m4i7ab(t$oFNl$BJMz6LT@2^Q6ht-@}SV+ZVpAK6aNWC3chv3>oG zeB5=edHK1DoAw{;!F+w7^nn(sK_g7E;Y511kyMKQ)4)eW3XlO%)hB=HW+fnVQ8M>& zv~u)YvP043kJSF^bCFDLXJmA5pz3hsWyD|7PH77bhGbn-YWXKBY3 zJ71hRIe#oSmdi;nIjRNR)}L+lK29x~L%pm`D@58?4DV06K@t4#Xldi{E&~0Ix$L<9 zhSeNgix7LRER^f6B*op?rJB!nygr{jKB!zRzfZ)!qp@_3C(*d`zKbh9hOE7}5Xx)_S-z*mfQKjoD7 z1Gne5hd3DD=2ms5^=};VyZ475JLD_ts{7uGrc?f!uVGsNYo~k(K(js(9@AJF!uz4B z0^0Snf?-Z!ga!UUjPQb@8KUY*zpiN{0&l8h>GBew_FjPZ$e%XbN!_(AKSg^CQ@X9) zS*bdGBMBB?D4+9&g$T5|J$*dCeb(jO)C+lTr{!JZP}=E)KCijT0X3((CE+;Lu%k-G^V$n95KU}$5>P%Wq5*P#A0%UW)w0boAx-aHti|Of(Gx(rtZn-V~-pL%qF#V z86C$r*UF~mH}UKh{9H$0wiNn*w)AW$(8pA&qPPP@zs!@i>^< z^g49OLa)Z%fS3ghPEd^JO0QE3sVU-ETr=mM#1-%5`*4M{ij>9C4ejH)tU% z&sjn~SpQl7VUxTSzG=uZ8#dMh!`H>TG2Sh#C)ZwZ^N;!5=;ds1T8yKZT``a ze88Vq*gKB4!W*ebZ%4)3{PEsYIX<1=hC=B|3m2LjDeiK6K9bMxUFBMfQg0rVjzfu} z1%?zd6fPY@zgBDE8pV*QTMb7&G`I*3Rbb9m{5!N9pUl~yMObPL_G>ZH`wU3Yj>QF|=I&%9Bx6z7_a zulak4$1&^hed=k#p|sQ06vcwO^IN4e<#>9X*{4PYD>X|)i@`|h@HwvL@aCB`lN`rz zn%C%HubJj`SdyX?%<_OmfdyXCZWFG?!Ji|lx{?3n)q@5b%(b?ZA zAwJK}TX|xpc-cO)RteFAKjWfup{li9j;N?ywIQ`w+`?T9!!754op1MYca$0J_-CON zPcBb^9#iw-ma7|=>$JV@s!Ija)UxqL%4uzpjkTiP_GRP9T3xO{SctYXN(EgsxG?dX0M_f+t*LXBWzXoA`o!Ofbd zB$cYsJBJy%OdM4nuG2V1X4-6>#eH0FtW>62KT6&40-$pBa|ZlEu+^}cEdAlh^#jh8zu?a z+DR5}12PHQ!=swJXMwqn^qo(kjhp!Qjkm`S8c|WyZzGwF*qghk%r0fpIEDIRC~^=k ztB03nt|D6pha)K8M%@xWvz+L)zPo!*NCKNTLC^i|j-2@+^PamA>@MFpJx(W(=GI!e<(-mHzef0Ypj>#TDqHCUS`D8a(%a&d1l~A? z05>ipChj{N>|aqIa{2aWGGe`z;|V)hPD>?z+uP*#4D}9anaekpgdLjNe8^dkG?SFphK1lfv51%K~<_}pxEckVVo!OKm zV=@=&RLXsS?2Fa{#mh?=$uYFWL>y!s)A|Xh=BB(qfmd|J0>=!6qGNv~Vru^Y23

4aj(lAK-(3Y@Ce+Y!*fnAB>d>YR6#VWe%Z-eZ7*4@2FLjdU_V2w+ zz7~@dSlmdKuE1MWm?SMT?;Qs%~HutBt`TJWY2*=1MKY4oQjfyBEjcK4$*!3VS zZ>`8F!>;`w4~6x%z@GdH{YN?(v9Eds2|GYRJu$L=H*x>*Fmwi(5lDlKzgv#@k&FME zmlw2{T~Nf*)|!G8&YA{1ft?l^c(5}+iMa!W&A-(s?at9f2JDV?_CInmCE=xyggUh# zq$g1%0RP8BV6{6S|2E9On_p1fMK$_^R!G!JYizAf8fn?mS2-RHP=}mOi&R4JB4jBo z6)6}m+YeYkz1+X_2KiH>=#Lv_q(tPR56l1Y(4hvn2_w32f0bcEZ9UeMKb|lLMEli7 zBR|jz#8=!O9Q1#T7dFZdEPUY~jR4d#LXw!@JvmSkS}D-#KOWYomw|u&1@iA{+ABE~ zXUvoU1QB0TU#idnn$MUt>I4+;|21B>>8W}}rNOpk2sBRkK8t0R>L?*5@P%pHtscEe zLj5jkYArDT!(yG%2iNybAnhjWQztk)ZkX*MonIV&Fl+ne!USI6W}6Rz!C}HJGRbQ_ zPm7{SjD_{+++QiDN;JaXHW5V*VXW@@z^vWQjhd4?gIN+3`WB69;G)~L ze&<-{_lQZeX@k|M+yoXkcK4{e42f`U(`iyDDQu5mm2&M&<5d5hiADdY)=L3J-p96@(IZm(`pE z^g-57G}^lwh@a^2pre=A34g!UA^>x}stQZ`1a=Vgf8WUO?)J(MkBQHnL)c&CdnnJ- z11yWxlHjg9iBP!oB;X~tsg#UKMg7WM0sZ_dyvfrzr zuX7M^a2i(BCRMJJelqCdwcni}K+tJ5UkYa0XVE<>*in!&nGoZ;$z`Y4a&tp=?eg7w-)3yJNBTM^TQ958f z5X&KP7`6w5AYd7*42MFFw+4lQ?&clzg-*e-zku$@CCf@39UU3Ek!!4OXXJT<3Gg3v zK3+f{=}jhhEuY>wz=3yAaNoa6Ybekz#z#0@^f?Jqh+wlLWwf05P&-3k;?Y#+^PYmO z%0%G4u5?}K7-*tRxW2j`tw_CncfX5w&{EY;mjmw}zj=?YQhc&=YH)LZ+t8@KIBf#a zve?#ubm16qipghneTKJ=pz5HJk94!O(vWoCvG)VZ&q~Ek+I`;>c72p7U#mVf&{aC`^u>N8UYS>* z+n+!pj@rtQxt`acu7lEk;|m-PRxtE>R~GM+yXjmNd{+p*&ud?SFBF^ucaQ!pyUm`D zN$=>&)n0Wxd%a7Wt$*Bz)1W@s9XAn8r1#U>*CMfJ zA8@2;#e;50B9k^i;t^9u>?i|K9^3j1CcBU}cgNv7{t?oSb$@+y|ei`+0JCDYV4dP+IjMa2V0{R&Fa>ek~au5SuLb6;4PG zUeg!1Bli+b^!vfCn#AhPR+?P7>i?&&GmnP)`~P?)*^|Ur6S9S5X=E=#lC=`~$iBod zgdqlH>?(Rw5SaTpjovvV>{G0^aro*Rnl3SwW%M<^LxyDY*vho z28*VC#3MbJfq`GWIzNI@4x4g?;s$JWIbYwdO`$Ii@YF-Pa-ebfxb%giuH z?)=fnIjV`IVKOE+7hluh3OnNDiIICtqZ<6p(g z+b>Sg^T)n$GEe@mx5wi8nNt@)fpq@vslN$#PBWUKk{R!abNyD=$^XBRgkLed)hNFm z=34sWw=^-zimmPmN(dL7du^Nv8Hxz>*9NFpt#xNe*u?%*-K zzw_r~=d=kjZQl#N3XVLXMIW$=*PV}>*9|)umr8C}A57h2=L>Js57o7cx>*_N)2LcA zM=CbgU>DMCe`Q~Q3SQGZU1*ZjLbpf5)sp8rHXaD5^*n9)wQmWCM>#BsTnIbT@G&y9 zg?HBI?rica3;Th1>Ow|%r|SI9YuRF+ap~wMv{ueZyh;;XRe^Tr1+q{4s?wXw+ENk5 z%D$Pz`Z736TO2QI@}TaU_Y*wD8f_bcB2nuGmi3{q8#ik1N26!=3%27pNb17an%<{+ zL&2JeeysDN9eFCseRQ9vKtICLg#OWBS9)*(+()c2U~N2_w%-4IJaoO`=!tv+KW7|~ zc1ZQCO7Slfw;5}rkvpOBnYqpi>VzGubOH?ie@5f(phW!@D$=YHszSzfOv~l_*3vFs z#-7Ib&L=3^lNkd0RM|RU2qA~hkORG5Yt$>jz-}Rerm08QAbx9J>+pgEwm(OcUN^2G z-pn=x&I56|6F9~*kVZw;Gf8Bt3DbtS&SJSnC1hCP*H_+0;F_Q_tY(_e@&32&OH4~;Y{CdBv z4c2Jq`_BoeQDSoK+u6lP1NCx`Ye0#`@VOitCt#!VqLS#cAjT2UDt@muTt$&=F;RQ% zveRW|lrM@Fu~@H%fEX|Xou50U>01E9K%K`vY^kAI}yMI&&giLAV_ z-@c(~%X)LBH8Mg6>$(ui=hoF$Y!U~j6p7g;h^ za{y>S0?XbSLA1e2x=eB;ss3Mn%*Qyfmw*?) z0R=|6q3ilb=Rxy{ji)r5JyiiL7wvOicE5io4pTq+0>u!UU>d&G2Ci% zA?WM3r+SDUjF;A7LuR9lVccVxf|i~{WW6pr6rasoLP;a}_PS9pDR^jP%Lpl%aTWj2 zPGn*G&xhW#gj>m^{lZY<{*l>I%C#)~-NT5igREh&L9#mLDQzko7T|N8V>QPlE#4|Y z^t~!o+t@Fv-`@E;+!4`U@TH(hkdt$mW;w>6y*iVk|i|NdFU_}?!+Zo1!c|(a3NKZrS+)7p>N-^B5TF>8u!p5iE^=3dcGIeJ@ zFTcB8;0Io#er_XEkuY4Vbyy8aW2^#gT9h!vaP)iymL+MJZcoRKrlF` z#L>I4N1+G5j*RpDO(DQTAb^g)-}(;Ar4c0L;?dmAf31gN9O?Fl}ZPW{US5 z#>;Gz4x>s(-+pX~%G^c|Bjgkb+>`nVA=v#01|dOQEtl^FIhSF?in!W4UeBvZ;EQZ5 zxiLeBA39R(u7~t0b$V{1yPSi68AI#c-ll@A>_08ONzuxW1Yfr* z*pWqNBbKa+J7F9-&atXvKf(UfQWmEcO$2pPE*nH3JO6fo?=1(lJQoC5UBE{OpjU*( z9@YN)tC_J+_Sb3dQNZ;o;=DYaCRUTcIoYM}ML)G(CR z>2m#hxxC;jxYS(=8ic%dXmJx=p*J^iaWo42*XBCa5wwObgWc<>(U{%`Yib^0 zj*uxCG&OKtcI%~I(Sl>UL-cc3q--gDj`)xwslpRgC;slPn*ArgtN4`5F#89tv0 z2LrCQq(bY_al5w8f*HN!yRx-D$wr37AjAy0JzPZdA?hS2 z3UwQiotNI%UL|Sx0f%C;Hs*}erl%j}2gNE2ZX{tdPUoTUA6h5TM zNhn~e*2Q0DOLDkPk-tqpgv*t8LJmk-sYBI^U>5%Nh2nJvM<;VOm=0Q+ym1 z27H+wdv>sjfz_cJhMtkK3y!u@W7kxt(Jh^{0Qz0#y=CP?K_Yp315NN>I}G{cj^miB zhv>nA{Eg&mrFLW=Z(F23bk_%mve+iK#PjAl-&NF#yn!^I zO68z9tbbaOp&oQD3t?*$OD{I=zI|52>^9!;!O|f`)`R=?m7RP`K+j;+zNXh*Bnp)oUWCQB z%YS${=Sc3$_so=E;jfg^Ug*tiSLSEZVBg#pxc{fbr}Z*%kIPMwVIE)qyF1}&rK1iz zaSz`pik^}i`zA9L9Fj=&d>kuSw2eb0d5r4!uiCUvTi5D}IJLX&e|jpoXCZ0|?P{LB z7=kZ-H#S_PhdeXuNZ#FCw%g3|2Zc!XBK)ek`&(RYDod zr)J#1@v~lWs&P-bWVl2cQao!t`=rT-j&|3JcH6H1K)wy)Mv(NdsQB7To#bqK093i; z2&oFsA(r9pqNhMK7;VUhWCKZ~>(ZNPO$SpA{-u^{9!llI;cyjSfSnpH)8kNCu|=zD zE)pkW(myfrIT?dy2YwzrAPk5~D~VKda7BLSXlsuibeYBBzdeVfyZUYe@w9w}PS@2b zG8Znh$lssv|1m&`cvEYQsf-Rddi!hff%zFS*}aYqLHk?{gcYXl~)}@#~eB`p&rP&$LP* zd4|e_NRqL^asH$Gd!<8ZceaY}r3g}6f2|R`VqH03okt~X<`grx)(543xw%+K;xU^WcC!ih>z~1ibGV!X~4tr4U#xp4<;iEvy{AIu#K>}hE4Q_)rX6k z=gvg4x-No<0HF+a;&g^=U&#`i|Fqj6o@wyMqXHk#(gsf#SgwX?9rUBGgSEE)*u(!s#~7%xl<7*gl{tbT`cIl z2xp`C+~wo=O{lv`6KuZUME-|?{?$rvG!29n=3Xm_{0&F{68v7$gu1+!s<-}A{XZLf z!+w$(*xM!b^!9H8;S^2qyG7?8{u?@76QVKIo22_cVty08)1ML Date: Wed, 13 Sep 2023 14:19:59 +0200 Subject: [PATCH 53/57] Specify the space available in /ceph --- web/content/posts/2023-09-12/_index.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/content/posts/2023-09-12/_index.md b/web/content/posts/2023-09-12/_index.md index eff289a..43d0d8a 100644 --- a/web/content/posts/2023-09-12/_index.md +++ b/web/content/posts/2023-09-12/_index.md @@ -11,7 +11,9 @@ last months. We have installed the latest [Ceph filesystem][1] (18.2.0) which stores three redundant copies of the data so a failure in one disk doesn't cause data loss. -It is mounted in /ceph and available for use in the owl1, owl2 and hut nodes. +It is mounted in /ceph and available for use in the owl1, owl2 and hut +nodes. For now it provides 2.8 TiB of space and it is expected to +increase when the last storage node is installed. [1]: https://en.wikipedia.org/wiki/Ceph_(software) -- GitLab From d9d249411d1cc32392e4b59b1d6742a3c2df3950 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 13 Sep 2023 15:57:13 +0200 Subject: [PATCH 54/57] Monitor storage nodes via IPMI too --- m/hut/targets.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/m/hut/targets.yml b/m/hut/targets.yml index 2cecd66..8924735 100644 --- a/m/hut/targets.yml +++ b/m/hut/targets.yml @@ -7,5 +7,9 @@ - 10.0.40.106 - 10.0.40.107 - 10.0.40.108 + # Storage + - 10.0.40.141 + - 10.0.40.142 + - 10.0.40.143 labels: job: ipmi-lan -- GitLab From aaefddc44a9073166ac52b8bd56ac96258d3b053 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 13 Sep 2023 17:44:24 +0200 Subject: [PATCH 55/57] Update slurm to 23.02.05.1 --- pkgs/overlay.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix index 03208c5..de16ffe 100644 --- a/pkgs/overlay.nix +++ b/pkgs/overlay.nix @@ -32,4 +32,16 @@ final: prev: lua = prev.lua5_4; fmt = prev.fmt_8; }) ceph ceph-client; + + # Update slurm to 23.02.5 to fix the firewall problem with pmix + slurm = prev.slurm.overrideAttrs (old: rec { + version = "23.02.5.1"; + src = prev.fetchFromGitHub { + owner = "SchedMD"; + repo = "slurm"; + # The release tags use - instead of . + rev = "slurm-${builtins.replaceStrings ["."] ["-"] version}"; + sha256 = "sha256-9VvZ8xySYFyBa5tZzf5WCShbEDpqE1/5t76jXX6t+bc="; + }; + }); } -- GitLab From e88805947e1e4b9396f63969fc32a1aca0425d2f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 14 Sep 2023 15:45:43 +0200 Subject: [PATCH 56/57] Open ports in firewall of compute nodes --- m/module/slurm-firewall.nix | 8 ++++++++ m/owl1/configuration.nix | 1 + m/owl2/configuration.nix | 1 + 3 files changed, 10 insertions(+) create mode 100644 m/module/slurm-firewall.nix diff --git a/m/module/slurm-firewall.nix b/m/module/slurm-firewall.nix new file mode 100644 index 0000000..8f52022 --- /dev/null +++ b/m/module/slurm-firewall.nix @@ -0,0 +1,8 @@ +{ ... }: + +{ + networking.firewall = { + # Required for PMIx in SLURM, we should find a better way + allowedTCPPortRanges = [ { from=1024; to=65535; } ]; + }; +} diff --git a/m/owl1/configuration.nix b/m/owl1/configuration.nix index 8255898..5afda72 100644 --- a/m/owl1/configuration.nix +++ b/m/owl1/configuration.nix @@ -4,6 +4,7 @@ imports = [ ../common/main.nix ../module/ceph.nix + ../module/slurm-firewall.nix ]; # Select the this using the ID to avoid mismatches diff --git a/m/owl2/configuration.nix b/m/owl2/configuration.nix index 0af3c40..9636760 100644 --- a/m/owl2/configuration.nix +++ b/m/owl2/configuration.nix @@ -4,6 +4,7 @@ imports = [ ../common/main.nix ../module/ceph.nix + ../module/slurm-firewall.nix ]; # Select the this using the ID to avoid mismatches -- GitLab From fdc6445d47fbfb9c92f00e9dc03344fd6c153369 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 14 Sep 2023 15:46:18 +0200 Subject: [PATCH 57/57] Revert "Update slurm to 23.02.05.1" This reverts commit aaefddc44a9073166ac52b8bd56ac96258d3b053. --- pkgs/overlay.nix | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix index de16ffe..03208c5 100644 --- a/pkgs/overlay.nix +++ b/pkgs/overlay.nix @@ -32,16 +32,4 @@ final: prev: lua = prev.lua5_4; fmt = prev.fmt_8; }) ceph ceph-client; - - # Update slurm to 23.02.5 to fix the firewall problem with pmix - slurm = prev.slurm.overrideAttrs (old: rec { - version = "23.02.5.1"; - src = prev.fetchFromGitHub { - owner = "SchedMD"; - repo = "slurm"; - # The release tags use - instead of . - rev = "slurm-${builtins.replaceStrings ["."] ["-"] version}"; - sha256 = "sha256-9VvZ8xySYFyBa5tZzf5WCShbEDpqE1/5t76jXX6t+bc="; - }; - }); } -- GitLab