From c4676535ec6bc8a343df022493b9a4ebac039c29 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 8 May 2024 16:44:28 +0200 Subject: [PATCH 001/310] Set Hart count to 20 --- lagarto-hun.nix | 7 +++++++ opensbi-lagarto-hun.patch | 13 +++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 opensbi-lagarto-hun.patch diff --git a/lagarto-hun.nix b/lagarto-hun.nix index 0e866ba..d366809 100644 --- a/lagarto-hun.nix +++ b/lagarto-hun.nix @@ -57,6 +57,12 @@ serviceConfig.Restart = "always"; }; + # Disable hvc0 as it is racing for the same console + systemd.services."serial-getty@hvc0" = { + enable = lib.mkForce false; + wantedBy = lib.mkForce [ ]; + }; + sdImage = { # The image will be loaded as-is in memory, so no compression compressImage = false; @@ -120,6 +126,7 @@ "PLATFORM=fpga/openpiton" "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin" ]; + patches = [ ./opensbi-lagarto-hun.patch ]; }); }) ]; } diff --git a/opensbi-lagarto-hun.patch b/opensbi-lagarto-hun.patch new file mode 100644 index 0000000..cbc1dc6 --- /dev/null +++ b/opensbi-lagarto-hun.patch @@ -0,0 +1,13 @@ +--- a/platform/fpga/openpiton/platform.c 2024-03-12 16:27:13.886525365 +0100 ++++ b/platform/fpga/openpiton/platform.c 2024-03-12 16:28:40.598403778 +0100 +@@ -26,8 +26,8 @@ + #define OPENPITON_DEFAULT_PLIC_ADDR 0xfff1100000 + #define OPENPITON_DEFAULT_PLIC_SIZE (0x200000 + \ + (OPENPITON_DEFAULT_HART_COUNT * 0x1000)) +-#define OPENPITON_DEFAULT_PLIC_NUM_SOURCES 2 +-#define OPENPITON_DEFAULT_HART_COUNT 3 ++#define OPENPITON_DEFAULT_PLIC_NUM_SOURCES 3 ++#define OPENPITON_DEFAULT_HART_COUNT 20 + #define OPENPITON_DEFAULT_CLINT_ADDR 0xfff1020000 + #define OPENPITON_DEFAULT_ACLINT_MTIMER_FREQ 1000000 + #define OPENPITON_DEFAULT_ACLINT_MSWI_ADDR \ -- GitLab From f69e04ccbe2a47d6c7a11872e1ac8d5905da9373 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 13 May 2024 18:59:00 +0200 Subject: [PATCH 002/310] WIP: Working spec cpu with clang epi --- configuration.nix | 6 +++++- flake.lock | 33 ++++++++++++++++++++++++++++++++- flake.nix | 19 ++++++++++++------- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/configuration.nix b/configuration.nix index 85296fa..92656d0 100644 --- a/configuration.nix +++ b/configuration.nix @@ -1,4 +1,4 @@ -{ config, lib, pkgs, modulesPath, ... }: +{ config, lib, pkgs, modulesPath, self, ... }: { imports = [ @@ -10,6 +10,10 @@ system = "riscv64-linux"; }; + nixpkgs.overlays = [ + self.inputs.bscpkgs.bscOverlay + ]; + networking.hostName = "nixos-riscv"; system.stateVersion = "24.05"; diff --git a/flake.lock b/flake.lock index 1a56d89..c5ce3d9 100644 --- a/flake.lock +++ b/flake.lock @@ -1,6 +1,36 @@ { "nodes": { + "bscpkgs": { + "inputs": { + "nixpkgs": "nixpkgs" + }, + "locked": { + "dirtyRev": "e605f7dfad67a6bf15c1f1ff83e0c8a1c9ee42b9-dirty", + "dirtyShortRev": "e605f7d-dirty", + "lastModified": 1715179381, + "narHash": "sha256-487pg/odvGPcEVpCBDS3gVmK/4o3Yv7S/uvI2HYECGM=", + "type": "git", + "url": "file:///home/Computational/rarias/bscpkgs" + }, + "original": { + "type": "git", + "url": "file:///home/Computational/rarias/bscpkgs" + } + }, "nixpkgs": { + "locked": { + "lastModified": 1700390070, + "narHash": "sha256-de9KYi8rSJpqvBfNwscWdalIJXPo8NjdIZcEJum1mH0=", + "path": "/nix/store/z7y28qzhk7driiwcw78k0mb24laknm0f-source", + "rev": "e4ad989506ec7d71f7302cc3067abd82730a4beb", + "type": "path" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs_2": { "locked": { "lastModified": 1706092046, "narHash": "sha256-Cbethl95Cu+WVIWfaAnRRBZiz5PmXxQvg4vXNqIZQUg=", @@ -18,7 +48,8 @@ }, "root": { "inputs": { - "nixpkgs": "nixpkgs" + "bscpkgs": "bscpkgs", + "nixpkgs": "nixpkgs_2" } } }, diff --git a/flake.nix b/flake.nix index baeeb65..9ffb77f 100644 --- a/flake.nix +++ b/flake.nix @@ -1,17 +1,20 @@ { inputs.nixpkgs.url = "github:rodarima/nixpkgs/fix-pkgs-static-gcc-march"; + #inputs.bscpkgs.url = "path:/home/Computational/rarias/bscpkgs"; + inputs.bscpkgs.url = "/home/Computational/rarias/bscpkgs"; - outputs = { self, nixpkgs, ... }: + outputs = { self, nixpkgs, bscpkgs, ... }: let system = "x86_64-linux"; - overlay = import ./overlay.nix; - pkgs = import nixpkgs { inherit system; }; + #overlay = import ./overlay.nix; + #pkgs = import nixpkgs { inherit system; }; in { - overlay = import ./overlay.nix; + #overlay = import ./overlay.nix; nixosConfigurations = { # The qemu configuration defines a system that runs in the RISC-V # architecture, but is build from an x86 host machine. qemu = nixpkgs.lib.nixosSystem { + specialArgs = { inherit self; }; system = "${system}"; modules = [ ./configuration.nix @@ -21,6 +24,7 @@ # Same, but disable compressed instructions qemu-nc = nixpkgs.lib.nixosSystem { + specialArgs = { inherit self; }; system = "${system}"; modules = [ ./configuration.nix @@ -31,6 +35,7 @@ # FPGA Lagarto Hun CPU lagarto-hun = nixpkgs.lib.nixosSystem { + specialArgs = { inherit self; }; system = "${system}"; modules = [ ./configuration.nix @@ -47,9 +52,9 @@ nixosconf = self.nixosConfigurations.qemu-nc; syspkgs = nixosconf.pkgs; toplevel = nixosconf.config.system.build.toplevel; - in pkgs.mkShell { + in syspkgs.mkShell { pname = "qemu-shell"; - buildInputs = with pkgs; [ qemu e2fsprogs ]; + buildInputs = with syspkgs; [ qemu e2fsprogs ]; # Here we tell the run script where to find the system NIXOS_SYSTEM_TOPLEVEL = toplevel; OPENSBI = syspkgs.opensbi-uboot; @@ -59,7 +64,7 @@ nixosconf = self.nixosConfigurations.lagarto-hun; syspkgs = nixosconf.pkgs; build = nixosconf.config.system.build; - in pkgs.mkShell { + in syspkgs.mkShell { pname = "lagarto-hun-shell"; TOPLEVEL = build.toplevel; OPENSBI = syspkgs.opensbi; -- GitLab From f554a154b7bd8da51ffc211aed52be64882f1b14 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 28 May 2024 18:12:14 +0200 Subject: [PATCH 003/310] WIP --- configuration.nix | 18 ++- flake.lock | 20 ++-- flake.nix | 30 +++-- fpga/fpgactl | 2 +- fpga/upload.sh | 4 + lagarto-hun.nix | 12 +- opensbi-lagarto-hun.patch | 8 +- overlay.nix | 32 +++++ unalign.c | 239 ++++++++++++++++++++++++++++++++++++++ vm.nix | 14 ++- 10 files changed, 341 insertions(+), 38 deletions(-) create mode 100644 unalign.c diff --git a/configuration.nix b/configuration.nix index 92656d0..7337141 100644 --- a/configuration.nix +++ b/configuration.nix @@ -6,13 +6,18 @@ #"${modulesPath}/profiles/minimal.nix" ]; - nixpkgs.crossSystem = { - system = "riscv64-linux"; - }; + nixpkgs = { + crossSystem = { + system = "riscv64-linux"; + }; - nixpkgs.overlays = [ - self.inputs.bscpkgs.bscOverlay - ]; + overlays = [ + self.inputs.bscpkgs.bscOverlay + (import ./overlay.nix) + ]; + + config.allowUnsupportedSystem = true; + }; networking.hostName = "nixos-riscv"; @@ -31,4 +36,5 @@ }; #environment.systemPackages = with pkgs; [ vim gdb neofetch gcc bintools ]; + environment.systemPackages = with pkgs; [ rvb unalignedCheck ]; } diff --git a/flake.lock b/flake.lock index c5ce3d9..5e898fe 100644 --- a/flake.lock +++ b/flake.lock @@ -5,10 +5,10 @@ "nixpkgs": "nixpkgs" }, "locked": { - "dirtyRev": "e605f7dfad67a6bf15c1f1ff83e0c8a1c9ee42b9-dirty", - "dirtyShortRev": "e605f7d-dirty", - "lastModified": 1715179381, - "narHash": "sha256-487pg/odvGPcEVpCBDS3gVmK/4o3Yv7S/uvI2HYECGM=", + "dirtyRev": "15c8a61d1787d52f904755bee9b1840ce8823525-dirty", + "dirtyShortRev": "15c8a61-dirty", + "lastModified": 1715619485, + "narHash": "sha256-zgwnjGVBXTvItXIYtaMIdf2acHiNQOJ6Fba4OFZRYT4=", "type": "git", "url": "file:///home/Computational/rarias/bscpkgs" }, @@ -32,17 +32,17 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1706092046, - "narHash": "sha256-Cbethl95Cu+WVIWfaAnRRBZiz5PmXxQvg4vXNqIZQUg=", - "owner": "rodarima", + "lastModified": 1700390070, + "narHash": "sha256-de9KYi8rSJpqvBfNwscWdalIJXPo8NjdIZcEJum1mH0=", + "owner": "NixOS", "repo": "nixpkgs", - "rev": "57e7c8fa4fdc414a936ce83afd0c70fb0a3a31d5", + "rev": "e4ad989506ec7d71f7302cc3067abd82730a4beb", "type": "github" }, "original": { - "owner": "rodarima", - "ref": "fix-pkgs-static-gcc-march", + "owner": "NixOS", "repo": "nixpkgs", + "rev": "e4ad989506ec7d71f7302cc3067abd82730a4beb", "type": "github" } }, diff --git a/flake.nix b/flake.nix index 9ffb77f..3dc1bfb 100644 --- a/flake.nix +++ b/flake.nix @@ -1,19 +1,33 @@ { - inputs.nixpkgs.url = "github:rodarima/nixpkgs/fix-pkgs-static-gcc-march"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/e4ad989506ec7d71f7302cc3067abd82730a4beb"; #inputs.bscpkgs.url = "path:/home/Computational/rarias/bscpkgs"; inputs.bscpkgs.url = "/home/Computational/rarias/bscpkgs"; - outputs = { self, nixpkgs, bscpkgs, ... }: + outputs = {self,...}@inputs: let system = "x86_64-linux"; - #overlay = import ./overlay.nix; - #pkgs = import nixpkgs { inherit system; }; + remoteNixpkgsPatches = [ + { + meta.description = "sha256-ZCDQ7SpGhH8JvAwWzdcyrc68RFEWHxxAj0M2+AvEzIg="; + url = "https://github.com/NixOS/nixpkgs/pull/283460.diff"; + sha256 = "sha256-g6o4rqkOOYZ6OJTzv9kTPq9Zsu5Z1QXZmPLC3Q7Sq6w="; + } + ]; + originPkgs = inputs.nixpkgs.legacyPackages."x86_64-linux"; + nixpkgs = originPkgs.applyPatches { + name = "nixpkgs-patched"; + src = inputs.nixpkgs; + patches = map originPkgs.fetchpatch remoteNixpkgsPatches; + }; + nixosSystem = import (nixpkgs + "/nixos/lib/eval-config.nix"); + + in { #overlay = import ./overlay.nix; nixosConfigurations = { # The qemu configuration defines a system that runs in the RISC-V # architecture, but is build from an x86 host machine. - qemu = nixpkgs.lib.nixosSystem { + qemu = nixosSystem { specialArgs = { inherit self; }; system = "${system}"; modules = [ @@ -23,7 +37,7 @@ }; # Same, but disable compressed instructions - qemu-nc = nixpkgs.lib.nixosSystem { + qemu-nc = nixosSystem { specialArgs = { inherit self; }; system = "${system}"; modules = [ @@ -34,7 +48,7 @@ }; # FPGA Lagarto Hun CPU - lagarto-hun = nixpkgs.lib.nixosSystem { + lagarto-hun = nixosSystem { specialArgs = { inherit self; }; system = "${system}"; modules = [ @@ -54,7 +68,7 @@ toplevel = nixosconf.config.system.build.toplevel; in syspkgs.mkShell { pname = "qemu-shell"; - buildInputs = with syspkgs; [ qemu e2fsprogs ]; + nativeBuildInputs = with syspkgs; [ qemu e2fsprogs ]; # Here we tell the run script where to find the system NIXOS_SYSTEM_TOPLEVEL = toplevel; OPENSBI = syspkgs.opensbi-uboot; diff --git a/fpga/fpgactl b/fpga/fpgactl index 2e0ac2c..c14c323 100755 --- a/fpga/fpgactl +++ b/fpga/fpgactl @@ -273,7 +273,7 @@ verbose= bootloader_addr=0x80000000 kernel_addr=0x84000000 initrd_addr=0x8c300000 -rootfs_addr=0x180000000 +rootfs_addr=0x140000000 function usage() { diff --git a/fpga/upload.sh b/fpga/upload.sh index 191e3cc..a47a667 100755 --- a/fpga/upload.sh +++ b/fpga/upload.sh @@ -5,6 +5,10 @@ set -x dst=femu:nixos/ +if [ "$1" != "" ]; then + dst="$1" +fi + rsync -a fpga/fpgactl "$dst" rsync -a fpga/boot.sh "$dst" rsync -a fpga/env.sh "$dst" diff --git a/lagarto-hun.nix b/lagarto-hun.nix index d366809..51359ba 100644 --- a/lagarto-hun.nix +++ b/lagarto-hun.nix @@ -102,13 +102,13 @@ initrd = "${config.system.build.initialRamdisk}/initrd"; in prev.runCommand "uboot.txt" {} '' cat > $out < + # Create pmem of 3 GiB [0x140000000, 0x200000000) + fdt mknode / pmem@0x140000000 + fdt set /pmem@0x140000000 compatible "pmem-region" + fdt set /pmem@0x140000000 reg <0x1 0x40000000 0x0 0xc0000000> - # Reduce memory - fdt set /memory@80000000 reg <0x00000000 0x80000000 0x00000001 0x00000000> + # Reduce memory to 3 GiB [0x80000000, 0x140000000) + fdt set /memory@80000000 reg <0x0 0x80000000 0x0 0xc0000000> # Set kernel options setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi boot.trace console=hvc0 init=${init}" diff --git a/opensbi-lagarto-hun.patch b/opensbi-lagarto-hun.patch index cbc1dc6..f7e938c 100644 --- a/opensbi-lagarto-hun.patch +++ b/opensbi-lagarto-hun.patch @@ -1,9 +1,9 @@ --- a/platform/fpga/openpiton/platform.c 2024-03-12 16:27:13.886525365 +0100 -+++ b/platform/fpga/openpiton/platform.c 2024-03-12 16:28:40.598403778 +0100 -@@ -26,8 +26,8 @@ ++++ b/platform/fpga/openpiton/platform.c 2024-05-27 11:42:47.748244398 +0200 +@@ -24,8 +24,8 @@ + #define OPENPITON_DEFAULT_UART_REG_WIDTH 1 + #define OPENPITON_DEFAULT_UART_REG_OFFSET 0 #define OPENPITON_DEFAULT_PLIC_ADDR 0xfff1100000 - #define OPENPITON_DEFAULT_PLIC_SIZE (0x200000 + \ - (OPENPITON_DEFAULT_HART_COUNT * 0x1000)) -#define OPENPITON_DEFAULT_PLIC_NUM_SOURCES 2 -#define OPENPITON_DEFAULT_HART_COUNT 3 +#define OPENPITON_DEFAULT_PLIC_NUM_SOURCES 3 diff --git a/overlay.nix b/overlay.nix index b3686b6..55d58ec 100644 --- a/overlay.nix +++ b/overlay.nix @@ -3,5 +3,37 @@ final: prev: # Changes to packages from nixpkgs { + blis = ((prev.blis.override { + blas64 = true; + withArchitecture = "generic"; + }).overrideAttrs (old: { + nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ + prev.buildPackages.gfortran + ]; + })).overrideDerivation (old : { + configureFlags = [ + "--enable-cblas" + "--blas-int-size=64" + "--enable-threading=openmp" + #"--build=x86_64-unknown-linux-gnu" + #"--host=riscv64-unknown-linux-gnu" + "generic" + ]; + }); + unalignedCheck = prev.stdenv.mkDerivation { + name = "unaligned-check"; + src = ./unalign.c; + unpackPhase = '' + cp ${./unalign.c} unalign.c + ''; + dontConfigure = true; + buildPhase = '' + $CC unalign.c -o unalign_check + ''; + installPhase = '' + mkdir -p $out/bin + cp unalign_check $out/bin/ + ''; + }; } diff --git a/unalign.c b/unalign.c new file mode 100644 index 0000000..8e575cd --- /dev/null +++ b/unalign.c @@ -0,0 +1,239 @@ +/* + * unalign_check - check the CPU behaviour on different alignments + * Copyright (C) 2021 Matteo Croce + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include + +#define ACT_READ 0 +#define ACT_WRITE 1 +#define ACT_XOR 2 +#define ACT_COPY 3 + +#define READ(SIZE) \ + case SIZE / 8: { \ + volatile uint##SIZE##_t *buf2 = (uint##SIZE##_t *)buf; \ + int i; \ + for (i = 0; i < count; i++) \ + (void)buf2[i]; \ + break; \ + } + +#define WRITE(SIZE) \ + case SIZE / 8: { \ + volatile uint##SIZE##_t *buf2 = (uint##SIZE##_t *)buf; \ + int i; \ + for (i = 0; i < count; i++) \ + buf2[i] = (uint##SIZE##_t)0xaabbccdd11223344; \ + break; \ + } + +#define XOR(SIZE) \ + case SIZE / 8: { \ + volatile uint##SIZE##_t *buf2 = (uint##SIZE##_t *)buf; \ + int i; \ + for (i = 0; i < count; i++) \ + buf2[i] = ~buf2[i]; \ + break; \ + } + +#define COPY(SIZE) \ + case SIZE / 8: { \ + volatile uint##SIZE##_t *buf2 = (uint##SIZE##_t *)buf; \ + int i; \ + for (i = 0; i < count / 2; i++) \ + buf2[i] = buf2[i + count / 2]; \ + for (i = count / 2; i < count; i++) \ + buf2[i] = buf2[i - count / 2]; \ + break; \ + } + +static void do_read(void *buf, size_t count, int size) +{ + switch (size) { + READ(8); + READ(16); + READ(32); + READ(64); + } +} + +static void do_write(void *buf, size_t count, int size) +{ + switch (size) { + WRITE(8); + WRITE(16); + WRITE(32); + WRITE(64); + } +} + +static void do_xor(void *buf, size_t count, int size) +{ + switch (size) { + XOR(8); + XOR(16); + XOR(32); + XOR(64); + } +} + +static void do_copy(void *buf, size_t count, int size) +{ + switch (size) { + COPY(8); + COPY(16); + COPY(32); + COPY(64); + } +} + +static uint64_t time_sub(struct timespec *since, struct timespec *to) +{ + if (to->tv_sec == since->tv_sec) + return to->tv_nsec - since->tv_nsec; + + return (to->tv_sec - since->tv_sec) * 1000000000 + to->tv_nsec - since->tv_nsec; +} + +static void __attribute__ ((noreturn)) usage(char *argv0, int ret) +{ + fprintf(ret ? stderr : stdout, + "usage: %s [-rwxc1234h] [-l length] [-u unalignment]\n" + "\n" + "Options:\n" + " -r read memory (default)\n" + " -w write memory\n" + " -x xor memory\n" + " -c copy memory\n" + " -l SIZE use SIZE Mb for the test (default 100)\n" + " -u BYTE unalign buffer by BYTE bytes (default 0)\n" + " -1 read 1 byte at time\n" + " -2 read 2 bytes at time\n" + " -4 read 4 bytes at time (default)\n" + " -8 read 8 bytes at time\n" + " -h this help\n", + argv0); + exit(ret); +} + +static const char *actions[] = { + "read", + "write", + "xor", + "copy", +}; + +int main(int argc, char *argv[]) +{ + struct timespec before, after; + uint64_t elapsed; + int action = ACT_READ; + size_t len = 100 * 1024 * 1024; + int shift = 0; + int size = sizeof(long); + char *buf; + int c; + + while((c = getopt(argc, argv, "hrwxc1248l:u:")) != -1) { + switch (c) { + case 'r': + action = ACT_READ; + break; + case 'w': + action = ACT_WRITE; + break; + case 'x': + action = ACT_XOR; + break; + case 'c': + action = ACT_COPY; + break; + case 'l': + len = atol(optarg) * 1024 * 1024; + if (len <= 0) { + fprintf(stderr, "Invalid size %s\n", optarg); + return 1; + } + break; + case 'u': + shift = atoi(optarg); + break; + case '1': + case '2': + case '4': + case '8': + size = c - '0'; + break; + case 'h': + default: + usage(argv[0], c != 'h'); + } + } + + shift %= size; + + if (optind != argc) + usage(argv[0], 1); + + buf = malloc(len); + if (!buf) { + perror("malloc"); + return 1; + } + + if (mlock(buf, len)) { + perror("mlock"); + return 1; + } + + clock_gettime(CLOCK_MONOTONIC, &before); + switch (action) { + case ACT_READ: + do_read(buf + shift, (len - shift) / size, size); + break; + case ACT_WRITE: + do_write(buf + shift, (len - shift) / size, size); + break; + case ACT_XOR: + do_xor(buf + shift, (len - shift) / size, size); + break; + case ACT_COPY: + do_copy(buf + shift, (len - shift) / size, size); + break; + } + clock_gettime(CLOCK_MONOTONIC, &after); + + elapsed = time_sub(&before, &after); + + printf( "size: %lu Mb\n" + "%s size: %d bit\n" + "unalignment: %d byte\n" + "elapsed time: %.2f sec\n" + "throughput: %.2f Mb/s\n", + len / 1024 / 1024, + actions[action], size * 8, + shift, + elapsed / 1E9, + (len / 1024 / 1024) / (elapsed / 1E9)); + + return 0; +} diff --git a/vm.nix b/vm.nix index ff03e65..b6a356f 100644 --- a/vm.nix +++ b/vm.nix @@ -47,8 +47,8 @@ INET y NETWORK_FILESYSTEMS y OVERLAY_FS y - "9P_FS" y - "9P_FS_POSIX_ACL" y + #"9P_FS" y + #"9P_FS_POSIX_ACL" y PCI y VIRTIO_PCI y PCI_HOST_GENERIC y @@ -63,7 +63,15 @@ nixpkgs.overlays = [ (final: prev: { - qemu = prev.qemu.override { rutabagaSupport = false; }; + qemu = prev.qemu.override { + pulseSupport = false; + pipewireSupport = false; + sdlSupport = false; + jackSupport = false; + gtkSupport = false; + vncSupport = false; + smartcardSupport = false; + }; uboot-custom = prev.ubootQemuRiscv64Smode.override { # Override preboot to set 'bootcmd' directly to the kernel address in RAM -- GitLab From 8bd4550bc6afd5f6a605d8d069b492cf9ee792d6 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 5 Jun 2024 14:24:29 +0200 Subject: [PATCH 004/310] Use remote bscpkgs input for riscv benchmarks --- flake.lock | 12 +++++++----- flake.nix | 5 +++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 5e898fe..cf69c3c 100644 --- a/flake.lock +++ b/flake.lock @@ -5,16 +5,18 @@ "nixpkgs": "nixpkgs" }, "locked": { - "dirtyRev": "15c8a61d1787d52f904755bee9b1840ce8823525-dirty", - "dirtyShortRev": "15c8a61-dirty", - "lastModified": 1715619485, + "lastModified": 1717064246, "narHash": "sha256-zgwnjGVBXTvItXIYtaMIdf2acHiNQOJ6Fba4OFZRYT4=", + "ref": "riscv-benchmarks", + "rev": "f5515a80bfed28cd4e2e737858ba4124a7e53333", + "revCount": 939, "type": "git", - "url": "file:///home/Computational/rarias/bscpkgs" + "url": "https://git.sr.ht/~rodarima/bscpkgs" }, "original": { + "ref": "riscv-benchmarks", "type": "git", - "url": "file:///home/Computational/rarias/bscpkgs" + "url": "https://git.sr.ht/~rodarima/bscpkgs" } }, "nixpkgs": { diff --git a/flake.nix b/flake.nix index 3dc1bfb..c1ede80 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,8 @@ { inputs.nixpkgs.url = "github:NixOS/nixpkgs/e4ad989506ec7d71f7302cc3067abd82730a4beb"; - #inputs.bscpkgs.url = "path:/home/Computational/rarias/bscpkgs"; - inputs.bscpkgs.url = "/home/Computational/rarias/bscpkgs"; + inputs.bscpkgs.url = "git+https://git.sr.ht/~rodarima/bscpkgs?ref=riscv-benchmarks"; + # When changing bscpkgs, take directly from disk to avoid having to push: + #inputs.bscpkgs.url = "/home/Computational/rarias/bscpkgs"; outputs = {self,...}@inputs: let -- GitLab From 4344f73d8732fd14ef117bb635dd5d9a3d3289f7 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 5 Jun 2024 14:49:51 +0200 Subject: [PATCH 005/310] Dump system information when entering shell --- flake.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/flake.nix b/flake.nix index c1ede80..f549cf1 100644 --- a/flake.nix +++ b/flake.nix @@ -87,6 +87,15 @@ INITRD = build.initialRamdisk; ROOTFS = build.sdImage; UBOOT_ENV = syspkgs.uboot-env; + shellHook = '' + echo "Here are the current system pieces:" + echo " TOPLEVEL = $TOPLEVEL" + echo " KERNEL = $KERNEL" + echo " OPENSBI = $OPENSBI" + echo " INITRD = $INITRD" + echo " ROOTFS = $ROOTFS" + echo " UBOOT_ENV = $UBOOT_ENV" + ''; }; }; } -- GitLab From c8efa952f9d76c058972f12007f9945ff7bf3ee8 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 25 Jun 2024 11:11:05 +0200 Subject: [PATCH 006/310] Add configuration for Lagarto Ox --- flake.nix | 35 +++++++++++++ lagarto-ox.nix | 132 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 lagarto-ox.nix diff --git a/flake.nix b/flake.nix index f549cf1..cef84f7 100644 --- a/flake.nix +++ b/flake.nix @@ -58,6 +58,17 @@ ./no-compressed.nix ]; }; + + # FPGA Lagarto Ox CPU + lagarto-ox = nixosSystem { + specialArgs = { inherit self; }; + system = "${system}"; + modules = [ + ./configuration.nix + ./lagarto-ox.nix + ./no-compressed.nix + ]; + }; }; # A development shell with QEMU ready to boot the RISC-V system in an x86 @@ -97,5 +108,29 @@ echo " UBOOT_ENV = $UBOOT_ENV" ''; }; + + devShells.x86_64-linux.lagarto-ox = + let + nixosconf = self.nixosConfigurations.lagarto-ox; + syspkgs = nixosconf.pkgs; + build = nixosconf.config.system.build; + in syspkgs.mkShell { + pname = "lagarto-ox-shell"; + TOPLEVEL = build.toplevel; + OPENSBI = syspkgs.opensbi; + KERNEL = build.kernel; + INITRD = build.initialRamdisk; + ROOTFS = build.sdImage; + UBOOT_ENV = syspkgs.uboot-env; + shellHook = '' + echo "Here are the current system pieces:" + echo " TOPLEVEL = $TOPLEVEL" + echo " KERNEL = $KERNEL" + echo " OPENSBI = $OPENSBI" + echo " INITRD = $INITRD" + echo " ROOTFS = $ROOTFS" + echo " UBOOT_ENV = $UBOOT_ENV" + ''; + }; }; } diff --git a/lagarto-ox.nix b/lagarto-ox.nix new file mode 100644 index 0000000..51359ba --- /dev/null +++ b/lagarto-ox.nix @@ -0,0 +1,132 @@ +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = [ + "${modulesPath}/installer/sd-card/sd-image.nix" + ]; + + #boot.kernelPackages = pkgs.linuxPackages_latest; + boot = { + kernelPatches = [ + { + name = assert false; "sbi-early-console"; + patch = null; + extraConfig = + # Early console via SBI + '' + RISCV_SBI y + RISCV_SBI_V01 y + SERIAL_EARLYCON y + SERIAL_EARLYCON_RISCV_SBI y + HVC_DRIVER y + HVC_RISCV_SBI y + '' + # Allows regions of persistent memory to be described in the device-tree. + + '' + OF_PMEM y + '' + # Allow you to use a contiguous range of reserved memory as one or more + # persistent block devices (/dev/pmem0) + + '' + LIBNVDIMM y + BLK_DEV_PMEM y + '' + ; + } + ]; + + initrd = { + # Avoid zstd as we don't have the tools in "cucu" machine + compressor = "gzip"; + kernelModules = [ ]; + }; + + loader = { + grub.enable = false; + generic-extlinux-compatible.enable = true; + }; + }; + + # No network + services.openssh.enable = false; + + # Run getty on /dev/console and restartt until it works + systemd.services."serial-getty@console" = { + enable = true; + wantedBy = [ "getty.target" ]; # to start at boot + serviceConfig.Restart = "always"; + }; + + # Disable hvc0 as it is racing for the same console + systemd.services."serial-getty@hvc0" = { + enable = lib.mkForce false; + wantedBy = lib.mkForce [ ]; + }; + + sdImage = { + # The image will be loaded as-is in memory, so no compression + compressImage = false; + imageName = "rootfs.img"; + # Not needed for now + expandOnBoot = false; + populateFirmwareCommands = ""; + populateRootCommands = '' + mkdir -p ./files/boot + ${config.boot.loader.generic-extlinux-compatible.populateCmd} \ + -c ${config.system.build.toplevel} \ + -d ./files/boot + ''; + }; + + nixpkgs.overlays = [ (final: prev: { + uboot = prev.ubootQemuRiscv64Smode.override { + filesToInstall = [ "u-boot.bin" "u-boot-nodtb.bin" ]; + extraConfig = '' + CONFIG_RISCV_ISA_C=n + CONFIG_REQUIRE_SERIAL_CONSOLE=n + CONFIG_SERIAL_SEARCH_ALL=y + CONFIG_OF_CONTROL=y + CONFIG_OF_BOARD=y + CONFIG_OF_HAS_PRIOR_STAGE=y + CONFIG_BLKMAP=y + CONFIG_CMD_BLKMAP=y + CONFIG_SBI_V01=y + CONFIG_DEBUG_UART=y + CONFIG_DEBUG_UART_ANNOUNCE=y + CONFIG_DEBUG_SBI_CONSOLE=y + ''; + }; + + uboot-env = let + init = "${config.system.build.toplevel}/init"; + initrd = "${config.system.build.initialRamdisk}/initrd"; + in prev.runCommand "uboot.txt" {} '' + cat > $out < + + # Reduce memory to 3 GiB [0x80000000, 0x140000000) + fdt set /memory@80000000 reg <0x0 0x80000000 0x0 0xc0000000> + + # Set kernel options + setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi boot.trace console=hvc0 init=${init}" + + EOF + + # Populate the bood commands from the initrd size + echo "# Boot system" >> $out + echo "setenv ramdisk_size $(stat --format %s $(readlink -f ${initrd}))" >> $out + echo 'booti ''${kernel_addr_r} ''${ramdisk_addr_r}:''${ramdisk_size} ''${fdtcontroladdr}' >> $out + ''; + + opensbi = prev.opensbi.overrideAttrs (old: { + makeFlags = old.makeFlags ++ [ + "PLATFORM=fpga/openpiton" + "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin" + ]; + patches = [ ./opensbi-lagarto-hun.patch ]; + }); + }) ]; +} -- GitLab From 1bfc32e91a48065b47edc39c0dd3dd38bbfd7495 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 27 Jun 2024 11:29:15 +0200 Subject: [PATCH 007/310] Working OpenSBI 1.2 --- alveo-uart-sbi-hack.patch | 438 +++++++++++++++++++++++++++++++++++ lagarto-ox.nix | 64 ++++- ox-alveo-platform-plic.patch | 166 +++++++++++++ ox-alveo-platform.patch | 148 ++++++++++++ ox-plic.dts | 143 ++++++++++++ ox.dts | 58 +++++ 6 files changed, 1013 insertions(+), 4 deletions(-) create mode 100644 alveo-uart-sbi-hack.patch create mode 100644 ox-alveo-platform-plic.patch create mode 100644 ox-alveo-platform.patch create mode 100644 ox-plic.dts create mode 100644 ox.dts diff --git a/alveo-uart-sbi-hack.patch b/alveo-uart-sbi-hack.patch new file mode 100644 index 0000000..7de4768 --- /dev/null +++ b/alveo-uart-sbi-hack.patch @@ -0,0 +1,438 @@ +diff --git a/include/sbi_utils/serial/alveo_uart.h b/include/sbi_utils/serial/alveo_uart.h +new file mode 100644 +index 0000000..0b6408c +--- /dev/null ++++ b/include/sbi_utils/serial/alveo_uart.h +@@ -0,0 +1,13 @@ ++/* ++ * SPDX-License-Identifier: BSD-2-Clause ++ */ ++ ++#ifndef __SERIAL_ALVEO_UART_H__ ++#define __SERIAL_ALVEO_UART_H__ ++ ++#include ++ ++int alveo_uart_init(unsigned long base, u32 in_freq, u32 baudrate, u32 reg_shift, ++ u32 reg_width, u32 reg_offset); ++ ++#endif +diff --git a/lib/utils/serial/Kconfig b/lib/utils/serial/Kconfig +index e3589ca..b754c7c 100644 +--- a/lib/utils/serial/Kconfig ++++ b/lib/utils/serial/Kconfig +@@ -84,6 +84,10 @@ config SERIAL_UART8250 + bool "8250 UART support" + default n + ++config SERIAL_ALVEO_UART ++ bool "ALveo UART support" ++ default n ++ + config SERIAL_XILINX_UARTLITE + bool "Xilinx UART Lite support" + default n +diff --git a/lib/utils/serial/alveo_uart.c b/lib/utils/serial/alveo_uart.c +new file mode 100644 +index 0000000..a351741 +--- /dev/null ++++ b/lib/utils/serial/alveo_uart.c +@@ -0,0 +1,123 @@ ++/* ++ * SPDX-License-Identifier: BSD-2-Clause ++ */ ++ ++#include ++#include ++#include ++ ++/* clang-format off */ ++ ++#define UART_RBR_OFFSET 0 /* In: Recieve Buffer Register */ ++#define UART_THR_OFFSET 0 /* Out: Transmitter Holding Register */ ++#define UART_DLL_OFFSET 0 /* Out: Divisor Latch Low */ ++#define UART_IER_OFFSET 1 /* I/O: Interrupt Enable Register */ ++#define UART_DLM_OFFSET 1 /* Out: Divisor Latch High */ ++#define UART_FCR_OFFSET 2 /* Out: FIFO Control Register */ ++#define UART_IIR_OFFSET 2 /* I/O: Interrupt Identification Register */ ++#define UART_LCR_OFFSET 3 /* Out: Line Control Register */ ++#define UART_MCR_OFFSET 4 /* Out: Modem Control Register */ ++#define UART_LSR_OFFSET 5 /* In: Line Status Register */ ++#define UART_MSR_OFFSET 6 /* In: Modem Status Register */ ++#define UART_SCR_OFFSET 7 /* I/O: Scratch Register */ ++#define UART_MDR1_OFFSET 8 /* I/O: Mode Register */ ++ ++#define UART_LSR_FIFOE 0x80 /* Fifo error */ ++#define UART_LSR_TEMT 0x40 /* Transmitter empty */ ++#define UART_LSR_THRE 0x20 /* Transmit-hold-register empty */ ++#define UART_LSR_BI 0x10 /* Break interrupt indicator */ ++#define UART_LSR_FE 0x08 /* Frame error indicator */ ++#define UART_LSR_PE 0x04 /* Parity error indicator */ ++#define UART_LSR_OE 0x02 /* Overrun error indicator */ ++#define UART_LSR_DR 0x01 /* Receiver data ready */ ++#define UART_LSR_BRK_ERROR_BITS 0x1E /* BI, FE, PE, OE bits */ ++ ++/* clang-format on */ ++ ++static volatile char *alveo_uart_base; ++static u32 alveo_uart_in_freq; ++static u32 alveo_uart_baudrate; ++static u32 alveo_uart_reg_width; ++static u32 alveo_uart_reg_shift; ++ ++static u32 get_reg(u32 num) ++{ ++ u32 offset = num << alveo_uart_reg_shift; ++ ++ if (alveo_uart_reg_width == 1) ++ return readb(alveo_uart_base + offset); ++ else if (alveo_uart_reg_width == 2) ++ return readw(alveo_uart_base + offset); ++ else ++ return readl(alveo_uart_base + offset); ++} ++ ++static void set_reg(u32 num, u32 val) ++{ ++ u32 offset = num << alveo_uart_reg_shift; ++ ++ if (alveo_uart_reg_width == 1) ++ writeb(val, alveo_uart_base + offset); ++ else if (alveo_uart_reg_width == 2) ++ writew(val, alveo_uart_base + offset); ++ else ++ writel(val, alveo_uart_base + offset); ++} ++ ++static void alveo_uart_putc(char ch) ++{ ++ while (!(get_reg(UART_LSR_OFFSET) & UART_LSR_TEMT)) ++ ; ++ ++ set_reg(UART_THR_OFFSET, ch); ++} ++ ++static int alveo_uart_getc(void) ++{ ++ if (get_reg(UART_LSR_OFFSET) & UART_LSR_DR) ++ return get_reg(UART_RBR_OFFSET); ++ return -1; ++} ++ ++static struct sbi_console_device alveo_uart_console = { ++ .name = "alveo_uart", ++ .console_putc = alveo_uart_putc, ++ .console_getc = alveo_uart_getc ++}; ++ ++int alveo_uart_init(unsigned long base, u32 in_freq, u32 baudrate, u32 reg_shift, ++ u32 reg_width, u32 reg_offset) ++{ ++ u16 bdiv = 0; ++ ++ alveo_uart_base = (volatile char *)base + reg_offset; ++ alveo_uart_reg_shift = reg_shift; ++ alveo_uart_reg_width = reg_width; ++ alveo_uart_in_freq = in_freq; ++ alveo_uart_baudrate = baudrate; ++ ++ if (alveo_uart_baudrate) { ++ bdiv = alveo_uart_in_freq / (16 * alveo_uart_baudrate); ++ } ++ ++ /* Disable all interrupts */ ++ set_reg(UART_IER_OFFSET, 0x00); ++ /* Enable DLAB */ ++ set_reg(UART_LCR_OFFSET, 0x80); ++ ++ if (bdiv) { ++ /* Set divisor low byte */ ++ set_reg(UART_DLL_OFFSET, bdiv & 0xff); ++ /* Set divisor high byte */ ++ set_reg(UART_DLM_OFFSET, (bdiv >> 8) & 0xff); ++ } ++ ++ /* 8 bits, no parity, one stop bit */ ++ set_reg(UART_LCR_OFFSET, 0x03); // previous was 0x0B ++ /* Disable all interrupts*/ ++ set_reg(UART_IER_OFFSET, 0x00); ++ ++ sbi_console_set_device(&alveo_uart_console); ++ ++ return 0; ++} +diff --git a/lib/utils/serial/objects.mk b/lib/utils/serial/objects.mk +index 1e6bd2e..0268bdf 100644 +--- a/lib/utils/serial/objects.mk ++++ b/lib/utils/serial/objects.mk +@@ -44,5 +44,6 @@ libsbiutils-objs-$(CONFIG_SERIAL_SHAKTI) += serial/shakti-uart.o + libsbiutils-objs-$(CONFIG_SERIAL_SIFIVE) += serial/sifive-uart.o + libsbiutils-objs-$(CONFIG_SERIAL_LITEX) += serial/litex-uart.o + libsbiutils-objs-$(CONFIG_SERIAL_UART8250) += serial/uart8250.o ++libsbiutils-objs-$(CONFIG_SERIAL_ALVEO_UART) += serial/alveo_uart.o + libsbiutils-objs-$(CONFIG_SERIAL_XILINX_UARTLITE) += serial/xlnx-uartlite.o + libsbiutils-objs-$(CONFIG_SERIAL_SEMIHOSTING) += serial/semihosting.o +diff --git a/platform/fpga/sargantana_alveo/Kconfig b/platform/fpga/sargantana_alveo/Kconfig +new file mode 100644 +index 0000000..bf3e7e6 +--- /dev/null ++++ b/platform/fpga/sargantana_alveo/Kconfig +@@ -0,0 +1,10 @@ ++# SPDX-License-Identifier: BSD-2-Clause ++ ++config PLATFORM_SARGANTANA_ALVEO_FPGA ++ bool ++ select FDT ++ select IPI_MSWI ++ select IRQCHIP_PLIC ++ select SERIAL_ALVEO_UART ++ select TIMER_MTIMER ++ default y +diff --git a/platform/fpga/sargantana_alveo/configs/defconfig b/platform/fpga/sargantana_alveo/configs/defconfig +new file mode 100644 +index 0000000..e69de29 +diff --git a/platform/fpga/sargantana_alveo/objects.mk b/platform/fpga/sargantana_alveo/objects.mk +new file mode 100644 +index 0000000..d444abe +--- /dev/null ++++ b/platform/fpga/sargantana_alveo/objects.mk +@@ -0,0 +1,87 @@ ++# ++# SPDX-License-Identifier: BSD-2-Clause ++# ++# Copyright (c) 2019 Western Digital Corporation or its affiliates. ++# ++ ++# Compiler pre-processor flags ++platform-cppflags-y = ++ ++# C Compiler and assembler flags. ++platform-cflags-y = ++platform-asflags-y = ++ ++# Linker flags: additional libraries and object files that the platform ++# code needs can be added here ++platform-ldflags-y = ++ ++# ++# Command for platform specific "make run" ++# Useful for development and debugging on plaftform simulator (such as QEMU) ++# ++# platform-runcmd = your_platform_run.sh ++ ++# ++# Platform RISC-V XLEN, ABI, ISA and Code Model configuration. ++# These are optional parameters but platforms can optionaly provide it. ++# Some of these are guessed based on GCC compiler capabilities ++# ++PLATFORM_RISCV_XLEN = 64 ++PLATFORM_RISCV_ABI = lp64d ++PLATFORM_RISCV_ISA = rv64imafd ++PLATFORM_RISCV_CODE_MODEL = medany ++ ++# Space separated list of object file names to be compiled for the platform ++platform-objs-y += platform.o ++ ++# ++# If the platform support requires a builtin device tree file, the name of ++# the device tree compiled file should be specified here. The device tree ++# source file be in the form
.dts ++# ++# platform-objs-y +=
.o ++ ++# Firmware load address configuration. This is mandatory. ++FW_TEXT_START=0x80000000 ++ ++# Optional parameter for path to external FDT ++# FW_FDT_PATH="path to platform flattened device tree file" ++ ++# ++# Dynamic firmware configuration. ++# Optional parameters are commented out. Uncomment and define these parameters ++# as needed. ++# ++FW_DYNAMIC=n ++ ++# ++# Jump firmware configuration. ++# Optional parameters are commented out. Uncomment and define these parameters ++# as needed. ++# ++FW_JUMP=n ++# This needs to be 4MB aligned for 32-bit support ++# This needs to be 2MB aligned for 64-bit support ++# ifeq ($(PLATFORM_RISCV_XLEN), 32) ++# FW_JUMP_ADDR=0x80400000 ++# else ++# FW_JUMP_ADDR=0x80200000 ++# endif ++# FW_JUMP_FDT_ADDR=0x82200000 ++ ++# ++# Firmware with payload configuration. ++# Optional parameters are commented out. Uncomment and define these parameters ++# as needed. ++# ++FW_PAYLOAD=y ++# This needs to be 4MB aligned for 32-bit support ++# This needs to be 2MB aligned for 64-bit support ++ifeq ($(PLATFORM_RISCV_XLEN), 32) ++FW_PAYLOAD_OFFSET=0x400000 ++else ++FW_PAYLOAD_OFFSET=0x200000 ++endif ++FW_PAYLOAD_ALIGN=0x1000 ++# FW_PAYLOAD_PATH="path to next boot stage binary image file" ++# FW_PAYLOAD_FDT_ADDR=0x82200000 +diff --git a/platform/fpga/sargantana_alveo/platform.c b/platform/fpga/sargantana_alveo/platform.c +new file mode 100644 +index 0000000..a359b34 +--- /dev/null ++++ b/platform/fpga/sargantana_alveo/platform.c +@@ -0,0 +1,146 @@ ++/* ++ * SPDX-License-Identifier: BSD-2-Clause ++ * ++ * Copyright (c) 2019 Western Digital Corporation or its affiliates. ++ */ ++ ++#include ++#include ++#include ++#include ++ ++/* ++ * Include these files as needed. ++ * See objects.mk SARGANTANA_ALVEO_xxx configuration parameters. ++ */ ++ ++#include ++#include ++ ++#define SARGANTANA_ALVEO_HART_COUNT 1 ++ ++#define SARGANTANA_ALVEO_UART_BASE_ADDR 0x40000000 ++#define SARGANTANA_ALVEO_UART_OFFSET 0x1000 ++//#define SARGANTANA_ALVEO_UART_ADDR SARGANTANA_ALVEO_UART_BASE_ADDR + SARGANTANA_ALVEO_UART_XLNX_OFFSET ++#define SARGANTANA_ALVEO_UART_INPUT_FREQ 50000000 ++#define SARGANTANA_ALVEO_UART_BAUDRATE 115200 ++ ++#define SARGANTANA_ALVEO_TIMER_BASE 0x40170000 ++#define ADDR_TIME_L 0x0u // 32 lower bits of the time register ++#define ADDR_TIME_H 0x1u // 32 higher bits of the time register ++#define ADDR_TIMECMP_L 0x2u // 32 lower bits of the time comparator ++#define ADDR_TIMECMP_H 0x3u // 32 higher bits of the time comparator ++ ++volatile uint32_t *timer_base_ptr = (uint32_t *)(SARGANTANA_ALVEO_TIMER_BASE); ++ ++/* ++ * Platform early initialization. ++ */ ++static int sargantana_alveo_early_init(bool cold_boot) ++{ ++ return 0; ++} ++ ++/* ++ * Platform final initialization. ++ */ ++static int sargantana_alveo_final_init(bool cold_boot) ++{ ++ return 0; ++} ++ ++/* ++ * Initialize the platform console. ++ */ ++static int sargantana_alveo_console_init(void) ++{ ++ return alveo_uart_init(SARGANTANA_ALVEO_UART_BASE_ADDR, ++ SARGANTANA_ALVEO_UART_INPUT_FREQ, ++ SARGANTANA_ALVEO_UART_BAUDRATE, ++ 2, 4, ++ SARGANTANA_ALVEO_UART_OFFSET); ++} ++ ++/* ++ * Initialize the platform interrupt controller for current HART. ++ */ ++static int sargantana_alveo_irqchip_init(bool cold_boot) ++{ ++ u32 hartid = current_hartid(); ++ return hartid; ++} ++ ++/* ++ * Initialize IPI for current HART. ++ */ ++static int sargantana_alveo_ipi_init(bool cold_boot) ++{ ++ return 0; ++} ++ ++/* ++ * Get platform timer value. ++ */ ++static u64 sargantana_alveo_timer_value(void) ++{ ++ return ((u64)*(timer_base_ptr + ADDR_TIME_H) << 32) + *(timer_base_ptr + ADDR_TIME_L); ++} ++ ++/* ++ * Start platform timer event for current HART. ++ */ ++static void sargantana_alveo_timer_event_start(u64 next_event) ++{ ++ *(timer_base_ptr + ADDR_TIMECMP_H) = next_event >> 32; ++ *(timer_base_ptr + ADDR_TIMECMP_L) = next_event; ++} ++ ++/* ++ * Stop platform timer event for current HART. ++ */ ++static void sargantana_alveo_timer_event_stop(void) ++{ ++ ++ *(timer_base_ptr + ADDR_TIMECMP_H) = 0; ++ *(timer_base_ptr + ADDR_TIMECMP_L) = 0; ++} ++ ++static struct sbi_timer_device mtimer = { ++ .name = "generic_timer", // TODO Where the timer comes from? I would prefer a better name :p ++ .timer_freq = SARGANTANA_ALVEO_UART_INPUT_FREQ, ++ .timer_value = sargantana_alveo_timer_value, ++ .timer_event_start = sargantana_alveo_timer_event_start, ++ .timer_event_stop = sargantana_alveo_timer_event_stop ++}; ++ ++/* ++ * Initialize platform timer for current HART. ++ */ ++static int sargantana_alveo_timer_init(bool cold_boot) ++{ ++ *(timer_base_ptr + ADDR_TIMECMP_H) = 0; ++ *(timer_base_ptr + ADDR_TIMECMP_L) = 0; ++ sbi_timer_set_device(&mtimer); ++ return 0; ++} ++ ++/* ++ * Platform descriptor. ++ */ ++const struct sbi_platform_operations sargantana_alveo_ops = { ++ .early_init = sargantana_alveo_early_init, ++ .final_init = sargantana_alveo_final_init, ++ .console_init = sargantana_alveo_console_init, ++ .irqchip_init = sargantana_alveo_irqchip_init, ++ .ipi_init = sargantana_alveo_ipi_init, ++ .timer_init = sargantana_alveo_timer_init ++}; ++const struct sbi_platform platform = { ++ .opensbi_version = OPENSBI_VERSION, ++ .platform_version = SBI_PLATFORM_VERSION(0x0, 0x01), ++ .name = "Sargantana (for Xilinx Alveo FPGA)", ++ .features = SBI_PLATFORM_DEFAULT_FEATURES, ++ .hart_count = SARGANTANA_ALVEO_HART_COUNT, ++ .hart_stack_size = SBI_PLATFORM_DEFAULT_HART_STACK_SIZE, ++ .platform_ops_addr = (unsigned long)&sargantana_alveo_ops ++}; diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 51359ba..5f4442b 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -5,6 +5,13 @@ "${modulesPath}/installer/sd-card/sd-image.nix" ]; + # No need, Lagarto OX has support for rv64gc. + #nixpkgs.crossSystem = { + # system = "riscv64-linux"; + # gcc.arch = "rv64imafd"; + # gcc.tune = "rv64imafd"; + #}; + #boot.kernelPackages = pkgs.linuxPackages_latest; boot = { kernelPatches = [ @@ -121,12 +128,61 @@ echo 'booti ''${kernel_addr_r} ''${ramdisk_addr_r}:''${ramdisk_size} ''${fdtcontroladdr}' >> $out ''; - opensbi = prev.opensbi.overrideAttrs (old: { - makeFlags = old.makeFlags ++ [ - "PLATFORM=fpga/openpiton" + # Lagarto OX doesn't work with newer versions of OpenSBI (1.3, 1.4), not + # sure why. + opensbi = prev.opensbi.overrideAttrs (old: rec { + version = "1.2"; # ok + #version = "1.3"; # bad + #version = "1.4"; # bad + src = prev.fetchFromGitHub { + owner = "riscv-software-src"; + repo = "opensbi"; + rev = "v${version}"; + hash = "sha256-Zcl+SE2nySMycV/ozsl4AvGipRsMblA5mt3oVZ81Z44="; #1.2 + #hash = "sha256-Dr16fVUGLYGnGYHkjAyqpJxt8p95F0CJIU9ESGWKGWo="; #1.3 + #hash = "sha256-T8ZeAzjM9aeTXitjE7s+m+jjGGtDo2jK1qO5EuKiVLU="; #1.4 + }; + #NIX_DEBUG=5; + # Compile dts to dtb form + nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ prev.buildPackages.dtc ]; + dtsFile = ./ox-plic.dts; + preBuild = '' + dtc -O dtb -o ox.dtb $dtsFile + ''; + makeFlags = [ + "PLATFORM=fpga/ox_alveo" + #"CONFIG_SBI_ECALL_RFENCE=n" + #"PLATFORM_RISCV_ISA=rv64imafd" # No compressed instructions + #"PLATFORM_RISCV_ISA=rv64g" # No compressed instructions + #"PLATFORM_RISCV_ABI=lp64d" "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin" + "FW_FDT_PATH=ox.dtb" ]; - patches = [ ./opensbi-lagarto-hun.patch ]; + patches = [ ./ox-alveo-platform-plic.patch ]; }); +# opensbi = prev.opensbi.overrideAttrs (old: { +# #NIX_DEBUG=5; +# # Compile dts to dtb form +# #nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ prev.buildPackages.dtc ]; +# #preBuild = '' +# # dtc -O dtb -o ox.dtb ${./ox.dts} +# #''; +# makeFlags = old.makeFlags ++ [ +# "PLATFORM=fpga/sargantana_alveo" +# #"PLATFORM_RISCV_ISA=rv64imafd" # No compressed instructions +# "PLATFORM_RISCV_ISA=rv64g" # No compressed instructions +# #"PLATFORM_RISCV_ABI=lp64d" +# "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin" +# #"FW_FDT_PATH=ox.dtb" +# ]; +# patches = [ ./alveo-uart-sbi-hack.patch ]; +# }); +# opensbi = prev.opensbi.overrideAttrs (old: { +# makeFlags = old.makeFlags ++ [ +# "PLATFORM=fpga/openpiton" +# "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin" +# ]; +# patches = [ ./opensbi-lagarto-hun.patch ]; +# }); }) ]; } diff --git a/ox-alveo-platform-plic.patch b/ox-alveo-platform-plic.patch new file mode 100644 index 0000000..d906a6f --- /dev/null +++ b/ox-alveo-platform-plic.patch @@ -0,0 +1,166 @@ +diff --git a/platform/fpga/ox_alveo/Kconfig b/platform/fpga/ox_alveo/Kconfig +new file mode 100644 +index 0000000..bf3e7e6 +--- /dev/null ++++ b/platform/fpga/ox_alveo/Kconfig +@@ -0,0 +1,5 @@ ++config PLATFORM_OX_ALVEO_FPGA ++ bool ++ select SERIAL_UART8250 ++ select IRQCHIP_PLIC ++ default y +diff --git a/platform/fpga/ox_alveo/configs/defconfig b/platform/fpga/ox_alveo/configs/defconfig +new file mode 100644 +index 0000000..e69de29 +diff --git a/platform/fpga/ox_alveo/objects.mk b/platform/fpga/ox_alveo/objects.mk +new file mode 100644 +index 0000000..d444abe +--- /dev/null ++++ b/platform/fpga/ox_alveo/objects.mk +@@ -0,0 +1,19 @@ ++platform-cppflags-y = ++platform-cflags-y = ++platform-asflags-y = ++platform-ldflags-y = ++PLATFORM_RISCV_XLEN = 64 ++PLATFORM_RISCV_ABI = lp64d ++PLATFORM_RISCV_ISA = rv64g ++PLATFORM_RISCV_CODE_MODEL = medany ++ ++platform-objs-y += platform.o ++ ++FW_TEXT_START=0x80000000 ++ ++FW_DYNAMIC=n ++FW_JUMP=n ++FW_PAYLOAD=y ++ ++FW_PAYLOAD_OFFSET=0x200000 ++FW_PAYLOAD_ALIGN=0x1000 +diff --git a/platform/fpga/ox_alveo/platform.c b/platform/fpga/ox_alveo/platform.c +new file mode 100644 +index 0000000..a359b34 +--- /dev/null ++++ b/platform/fpga/ox_alveo/platform.c +@@ -0,0 +1,121 @@ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define OX_ALVEO_HART_COUNT 1 ++ ++#define OX_ALVEO_UART_BASE_ADDR 0x40000000 ++#define OX_ALVEO_UART_OFFSET 0x1000 ++#define OX_ALVEO_UART_INPUT_FREQ 50000000 ++#define OX_ALVEO_UART_BAUDRATE 115200 ++#define OX_ALVEO_PLIC_ADDR 0x40800000 ++#define OX_ALVEO_PLIC_NUM_SOURCES 3 ++ ++#define OX_ALVEO_TIMER_BASE 0x40170000 ++#define ADDR_TIME_L 0x0u // 32 lower bits of the time register ++#define ADDR_TIME_H 0x1u // 32 higher bits of the time register ++#define ADDR_TIMECMP_L 0x2u // 32 lower bits of the time comparator ++#define ADDR_TIMECMP_H 0x3u // 32 higher bits of the time comparator ++ ++volatile uint32_t *timer_base_ptr = (uint32_t *)(OX_ALVEO_TIMER_BASE); ++ ++static struct plic_data plic = { ++ .addr = OX_ALVEO_PLIC_ADDR, ++ .num_src = OX_ALVEO_PLIC_NUM_SOURCES, ++}; ++ ++static int ox_alveo_early_init(bool cold_boot) // Platform early initialization. ++{ ++ return 0; ++} ++ ++static int ox_alveo_final_init(bool cold_boot) // Platform final initialization. ++{ ++ return 0; ++} ++ ++static int ox_alveo_console_init(void) // Initialize the platform console. ++{ ++ return uart8250_init(OX_ALVEO_UART_BASE_ADDR, ++ OX_ALVEO_UART_INPUT_FREQ, ++ OX_ALVEO_UART_BAUDRATE, ++ 2, 4, ++ OX_ALVEO_UART_OFFSET); ++} ++ ++static int ox_alveo_irqchip_init(bool cold_boot) // Initialize the platform interrupt controller for current HART. ++{ ++ u32 hartid = current_hartid(); ++ int ret; ++ ++ /* Example if the generic PLIC driver is used */ ++ if (cold_boot) { ++ ret = plic_cold_irqchip_init(&plic); ++ if (ret) ++ return ret; ++ } ++ ++ return plic_warm_irqchip_init(&plic, 2 * hartid, -1); ++} ++ ++static int ox_alveo_ipi_init(bool cold_boot) // Initialize IPI for current HART. ++{ ++ return 0; ++} ++ ++static u64 ox_alveo_timer_value(void) // Get platform timer value. ++{ ++ return ((u64)*(timer_base_ptr + ADDR_TIME_H) << 32) + *(timer_base_ptr + ADDR_TIME_L); ++} ++ ++static void ox_alveo_timer_event_start(u64 next_event) // Start platform timer event for current HART. ++{ ++ *(timer_base_ptr + ADDR_TIMECMP_H) = next_event >> 32; ++ *(timer_base_ptr + ADDR_TIMECMP_L) = next_event; ++} ++ ++static void ox_alveo_timer_event_stop(void) // Stop platform timer event for current HART. ++{ ++ ++ *(timer_base_ptr + ADDR_TIMECMP_H) = 0; ++ *(timer_base_ptr + ADDR_TIMECMP_L) = 0; ++} ++ ++static struct sbi_timer_device mtimer = { ++ .name = "axi_timer", ++ .timer_freq = OX_ALVEO_UART_INPUT_FREQ, ++ .timer_value = ox_alveo_timer_value, ++ .timer_event_start = ox_alveo_timer_event_start, ++ .timer_event_stop = ox_alveo_timer_event_stop ++}; ++ ++static int ox_alveo_timer_init(bool cold_boot) // Initialize platform timer for current HART. ++{ ++ *(timer_base_ptr + ADDR_TIMECMP_H) = 0; ++ *(timer_base_ptr + ADDR_TIMECMP_L) = 0; ++ sbi_timer_set_device(&mtimer); ++ return 0; ++} ++ ++const struct sbi_platform_operations ox_alveo_ops = { // Platform descriptor. ++ .early_init = ox_alveo_early_init, ++ .final_init = ox_alveo_final_init, ++ .console_init = ox_alveo_console_init, ++ .irqchip_init = ox_alveo_irqchip_init, ++ .ipi_init = ox_alveo_ipi_init, ++ .timer_init = ox_alveo_timer_init ++}; ++ ++const struct sbi_platform platform = { ++ .opensbi_version = OPENSBI_VERSION, ++ .platform_version = SBI_PLATFORM_VERSION(0x0, 0x01), ++ .name = "ox (Rodrigo NixOS version)", ++ .features = SBI_PLATFORM_DEFAULT_FEATURES, ++ .hart_count = OX_ALVEO_HART_COUNT, ++ .hart_stack_size = SBI_PLATFORM_DEFAULT_HART_STACK_SIZE, ++ .platform_ops_addr = (unsigned long)&ox_alveo_ops ++}; diff --git a/ox-alveo-platform.patch b/ox-alveo-platform.patch new file mode 100644 index 0000000..347d0d4 --- /dev/null +++ b/ox-alveo-platform.patch @@ -0,0 +1,148 @@ +diff --git a/platform/fpga/ox_alveo/Kconfig b/platform/fpga/ox_alveo/Kconfig +new file mode 100644 +index 0000000..bf3e7e6 +--- /dev/null ++++ b/platform/fpga/ox_alveo/Kconfig +@@ -0,0 +1,4 @@ ++config PLATFORM_OX_ALVEO_FPGA ++ bool ++ select SERIAL_UART8250 ++ default y +diff --git a/platform/fpga/ox_alveo/configs/defconfig b/platform/fpga/ox_alveo/configs/defconfig +new file mode 100644 +index 0000000..e69de29 +diff --git a/platform/fpga/ox_alveo/objects.mk b/platform/fpga/ox_alveo/objects.mk +new file mode 100644 +index 0000000..d444abe +--- /dev/null ++++ b/platform/fpga/ox_alveo/objects.mk +@@ -0,0 +1,19 @@ ++platform-cppflags-y = ++platform-cflags-y = ++platform-asflags-y = ++platform-ldflags-y = ++PLATFORM_RISCV_XLEN = 64 ++PLATFORM_RISCV_ABI = lp64d ++PLATFORM_RISCV_ISA = rv64imafd ++PLATFORM_RISCV_CODE_MODEL = medany ++ ++platform-objs-y += platform.o ++ ++FW_TEXT_START=0x80000000 ++ ++FW_DYNAMIC=n ++FW_JUMP=n ++FW_PAYLOAD=y ++ ++FW_PAYLOAD_OFFSET=0x200000 ++FW_PAYLOAD_ALIGN=0x1000 +diff --git a/platform/fpga/ox_alveo/platform.c b/platform/fpga/ox_alveo/platform.c +new file mode 100644 +index 0000000..a359b34 +--- /dev/null ++++ b/platform/fpga/ox_alveo/platform.c +@@ -0,0 +1,104 @@ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define OX_ALVEO_HART_COUNT 1 ++ ++#define OX_ALVEO_UART_BASE_ADDR 0x40000000 ++#define OX_ALVEO_UART_OFFSET 0x1000 ++#define OX_ALVEO_UART_INPUT_FREQ 50000000 ++#define OX_ALVEO_UART_BAUDRATE 115200 ++ ++#define OX_ALVEO_TIMER_BASE 0x40170000 ++#define ADDR_TIME_L 0x0u // 32 lower bits of the time register ++#define ADDR_TIME_H 0x1u // 32 higher bits of the time register ++#define ADDR_TIMECMP_L 0x2u // 32 lower bits of the time comparator ++#define ADDR_TIMECMP_H 0x3u // 32 higher bits of the time comparator ++ ++volatile uint32_t *timer_base_ptr = (uint32_t *)(OX_ALVEO_TIMER_BASE); ++ ++static int ox_alveo_early_init(bool cold_boot) // Platform early initialization. ++{ ++ return 0; ++} ++ ++static int ox_alveo_final_init(bool cold_boot) // Platform final initialization. ++{ ++ return 0; ++} ++ ++static int ox_alveo_console_init(void) // Initialize the platform console. ++{ ++ return uart8250_init(OX_ALVEO_UART_BASE_ADDR, ++ OX_ALVEO_UART_INPUT_FREQ, ++ OX_ALVEO_UART_BAUDRATE, ++ 2, 4, ++ OX_ALVEO_UART_OFFSET); ++} ++ ++static int ox_alveo_irqchip_init(bool cold_boot) // Initialize the platform interrupt controller for current HART. ++{ ++ u32 hartid = current_hartid(); ++ return hartid; ++} ++ ++static int ox_alveo_ipi_init(bool cold_boot) // Initialize IPI for current HART. ++{ ++ return 0; ++} ++ ++static u64 ox_alveo_timer_value(void) // Get platform timer value. ++{ ++ return ((u64)*(timer_base_ptr + ADDR_TIME_H) << 32) + *(timer_base_ptr + ADDR_TIME_L); ++} ++ ++static void ox_alveo_timer_event_start(u64 next_event) // Start platform timer event for current HART. ++{ ++ *(timer_base_ptr + ADDR_TIMECMP_H) = next_event >> 32; ++ *(timer_base_ptr + ADDR_TIMECMP_L) = next_event; ++} ++ ++static void ox_alveo_timer_event_stop(void) // Stop platform timer event for current HART. ++{ ++ ++ *(timer_base_ptr + ADDR_TIMECMP_H) = 0; ++ *(timer_base_ptr + ADDR_TIMECMP_L) = 0; ++} ++ ++static struct sbi_timer_device mtimer = { ++ .name = "axi_timer", ++ .timer_freq = OX_ALVEO_UART_INPUT_FREQ, ++ .timer_value = ox_alveo_timer_value, ++ .timer_event_start = ox_alveo_timer_event_start, ++ .timer_event_stop = ox_alveo_timer_event_stop ++}; ++ ++static int ox_alveo_timer_init(bool cold_boot) // Initialize platform timer for current HART. ++{ ++ *(timer_base_ptr + ADDR_TIMECMP_H) = 0; ++ *(timer_base_ptr + ADDR_TIMECMP_L) = 0; ++ sbi_timer_set_device(&mtimer); ++ return 0; ++} ++ ++const struct sbi_platform_operations ox_alveo_ops = { // Platform descriptor. ++ .early_init = ox_alveo_early_init, ++ .final_init = ox_alveo_final_init, ++ .console_init = ox_alveo_console_init, ++ .irqchip_init = ox_alveo_irqchip_init, ++ .ipi_init = ox_alveo_ipi_init, ++ .timer_init = ox_alveo_timer_init ++}; ++ ++const struct sbi_platform platform = { ++ .opensbi_version = OPENSBI_VERSION, ++ .platform_version = SBI_PLATFORM_VERSION(0x0, 0x01), ++ .name = "ox (for Xilinx Alveo FPGA)", ++ .features = SBI_PLATFORM_DEFAULT_FEATURES, ++ .hart_count = OX_ALVEO_HART_COUNT, ++ .hart_stack_size = SBI_PLATFORM_DEFAULT_HART_STACK_SIZE, ++ .platform_ops_addr = (unsigned long)&ox_alveo_ops ++}; diff --git a/ox-plic.dts b/ox-plic.dts new file mode 100644 index 0000000..c91c375 --- /dev/null +++ b/ox-plic.dts @@ -0,0 +1,143 @@ +/dts-v1/; +/ { + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + compatible = "riscv,rv64i"; + model = "Barcelona Supercomputing Center - Lagarto Ox"; + chosen { + bootargs = "earlyprintk ignore_loglevel earlycon=sbi console=hvc0 root=/dev/pmem0p1 ro init=/bin/bash"; + }; + cpus { + #address-cells = <0x00000001>; + #size-cells = <0x00000000>; + timebase-frequency = <0x0000c350>; + cpu@0 { + clock-frequency = <0x02FAF080>; + device_type = "cpu"; + reg = <0x00000000>; + status = "okay"; + compatible = "riscv"; + riscv,isa = "rv64imafd"; + mmu-type = "riscv,sv39"; + tlb-split; + L3: interrupt-controller { + #interrupt-cells = <0x00000001>; + interrupt-controller; + compatible = "riscv,cpu-intc"; + }; + }; + }; + memory@80000000 { + device_type = "memory"; + reg = <0x00000000 0x80000000 0x00000000 0x77e00000>; + }; + reserved-memory { + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + ranges; + eth_pool: dma_pool@60000000 { + reg = <0x00000000 0x60000000 0x00000000 0x10000000>; + compatible = "shared-dma-pool"; + }; + onic_pool: dma_pool@70000000 { + reg = <0x00000000 0x70000000 0x00000000 0x10000000>; + compatible = "shared-dma-pool"; + }; + }; + eth0_clk: eth0_clk { + compatible = "fixed-clock"; + #clock-cells = <0x00000000>; + clock-frequency = <0x09502f90>; + }; + pmem@1bff00000 { + volatile; + compatible = "pmem-region"; + reg = <0x00000001 0xbff00000 0x00000000 0xc0100000>; + }; + pmem2@1b7f00000 { + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + volatile; + compatible = "pmem-region"; + reg = <0x00000001 0xb7f00000 0x00000000 0x08000000>; + }; + pmem3@f7e00000 { + volatile; + compatible = "pmem-region"; + reg = <0x00000000 0xf7e00000 0x00000000 0xc0100000>; + }; + soc { + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + compatible = "BSC,Lagarto-bare-soc", "simple-bus"; + ranges; + serial@40001000 { + compatible = "ns16750"; + reg = <0x00000000 0x40001000 0x00000000 0x00001000>; + interrupts = <0x00000000>; + port-number = <0x00000000>; + reg-shift = <0x00000002>; + clock-frequency = <0x02FAF080>; + current-speed = <0x0001c200>; + status = "okay"; + }; + ethernet0 { + xlnx,rxmem = <0x000005f2>; + carv,mtu = <0x000005dc>; + carv,no-mac; + device_type = "network"; + local-mac-address = [02 05 00 01 00 05]; + axistream-connected = <0x000000fe>; + compatible = "xlnx,xxv-ethernet-1.0-carv"; + memory-region = <ð_pool>; + }; + dma@40400000 { + xlnx,include-dre; + phandle = <0x000000fe>; + #dma-cells = <0x00000001>; + compatible = "xlnx,axi-dma-1.00.a"; + clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk", "m_axi_s2mm_aclk", "m_axi_sg_aclk"; + clocks = <ð0_clk>, <ð0_clk>, <ð0_clk>, <ð0_clk>; + reg = <0x00000000 0x40400000 0x00000000 0x00400000>; + interrupt-names = "mm2s_introut", "s2mm_introut"; + interrupt-parent = <&PLIC0>; + interrupts = <0x00000002 0x00000003>; + xlnx,addrwidth = <0x00000028>; + xlnx,include-sg; + xlnx,sg-length-width = <0x00000017>; + dma-channel@40400000 { + compatible = "xlnx,axi-dma-mm2s-channel"; + dma-channels = <0x00000000>; + interrupts = <0x00000002>; + xlnx,datawidth = <0x00000040>; + xlnx,device-id = <0x00000000>; + xlnx,include-dre; + }; + dma-channel@40400030 { + compatible = "xlnx,axi-dma-s2mm-channel"; + dma-channels = <0x00000001>; + interrupts = <0x00000003>; + xlnx,datawidth = <0x00000040>; + xlnx,device-id = <0x00000000>; + xlnx,include-dre; + }; + }; + clint@40002000 { + reg-names = "control"; + interrupts-extended = <0x00000004 0x00000003 0x00000004 0x00000007 0x00000005 0x00000003 0x00000005 0x00000007 0x00000006 0x00000003 0x00000006 0x00000007 0x00000007 0x00000003 0x00000007 0x00000007>; + compatible = "riscv,clint0"; + #interrupt-cells = <0x00000001>; + reg = <0x00000000 0x40002000 0x00000000 0x000c0000>; + }; + PLIC0: plic@40800000 { + #interrupt-cells = <1>; + compatible = "riscv,plic0"; + interrupt-controller; + interrupts-extended = <&L3 3 &L3 7>; + reg = < 0x00000000 0x40800000 0x00000000 0x00400000>; + riscv,ndev = <0x00000003>; + riscv,max-priority = <0x00000007>; + }; + }; + +}; diff --git a/ox.dts b/ox.dts new file mode 100644 index 0000000..8fd11b8 --- /dev/null +++ b/ox.dts @@ -0,0 +1,58 @@ +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + compatible = "riscv,rv64i"; + model = "Barcelona Supercomputing Center - OX"; + chosen { + bootargs = "earlycon=sbi console=ttyS0,115200n8"; + }; + cpus { + #address-cells = <1>; + #size-cells = <0>; + timebase-frequency = <50000>; // 32.768 kHz + CPU0: cpu@0 { + //clock-frequency = <25000000>; // 25 MHz + clock-frequency = <50000000>; // 50 MHz + device_type = "cpu"; + reg = <0>; + status = "okay"; + compatible = "riscv"; + riscv,isa = "rv64imafd"; + mmu-type = "riscv,sv39"; + tlb-split; + // HLIC - hart local interrupt controller + CPU0_intc: interrupt-controller { + #address-cells = <1>; + #interrupt-cells = <1>; + interrupt-controller; + compatible = "riscv,cpu-intc"; + }; + }; + }; + + uart0: serial@40001000 { + compatible = "ns16750"; + //reg = <0x40001000 0x1000>; + reg = <0x00000000 0x40001000 0x00000000 0x00000100>; + interrupts = <0>; + port-number = <0>; + reg-shift = <2>; + reg-io-width = <4>; + clock-frequency = <25000000>; + current-speed = <115200>; + status = "okay"; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x80000000 0x40000000>; + }; + soc { + #address-cells = <1>; + #size-cells = <1>; + compatible = "simple-bus"; + ranges; + }; +}; -- GitLab From 72b4788888d1569c61aa2c7bc5da4ddc6eb8cf72 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 27 Jun 2024 16:08:30 +0200 Subject: [PATCH 008/310] Reached U-Boot console with FDT parsing error --- lagarto-ox.nix | 33 +++++- ox-plic.dts | 229 +++++++++++++++++++------------------ ox.dts | 2 +- u-boot-debug.patch | 83 ++++++++++++++ xavi-good-ftd.dts | 278 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 506 insertions(+), 119 deletions(-) create mode 100644 u-boot-debug.patch create mode 100644 xavi-good-ftd.dts diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 5f4442b..36e04e3 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -86,14 +86,30 @@ }; nixpkgs.overlays = [ (final: prev: { + ox-dtb = prev.runCommand "ox.dtb" { + nativeBuildInputs = [ prev.buildPackages.dtc ]; + } "dtc -O dtb -o $out ${./ox-plic.dts}"; + uboot = prev.ubootQemuRiscv64Smode.override { filesToInstall = [ "u-boot.bin" "u-boot-nodtb.bin" ]; + version = "2023.07.02-print-cpu-probe"; + src = builtins.fetchGit { + url = "file:///home/Computational/rarias/riscv/u-boot"; + rev = "b0868e84c392daadcaf611f92209d0c82203386f"; + }; + #patches = [ ./u-boot-debug.patch ]; extraConfig = '' CONFIG_RISCV_ISA_C=n CONFIG_REQUIRE_SERIAL_CONSOLE=n - CONFIG_SERIAL_SEARCH_ALL=y + CONFIG_SERIAL=y + CONFIG_SERIAL_PUTS=y + CONFIG_SHOW_BOOT_PROGRESS=y + CONFIG_SHOW_REGS=y + CONFIG_LIBCOMMON_SUPPORT=y + CONFIG_SERIAL_SEARCH_ALL=n + CONFIG_SERIAL_PROBE_ALL=n CONFIG_OF_CONTROL=y - CONFIG_OF_BOARD=y + CONFIG_OF_EMBED=y CONFIG_OF_HAS_PRIOR_STAGE=y CONFIG_BLKMAP=y CONFIG_CMD_BLKMAP=y @@ -101,7 +117,18 @@ CONFIG_DEBUG_UART=y CONFIG_DEBUG_UART_ANNOUNCE=y CONFIG_DEBUG_SBI_CONSOLE=y + CONFIG_SYS_NS16550=n + CONFIG_SMP=n + CONFIG_TRACE_EARLY=y + CONFIG_LOG=y + CONFIG_LOGLEVEL=9 + CONFIG_LOG_MAX_LEVEL=9 + CONFIG_LOG_DEFAULT_LEVEL=9 ''; + extraMakeFlags = [ + "KCPPFLAGS=-DLOG_DEBUG" + #"EXT_DTB=${final.ox-dtb}" + ]; }; uboot-env = let @@ -155,7 +182,7 @@ #"PLATFORM_RISCV_ISA=rv64imafd" # No compressed instructions #"PLATFORM_RISCV_ISA=rv64g" # No compressed instructions #"PLATFORM_RISCV_ABI=lp64d" - "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin" + "FW_PAYLOAD_PATH=${final.uboot}/u-boot.bin" "FW_FDT_PATH=ox.dtb" ]; patches = [ ./ox-alveo-platform-plic.patch ]; diff --git a/ox-plic.dts b/ox-plic.dts index c91c375..f22607e 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -5,139 +5,138 @@ compatible = "riscv,rv64i"; model = "Barcelona Supercomputing Center - Lagarto Ox"; chosen { - bootargs = "earlyprintk ignore_loglevel earlycon=sbi console=hvc0 root=/dev/pmem0p1 ro init=/bin/bash"; + bootargs = "earlyprintk ignore_loglevel earlycon=sbi console=hvc0 root=/dev/pmem0p1 ro init=/bin/bash"; }; cpus { - #address-cells = <0x00000001>; - #size-cells = <0x00000000>; - timebase-frequency = <0x0000c350>; - cpu@0 { - clock-frequency = <0x02FAF080>; - device_type = "cpu"; - reg = <0x00000000>; - status = "okay"; - compatible = "riscv"; - riscv,isa = "rv64imafd"; - mmu-type = "riscv,sv39"; - tlb-split; - L3: interrupt-controller { - #interrupt-cells = <0x00000001>; - interrupt-controller; - compatible = "riscv,cpu-intc"; - }; + #address-cells = <0x00000001>; + #size-cells = <0x00000000>; + timebase-frequency = <0x0000c350>; + cpu@0 { + clock-frequency = <0x02FAF080>; + device_type = "cpu"; + reg = <0x00000000>; + status = "okay"; + compatible = "riscv"; + riscv,isa = "rv64imafd"; + mmu-type = "riscv,sv39"; + tlb-split; + L3: interrupt-controller { + #interrupt-cells = <0x00000001>; + interrupt-controller; + compatible = "riscv,cpu-intc"; }; + }; }; memory@80000000 { - device_type = "memory"; - reg = <0x00000000 0x80000000 0x00000000 0x77e00000>; + device_type = "memory"; + reg = <0x00000000 0x80000000 0x00000000 0x77e00000>; }; reserved-memory { - #address-cells = <0x00000002>; - #size-cells = <0x00000002>; - ranges; - eth_pool: dma_pool@60000000 { - reg = <0x00000000 0x60000000 0x00000000 0x10000000>; - compatible = "shared-dma-pool"; - }; - onic_pool: dma_pool@70000000 { - reg = <0x00000000 0x70000000 0x00000000 0x10000000>; - compatible = "shared-dma-pool"; - }; + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + ranges; + eth_pool: dma_pool@60000000 { + reg = <0x00000000 0x60000000 0x00000000 0x10000000>; + compatible = "shared-dma-pool"; + }; + onic_pool: dma_pool@70000000 { + reg = <0x00000000 0x70000000 0x00000000 0x10000000>; + compatible = "shared-dma-pool"; + }; }; eth0_clk: eth0_clk { - compatible = "fixed-clock"; - #clock-cells = <0x00000000>; - clock-frequency = <0x09502f90>; + compatible = "fixed-clock"; + #clock-cells = <0x00000000>; + clock-frequency = <0x09502f90>; }; pmem@1bff00000 { - volatile; - compatible = "pmem-region"; - reg = <0x00000001 0xbff00000 0x00000000 0xc0100000>; + volatile; + compatible = "pmem-region"; + reg = <0x00000001 0xbff00000 0x00000000 0xc0100000>; }; pmem2@1b7f00000 { - #address-cells = <0x00000002>; - #size-cells = <0x00000002>; - volatile; - compatible = "pmem-region"; - reg = <0x00000001 0xb7f00000 0x00000000 0x08000000>; + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + volatile; + compatible = "pmem-region"; + reg = <0x00000001 0xb7f00000 0x00000000 0x08000000>; }; pmem3@f7e00000 { - volatile; - compatible = "pmem-region"; - reg = <0x00000000 0xf7e00000 0x00000000 0xc0100000>; + volatile; + compatible = "pmem-region"; + reg = <0x00000000 0xf7e00000 0x00000000 0xc0100000>; }; soc { - #address-cells = <0x00000002>; - #size-cells = <0x00000002>; - compatible = "BSC,Lagarto-bare-soc", "simple-bus"; - ranges; - serial@40001000 { - compatible = "ns16750"; - reg = <0x00000000 0x40001000 0x00000000 0x00001000>; - interrupts = <0x00000000>; - port-number = <0x00000000>; - reg-shift = <0x00000002>; - clock-frequency = <0x02FAF080>; - current-speed = <0x0001c200>; - status = "okay"; - }; - ethernet0 { - xlnx,rxmem = <0x000005f2>; - carv,mtu = <0x000005dc>; - carv,no-mac; - device_type = "network"; - local-mac-address = [02 05 00 01 00 05]; - axistream-connected = <0x000000fe>; - compatible = "xlnx,xxv-ethernet-1.0-carv"; - memory-region = <ð_pool>; - }; - dma@40400000 { - xlnx,include-dre; - phandle = <0x000000fe>; - #dma-cells = <0x00000001>; - compatible = "xlnx,axi-dma-1.00.a"; - clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk", "m_axi_s2mm_aclk", "m_axi_sg_aclk"; - clocks = <ð0_clk>, <ð0_clk>, <ð0_clk>, <ð0_clk>; - reg = <0x00000000 0x40400000 0x00000000 0x00400000>; - interrupt-names = "mm2s_introut", "s2mm_introut"; - interrupt-parent = <&PLIC0>; - interrupts = <0x00000002 0x00000003>; - xlnx,addrwidth = <0x00000028>; - xlnx,include-sg; - xlnx,sg-length-width = <0x00000017>; - dma-channel@40400000 { - compatible = "xlnx,axi-dma-mm2s-channel"; - dma-channels = <0x00000000>; - interrupts = <0x00000002>; - xlnx,datawidth = <0x00000040>; - xlnx,device-id = <0x00000000>; - xlnx,include-dre; - }; - dma-channel@40400030 { - compatible = "xlnx,axi-dma-s2mm-channel"; - dma-channels = <0x00000001>; - interrupts = <0x00000003>; - xlnx,datawidth = <0x00000040>; - xlnx,device-id = <0x00000000>; - xlnx,include-dre; - }; - }; - clint@40002000 { - reg-names = "control"; - interrupts-extended = <0x00000004 0x00000003 0x00000004 0x00000007 0x00000005 0x00000003 0x00000005 0x00000007 0x00000006 0x00000003 0x00000006 0x00000007 0x00000007 0x00000003 0x00000007 0x00000007>; - compatible = "riscv,clint0"; - #interrupt-cells = <0x00000001>; - reg = <0x00000000 0x40002000 0x00000000 0x000c0000>; + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + compatible = "BSC,Lagarto-ox-soc", "simple-bus"; + ranges; + serial@40001000 { + compatible = "ns16750"; + reg = <0x00000000 0x40001000 0x00000000 0x00001000>; + interrupts = <0x00000000>; + port-number = <0x00000000>; + reg-shift = <0x00000002>; + clock-frequency = <0x02FAF080>; + current-speed = <0x0001c200>; + status = "okay"; + }; + ethernet0 { + xlnx,rxmem = <0x000005f2>; + carv,mtu = <0x000005dc>; + carv,no-mac; + device_type = "network"; + local-mac-address = [02 05 00 01 00 05]; + axistream-connected = <0x000000fe>; + compatible = "xlnx,xxv-ethernet-1.0-carv"; + memory-region = <ð_pool>; + }; + dma@40400000 { + xlnx,include-dre; + phandle = <0x000000fe>; + #dma-cells = <0x00000001>; + compatible = "xlnx,axi-dma-1.00.a"; + clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk", "m_axi_s2mm_aclk", "m_axi_sg_aclk"; + clocks = <ð0_clk>, <ð0_clk>, <ð0_clk>, <ð0_clk>; + reg = <0x00000000 0x40400000 0x00000000 0x00400000>; + interrupt-names = "mm2s_introut", "s2mm_introut"; + interrupt-parent = <&PLIC0>; + interrupts = <0x00000002 0x00000003>; + xlnx,addrwidth = <0x00000028>; + xlnx,include-sg; + xlnx,sg-length-width = <0x00000017>; + dma-channel@40400000 { + compatible = "xlnx,axi-dma-mm2s-channel"; + dma-channels = <0x00000000>; + interrupts = <0x00000002>; + xlnx,datawidth = <0x00000040>; + xlnx,device-id = <0x00000000>; + xlnx,include-dre; }; - PLIC0: plic@40800000 { - #interrupt-cells = <1>; - compatible = "riscv,plic0"; - interrupt-controller; - interrupts-extended = <&L3 3 &L3 7>; - reg = < 0x00000000 0x40800000 0x00000000 0x00400000>; - riscv,ndev = <0x00000003>; - riscv,max-priority = <0x00000007>; + dma-channel@40400030 { + compatible = "xlnx,axi-dma-s2mm-channel"; + dma-channels = <0x00000001>; + interrupts = <0x00000003>; + xlnx,datawidth = <0x00000040>; + xlnx,device-id = <0x00000000>; + xlnx,include-dre; }; + }; + clint@40002000 { + reg-names = "control"; + interrupts-extended = <0x00000004 0x00000003 0x00000004 0x00000007 0x00000005 0x00000003 0x00000005 0x00000007 0x00000006 0x00000003 0x00000006 0x00000007 0x00000007 0x00000003 0x00000007 0x00000007>; + compatible = "riscv,clint0"; + #interrupt-cells = <0x00000001>; + reg = <0x00000000 0x40002000 0x00000000 0x000c0000>; + }; + PLIC0: plic@40800000 { + #interrupt-cells = <1>; + compatible = "riscv,plic0"; + interrupt-controller; + interrupts-extended = <&L3 3 &L3 7>; + reg = < 0x00000000 0x40800000 0x00000000 0x00400000>; + riscv,ndev = <0x00000003>; + riscv,max-priority = <0x00000007>; + }; }; - }; diff --git a/ox.dts b/ox.dts index 8fd11b8..f9e2e94 100644 --- a/ox.dts +++ b/ox.dts @@ -35,7 +35,7 @@ uart0: serial@40001000 { compatible = "ns16750"; //reg = <0x40001000 0x1000>; - reg = <0x00000000 0x40001000 0x00000000 0x00000100>; + reg = <0x00000000 0x40001000 0x00000000 0x00001000>; interrupts = <0>; port-number = <0>; reg-shift = <2>; diff --git a/u-boot-debug.patch b/u-boot-debug.patch new file mode 100644 index 0000000..2b0f46f --- /dev/null +++ b/u-boot-debug.patch @@ -0,0 +1,83 @@ +diff --git a/common/board_f.c b/common/board_f.c +index 1688e27071..216839febb 100644 +--- a/common/board_f.c ++++ b/common/board_f.c +@@ -978,6 +978,8 @@ static const init_fnc_t init_sequence_f[] = { + + void board_init_f(ulong boot_flags) + { ++ puts("board_init_f() called\n"); ++ + gd->flags = boot_flags; + gd->have_console = 0; + +@@ -990,6 +992,7 @@ void board_init_f(ulong boot_flags) + /* NOTREACHED - jump_to_copy() does not return */ + hang(); + #endif ++ puts("board_init_f() ends ok\n"); + } + + #if defined(CONFIG_X86) || defined(CONFIG_ARC) +diff --git a/common/board_r.c b/common/board_r.c +index d798c00a80..cb8119a603 100644 +--- a/common/board_r.c ++++ b/common/board_r.c +@@ -786,6 +786,8 @@ static init_fnc_t init_sequence_r[] = { + + void board_init_r(gd_t *new_gd, ulong dest_addr) + { ++ puts("board_init_r called\n"); ++ + /* + * The pre-relocation drivers may be using memory that has now gone + * away. Mark serial as unavailable - this will fall back to the debug +diff --git a/drivers/cpu/riscv_cpu.c b/drivers/cpu/riscv_cpu.c +index d6484d7f4b..64a507248d 100644 +--- a/drivers/cpu/riscv_cpu.c ++++ b/drivers/cpu/riscv_cpu.c +@@ -92,10 +92,13 @@ static int riscv_cpu_get_count(const struct udevice *dev) + + static int riscv_cpu_bind(struct udevice *dev) + { ++ puts("riscv_cpu_bind called()\n"); ++ + struct cpu_plat *plat = dev_get_parent_plat(dev); + struct driver *drv; + int ret; + ++ puts("looking for timebase-frequency\n"); + /* save the hart id */ + plat->cpu_id = dev_read_addr(dev); + /* first examine the property in current cpu node */ +@@ -105,6 +108,8 @@ static int riscv_cpu_bind(struct udevice *dev) + dev_read_u32(dev->parent, "timebase-frequency", + &plat->timebase_freq); + ++ printf("timebase-frequency=%lu\n", (unsigned long) plat->timebase_freq); ++ + /* + * Bind riscv-timer driver on boot hart. + * +@@ -125,6 +130,8 @@ static int riscv_cpu_bind(struct udevice *dev) + device_bind_with_driver_data(dev, drv, "riscv_timer", + plat->timebase_freq, ofnode_null(), + NULL); ++ } else { ++ printf("ignoring cpu_id=%d\n", plat->cpu_id); + } + + return 0; +diff --git a/lib/hang.c b/lib/hang.c +index 2735774f9a..84eff21ffc 100644 +--- a/lib/hang.c ++++ b/lib/hang.c +@@ -22,6 +22,8 @@ + */ + void hang(void) + { ++ puts("oh no, we are in hang()\n"); ++ + #if !defined(CONFIG_SPL_BUILD) || \ + (CONFIG_IS_ENABLED(LIBCOMMON_SUPPORT) && \ + CONFIG_IS_ENABLED(SERIAL)) diff --git a/xavi-good-ftd.dts b/xavi-good-ftd.dts new file mode 100644 index 0000000..885ec8c --- /dev/null +++ b/xavi-good-ftd.dts @@ -0,0 +1,278 @@ +LOx$ +OpenSBI v1.2 + ____ _____ ____ _____ + / __ \ / ____| _ \_ _| + | | | |_ __ ___ _ __ | (___ | |_) || | + | | | | '_ \ / _ \ '_ \ \___ \| _ < | | + | |__| | |_) | __/ | | |____) | |_) || |_ + \____/| .__/ \___|_| |_|_____/|____/_____| + | | + |_| + +Platform Name : Sargantana (for Xilinx Alveo FPGA) +Platform Features : medeleg +Platform HART Count : 1 +Platform IPI Device : --- +Platform Timer Device : generic_timer @ 25000000Hz +Platform Console Device : alveo_uart +Platform HSM Device : --- +Platform PMU Device : --- +Platform Reboot Device : --- +Platform Shutdown Device : --- +Firmware Base : 0x80000000 +Firmware Size : 148 KB +Runtime SBI Version : 1.0 + +Domain0 Name : root +Domain0 Boot HART : 0 +Domain0 HARTs : 0* +Domain0 Region00 : 0x0000000080000000-0x000000008003ffff () +Domain0 Region01 : 0x0000000000000000-0xffffffffffffffff (R,W,X) +Domain0 Next Address : 0x0000000080200000 +Domain0 Next Arg1 : 0x0000000080010000 +Domain0 Next Mode : S-mode +Domain0 SysReset : yes + +Boot HART ID : 0 +Boot HART Domain : root +Boot HART Priv Version : v1.10 +Boot HART Base ISA : rv64imafdc +Boot HART ISA Extensions : time +Boot HART PMP Count : 0 +Boot HART PMP Granularity : 0 +Boot HART PMP Address Bits: 0 +Boot HART MHPM Count : 29 +Boot HART MIDELEG : 0x0000000000000222 +Boot HART MEDELEG : 0x000000000000b109 + + +Core: 9 devices, 8 uclasses, devicetree: separate +MMC: +Loading Environment from ... OK +In: serial@40001000 +Out: serial@40001000 +Err: serial@40001000 +Model: Barcelona Supercomputing Center - Lagarto Ox +Hit any key to stop autoboot: 0 +LOx$ fdt +fdt - flattened device tree utility commands + +Usage: +fdt addr [-c] [-q] [] - Set the [control] fdt location to +fdt apply - Apply overlay to the DT +fdt move - Copy the fdt to and make it active +fdt resize [] - Resize fdt to size + padding to 4k addr + some optional if needed +fdt print [] - Recursive print starting at +fdt list [] - Print one level starting at +fdt get value [] - Get and store in + In case of stringlist property, use optional + to select string within the stringlist. Default is 0. +fdt get name - Get name of node and store in +fdt get addr - Get start address of and store in +fdt get size [] - Get size of [] or num nodes and store in +fdt set [] - Set [to ] +fdt mknode - Create a new node after +fdt rm [] - Delete the node or +fdt header [get ] - Display header info + get - get header member and store it in +fdt bootcpu - Set boot cpuid +fdt memory - Add/Update memory node +fdt rsvmem print - Show current mem reserves +fdt rsvmem add - Add a mem reserve +fdt rsvmem delete - Delete a mem reserves +fdt chosen [ ] - Add/update the /chosen branch in the tree + / - initrd start addr/size +NOTE: Dereference aliases by omitting the leading '/', e.g. fdt print ethernet0. +LOx$ fdt print +No FDT memory address configured. Please configure +the FDT address via "fdt addr
" command. +Aborting! +LOx$ fdt list +No FDT memory address configured. Please configure +the FDT address via "fdt addr
" command. +Aborting! +LOx$ printenv +arch=riscv +baudrate=115200 +board=riscv64 +board_name=riscv64 +bootcmd=fdt addr ${fdtcontroladdr}; fdt move ${fdtcontroladdr} ${fdt_addr_r}; blkmap create pmem2; blkmap map pmem2 0 0x40000 mem 0x1b7f00000; blkmap info; ls blkmap 0; load blkmap 0 ${kernel_addr_r} Image; setenv bootargs ignore_loglevel earlycon=sbi console=hvc0 root=/dev/pmem0p1 ro init=/bin/bash; booti ${kernel_addr_r} - ${fdt_addr_r}; +bootdelay=10 +cpu=generic +fdt_addr_r=0x86000000 +fdtcontroladdr=e7d87e60 +image=boot/Image +kernel_addr_r=0x80200000 +loadaddr=0x87000000 +mmcdev=0 +mmcpart=1 +stderr=serial@40001000 +stdin=serial@40001000 +stdout=serial@40001000 +vendor=openpiton + +Environment size: 640/126972 bytes +LOx$ fdt addr ${fdt_addr_r} +libfdt fdt_check_header(): FDT_ERR_BADMAGIC +LOx$ fdt addr ${fdtcontroladdr} +Working FDT set to e7d87e60 +LOx$ fdt print +/ { + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + compatible = "riscv,rv64i"; + model = "Barcelona Supercomputing Center - Lagarto Ox"; + chosen { + bootargs = "earlyprintk ignore_loglevel earlycon=sbi console=hvc0 root=/dev/pmem0p1 ro init=/bin/bash"; + }; + cpus { + #address-cells = <0x00000001>; + #size-cells = <0x00000000>; + timebase-frequency = <0x0000c350>; + cpu@0 { + clock-frequency = <0x02faf080>; + device_type = "cpu"; + reg = <0x00000000>; + status = "okay"; + compatible = "riscv"; + riscv,isa = "rv64imafd"; + mmu-type = "riscv,sv39"; + tlb-split; + phandle = <0x00000004>; + interrupt-controller { + #interrupt-cells = <0x00000001>; + interrupt-controller; + compatible = "riscv,cpu-intc"; + phandle = <0x00000005>; + }; + }; + }; + memory@80000000 { + device_type = "memory"; + reg = <0x00000000 0x80000000 0x00000000 0x77e00000>; + }; + reserved-memory { + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + ranges; + dma_pool@60000000 { + reg = <0x00000000 0x60000000 0x00000000 0x10000000>; + compatible = "shared-dma-pool"; + phandle = <0x00000001>; + }; + dma_pool@70000000 { + reg = <0x00000000 0x70000000 0x00000000 0x10000000>; + compatible = "shared-dma-pool"; + phandle = <0x00000006>; + }; + }; + eth0_clk { + compatible = "fixed-clock"; + #clock-cells = <0x00000000>; + clock-frequency = <0x09502f90>; + phandle = <0x00000002>; + }; + pmem@1bff00000 { + volatile; + compatible = "pmem-region"; + reg = <0x00000001 0xbff00000 0x00000000 0xc0100000>; + }; + pmem2@1b7f00000 { + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + volatile; + compatible = "pmem-region"; + reg = <0x00000001 0xb7f00000 0x00000000 0x08000000>; + }; + pmem3@f7e00000 { + volatile; + compatible = "pmem-region"; + reg = <0x00000000 0xf7e00000 0x00000000 0xc0100000>; + }; + soc { + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + compatible = "BSC,Lagarto-bare-soc", "simple-bus"; + ranges; + serial@40001000 { + compatible = "ns16750"; + reg = <0x00000000 0x40001000 0x00000000 0x00000100>; + interrupts = <0x00000000>; + port-number = <0x00000000>; + reg-shift = <0x00000002>; + clock-frequency = <0x017d7840>; + current-speed = <0x0001c200>; + status = "okay"; + phandle = <0x00000007>; + }; + ethernet0 { + xlnx,rxmem = <0x000005f2>; + carv,mtu = <0x000005dc>; + carv,no-mac; + device_type = "network"; + local-mac-address = [02 05 00 01 00 05]; + axistream-connected = <0x000000fe>; + compatible = "xlnx,xxv-ethernet-1.0-carv"; + memory-region = <0x00000001>; + }; + dma@40020000 { + xlnx,include-dre; + phandle = <0x000000fe>; + #dma-cells = <0x00000001>; + compatible = "xlnx,axi-dma-1.00.a"; + clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk", "m_axi_s2mm_aclk", "m_axi_sg_aclk"; + clocks = <0x00000002 0x00000002 0x00000002 0x00000002>; + reg = <0x00000000 0x40200000 0x00000000 0x00400000>; + interrupt-names = "mm2s_introut", "s2mm_introut"; + interrupt-parent = <0x00000003>; + interrupts = <0x00000002 0x00000003>; + xlnx,addrwidth = <0x00000028>; + xlnx,include-sg; + xlnx,sg-length-width = <0x00000017>; + dma-channel@40020000 { + compatible = "xlnx,axi-dma-mm2s-channel"; + dma-channels = <0x00000001>; + interrupts = <0x00000002>; + xlnx,datawidth = <0x00000040>; + xlnx,device-id = <0x00000000>; + xlnx,include-dre; + }; + dma-channel@40020030 { + compatible = "xlnx,axi-dma-s2mm-channel"; + dma-channels = <0x00000001>; + interrupts = <0x00000003>; + xlnx,datawidth = <0x00000040>; + xlnx,device-id = <0x00000000>; + xlnx,include-dre; + }; + }; + clint@40002000 { + reg-names = "control"; + interrupts-extended = <0x00000004 0x00000003 0x00000004 0x00000007 0x00000005 0x00000003 0x00000005 0x00000007 0x00000006 0x00000003 0x00000006 0x00000007 0x00000007 0x00000003 0x00000007 0x00000007>; + compatible = "riscv,clint0"; + #interrupt-cells = <0x00000001>; + reg = <0x00000000 0x40002000 0x00000000 0x000c0000>; + }; + plic@fff1100000 { + #address-cells = <0x00000000>; + compatible = "riscv,plic0"; + #interrupt-cells = <0x00000001>; + reg = <0x000000ff 0xf1100000 0x00000000 0x04000000>; + riscv,ndev = <0x00000003>; + riscv,max-priority = <0x00000007>; + interrupt-controller; + phandle = <0x00000003>; + }; + }; + __symbols__ { + CPU0 = "/cpus/cpu@0"; + CPU0_intc = "/cpus/cpu@0/interrupt-controller"; + eth_pool = "/reserved-memory/dma_pool@60000000"; + meep_pool = "/reserved-memory/dma_pool@70000000"; + eth0_clk = "/eth0_clk"; + uart0 = "/soc/serial@40001000"; + dma_eth = "/soc/dma@40020000"; + PLIC0 = "/soc/plic@fff1100000"; + }; +}; +LOx$ -- GitLab From a8f323403117dbd05b89f5383084a41ab1d91836 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 27 Jun 2024 17:58:21 +0200 Subject: [PATCH 009/310] Reached working U-Boot prompt The serial compatible string must be "ns16550" to be recognized by U-Boot. --- lagarto-ox.nix | 12 +- ox-plic.dts | 33 ++++-- xavi-good-ftd.dts | 278 ---------------------------------------------- xavi.dts | 159 ++++++++++++++++++++++++++ 4 files changed, 189 insertions(+), 293 deletions(-) delete mode 100644 xavi-good-ftd.dts create mode 100644 xavi.dts diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 36e04e3..92355ec 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -87,17 +87,22 @@ nixpkgs.overlays = [ (final: prev: { ox-dtb = prev.runCommand "ox.dtb" { + dtsFile = ./ox-plic.dts; + #dtsFile = ./xavi.dts; nativeBuildInputs = [ prev.buildPackages.dtc ]; - } "dtc -O dtb -o $out ${./ox-plic.dts}"; + } "dtc -O dtb -o $out $dtsFile"; uboot = prev.ubootQemuRiscv64Smode.override { filesToInstall = [ "u-boot.bin" "u-boot-nodtb.bin" ]; version = "2023.07.02-print-cpu-probe"; src = builtins.fetchGit { url = "file:///home/Computational/rarias/riscv/u-boot"; - rev = "b0868e84c392daadcaf611f92209d0c82203386f"; + rev = "6fb984f14607c53878ba0ed82425d77e396b727b"; }; #patches = [ ./u-boot-debug.patch ]; + # + # CONFIG_SERIAL_PRESENT=n + # CONFIG_SYS_NS16550=n extraConfig = '' CONFIG_RISCV_ISA_C=n CONFIG_REQUIRE_SERIAL_CONSOLE=n @@ -117,7 +122,6 @@ CONFIG_DEBUG_UART=y CONFIG_DEBUG_UART_ANNOUNCE=y CONFIG_DEBUG_SBI_CONSOLE=y - CONFIG_SYS_NS16550=n CONFIG_SMP=n CONFIG_TRACE_EARLY=y CONFIG_LOG=y @@ -183,7 +187,7 @@ #"PLATFORM_RISCV_ISA=rv64g" # No compressed instructions #"PLATFORM_RISCV_ABI=lp64d" "FW_PAYLOAD_PATH=${final.uboot}/u-boot.bin" - "FW_FDT_PATH=ox.dtb" + "FW_FDT_PATH=${final.ox-dtb}" ]; patches = [ ./ox-alveo-platform-plic.patch ]; }); diff --git a/ox-plic.dts b/ox-plic.dts index f22607e..3611dc4 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -3,7 +3,7 @@ #address-cells = <0x00000002>; #size-cells = <0x00000002>; compatible = "riscv,rv64i"; - model = "Barcelona Supercomputing Center - Lagarto Ox"; + model = "Barcelona Supercomputing Center - Lagarto Ox (NixOS)"; chosen { bootargs = "earlyprintk ignore_loglevel earlycon=sbi console=hvc0 root=/dev/pmem0p1 ro init=/bin/bash"; }; @@ -11,7 +11,7 @@ #address-cells = <0x00000001>; #size-cells = <0x00000000>; timebase-frequency = <0x0000c350>; - cpu@0 { + CPU0: cpu@0 { clock-frequency = <0x02FAF080>; device_type = "cpu"; reg = <0x00000000>; @@ -20,10 +20,12 @@ riscv,isa = "rv64imafd"; mmu-type = "riscv,sv39"; tlb-split; + phandle = <0x00000004>; L3: interrupt-controller { #interrupt-cells = <0x00000001>; interrupt-controller; compatible = "riscv,cpu-intc"; + phandle = <0x00000005>; }; }; }; @@ -38,16 +40,19 @@ eth_pool: dma_pool@60000000 { reg = <0x00000000 0x60000000 0x00000000 0x10000000>; compatible = "shared-dma-pool"; + phandle = <0x00000001>; }; onic_pool: dma_pool@70000000 { reg = <0x00000000 0x70000000 0x00000000 0x10000000>; compatible = "shared-dma-pool"; + phandle = <0x00000006>; }; }; eth0_clk: eth0_clk { compatible = "fixed-clock"; #clock-cells = <0x00000000>; clock-frequency = <0x09502f90>; + phandle = <0x00000002>; }; pmem@1bff00000 { volatile; @@ -71,15 +76,16 @@ #size-cells = <0x00000002>; compatible = "BSC,Lagarto-ox-soc", "simple-bus"; ranges; - serial@40001000 { - compatible = "ns16750"; - reg = <0x00000000 0x40001000 0x00000000 0x00001000>; - interrupts = <0x00000000>; - port-number = <0x00000000>; - reg-shift = <0x00000002>; - clock-frequency = <0x02FAF080>; - current-speed = <0x0001c200>; + SERIAL: serial@40001000 { + compatible = "ns16550"; + reg = <0x0 0x40001000 0x0 0x00000100>; + interrupts = <0>; + /*port-number = <0>;*/ + reg-shift = <2>; + clock-frequency = <50000000>; + current-speed = <115200>; status = "okay"; + phandle = <0x00000007>; }; ethernet0 { xlnx,rxmem = <0x000005f2>; @@ -124,7 +130,11 @@ }; clint@40002000 { reg-names = "control"; - interrupts-extended = <0x00000004 0x00000003 0x00000004 0x00000007 0x00000005 0x00000003 0x00000005 0x00000007 0x00000006 0x00000003 0x00000006 0x00000007 0x00000007 0x00000003 0x00000007 0x00000007>; + interrupts-extended = + <&CPU0 0x3>, <&CPU0 0x7>, + <&L3 0x3>, <&L3 0x7>, + <&onic_pool 0x3>, <&onic_pool 0x7>, + <&SERIAL 0x3>, <&SERIAL 0x7>; compatible = "riscv,clint0"; #interrupt-cells = <0x00000001>; reg = <0x00000000 0x40002000 0x00000000 0x000c0000>; @@ -137,6 +147,7 @@ reg = < 0x00000000 0x40800000 0x00000000 0x00400000>; riscv,ndev = <0x00000003>; riscv,max-priority = <0x00000007>; + phandle = <0x00000003>; }; }; }; diff --git a/xavi-good-ftd.dts b/xavi-good-ftd.dts deleted file mode 100644 index 885ec8c..0000000 --- a/xavi-good-ftd.dts +++ /dev/null @@ -1,278 +0,0 @@ -LOx$ -OpenSBI v1.2 - ____ _____ ____ _____ - / __ \ / ____| _ \_ _| - | | | |_ __ ___ _ __ | (___ | |_) || | - | | | | '_ \ / _ \ '_ \ \___ \| _ < | | - | |__| | |_) | __/ | | |____) | |_) || |_ - \____/| .__/ \___|_| |_|_____/|____/_____| - | | - |_| - -Platform Name : Sargantana (for Xilinx Alveo FPGA) -Platform Features : medeleg -Platform HART Count : 1 -Platform IPI Device : --- -Platform Timer Device : generic_timer @ 25000000Hz -Platform Console Device : alveo_uart -Platform HSM Device : --- -Platform PMU Device : --- -Platform Reboot Device : --- -Platform Shutdown Device : --- -Firmware Base : 0x80000000 -Firmware Size : 148 KB -Runtime SBI Version : 1.0 - -Domain0 Name : root -Domain0 Boot HART : 0 -Domain0 HARTs : 0* -Domain0 Region00 : 0x0000000080000000-0x000000008003ffff () -Domain0 Region01 : 0x0000000000000000-0xffffffffffffffff (R,W,X) -Domain0 Next Address : 0x0000000080200000 -Domain0 Next Arg1 : 0x0000000080010000 -Domain0 Next Mode : S-mode -Domain0 SysReset : yes - -Boot HART ID : 0 -Boot HART Domain : root -Boot HART Priv Version : v1.10 -Boot HART Base ISA : rv64imafdc -Boot HART ISA Extensions : time -Boot HART PMP Count : 0 -Boot HART PMP Granularity : 0 -Boot HART PMP Address Bits: 0 -Boot HART MHPM Count : 29 -Boot HART MIDELEG : 0x0000000000000222 -Boot HART MEDELEG : 0x000000000000b109 - - -Core: 9 devices, 8 uclasses, devicetree: separate -MMC: -Loading Environment from ... OK -In: serial@40001000 -Out: serial@40001000 -Err: serial@40001000 -Model: Barcelona Supercomputing Center - Lagarto Ox -Hit any key to stop autoboot: 0 -LOx$ fdt -fdt - flattened device tree utility commands - -Usage: -fdt addr [-c] [-q] [] - Set the [control] fdt location to -fdt apply - Apply overlay to the DT -fdt move - Copy the fdt to and make it active -fdt resize [] - Resize fdt to size + padding to 4k addr + some optional if needed -fdt print [] - Recursive print starting at -fdt list [] - Print one level starting at -fdt get value [] - Get and store in - In case of stringlist property, use optional - to select string within the stringlist. Default is 0. -fdt get name - Get name of node and store in -fdt get addr - Get start address of and store in -fdt get size [] - Get size of [] or num nodes and store in -fdt set [] - Set [to ] -fdt mknode - Create a new node after -fdt rm [] - Delete the node or -fdt header [get ] - Display header info - get - get header member and store it in -fdt bootcpu - Set boot cpuid -fdt memory - Add/Update memory node -fdt rsvmem print - Show current mem reserves -fdt rsvmem add - Add a mem reserve -fdt rsvmem delete - Delete a mem reserves -fdt chosen [ ] - Add/update the /chosen branch in the tree - / - initrd start addr/size -NOTE: Dereference aliases by omitting the leading '/', e.g. fdt print ethernet0. -LOx$ fdt print -No FDT memory address configured. Please configure -the FDT address via "fdt addr
" command. -Aborting! -LOx$ fdt list -No FDT memory address configured. Please configure -the FDT address via "fdt addr
" command. -Aborting! -LOx$ printenv -arch=riscv -baudrate=115200 -board=riscv64 -board_name=riscv64 -bootcmd=fdt addr ${fdtcontroladdr}; fdt move ${fdtcontroladdr} ${fdt_addr_r}; blkmap create pmem2; blkmap map pmem2 0 0x40000 mem 0x1b7f00000; blkmap info; ls blkmap 0; load blkmap 0 ${kernel_addr_r} Image; setenv bootargs ignore_loglevel earlycon=sbi console=hvc0 root=/dev/pmem0p1 ro init=/bin/bash; booti ${kernel_addr_r} - ${fdt_addr_r}; -bootdelay=10 -cpu=generic -fdt_addr_r=0x86000000 -fdtcontroladdr=e7d87e60 -image=boot/Image -kernel_addr_r=0x80200000 -loadaddr=0x87000000 -mmcdev=0 -mmcpart=1 -stderr=serial@40001000 -stdin=serial@40001000 -stdout=serial@40001000 -vendor=openpiton - -Environment size: 640/126972 bytes -LOx$ fdt addr ${fdt_addr_r} -libfdt fdt_check_header(): FDT_ERR_BADMAGIC -LOx$ fdt addr ${fdtcontroladdr} -Working FDT set to e7d87e60 -LOx$ fdt print -/ { - #address-cells = <0x00000002>; - #size-cells = <0x00000002>; - compatible = "riscv,rv64i"; - model = "Barcelona Supercomputing Center - Lagarto Ox"; - chosen { - bootargs = "earlyprintk ignore_loglevel earlycon=sbi console=hvc0 root=/dev/pmem0p1 ro init=/bin/bash"; - }; - cpus { - #address-cells = <0x00000001>; - #size-cells = <0x00000000>; - timebase-frequency = <0x0000c350>; - cpu@0 { - clock-frequency = <0x02faf080>; - device_type = "cpu"; - reg = <0x00000000>; - status = "okay"; - compatible = "riscv"; - riscv,isa = "rv64imafd"; - mmu-type = "riscv,sv39"; - tlb-split; - phandle = <0x00000004>; - interrupt-controller { - #interrupt-cells = <0x00000001>; - interrupt-controller; - compatible = "riscv,cpu-intc"; - phandle = <0x00000005>; - }; - }; - }; - memory@80000000 { - device_type = "memory"; - reg = <0x00000000 0x80000000 0x00000000 0x77e00000>; - }; - reserved-memory { - #address-cells = <0x00000002>; - #size-cells = <0x00000002>; - ranges; - dma_pool@60000000 { - reg = <0x00000000 0x60000000 0x00000000 0x10000000>; - compatible = "shared-dma-pool"; - phandle = <0x00000001>; - }; - dma_pool@70000000 { - reg = <0x00000000 0x70000000 0x00000000 0x10000000>; - compatible = "shared-dma-pool"; - phandle = <0x00000006>; - }; - }; - eth0_clk { - compatible = "fixed-clock"; - #clock-cells = <0x00000000>; - clock-frequency = <0x09502f90>; - phandle = <0x00000002>; - }; - pmem@1bff00000 { - volatile; - compatible = "pmem-region"; - reg = <0x00000001 0xbff00000 0x00000000 0xc0100000>; - }; - pmem2@1b7f00000 { - #address-cells = <0x00000002>; - #size-cells = <0x00000002>; - volatile; - compatible = "pmem-region"; - reg = <0x00000001 0xb7f00000 0x00000000 0x08000000>; - }; - pmem3@f7e00000 { - volatile; - compatible = "pmem-region"; - reg = <0x00000000 0xf7e00000 0x00000000 0xc0100000>; - }; - soc { - #address-cells = <0x00000002>; - #size-cells = <0x00000002>; - compatible = "BSC,Lagarto-bare-soc", "simple-bus"; - ranges; - serial@40001000 { - compatible = "ns16750"; - reg = <0x00000000 0x40001000 0x00000000 0x00000100>; - interrupts = <0x00000000>; - port-number = <0x00000000>; - reg-shift = <0x00000002>; - clock-frequency = <0x017d7840>; - current-speed = <0x0001c200>; - status = "okay"; - phandle = <0x00000007>; - }; - ethernet0 { - xlnx,rxmem = <0x000005f2>; - carv,mtu = <0x000005dc>; - carv,no-mac; - device_type = "network"; - local-mac-address = [02 05 00 01 00 05]; - axistream-connected = <0x000000fe>; - compatible = "xlnx,xxv-ethernet-1.0-carv"; - memory-region = <0x00000001>; - }; - dma@40020000 { - xlnx,include-dre; - phandle = <0x000000fe>; - #dma-cells = <0x00000001>; - compatible = "xlnx,axi-dma-1.00.a"; - clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk", "m_axi_s2mm_aclk", "m_axi_sg_aclk"; - clocks = <0x00000002 0x00000002 0x00000002 0x00000002>; - reg = <0x00000000 0x40200000 0x00000000 0x00400000>; - interrupt-names = "mm2s_introut", "s2mm_introut"; - interrupt-parent = <0x00000003>; - interrupts = <0x00000002 0x00000003>; - xlnx,addrwidth = <0x00000028>; - xlnx,include-sg; - xlnx,sg-length-width = <0x00000017>; - dma-channel@40020000 { - compatible = "xlnx,axi-dma-mm2s-channel"; - dma-channels = <0x00000001>; - interrupts = <0x00000002>; - xlnx,datawidth = <0x00000040>; - xlnx,device-id = <0x00000000>; - xlnx,include-dre; - }; - dma-channel@40020030 { - compatible = "xlnx,axi-dma-s2mm-channel"; - dma-channels = <0x00000001>; - interrupts = <0x00000003>; - xlnx,datawidth = <0x00000040>; - xlnx,device-id = <0x00000000>; - xlnx,include-dre; - }; - }; - clint@40002000 { - reg-names = "control"; - interrupts-extended = <0x00000004 0x00000003 0x00000004 0x00000007 0x00000005 0x00000003 0x00000005 0x00000007 0x00000006 0x00000003 0x00000006 0x00000007 0x00000007 0x00000003 0x00000007 0x00000007>; - compatible = "riscv,clint0"; - #interrupt-cells = <0x00000001>; - reg = <0x00000000 0x40002000 0x00000000 0x000c0000>; - }; - plic@fff1100000 { - #address-cells = <0x00000000>; - compatible = "riscv,plic0"; - #interrupt-cells = <0x00000001>; - reg = <0x000000ff 0xf1100000 0x00000000 0x04000000>; - riscv,ndev = <0x00000003>; - riscv,max-priority = <0x00000007>; - interrupt-controller; - phandle = <0x00000003>; - }; - }; - __symbols__ { - CPU0 = "/cpus/cpu@0"; - CPU0_intc = "/cpus/cpu@0/interrupt-controller"; - eth_pool = "/reserved-memory/dma_pool@60000000"; - meep_pool = "/reserved-memory/dma_pool@70000000"; - eth0_clk = "/eth0_clk"; - uart0 = "/soc/serial@40001000"; - dma_eth = "/soc/dma@40020000"; - PLIC0 = "/soc/plic@fff1100000"; - }; -}; -LOx$ diff --git a/xavi.dts b/xavi.dts new file mode 100644 index 0000000..b7a7936 --- /dev/null +++ b/xavi.dts @@ -0,0 +1,159 @@ +/dts-v1/; +/ { + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + compatible = "riscv,rv64i"; + model = "Barcelona Supercomputing Center - Lagarto Ox"; + chosen { + bootargs = "earlyprintk ignore_loglevel earlycon=sbi console=hvc0 root=/dev/pmem0p1 ro init=/bin/bash"; + }; + cpus { + #address-cells = <0x00000001>; + #size-cells = <0x00000000>; + timebase-frequency = <0x0000c350>; + cpu@0 { + clock-frequency = <0x02faf080>; + device_type = "cpu"; + reg = <0x00000000>; + status = "okay"; + compatible = "riscv"; + riscv,isa = "rv64imafd"; + mmu-type = "riscv,sv39"; + tlb-split; + phandle = <0x00000004>; + interrupt-controller { + #interrupt-cells = <0x00000001>; + interrupt-controller; + compatible = "riscv,cpu-intc"; + phandle = <0x00000005>; + }; + }; + }; + memory@80000000 { + device_type = "memory"; + reg = <0x00000000 0x80000000 0x00000000 0x77e00000>; + }; + reserved-memory { + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + ranges; + dma_pool@60000000 { + reg = <0x00000000 0x60000000 0x00000000 0x10000000>; + compatible = "shared-dma-pool"; + phandle = <0x00000001>; + }; + dma_pool@70000000 { + reg = <0x00000000 0x70000000 0x00000000 0x10000000>; + compatible = "shared-dma-pool"; + phandle = <0x00000006>; + }; + }; + eth0_clk { + compatible = "fixed-clock"; + #clock-cells = <0x00000000>; + clock-frequency = <0x09502f90>; + phandle = <0x00000002>; + }; + pmem@1bff00000 { + volatile; + compatible = "pmem-region"; + reg = <0x00000001 0xbff00000 0x00000000 0xc0100000>; + }; + pmem2@1b7f00000 { + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + volatile; + compatible = "pmem-region"; + reg = <0x00000001 0xb7f00000 0x00000000 0x08000000>; + }; + pmem3@f7e00000 { + volatile; + compatible = "pmem-region"; + reg = <0x00000000 0xf7e00000 0x00000000 0xc0100000>; + }; + soc { + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + compatible = "BSC,Lagarto-bare-soc", "simple-bus"; + ranges; + serial@40001000 { + compatible = "ns16750"; + reg = <0x00000000 0x40001000 0x00000000 0x00000100>; + interrupts = <0x00000000>; + port-number = <0x00000000>; + reg-shift = <0x00000002>; + clock-frequency = <0x017d7840>; + current-speed = <0x0001c200>; + status = "okay"; + phandle = <0x00000007>; + }; + ethernet0 { + xlnx,rxmem = <0x000005f2>; + carv,mtu = <0x000005dc>; + carv,no-mac; + device_type = "network"; + local-mac-address = [02 05 00 01 00 05]; + axistream-connected = <0x000000fe>; + compatible = "xlnx,xxv-ethernet-1.0-carv"; + memory-region = <0x00000001>; + }; + dma@40020000 { + xlnx,include-dre; + phandle = <0x000000fe>; + #dma-cells = <0x00000001>; + compatible = "xlnx,axi-dma-1.00.a"; + clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk", "m_axi_s2mm_aclk", "m_axi_sg_aclk"; + clocks = <0x00000002 0x00000002 0x00000002 0x00000002>; + reg = <0x00000000 0x40200000 0x00000000 0x00400000>; + interrupt-names = "mm2s_introut", "s2mm_introut"; + interrupt-parent = <0x00000003>; + interrupts = <0x00000002 0x00000003>; + xlnx,addrwidth = <0x00000028>; + xlnx,include-sg; + xlnx,sg-length-width = <0x00000017>; + dma-channel@40020000 { + compatible = "xlnx,axi-dma-mm2s-channel"; + dma-channels = <0x00000001>; + interrupts = <0x00000002>; + xlnx,datawidth = <0x00000040>; + xlnx,device-id = <0x00000000>; + xlnx,include-dre; + }; + dma-channel@40020030 { + compatible = "xlnx,axi-dma-s2mm-channel"; + dma-channels = <0x00000001>; + interrupts = <0x00000003>; + xlnx,datawidth = <0x00000040>; + xlnx,device-id = <0x00000000>; + xlnx,include-dre; + }; + }; + clint@40002000 { + reg-names = "control"; + interrupts-extended = <0x00000004 0x00000003 0x00000004 0x00000007 0x00000005 0x00000003 0x00000005 0x00000007 0x00000006 0x00000003 0x00000006 0x00000007 0x00000007 0x00000003 0x00000007 0x00000007>; + compatible = "riscv,clint0"; + #interrupt-cells = <0x00000001>; + reg = <0x00000000 0x40002000 0x00000000 0x000c0000>; + }; + plic@fff1100000 { + #address-cells = <0x00000000>; + compatible = "riscv,plic0"; + #interrupt-cells = <0x00000001>; + reg = <0x000000ff 0xf1100000 0x00000000 0x04000000>; + riscv,ndev = <0x00000003>; + riscv,max-priority = <0x00000007>; + interrupt-controller; + phandle = <0x00000003>; + }; + }; + __symbols__ { + CPU0 = "/cpus/cpu@0"; + CPU0_intc = "/cpus/cpu@0/interrupt-controller"; + eth_pool = "/reserved-memory/dma_pool@60000000"; + meep_pool = "/reserved-memory/dma_pool@70000000"; + eth0_clk = "/eth0_clk"; + uart0 = "/soc/serial@40001000"; + dma_eth = "/soc/dma@40020000"; + PLIC0 = "/soc/plic@fff1100000"; + }; +}; -- GitLab From fc4b8126f66ef8d9422cfb9a3623f8a7ff4afaf3 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 28 Jun 2024 10:39:10 +0200 Subject: [PATCH 010/310] Document memory layout --- lagarto-ox.nix | 2 +- ox-plic.dts | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 92355ec..27f06c9 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -97,7 +97,7 @@ version = "2023.07.02-print-cpu-probe"; src = builtins.fetchGit { url = "file:///home/Computational/rarias/riscv/u-boot"; - rev = "6fb984f14607c53878ba0ed82425d77e396b727b"; + rev = "f80a22a480f0e4157647bacf90e663be457c72c4"; }; #patches = [ ./u-boot-debug.patch ]; # diff --git a/ox-plic.dts b/ox-plic.dts index 3611dc4..43cbfcc 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -29,6 +29,15 @@ }; }; }; + /* Memory layout: + * + * [0x0_6000_0000, 0x0_7000_0000) -> DMA pool (256 MiB) + * [0x0_7000_0000, 0x0_8000_0000) -> DMA pool (256 MiB) + * [0x0_8000_0000, 0x0_f7e0_0000) -> RAM memory (~1.8 GiB) + * [0x0_f7e0_0000, 0x1_b7f0_0000) -> PMEM3 (3 GiB) + * [0x1_b7f0_0000, 0x1_bff0_0000) -> PMEM2 (128 MiB) + * [0x1_bff0_0000, 0x2_8000_0000) -> PMEM (3 GiB) + */ memory@80000000 { device_type = "memory"; reg = <0x00000000 0x80000000 0x00000000 0x77e00000>; -- GitLab From 7b9eb7366ef9ab334abecc46d5e7bb6283ebc669 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 28 Jun 2024 14:36:32 +0200 Subject: [PATCH 011/310] Remove unused PMEM regions Boots until the init, then fails with: [ 42.561840] Freeing unused kernel image (initmem) memory: 2448K [ 42.572360] Run /init as init process [ 42.577400] with arguments: [ 42.581780] /init [ 42.584520] with environment: [ 42.589320] HOME=/ [ 42.592140] TERM=linux [ 42.637580] init[1]: unhandled signal 4 code 0x1 at 0x0000003f966980d8 in ld-linux-riscv64-lp64d.so.1[3f96683000+23000] [ 42.650580] CPU: 0 PID: 1 Comm: init Not tainted 6.1.62 #1-NixOS [ 42.657720] Hardware name: Barcelona Supercomputing Center - Lagarto Ox (NixOS) (DT) [ 42.666760] epc : 0000003f966980d8 ra : 0000000000000000 sp : 0000003fe5c95db0 [ 42.675040] gp : ffffffff8197ea48 tp : 0000000000000000 t0 : 0000000000000000 [ 42.683320] t1 : 0000000000000000 t2 : 0000000000000000 s0 : 0000000000000000 [ 42.691600] s1 : 0000000000000000 a0 : 0000000000000000 a1 : 0000000000000000 [ 42.699880] a2 : 0000000000000000 a3 : 0000000000000000 a4 : 0000000000000000 [ 42.708160] a5 : 0000000000000000 a6 : 0000000000000000 a7 : 0000000000000000 [ 42.716420] s2 : 0000000000000000 s3 : 0000000000000000 s4 : 0000000000000000 [ 42.724700] s5 : 0000000000000000 s6 : 0000000000000000 s7 : 0000000000000000 [ 42.733380] s8 : 0000000000000000 s9 : 0000000000000000 s10: 0000000000000000 [ 42.741660] s11: 0000000000000000 t3 : 0000000000000000 t4 : 0000000000000000 [ 42.749920] t5 : 0000000000000000 t6 : 0000000000000000 [ 42.756440] status: 0000000200004020 badaddr: 0000000000010513 cause: 0000000000000002 [ 42.767620] Kernel panic - not syncing: Attempted to kill init! exitcode=0x00000004 [ 42.775720] CPU: 0 PID: 1 Comm: init Not tainted 6.1.62 #1-NixOS [ 42.782320] Hardware name: Barcelona Supercomputing Center - Lagarto Ox (NixOS) (DT) [ 42.790460] Call Trace: [ 42.793380] [] dump_backtrace+0x38/0x48 [ 42.799520] [] show_stack+0x50/0x68 [ 42.805280] [] dump_stack_lvl+0x60/0x84 [ 42.811400] [] dump_stack+0x20/0x30 [ 42.817140] [] panic+0x160/0x390 [ 42.822620] [] do_exit+0xa70/0xa78 [ 42.828260] [] do_group_exit+0x44/0xb0 [ 42.834240] [] get_signal+0x9b4/0xa00 [ 42.840140] [] do_work_pending+0x18c/0x610 [ 42.846480] [] resume_userspace_slow+0x10/0x14 [ 42.853420] ---[ end Kernel panic - not syncing: Attempted to kill init! exitcode=0x00000004 ]--- --- flake.nix | 4 ++-- fpga/upload.sh | 12 +++++++----- ox-plic.dts | 22 ++++++---------------- 3 files changed, 15 insertions(+), 23 deletions(-) diff --git a/flake.nix b/flake.nix index cef84f7..a8a68fc 100644 --- a/flake.nix +++ b/flake.nix @@ -116,11 +116,11 @@ build = nixosconf.config.system.build; in syspkgs.mkShell { pname = "lagarto-ox-shell"; - TOPLEVEL = build.toplevel; + #TOPLEVEL = build.toplevel; OPENSBI = syspkgs.opensbi; KERNEL = build.kernel; INITRD = build.initialRamdisk; - ROOTFS = build.sdImage; + #ROOTFS = build.sdImage; UBOOT_ENV = syspkgs.uboot-env; shellHook = '' echo "Here are the current system pieces:" diff --git a/fpga/upload.sh b/fpga/upload.sh index a47a667..f36d23b 100755 --- a/fpga/upload.sh +++ b/fpga/upload.sh @@ -3,19 +3,21 @@ set -e set -x -dst=femu:nixos/ +#dst=femu:nixos/ +dst=fpgalogin1:nixos/ if [ "$1" != "" ]; then dst="$1" fi -rsync -a fpga/fpgactl "$dst" -rsync -a fpga/boot.sh "$dst" +#rsync -a fpga/fpgactl "$dst" +#rsync -a fpga/boot.sh "$dst" rsync -a fpga/env.sh "$dst" -rsync "$OPENSBI/share/opensbi/lp64/fpga/openpiton/firmware/fw_payload.bin" "$dst/opensbi.bin" +#rsync "$OPENSBI/share/opensbi/lp64/fpga/openpiton/firmware/fw_payload.bin" "$dst/opensbi.bin" +rsync "$OPENSBI/share/opensbi/lp64d/fpga/ox_alveo/firmware/fw_payload.bin" "$dst/opensbi.bin" rsync "$KERNEL/Image" "$dst/kernel.bin" rsync "$INITRD/initrd" "$dst/initrd.bin" -rsync "$ROOTFS/sd-image/rootfs.img" "$dst/rootfs.img" +#rsync "$ROOTFS/sd-image/rootfs.img" "$dst/rootfs.img" rsync "$UBOOT_ENV" "$dst/uboot.env" echo "Now go to $dst and run ./boot.sh" diff --git a/ox-plic.dts b/ox-plic.dts index 43cbfcc..e9afd0c 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -33,14 +33,12 @@ * * [0x0_6000_0000, 0x0_7000_0000) -> DMA pool (256 MiB) * [0x0_7000_0000, 0x0_8000_0000) -> DMA pool (256 MiB) - * [0x0_8000_0000, 0x0_f7e0_0000) -> RAM memory (~1.8 GiB) - * [0x0_f7e0_0000, 0x1_b7f0_0000) -> PMEM3 (3 GiB) - * [0x1_b7f0_0000, 0x1_bff0_0000) -> PMEM2 (128 MiB) + * [0x0_8000_0000, 0x1_bff0_0000) -> RAM memory (~5 GiB) * [0x1_bff0_0000, 0x2_8000_0000) -> PMEM (3 GiB) */ memory@80000000 { device_type = "memory"; - reg = <0x00000000 0x80000000 0x00000000 0x77e00000>; + reg = <0x00000000 0x80000000 0x00000001 0x3ff00000>; }; reserved-memory { #address-cells = <0x00000002>; @@ -64,22 +62,14 @@ phandle = <0x00000002>; }; pmem@1bff00000 { + /* volatile; This property indicates that this region is + * actually backed by non-persistent memory. This lets the OS + * know that it may skip the cache flushes required to ensure + * data is made persistent after a write. */ volatile; compatible = "pmem-region"; reg = <0x00000001 0xbff00000 0x00000000 0xc0100000>; }; - pmem2@1b7f00000 { - #address-cells = <0x00000002>; - #size-cells = <0x00000002>; - volatile; - compatible = "pmem-region"; - reg = <0x00000001 0xb7f00000 0x00000000 0x08000000>; - }; - pmem3@f7e00000 { - volatile; - compatible = "pmem-region"; - reg = <0x00000000 0xf7e00000 0x00000000 0xc0100000>; - }; soc { #address-cells = <0x00000002>; #size-cells = <0x00000002>; -- GitLab From 2cefcef63d6e99cba57bf7b5aa72d1d8b025d5ae Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 28 Jun 2024 15:08:53 +0200 Subject: [PATCH 012/310] Make busybox static Now the problem is inside the busybox binary: [ 43.028000] Run /init as init process [ 43.033300] with arguments: [ 43.037260] /init [ 43.040000] with environment: [ 43.044180] HOME=/ [ 43.047600] TERM=linux [ 43.078020] init[1]: unhandled signal 4 code 0x1 at 0x0000000000010660 in busybox[10000+22d000] [ 43.090080] CPU: 0 PID: 1 Comm: init Not tainted 6.1.62 #1-NixOS [ 43.097580] Hardware name: Barcelona Supercomputing Center - Lagarto Ox (NixOS) (DT) [ 43.106480] epc : 0000000000010660 ra : 0000000000000000 sp : 0000003fd5489db0 [ 43.114780] gp : ffffffff8197ea48 tp : 0000000000000000 t0 : 0000000000000000 [ 43.123080] t1 : 0000000000000000 t2 : 0000000000000000 s0 : 0000000000000000 [ 43.131340] s1 : 0000000000000000 a0 : 0000000000000000 a1 : 0000000000000000 [ 43.139620] a2 : 0000000000000000 a3 : 0000000000000000 a4 : 0000000000000000 [ 43.147900] a5 : 0000000000000000 a6 : 0000000000000000 a7 : 0000000000000000 [ 43.156180] s2 : 0000000000000000 s3 : 0000000000000000 s4 : 0000000000000000 [ 43.164860] s5 : 0000000000000000 s6 : 0000000000000000 s7 : 0000000000000000 [ 43.173140] s8 : 0000000000000000 s9 : 0000000000000000 s10: 0000000000000000 [ 43.181420] s11: 0000000000000000 t3 : 0000000000000000 t4 : 0000000000000000 [ 43.189680] t5 : 0000000000000000 t6 : 0000000000000000 [ 43.196040] status: 0000000200004020 badaddr: 00000000034000ef cause: 0000000000000002 [ 43.208240] Kernel panic - not syncing: Attempted to kill init! exitcode=0x00000004 [ 43.216340] CPU: 0 PID: 1 Comm: init Not tainted 6.1.62 #1-NixOS [ 43.222940] Hardware name: Barcelona Supercomputing Center - Lagarto Ox (NixOS) (DT) [ 43.231100] Call Trace: [ 43.234000] [] dump_backtrace+0x38/0x48 [ 43.240140] [] show_stack+0x50/0x68 [ 43.245900] [] dump_stack_lvl+0x60/0x84 [ 43.252000] [] dump_stack+0x20/0x30 [ 43.257740] [] panic+0x160/0x390 [ 43.263200] [] do_exit+0xa70/0xa78 [ 43.268820] [] do_group_exit+0x44/0xb0 [ 43.274780] [] get_signal+0x9b4/0xa00 [ 43.280700] [] do_work_pending+0x18c/0x610 [ 43.287020] [] resume_userspace_slow+0x10/0x14 [ 43.293960] ---[ end Kernel panic - not syncing: Attempted to kill init! exitcode=0x00000004 ]--- --- lagarto-ox.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 27f06c9..6d853d2 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -86,6 +86,9 @@ }; nixpkgs.overlays = [ (final: prev: { + busybox = prev.busybox.override { + enableStatic = true; + }; ox-dtb = prev.runCommand "ox.dtb" { dtsFile = ./ox-plic.dts; #dtsFile = ./xavi.dts; -- GitLab From 975a1c0e870eddc1793fe010617d0c9063666754 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 1 Jul 2024 09:57:05 +0200 Subject: [PATCH 013/310] Update fpga tools for MEEP --- fpga/env.sh | 60 +++++++++--- fpga/fpgactl | 256 ++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 237 insertions(+), 79 deletions(-) diff --git a/fpga/env.sh b/fpga/env.sh index 573d4f6..b74331a 100644 --- a/fpga/env.sh +++ b/fpga/env.sh @@ -1,21 +1,53 @@ +#!/bin/bash # Source this file to setup the environment -INSTALL_PATH=/home/tools -LOAD_BITSTREAM=$INSTALL_PATH/scripts -HOSTNAME=$(hostname) +function setup_cucu() +{ + INSTALL_PATH=/home/tools + LOAD_BITSTREAM=$INSTALL_PATH/scripts + + export DMA_IP_DRIVERS="$INSTALL_PATH/drivers/$hostname/dma_ip_drivers-onic-gamma/xilinx_pcie_drivers" + + if [ ! -d $DMA_IP_DRIVERS ]; then + echo "error: DMA_IP_DRIVERS $DMA_IP_DRIVERS directory does not exist" >&2 + return + fi + + export PATH="$DMA_IP_DRIVERS/QDMA/linux-kernel/bin/:$PATH" + + if [ -x /opt/Xilinx/Vivado/2020.1/settings64.sh ]; then + source /opt/Xilinx/Vivado/2020.1/settings64.sh + elif [ -x /opt/Xilinx/Vivado/2021.2/settings64.sh ]; then + source /opt/Xilinx/Vivado/2021.2/settings64.sh + fi +} -export DMA_IP_DRIVERS="$INSTALL_PATH/drivers/$HOSTNAME/dma_ip_drivers-onic-gamma/xilinx_pcie_drivers" +function setup_meep() +{ + . /nfs/apps/XILINX/xilinx_22_env.sh + export PATH="$PATH:/apps/QDMA/meep-ionic/2022.1.4.4/linux-kernel/bin/" -if [ ! -d $DMA_IP_DRIVERS ]; then - echo "error: DMA_IP_DRIVERS $DMA_IP_DRIVERS directory does not exist" >&2 - return -fi + # Select the first FPGA in the node + local line=$(grep fpgan /etc/motd | head -1 | tr -d ' ') + export FPGACTL_PCIDEV=$(echo "$line" | awk -F'|' '{print $5}') + export FPGACTL_SERIAL=$(echo "$line" | awk -F'|' '{print $4}') + export FPGACTL_UART=$(echo "$line" | awk -F'|' '{print "/dev/"$7}') -export PATH="$DMA_IP_DRIVERS/QDMA/linux-kernel/bin/:$PATH" + # Setup mappings -if [ -x /opt/Xilinx/Vivado/2020.1/settings64.sh ]; then - source /opt/Xilinx/Vivado/2020.1/settings64.sh -elif [ -x /opt/Xilinx/Vivado/2021.2/settings64.sh ]; then - source /opt/Xilinx/Vivado/2021.2/settings64.sh -fi + # Delta between where we load in the dma device and RAM + local delta_addr=-0x60000000 + export FPGACTL_BOOTLOADER_ADDR=$((0x80000000+$delta_addr)) + export FPGACTL_KERNEL_ADDR=$((0x84000000+$delta_addr)) + export FPGACTL_INITRD_ADDR=$((0x8c300000+$delta_addr)) + export FPGACTL_ROOTFS_ADDR=$((0x1bff00000+$delta_addr)) +} + +hostname=$(hostname) + +case "$hostname" in + cucu) setup_cucu ;; + fpgan*) setup_meep ;; + *) echo "ERROR: unknown host $hostname";; +esac diff --git a/fpga/fpgactl b/fpga/fpgactl index c14c323..f208480 100755 --- a/fpga/fpgactl +++ b/fpga/fpgactl @@ -27,9 +27,7 @@ function check_environment() # {{{ } # }}} function create_qdma_queue() # {{{ { - pcidir="/sys/bus/pci/devices/0000:08:00.0" - - if [ ! -d "$pcidir/qdma" ]; then + if [ ! -d "$pcidir" ]; then echo "missing pci directory: $pcidir" >&2 exit 1 fi @@ -44,48 +42,72 @@ function create_qdma_queue() # {{{ exit 1 fi - if [ ! -c "/dev/qdma08000-MM-1" ]; then - echo 2 | sudo dd of="$pcidir/qdma/qmax" - - dma-ctl qdma08000 q add mode mm idx 1 dir bi - dma-ctl qdma08000 q start idx 1 dir bi + if [ ! -r "$pcidir/qdma/qmax" ]; then + echo "cannot read qmax file: $pcidir/qdma/qmax" >&2 + exit 1 + fi - sudo chmod go+rw "/dev/qdma08000-MM-1" - sudo chmod go+rw "$pcidir/resource0" - sudo chmod go+rw "$pcidir/resource0_wc" - sudo chmod go+rw "$pcidir/resource2" - sudo chmod go+rw "$pcidir/resource2_wc" + # There should be two queues + local qmax=$(cat "$pcidir/qdma/qmax") + if [ "$qmax" != 2 ]; then + if [ -w "$pcidir/qdma/qmax" ]; then + echo 2 | dd of="$pcidir/qdma/qmax" + else + echo 2 | sudo dd of="$pcidir/qdma/qmax" + fi fi - if [ ! -c "/dev/qdma08000-MM-0" ]; then - dma-ctl qdma08000 q add mode mm idx 0 dir bi - dma-ctl qdma08000 q start idx 0 dir bi - sudo chmod go+rw "/dev/qdma08000-MM-0" + # Create the two queues if they don't exist + if [ ! -c "/dev/${qdmadev}-MM-1" ]; then + dma-ctl "${qdmadev}" q add mode mm idx 1 dir bi + dma-ctl "${qdmadev}" q start idx 1 dir bi fi + if [ ! -c "/dev/${qdmadev}-MM-0" ]; then + dma-ctl "${qdmadev}" q add mode mm idx 0 dir bi + dma-ctl "${qdmadev}" q start idx 0 dir bi + fi + + # Wait for udev to process the new devices + udevadm settle + + # Ensure we have write access. On some clusters this is automatically done + # by udev rules, on others we are expect to use sudo. + for f in /dev/${qdmadev}-MM-{0,1} ${pcidir}/resource{0,0_wc,2,2_wc}; do + test -w "$f" || sudo chmod go+rw "$f" + done sleep 2 } # }}} -function do_system_reset() # {{{ +function do_cpu_reset() # {{{ { - # UartBootEn (bit2) + system reset (bit0) - dma-ctl qdma08000 reg write bar 2 0x0 0x0 > /dev/null - sleep 0.2 - # Release system reset, we must wait until the memory is filled with 0s - dma-ctl qdma08000 reg write bar 2 0x0 0x1 > /dev/null - #sleep 5 + if [ "$model" == "hun" ]; then + # UartBootEn (bit2) + system reset (bit0) + dma-ctl "${qdmadev}" reg write bar 2 0x0 0x0 + sleep 0.2 + dma-ctl "${qdmadev}" reg write bar 2 0x0 0x1 + elif [ "$model" == "ox" ]; then + dma-ctl "${qdmadev}" reg write bar 2 0x0 0x0 + sleep 0.2 + fi } # }}} -function do_system_release() # {{{ +function do_cpu_release() # {{{ { - # Release Ariane's reset - dma-ctl qdma08000 reg write bar 2 0x0 0x3 > /dev/null + if [ "$model" == "hun" ]; then + # Release Ariane's reset + dma-ctl "${qdmadev}" reg write bar 2 0x0 0x3 + elif [ "$model" == "ox" ]; then + dma-ctl "${qdmadev}" reg write bar 2 0x0 0x1 + sleep 1 + fi } # }}} function copy_by_dma() # {{{ { ifile="$1" address="$2" - ofile="/dev/qdma08000-MM-1" - bs=$((8*1024*1024)) # 8 MiB + ofile="/dev/${qdmadev}-MM-1" + #bs=$((8*1024*1024)) # 8 MiB + bs=$((1*1024*1024)) # 1 MiB total_size=$(stat --format "%s" "$ifile") @@ -101,6 +123,7 @@ function copy_by_dma() # {{{ dd if="$ifile" skip=$skip count=1 bs=$bs of="$ofile" seek=$dst oflag=seek_bytes status=none let skip=$skip+1 done + #dma-to-device -d "$ofile" -s "$total_size" -a "$address" -f "$ifile" } # }}} function load_file_in_memory() # {{{ @@ -112,21 +135,21 @@ function load_file_in_memory() # {{{ total_size=$(stat --format "%s" "$file") # Previous tests... - #strace -f dma-to-device -d /dev/qdma08000-MM-1 -a "$address" -s $((8*1024*1024)) -f "$file" - #strace -f dd if="$file" bs=16M seek="${address}" oflag=seek_bytes of=/dev/qdma08000-MM-1 status=progress conv=sync - #strace -f fpgakit/fpgadd -i "$file" -a "$address" -d /dev/qdma08000-MM-1 -c 1024 -s 1024 + #strace -f dma-to-device -d /dev/${qdmadev}-MM-1 -a "$address" -s $((8*1024*1024)) -f "$file" + #strace -f dd if="$file" bs=16M seek="${address}" oflag=seek_bytes of=/dev/${qdmadev}-MM-1 status=progress conv=sync + #strace -f fpgakit/fpgadd -i "$file" -a "$address" -d /dev/${qdmadev}-MM-1 -c 1024 -s 1024 #ID=08 ./load_image.sh "$file" "$address" # Now dd seems to work fine, but I will leave this as fallback: copy_by_dma "$file" "$address" - #dd if="$file" bs=8M seek="${address}" oflag=seek_bytes of=/dev/qdma08000-MM-1 status=none + #dd if="$file" bs=8M seek="${address}" oflag=seek_bytes of=/dev/${qdmadev}-MM-1 status=none printf "loaded '%s' at 0x%x with size %d\n" "$file" "$address" "$total_size" >&2 } # }}} function do_boot_only() # {{{ { - do_system_reset + do_cpu_reset ./load_image.sh ${OSBI} $((0x80000000)) && @@ -135,7 +158,7 @@ function do_boot_only() # {{{ sleep 2 && # #Release Ariane's reset - dma-ctl qdma08000 reg write bar 2 0x0 0x3 && + dma-ctl "${qdmadev}" reg write bar 2 0x0 0x3 && sleep 10 && @@ -143,7 +166,7 @@ function do_boot_only() # {{{ echo mount -o nolock -o rw -o retrans=10 192.168.0.16:/media/sda2/scratch/xavim/point /root && - if [ ! -c /dev/qdma08000-MM-0 ] ; then + if [ ! -c "/dev/${qdmadev}-MM-0" ] ; then /home/tools/drivers/`/bin/hostname`/dma_ip_drivers-onic-gamma/create-queue-qdma.sh -2 fi && @@ -155,7 +178,7 @@ function do_boot_only() # {{{ } # }}} function do_reload_fs() # {{{ { - do_system_reset + do_cpu_reset #~xavim/LAGARTO_LINUX-4.1/./load_image.sh \ # /home/xavim/ARIANE_LINUX-3.0/recovery/fedora-fs-dx-java-cucu-0.108.raw.recovered \ @@ -172,7 +195,7 @@ function do_reload_fs() # {{{ sleep 2 - do_system_release + do_cpu_release create_qdma_queue # uncomment to enable eth-over-pcie @@ -180,23 +203,25 @@ function do_reload_fs() # {{{ } # }}} function upload_bitstream_file() # {{{ { - bitfile="$1" + if [ -z "$jtagserial" ]; then + >&2 echo "JTAG serial required" + usage + fi - fpgajtag=$(lsusb -vd 0403: 2>&1 | grep iSerial | awk ' { print $3; }') - if [ -z "$fpgajtag" ]; then - echo "error: cannot find JTAG serial" >&2 - exit 1 + if [ -z "$bitstream" ]; then + >&2 echo "bitstream file required" + usage fi script=$(mktemp vivado-XXXXXXXXXX.tcl) cat > "$script" <&2 echo -e "error: missing PCI device (hint: lspci -d 10ee:902f)" + usage + fi - unload_modules "xocl xclmgmt qdma_pf xdma" # qdma_vf not removable - remove_pci_devices + # Ensure it is ok + local matches=$(lspci -s "$pcidev") + if [ -z "$matches" ]; then + >&2 echo "no match for PCI device '$pcidev'" + exit 1 + fi - upload_bitstream_file "$bitstream" + local n="$(echo "$matches" | wc -l)" + if [ "$n" -gt 1 ]; then + >&2 echo "multiple matches for PCI device '$pcidev'" + exit 1 + fi + # Fill the PCI device with the domain + local fulldev=$(lspci -s "$pcidev" -D | cut -d' ' -f1) + pcidir="/sys/bus/pci/devices/$fulldev" + + if [ ! -d "$pcidir" ]; then + >&2 echo "cannot find PCI dir: $pcidir" + exit 1 + fi + + # Set the PCI device to the full device + pcidev="$fulldev" + + # Find slot + slot=$(lspci -s "$pcidev" -vm | grep PhySlot | cut -f2) + + if [ -z "$slot" ]; then + >&2 echo "cannot find physical slot for PCI '$pcidev'" + exit 1 + fi + + local devid=$(echo "$pcidev" | cut -d: -f2- | tr -d ':.') + qdmadev="qdma${devid}" + +} # }}} +function preload_hook() #{{{ +{ + case "$hostname" in + cucu) + unload_modules "xocl xclmgmt qdma_pf xdma" # qdma_vf not removable + remove_pci_devices + ;; + fpgan*) + ;; + *) + echo "hostname $hostname not known" + exit 1 + ;; + esac +} #}}} +function postload_hook() #{{{ +{ rescan_pci_devices - unload_modules "qdma_pf xdma" # qdma_vf not removable - remove_pci_devices - load_qdma_modules - rescan_pci_devices + + case "$hostname" in + cucu) + unload_modules "qdma_pf xdma" # qdma_vf not removable + remove_pci_devices + load_qdma_modules + rescan_pci_devices + ;; + fpgan*) + ;; + *) + echo "hostname $hostname not known" + exit 1 + ;; + esac + create_qdma_queue +} #}}} +function load_bitstream() # {{{ +{ + preload_hook + upload_bitstream_file + postload_hook } # }}} bitstream= @@ -269,32 +372,46 @@ initrd= rootfs= resetcpu= verbose= +pcidev= +model=ox +# Internal +slot= +pcidir= +qdmadev= + +bootloader_addr="${FPGA_BOOTLOADER_ADDR:-0x80000000}" +kernel_addr="${FPGA_KERNEL_ADDR:-0x84000000}" +initrd_addr="${FPGA_INITRD_ADDR:-0x8c300000}" +rootfs_addr="${FPGA_ROOTFS_ADDR:-0x140000000}" -bootloader_addr=0x80000000 -kernel_addr=0x84000000 -initrd_addr=0x8c300000 -rootfs_addr=0x140000000 +hostname="${hostname:-$(hostname)}" +echo "hostname=$hostname" function usage() { echo "" >&2 - echo "Usage: $0 [-v] [-w bitstream] [-b bootloader] [-k kernel] [-i initrd]" >&2 + echo "Usage: $0 [-p pcidev] [-v] [-w bitstream] [-j serial] [-b bootloader] [-k kernel] [-i initrd]" >&2 echo "" >&2 echo "First writes the bitstream if given. Then loads the rest of files" >&2 echo "into memory and restarts the CPU." >&2 echo "" >&2 echo "Options" >&2 + echo " -p pcidev Select PCI device (same format as lspci -s)." >&2 + echo " Read from \$FPGACTL_PCIDEV if not given." >&2 echo " -w bitstream Write the bitstream file to the FPGA" >&2 + echo " -j serial JTAG serial (can be found by lsusb -v)" >&2 + echo " Read from \$FPGACTL_SERIAL if not given." >&2 echo " -b bootloader Load the bootloader file in $bootloader_addr" >&2 echo " -k kernel Load the kernel file in $kernel_addr" >&2 echo " -i initrd Load the initrd file in $initrd_addr" >&2 echo " -r rootfs Load the rootfs file in $rootfs_addr" >&2 + echo " -m model CPU model: Either 'hun' or 'ox' (default ox)" >&2 echo " -v Be verbose" >&2 echo "" >&2 exit 1 } -while getopts "hvw:b:k:i:r:" opt; do +while getopts "hvw:b:k:i:r:p:j:m:" opt; do case "${opt}" in v) verbose=1 ;; w) bitstream="${OPTARG}" ;; @@ -302,21 +419,30 @@ while getopts "hvw:b:k:i:r:" opt; do k) kernel="${OPTARG}"; resetcpu=1 ;; i) initrd="${OPTARG}"; resetcpu=1 ;; r) rootfs="${OPTARG}"; resetcpu=1 ;; + p) pcidev="${OPTARG}" ;; + j) jtagserial="${OPTARG}" ;; + m) model="${OPTARG}" ;; h) usage ;; *) usage ;; esac done +jtagserial="${jtagserial:-$FPGACTL_SERIAL}" +pcidev="${pcidev:-$FPGACTL_PCIDEV}" + test "$verbose" && set -x check_environment +select_pcidev test "$bitstream" && load_bitstream "$bitstream" -test "$resetcpu" && do_system_reset +test "$resetcpu" && do_cpu_reset test "$bootloader" && load_file_in_memory "$bootloader" $bootloader_addr test "$kernel" && load_file_in_memory "$kernel" $kernel_addr test "$initrd" && load_file_in_memory "$initrd" $initrd_addr test "$rootfs" && load_file_in_memory "$rootfs" $rootfs_addr -test "$resetcpu" && do_system_release +test "$resetcpu" && do_cpu_release + +exit 0 # vim:ts=2:sw=2:ai:foldmethod=marker:foldlevel=0: -- GitLab From 33b227f576111fae587a3e4a53478770173a3f7e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 1 Jul 2024 10:14:39 +0200 Subject: [PATCH 014/310] Hangs in swtich_root in stage1 --- flake.nix | 4 ++-- fpga/upload.sh | 2 +- lagarto-ox.nix | 31 ++++++++++++++++++++++++++----- ox-plic.dts | 5 +++-- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/flake.nix b/flake.nix index a8a68fc..cef84f7 100644 --- a/flake.nix +++ b/flake.nix @@ -116,11 +116,11 @@ build = nixosconf.config.system.build; in syspkgs.mkShell { pname = "lagarto-ox-shell"; - #TOPLEVEL = build.toplevel; + TOPLEVEL = build.toplevel; OPENSBI = syspkgs.opensbi; KERNEL = build.kernel; INITRD = build.initialRamdisk; - #ROOTFS = build.sdImage; + ROOTFS = build.sdImage; UBOOT_ENV = syspkgs.uboot-env; shellHook = '' echo "Here are the current system pieces:" diff --git a/fpga/upload.sh b/fpga/upload.sh index f36d23b..503f775 100755 --- a/fpga/upload.sh +++ b/fpga/upload.sh @@ -17,7 +17,7 @@ rsync -a fpga/env.sh "$dst" rsync "$OPENSBI/share/opensbi/lp64d/fpga/ox_alveo/firmware/fw_payload.bin" "$dst/opensbi.bin" rsync "$KERNEL/Image" "$dst/kernel.bin" rsync "$INITRD/initrd" "$dst/initrd.bin" -#rsync "$ROOTFS/sd-image/rootfs.img" "$dst/rootfs.img" +rsync "$ROOTFS/sd-image/rootfs.img" "$dst/rootfs.img" rsync "$UBOOT_ENV" "$dst/uboot.env" echo "Now go to $dst and run ./boot.sh" diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 6d853d2..15cc1e6 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -5,14 +5,13 @@ "${modulesPath}/installer/sd-card/sd-image.nix" ]; - # No need, Lagarto OX has support for rv64gc. #nixpkgs.crossSystem = { # system = "riscv64-linux"; # gcc.arch = "rv64imafd"; - # gcc.tune = "rv64imafd"; + # gcc.tune = "generic"; #}; - #boot.kernelPackages = pkgs.linuxPackages_latest; + boot.kernelPackages = pkgs.linuxPackages_latest; boot = { kernelPatches = [ { @@ -38,6 +37,16 @@ LIBNVDIMM y BLK_DEV_PMEM y '' + # No vector extensions + + '' + RISCV_ISA_V n + CONFIG_RISCV_ISA_V_DEFAULT_ENABLE n + '' + # Debugging + + '' + DEBUG_WX y + MAGIC_SYSRQ y + '' ; } ]; @@ -86,9 +95,21 @@ }; nixpkgs.overlays = [ (final: prev: { - busybox = prev.busybox.override { + + # FIXME: Fails on start + busybox = (prev.busybox.override { enableStatic = true; - }; + }).overrideAttrs (old: { + #env.NIX_CFLAGS_COMPILE = "-mtune=generic"; + #env.NIX_DEBUG = 5; + }); + + linuxPackages_latest = prev.linuxPackages_latest.extend (lib.const (ksuper: { + kernel = ksuper.kernel.override { + stdenv = prev.gcc8Stdenv; + }; + })); + ox-dtb = prev.runCommand "ox.dtb" { dtsFile = ./ox-plic.dts; #dtsFile = ./xavi.dts; diff --git a/ox-plic.dts b/ox-plic.dts index e9afd0c..2735352 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -33,12 +33,13 @@ * * [0x0_6000_0000, 0x0_7000_0000) -> DMA pool (256 MiB) * [0x0_7000_0000, 0x0_8000_0000) -> DMA pool (256 MiB) - * [0x0_8000_0000, 0x1_bff0_0000) -> RAM memory (~5 GiB) + * [0x0_8000_0000, 0x0_7ff0_0000) -> RAM memory (~2 GiB) + * [0x0_7ff0_0000, 0x1_bff0_0000) -> Empty * [0x1_bff0_0000, 0x2_8000_0000) -> PMEM (3 GiB) */ memory@80000000 { device_type = "memory"; - reg = <0x00000000 0x80000000 0x00000001 0x3ff00000>; + reg = <0x00000000 0x80000000 0x00000000 0x7ff00000>; }; reserved-memory { #address-cells = <0x00000002>; -- GitLab From 763f053f01439f8ee801658490daa965e8bbfc30 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 1 Jul 2024 11:25:59 +0200 Subject: [PATCH 015/310] Simplify, correct and comment DTS Disable unused devices and try to fix the interrupt connections. The compiler dtc doesn't report any warning now. --- ox-plic.dts | 194 +++++++++++++++++++++++++++++----------------------- 1 file changed, 107 insertions(+), 87 deletions(-) diff --git a/ox-plic.dts b/ox-plic.dts index 2735352..2423b87 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -1,31 +1,42 @@ /dts-v1/; / { - #address-cells = <0x00000002>; - #size-cells = <0x00000002>; + #address-cells = <2>; + #size-cells = <2>; /* 64 bits memory addresses */ compatible = "riscv,rv64i"; model = "Barcelona Supercomputing Center - Lagarto Ox (NixOS)"; chosen { bootargs = "earlyprintk ignore_loglevel earlycon=sbi console=hvc0 root=/dev/pmem0p1 ro init=/bin/bash"; }; cpus { - #address-cells = <0x00000001>; - #size-cells = <0x00000000>; - timebase-frequency = <0x0000c350>; + #address-cells = <1>; + #size-cells = <0>; + /* Timer */ + timebase-frequency = <50000>; /* 50 kHz */ CPU0: cpu@0 { - clock-frequency = <0x02FAF080>; + clock-frequency = <50000000>; /* 50 MHz */ device_type = "cpu"; - reg = <0x00000000>; + reg = <0>; status = "okay"; compatible = "riscv"; riscv,isa = "rv64imafd"; mmu-type = "riscv,sv39"; tlb-split; phandle = <0x00000004>; - L3: interrupt-controller { - #interrupt-cells = <0x00000001>; - interrupt-controller; + /* Hart-Level Interrupt Controller: Every interrupt is + * ultimately routed through a hart's HLIC before it + * interrupts that hart. */ + HLIC0: interrupt-controller { + #interrupt-cells = <1>; + interrupt-controller; /* Receives interrupts */ compatible = "riscv,cpu-intc"; - phandle = <0x00000005>; + phandle = <0x5>; + }; + }; + cpu-map { + cluster0 { + core0 { + cpu = <&CPU0>; + }; }; }; }; @@ -39,29 +50,27 @@ */ memory@80000000 { device_type = "memory"; - reg = <0x00000000 0x80000000 0x00000000 0x7ff00000>; + reg = <0x0 0x80000000 0x0 0x7ff00000>; }; reserved-memory { - #address-cells = <0x00000002>; - #size-cells = <0x00000002>; + #address-cells = <2>; /* Starting address and size */ + #size-cells = <2>; /* 64 bits memory addresses */ ranges; eth_pool: dma_pool@60000000 { - reg = <0x00000000 0x60000000 0x00000000 0x10000000>; + reg = <0x0 0x60000000 0x0 0x10000000>; compatible = "shared-dma-pool"; - phandle = <0x00000001>; }; onic_pool: dma_pool@70000000 { - reg = <0x00000000 0x70000000 0x00000000 0x10000000>; + reg = <0x0 0x70000000 0x0 0x10000000>; compatible = "shared-dma-pool"; - phandle = <0x00000006>; }; }; - eth0_clk: eth0_clk { - compatible = "fixed-clock"; - #clock-cells = <0x00000000>; - clock-frequency = <0x09502f90>; - phandle = <0x00000002>; - }; +// eth0_clk: eth0_clk { +// compatible = "fixed-clock"; +// #clock-cells = <0x00000000>; +// clock-frequency = <0x09502f90>; +// phandle = <0x00000002>; +// }; pmem@1bff00000 { /* volatile; This property indicates that this region is * actually backed by non-persistent memory. This lets the OS @@ -69,7 +78,7 @@ * data is made persistent after a write. */ volatile; compatible = "pmem-region"; - reg = <0x00000001 0xbff00000 0x00000000 0xc0100000>; + reg = <0x1 0xbff00000 0x0 0xc0100000>; }; soc { #address-cells = <0x00000002>; @@ -78,76 +87,87 @@ ranges; SERIAL: serial@40001000 { compatible = "ns16550"; - reg = <0x0 0x40001000 0x0 0x00000100>; - interrupts = <0>; - /*port-number = <0>;*/ + reg = <0x0 0x40001000 0x0 0x100>; + interrupts = <0>; /* Output interrupt 0 */ + interrupt-parent = <&PLIC>; reg-shift = <2>; clock-frequency = <50000000>; current-speed = <115200>; status = "okay"; - phandle = <0x00000007>; - }; - ethernet0 { - xlnx,rxmem = <0x000005f2>; - carv,mtu = <0x000005dc>; - carv,no-mac; - device_type = "network"; - local-mac-address = [02 05 00 01 00 05]; - axistream-connected = <0x000000fe>; - compatible = "xlnx,xxv-ethernet-1.0-carv"; - memory-region = <ð_pool>; }; - dma@40400000 { - xlnx,include-dre; - phandle = <0x000000fe>; - #dma-cells = <0x00000001>; - compatible = "xlnx,axi-dma-1.00.a"; - clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk", "m_axi_s2mm_aclk", "m_axi_sg_aclk"; - clocks = <ð0_clk>, <ð0_clk>, <ð0_clk>, <ð0_clk>; - reg = <0x00000000 0x40400000 0x00000000 0x00400000>; - interrupt-names = "mm2s_introut", "s2mm_introut"; - interrupt-parent = <&PLIC0>; - interrupts = <0x00000002 0x00000003>; - xlnx,addrwidth = <0x00000028>; - xlnx,include-sg; - xlnx,sg-length-width = <0x00000017>; - dma-channel@40400000 { - compatible = "xlnx,axi-dma-mm2s-channel"; - dma-channels = <0x00000000>; - interrupts = <0x00000002>; - xlnx,datawidth = <0x00000040>; - xlnx,device-id = <0x00000000>; - xlnx,include-dre; - }; - dma-channel@40400030 { - compatible = "xlnx,axi-dma-s2mm-channel"; - dma-channels = <0x00000001>; - interrupts = <0x00000003>; - xlnx,datawidth = <0x00000040>; - xlnx,device-id = <0x00000000>; - xlnx,include-dre; - }; + +// ethernet0 { +// xlnx,rxmem = <0x000005f2>; +// carv,mtu = <0x000005dc>; +// carv,no-mac; +// device_type = "network"; +// local-mac-address = [02 05 00 01 00 05]; +// axistream-connected = <0x000000fe>; +// compatible = "xlnx,xxv-ethernet-1.0-carv"; +// memory-region = <ð_pool>; +// }; +// dma@40400000 { +// xlnx,include-dre; +// phandle = <0x000000fe>; +// #dma-cells = <0x00000001>; +// compatible = "xlnx,axi-dma-1.00.a"; +// clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk", "m_axi_s2mm_aclk", "m_axi_sg_aclk"; +// clocks = <ð0_clk>, <ð0_clk>, <ð0_clk>, <ð0_clk>; +// reg = <0x00000000 0x40400000 0x00000000 0x00400000>; +// interrupt-names = "mm2s_introut", "s2mm_introut"; +// interrupt-parent = <&PLIC>; +// interrupts = <2 3>; +// xlnx,addrwidth = <0x00000028>; +// xlnx,include-sg; +// xlnx,sg-length-width = <0x00000017>; +// dma-channel@40400000 { +// compatible = "xlnx,axi-dma-mm2s-channel"; +// dma-channels = <0x00000000>; +// interrupts = <0x00000002>; +// xlnx,datawidth = <0x00000040>; +// xlnx,device-id = <0x00000000>; +// xlnx,include-dre; +// }; +// dma-channel@40400030 { +// compatible = "xlnx,axi-dma-s2mm-channel"; +// dma-channels = <0x00000001>; +// interrupts = <0x00000003>; +// xlnx,datawidth = <0x00000040>; +// xlnx,device-id = <0x00000000>; +// xlnx,include-dre; +// }; +// }; + + /* Platform-Level Interrupt Controller: Delivers interrupts to + * HARTs. */ + PLIC: plic@40800000 { + compatible = "riscv,plic0"; + interrupt-controller; /* Receives interrupts */ + #interrupt-cells = <1>; + /* Sends interrupts to HART interrupt controllers */ + interrupts-extended = <&HLIC0 3 &HLIC0 7>; + reg = < 0x0 0x40800000 0x0 0x00400000>; + riscv,ndev = <0x3>; + riscv,max-priority = <0x7>; + phandle = <0x3>; }; - clint@40002000 { + /* Core Local Interruptor: It directly connects to the timer and + * inter-processor interrupt lines of various HARTs (or CPUs) so + * RISC-V per-HART (or per-CPU) local interrupt controller is + * the parent interrupt controller for CLINT device. The clock + * frequency of CLINT is specified via "timebase-frequency" DT + * property of "/cpus" DT node. The "timebase-frequency" DT + * property is described in + * Documentation/devicetree/bindings/riscv/cpus.yaml + */ + timer@40002000 { + reg = <0x0 0x40002000 0x0 0x000c0000>; reg-names = "control"; - interrupts-extended = - <&CPU0 0x3>, <&CPU0 0x7>, - <&L3 0x3>, <&L3 0x7>, - <&onic_pool 0x3>, <&onic_pool 0x7>, - <&SERIAL 0x3>, <&SERIAL 0x7>; + interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; + /*<&CPU0 0x3>, <&CPU0 0x7>,*/ + /*<&onic_pool 0x3>, <&onic_pool 0x7>,*/ + /*<&SERIAL 0x3>, <&SERIAL 0x7>*/ compatible = "riscv,clint0"; - #interrupt-cells = <0x00000001>; - reg = <0x00000000 0x40002000 0x00000000 0x000c0000>; - }; - PLIC0: plic@40800000 { - #interrupt-cells = <1>; - compatible = "riscv,plic0"; - interrupt-controller; - interrupts-extended = <&L3 3 &L3 7>; - reg = < 0x00000000 0x40800000 0x00000000 0x00400000>; - riscv,ndev = <0x00000003>; - riscv,max-priority = <0x00000007>; - phandle = <0x00000003>; }; }; }; -- GitLab From a7e1a6a1009ff7e4cba415c6b777fee94b2a9d91 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 1 Jul 2024 11:30:12 +0200 Subject: [PATCH 016/310] Go back to default gcc for the kernel --- lagarto-ox.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 15cc1e6..f2880eb 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -104,11 +104,12 @@ #env.NIX_DEBUG = 5; }); - linuxPackages_latest = prev.linuxPackages_latest.extend (lib.const (ksuper: { - kernel = ksuper.kernel.override { - stdenv = prev.gcc8Stdenv; - }; - })); + linuxPackages_latest = prev.linuxPackages_latest; + #linuxPackages_latest = prev.linuxPackages_latest.extend (lib.const (ksuper: { + # kernel = ksuper.kernel.override { + # stdenv = prev.gcc8Stdenv; + # }; + #})); ox-dtb = prev.runCommand "ox.dtb" { dtsFile = ./ox-plic.dts; -- GitLab From a7cc8e2611729c0d57795e2caeb716684cafa73a Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 1 Jul 2024 11:47:16 +0200 Subject: [PATCH 017/310] Fix fpgactl wrong variable names --- fpga/fpgactl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fpga/fpgactl b/fpga/fpgactl index f208480..78b9c1b 100755 --- a/fpga/fpgactl +++ b/fpga/fpgactl @@ -379,10 +379,10 @@ slot= pcidir= qdmadev= -bootloader_addr="${FPGA_BOOTLOADER_ADDR:-0x80000000}" -kernel_addr="${FPGA_KERNEL_ADDR:-0x84000000}" -initrd_addr="${FPGA_INITRD_ADDR:-0x8c300000}" -rootfs_addr="${FPGA_ROOTFS_ADDR:-0x140000000}" +bootloader_addr="${FPGACTL_BOOTLOADER_ADDR:-0x80000000}" +kernel_addr="${FPGACTL_KERNEL_ADDR:-0x84000000}" +initrd_addr="${FPGACTL_INITRD_ADDR:-0x8c300000}" +rootfs_addr="${FPGACTL_ROOTFS_ADDR:-0x140000000}" hostname="${hostname:-$(hostname)}" echo "hostname=$hostname" -- GitLab From 869bd302a8232806889685022231912210e49ff9 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 1 Jul 2024 11:47:33 +0200 Subject: [PATCH 018/310] Find opensbi in any directory name --- fpga/upload.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fpga/upload.sh b/fpga/upload.sh index 503f775..d64bfa7 100755 --- a/fpga/upload.sh +++ b/fpga/upload.sh @@ -10,11 +10,10 @@ if [ "$1" != "" ]; then dst="$1" fi -#rsync -a fpga/fpgactl "$dst" +rsync -a fpga/fpgactl "$dst" #rsync -a fpga/boot.sh "$dst" rsync -a fpga/env.sh "$dst" -#rsync "$OPENSBI/share/opensbi/lp64/fpga/openpiton/firmware/fw_payload.bin" "$dst/opensbi.bin" -rsync "$OPENSBI/share/opensbi/lp64d/fpga/ox_alveo/firmware/fw_payload.bin" "$dst/opensbi.bin" +rsync $OPENSBI/share/opensbi/*/fpga/*/firmware/fw_payload.bin "$dst/opensbi.bin" rsync "$KERNEL/Image" "$dst/kernel.bin" rsync "$INITRD/initrd" "$dst/initrd.bin" rsync "$ROOTFS/sd-image/rootfs.img" "$dst/rootfs.img" -- GitLab From 67fedf4651ef8996b6729f0a41c0609920470da2 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 1 Jul 2024 11:51:20 +0200 Subject: [PATCH 019/310] Remove unused dtb compilation in OpenSBI --- lagarto-ox.nix | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index f2880eb..5350326 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -199,12 +199,6 @@ #hash = "sha256-T8ZeAzjM9aeTXitjE7s+m+jjGGtDo2jK1qO5EuKiVLU="; #1.4 }; #NIX_DEBUG=5; - # Compile dts to dtb form - nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ prev.buildPackages.dtc ]; - dtsFile = ./ox-plic.dts; - preBuild = '' - dtc -O dtb -o ox.dtb $dtsFile - ''; makeFlags = [ "PLATFORM=fpga/ox_alveo" #"CONFIG_SBI_ECALL_RFENCE=n" -- GitLab From 14f6528318bdc7baf48305810546c752a7434273 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 1 Jul 2024 12:10:19 +0200 Subject: [PATCH 020/310] Use upstream u-boot and disable debug logs --- lagarto-ox.nix | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 5350326..b160dda 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -119,11 +119,11 @@ uboot = prev.ubootQemuRiscv64Smode.override { filesToInstall = [ "u-boot.bin" "u-boot-nodtb.bin" ]; - version = "2023.07.02-print-cpu-probe"; - src = builtins.fetchGit { - url = "file:///home/Computational/rarias/riscv/u-boot"; - rev = "f80a22a480f0e4157647bacf90e663be457c72c4"; - }; + #version = "2023.07.02-print-cpu-probe"; + #src = builtins.fetchGit { + # url = "file:///home/Computational/rarias/riscv/u-boot"; + # rev = "f80a22a480f0e4157647bacf90e663be457c72c4"; + #}; #patches = [ ./u-boot-debug.patch ]; # # CONFIG_SERIAL_PRESENT=n @@ -149,13 +149,18 @@ CONFIG_DEBUG_SBI_CONSOLE=y CONFIG_SMP=n CONFIG_TRACE_EARLY=y - CONFIG_LOG=y - CONFIG_LOGLEVEL=9 - CONFIG_LOG_MAX_LEVEL=9 - CONFIG_LOG_DEFAULT_LEVEL=9 - ''; + '' +# # Enable debug logs +# + +# '' +# CONFIG_LOG=y +# CONFIG_LOGLEVEL=9 +# CONFIG_LOG_MAX_LEVEL=9 +# CONFIG_LOG_DEFAULT_LEVEL=9 +# '' + ; extraMakeFlags = [ - "KCPPFLAGS=-DLOG_DEBUG" + #"KCPPFLAGS=-DLOG_DEBUG" #"EXT_DTB=${final.ox-dtb}" ]; }; -- GitLab From 65c7eca2c8a5511f357dcc6c670e07ddc7279db6 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 1 Jul 2024 13:28:32 +0200 Subject: [PATCH 021/310] Fix OpenSBI configuration for newer versions OpenSBI 1.3 an newer require the heap_size to be specified in the sbi_platform structure, othewise the default 0 value causes a hangg at the start. I found the problem after bisecting OpenSBI: # Bisect opensbi version = src.shortRev; src = builtins.fetchGit { url = "file:///home/Computational/rarias/riscv/opensbi"; #rev = "908be1b85c8ff0695ea226fbbf0ff24a779cdece"; #good #rev = "6bc02dede86c47f87e65293b7099e9caf3b22c29"; #good #rev = "bbff53fe3b6cdd3c9bc084d489640d7ee2a3f831"; #bad #rev = "8b99a7f7d8294be29e18a667d51e13755ed2c0e0"; #good #rev = "bdde2ecd27af1ac158669809f6658376fb5137ab"; #good #rev = "5cf9a540164a018a31a679578a27eb964af0340d"; #good #rev = "2a04f7037317c6c5b591b160a074c700de9b3378"; #bad rev = "40d36a6673131e36075b1df78af4d7ab92e8cc01"; #bad }; --- lagarto-ox.nix | 10 ++-------- ox-alveo-platform-plic.patch | 3 ++- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index b160dda..3e89bc2 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -189,19 +189,13 @@ echo 'booti ''${kernel_addr_r} ''${ramdisk_addr_r}:''${ramdisk_size} ''${fdtcontroladdr}' >> $out ''; - # Lagarto OX doesn't work with newer versions of OpenSBI (1.3, 1.4), not - # sure why. opensbi = prev.opensbi.overrideAttrs (old: rec { - version = "1.2"; # ok - #version = "1.3"; # bad - #version = "1.4"; # bad + version = "1.4"; src = prev.fetchFromGitHub { owner = "riscv-software-src"; repo = "opensbi"; rev = "v${version}"; - hash = "sha256-Zcl+SE2nySMycV/ozsl4AvGipRsMblA5mt3oVZ81Z44="; #1.2 - #hash = "sha256-Dr16fVUGLYGnGYHkjAyqpJxt8p95F0CJIU9ESGWKGWo="; #1.3 - #hash = "sha256-T8ZeAzjM9aeTXitjE7s+m+jjGGtDo2jK1qO5EuKiVLU="; #1.4 + hash = "sha256-T8ZeAzjM9aeTXitjE7s+m+jjGGtDo2jK1qO5EuKiVLU="; }; #NIX_DEBUG=5; makeFlags = [ diff --git a/ox-alveo-platform-plic.patch b/ox-alveo-platform-plic.patch index d906a6f..cd54047 100644 --- a/ox-alveo-platform-plic.patch +++ b/ox-alveo-platform-plic.patch @@ -42,7 +42,7 @@ new file mode 100644 index 0000000..a359b34 --- /dev/null +++ b/platform/fpga/ox_alveo/platform.c -@@ -0,0 +1,121 @@ +@@ -0,0 +1,122 @@ +#include +#include +#include @@ -162,5 +162,6 @@ index 0000000..a359b34 + .features = SBI_PLATFORM_DEFAULT_FEATURES, + .hart_count = OX_ALVEO_HART_COUNT, + .hart_stack_size = SBI_PLATFORM_DEFAULT_HART_STACK_SIZE, ++ .heap_size = SBI_PLATFORM_DEFAULT_HEAP_SIZE(OX_ALVEO_HART_COUNT), + .platform_ops_addr = (unsigned long)&ox_alveo_ops +}; -- GitLab From 917cbf3439d49ffbab0007184fcf541d42487910 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 1 Jul 2024 14:53:08 +0200 Subject: [PATCH 022/310] Revert to upstream busybox --- lagarto-ox.nix | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 3e89bc2..78220fe 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -97,12 +97,12 @@ nixpkgs.overlays = [ (final: prev: { # FIXME: Fails on start - busybox = (prev.busybox.override { - enableStatic = true; - }).overrideAttrs (old: { - #env.NIX_CFLAGS_COMPILE = "-mtune=generic"; - #env.NIX_DEBUG = 5; - }); +# busybox = (prev.busybox.override { +# enableStatic = true; +# }).overrideAttrs (old: { +# #env.NIX_CFLAGS_COMPILE = "-mtune=generic"; +# #env.NIX_DEBUG = 5; +# }); linuxPackages_latest = prev.linuxPackages_latest; #linuxPackages_latest = prev.linuxPackages_latest.extend (lib.const (ksuper: { -- GitLab From 046f017b703b1f7c9b185b2ceda07701858e2f3f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 1 Jul 2024 14:53:37 +0200 Subject: [PATCH 023/310] Move the pmem closer to the RAM memory Continues stuck in switch_root. --- fpga/env.sh | 2 +- ox-plic.dts | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/fpga/env.sh b/fpga/env.sh index b74331a..580e3bd 100644 --- a/fpga/env.sh +++ b/fpga/env.sh @@ -41,7 +41,7 @@ function setup_meep() export FPGACTL_BOOTLOADER_ADDR=$((0x80000000+$delta_addr)) export FPGACTL_KERNEL_ADDR=$((0x84000000+$delta_addr)) export FPGACTL_INITRD_ADDR=$((0x8c300000+$delta_addr)) - export FPGACTL_ROOTFS_ADDR=$((0x1bff00000+$delta_addr)) + export FPGACTL_ROOTFS_ADDR=$((0x100000000+$delta_addr)) } hostname=$(hostname) diff --git a/ox-plic.dts b/ox-plic.dts index 2423b87..915fdec 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -4,9 +4,9 @@ #size-cells = <2>; /* 64 bits memory addresses */ compatible = "riscv,rv64i"; model = "Barcelona Supercomputing Center - Lagarto Ox (NixOS)"; - chosen { - bootargs = "earlyprintk ignore_loglevel earlycon=sbi console=hvc0 root=/dev/pmem0p1 ro init=/bin/bash"; - }; +// chosen { +// bootargs = "earlyprintk ignore_loglevel earlycon=sbi console=hvc0 root=/dev/pmem0p1 ro init=/bin/bash"; +// }; cpus { #address-cells = <1>; #size-cells = <0>; @@ -44,9 +44,10 @@ * * [0x0_6000_0000, 0x0_7000_0000) -> DMA pool (256 MiB) * [0x0_7000_0000, 0x0_8000_0000) -> DMA pool (256 MiB) - * [0x0_8000_0000, 0x0_7ff0_0000) -> RAM memory (~2 GiB) - * [0x0_7ff0_0000, 0x1_bff0_0000) -> Empty - * [0x1_bff0_0000, 0x2_8000_0000) -> PMEM (3 GiB) + * [0x0_8000_0000, 0x0_fff0_0000) -> RAM memory (2047 MiB) + * [0x0_fff0_0000, 0x1_0000_0000) -> Empty (1 MiB) + * [0x1_0000_0000, 0x1_c000_0000) -> PMEM (3072 MiB) + * [0x1_c000_0000, 0x2_8000_0000) -> Empty (3072 MiB) */ memory@80000000 { device_type = "memory"; @@ -71,14 +72,14 @@ // clock-frequency = <0x09502f90>; // phandle = <0x00000002>; // }; - pmem@1bff00000 { + pmem@100000000 { /* volatile; This property indicates that this region is * actually backed by non-persistent memory. This lets the OS * know that it may skip the cache flushes required to ensure * data is made persistent after a write. */ volatile; compatible = "pmem-region"; - reg = <0x1 0xbff00000 0x0 0xc0100000>; + reg = <0x1 0x00000000 0x0 0xc0000000>; }; soc { #address-cells = <0x00000002>; -- GitLab From 109dea5b4ed50a283833fbcc5a7937378efcf884 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 1 Jul 2024 18:00:54 +0200 Subject: [PATCH 024/310] Enable kernel and busybox debug Continues to hang in switch_root and no luck trying to send a magic sysrq via serial port. --- lagarto-ox.nix | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 78220fe..9350cd1 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -40,13 +40,25 @@ # No vector extensions + '' RISCV_ISA_V n - CONFIG_RISCV_ISA_V_DEFAULT_ENABLE n + RISCV_ISA_V_DEFAULT_ENABLE n '' # Debugging + '' + DEBUG_KERNEL y + DEBUG_MISC y DEBUG_WX y MAGIC_SYSRQ y + SYSRQ_SERIAL y + DEBUG_VM y + SOFTLOCKUP_DETECTOR y + SOFTLOCKUP_DETECTOR_INTR_STORM y + HARDLOCKUP_DETECTOR y + DETECT_HUNG_TASK y + WQ_WATCHDOG y + WQ_CPU_INTENSIVE_REPORT y + LOCK_TORTURE_TEST y '' + # RISCV_ISA_ZICBOM n ; } ]; @@ -95,14 +107,14 @@ }; nixpkgs.overlays = [ (final: prev: { - # FIXME: Fails on start -# busybox = (prev.busybox.override { + busybox = (prev.busybox.override { # enableStatic = true; -# }).overrideAttrs (old: { + }).overrideAttrs (old: { + src = /home/Computational/rarias/riscv/busybox-1.36.1; # #env.NIX_CFLAGS_COMPILE = "-mtune=generic"; # #env.NIX_DEBUG = 5; -# }); + }); linuxPackages_latest = prev.linuxPackages_latest; #linuxPackages_latest = prev.linuxPackages_latest.extend (lib.const (ksuper: { -- GitLab From 3229a3d194119265817a90f9bdb196c384328d51 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 3 Jul 2024 13:10:06 +0200 Subject: [PATCH 025/310] Ignore misc directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f8935b4..e2e5d7b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *result* nixos-riscv.qcow2 +misc/ -- GitLab From 93e83f52e82af21e1572601ea2d1c4a57dab2932 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 3 Jul 2024 13:21:04 +0200 Subject: [PATCH 026/310] Add busybox patch to see debug lines: initrd sh ok The switch_root command seems to hang in the execv() syscall. --- busybox-debug.patch | 68 +++++++++++++++++++++++++++++++++++++++++++++ lagarto-ox.nix | 12 +++----- 2 files changed, 72 insertions(+), 8 deletions(-) create mode 100644 busybox-debug.patch diff --git a/busybox-debug.patch b/busybox-debug.patch new file mode 100644 index 0000000..7e171c3 --- /dev/null +++ b/busybox-debug.patch @@ -0,0 +1,68 @@ +Only in busybox-1.36.1-mod: tags +diff -up -r busybox-1.36.1/util-linux/switch_root.c busybox-1.36.1-mod/util-linux/switch_root.c +--- busybox-1.36.1/util-linux/switch_root.c 2021-09-30 00:04:47.000000000 +0200 ++++ busybox-1.36.1-mod/util-linux/switch_root.c 2024-07-01 16:08:28.336541504 +0200 +@@ -181,6 +181,8 @@ int switch_root_main(int argc UNUSED_PAR + unsigned dry_run = 0; + dev_t rootdev; + ++ printf("HELLO THIS IS SWITCH ROOT STARTING\n"); ++ + // Parse args. '+': stop at first non-option + if (ENABLE_SWITCH_ROOT && (!ENABLE_RUN_INIT || applet_name[0] == 's')) { + //usage:#define switch_root_trivial_usage +@@ -241,12 +243,15 @@ int switch_root_main(int argc UNUSED_PAR + if (stat("/init", &st) != 0 || !S_ISREG(st.st_mode)) { + bb_error_msg_and_die("'%s' is not a regular file", "/init"); + } ++ printf("SWITCH ROOT LINE %d OK\n", __LINE__); + statfs("/", &stfs); // this never fails ++ printf("SWITCH ROOT LINE %d OK\n", __LINE__); + if ((unsigned)stfs.f_type != RAMFS_MAGIC + && (unsigned)stfs.f_type != TMPFS_MAGIC + ) { + bb_simple_error_msg_and_die("root filesystem is not ramfs/tmpfs"); + } ++ printf("SWITCH ROOT LINE %d OK\n", __LINE__); + + if (!dry_run) { + // Zap everything out of rootdev +@@ -258,19 +263,26 @@ int switch_root_main(int argc UNUSED_PAR + bb_simple_perror_msg_and_die("error moving root"); + } + } ++ printf("SWITCH ROOT LINE %d OK\n", __LINE__); ++ printf("XCHROOT\n"); + xchroot("."); ++ printf("SWITCH ROOT LINE %d OK\n", __LINE__); + // The chdir is needed to recalculate "." and ".." links + /*xchdir("/"); - done in xchroot */ + + // If a new console specified, redirect stdin/stdout/stderr to it + if (console) { ++ printf("REDIRECTING CONSOLE\n"); ++ printf("SWITCH ROOT LINE %d OK\n", __LINE__); + int fd = open_or_warn(console, O_RDWR); + if (fd >= 0) { + xmove_fd(fd, 0); + xdup2(0, 1); + xdup2(0, 2); + } ++ printf("SWITCH ROOT LINE %d OK\n", __LINE__); + } ++ printf("SWITCH ROOT LINE %d OK\n", __LINE__); + + if (dry_run) { + // Does NEW_INIT look like it can be executed? +@@ -280,8 +292,11 @@ int switch_root_main(int argc UNUSED_PAR + if (access(argv[0], X_OK) == 0) + return 0; + } else { ++ printf("SWITCH ROOT LINE %d OK\n", __LINE__); ++ printf("LAUNCHING EXECV\n"); + // Exec NEW_INIT + execv(argv[0], argv); ++ printf("RETURNED FROM EXECV???\n"); + } + bb_perror_msg_and_die("can't execute '%s'", argv[0]); + } diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 9350cd1..5c03360 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -107,14 +107,10 @@ }; nixpkgs.overlays = [ (final: prev: { - # FIXME: Fails on start - busybox = (prev.busybox.override { -# enableStatic = true; - }).overrideAttrs (old: { - src = /home/Computational/rarias/riscv/busybox-1.36.1; -# #env.NIX_CFLAGS_COMPILE = "-mtune=generic"; -# #env.NIX_DEBUG = 5; - }); + #busybox = prev.busybox.overrideAttrs (old: { + # # Print some debug lines on switch_root to see where it hangs. + # patches = (old.patches or []) ++ [ ./busybox-debug.patch ]; + #}); linuxPackages_latest = prev.linuxPackages_latest; #linuxPackages_latest = prev.linuxPackages_latest.extend (lib.const (ksuper: { -- GitLab From 2442bdf3f54a191b426658c8b13ffd6827a10905 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 3 Jul 2024 13:42:27 +0200 Subject: [PATCH 027/310] Disable lock torture test --- lagarto-ox.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 5c03360..14a0dec 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -56,7 +56,6 @@ DETECT_HUNG_TASK y WQ_WATCHDOG y WQ_CPU_INTENSIVE_REPORT y - LOCK_TORTURE_TEST y '' # RISCV_ISA_ZICBOM n ; -- GitLab From 7c25f76f8b9dc4a96e24e1bd7fa6571d7850b1a3 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 3 Jul 2024 13:58:18 +0200 Subject: [PATCH 028/310] Bake the git commit inside the uboot env file --- lagarto-ox.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 14a0dec..fbe3da0 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -1,4 +1,4 @@ -{ config, lib, pkgs, modulesPath, ... }: +{ config, lib, pkgs, modulesPath, self, ... }: { imports = [ @@ -175,8 +175,12 @@ uboot-env = let init = "${config.system.build.toplevel}/init"; initrd = "${config.system.build.initialRamdisk}/initrd"; + rev = if self ? rev then self.rev + else throw ("Refusing to build from a dirty Git tree!"); in prev.runCommand "uboot.txt" {} '' cat > $out < Date: Wed, 3 Jul 2024 14:01:40 +0200 Subject: [PATCH 029/310] Enable 8250 console driver --- lagarto-ox.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index fbe3da0..db51b7d 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -27,6 +27,12 @@ HVC_DRIVER y HVC_RISCV_SBI y '' + # Enable console driver + +'' + SERIAL_8250 y + SERIAL_8250_CONSOLE y + SERIAL_OF_PLATFORM y + '' # Allows regions of persistent memory to be described in the device-tree. + '' OF_PMEM y -- GitLab From a40414d08c8ec789832cba3585e57a55eca6753f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 3 Jul 2024 15:30:28 +0200 Subject: [PATCH 030/310] Add comment about the axi timer This timer is present and initialized in OpenSBI and seems to drive the UART device. Not sure if we need to see it from the kernel. --- ox-plic.dts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ox-plic.dts b/ox-plic.dts index 915fdec..11c801c 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -170,5 +170,18 @@ /*<&SERIAL 0x3>, <&SERIAL 0x7>*/ compatible = "riscv,clint0"; }; + + /* Guesswork: There must be a timer at 0x40170000 as it is + * initialized in OpenSBI. It seems to drive the console. */ + //axi_timer: timer@40170000 { + // clock-frequency = <100000000>; + // clocks = <&clk_bus_0>; + // compatible = "xlnx,xps-timer-1.00.a"; + // interrupt-parent = <&axi_intc_1>; + // interrupts = <2 2>; + // reg = <0x41c00000 0x10000>; + // xlnx,count-width = <0x20>; + // xlnx,one-timer-only = <0x0>; + //}; }; }; -- GitLab From 30703d1715e2a2ef184321863e88fae255b53fef Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 3 Jul 2024 16:09:38 +0200 Subject: [PATCH 031/310] Use the ttyS0 device for the console For now it seems to be outputting the log properly, but it doesn't read anything in the initrd input console. --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index db51b7d..95c43f2 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -196,7 +196,7 @@ fdt set /memory@80000000 reg <0x0 0x80000000 0x0 0xc0000000> # Set kernel options - setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi boot.trace console=hvc0 init=${init}" + setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi boot.trace console=ttyS0,115200n8 init=${init}" EOF -- GitLab From 0c4311e15c52a66ca70d1150e7cd51c9451c3706 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 3 Jul 2024 18:41:52 +0200 Subject: [PATCH 032/310] Remap interrupts to avoid 0 and duplicates Let see if we can guess which is the correct number for the interrupts. The plic should appear in the IRQ list but currently it doesn't. --- ox-plic.dts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ox-plic.dts b/ox-plic.dts index 11c801c..8e8d30d 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -89,9 +89,10 @@ SERIAL: serial@40001000 { compatible = "ns16550"; reg = <0x0 0x40001000 0x0 0x100>; - interrupts = <0>; /* Output interrupt 0 */ + interrupts = <1>; /* Output interrupt 1 */ interrupt-parent = <&PLIC>; reg-shift = <2>; + /* This clock is the SERIAL_CLK */ clock-frequency = <50000000>; current-speed = <115200>; status = "okay"; @@ -146,7 +147,7 @@ interrupt-controller; /* Receives interrupts */ #interrupt-cells = <1>; /* Sends interrupts to HART interrupt controllers */ - interrupts-extended = <&HLIC0 3 &HLIC0 7>; + interrupts-extended = <&HLIC0 2>; reg = < 0x0 0x40800000 0x0 0x00400000>; riscv,ndev = <0x3>; riscv,max-priority = <0x7>; @@ -173,7 +174,7 @@ /* Guesswork: There must be a timer at 0x40170000 as it is * initialized in OpenSBI. It seems to drive the console. */ - //axi_timer: timer@40170000 { + //SERIAL_CLK: timer@40170000 { // clock-frequency = <100000000>; // clocks = <&clk_bus_0>; // compatible = "xlnx,xps-timer-1.00.a"; -- GitLab From 4461686e80ffdab8212aa2cb58b0b7e502a8601a Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 3 Jul 2024 18:45:26 +0200 Subject: [PATCH 033/310] Add dev shell without rootfs Makes iteration quicker. --- flake.nix | 6 ++++++ fpga/upload.sh | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index cef84f7..e6590f5 100644 --- a/flake.nix +++ b/flake.nix @@ -132,5 +132,11 @@ echo " UBOOT_ENV = $UBOOT_ENV" ''; }; + + devShells.x86_64-linux.lagarto-ox-rd = + self.outputs.devShells.x86_64-linux.lagarto-ox.overrideAttrs (old:{ + TOPLEVEL = ""; + ROOTFS = ""; + }); }; } diff --git a/fpga/upload.sh b/fpga/upload.sh index d64bfa7..783b0ef 100755 --- a/fpga/upload.sh +++ b/fpga/upload.sh @@ -16,7 +16,11 @@ rsync -a fpga/env.sh "$dst" rsync $OPENSBI/share/opensbi/*/fpga/*/firmware/fw_payload.bin "$dst/opensbi.bin" rsync "$KERNEL/Image" "$dst/kernel.bin" rsync "$INITRD/initrd" "$dst/initrd.bin" -rsync "$ROOTFS/sd-image/rootfs.img" "$dst/rootfs.img" +if [ -n "$ROOTFS" ]; then + rsync "$ROOTFS/sd-image/rootfs.img" "$dst/rootfs.img" +else + echo "Skipping rootfs" +fi rsync "$UBOOT_ENV" "$dst/uboot.env" echo "Now go to $dst and run ./boot.sh" -- GitLab From 427859818c9d10cbf38b24cda2c3f6e63fd0a8bf Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 3 Jul 2024 19:31:30 +0200 Subject: [PATCH 034/310] Disable SMP The IPI extension is being used to multiplex IPI interruptions and we don't need it as we only have one CPU. --- lagarto-ox.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 95c43f2..18b024b 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -63,7 +63,10 @@ WQ_WATCHDOG y WQ_CPU_INTENSIVE_REPORT y '' - # RISCV_ISA_ZICBOM n + # Disable SMP so we don't have IPI + + '' + SMP n + '' ; } ]; -- GitLab From b1755354d022ffdba5d57892a3a69d9fc650923c Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 3 Jul 2024 19:35:23 +0200 Subject: [PATCH 035/310] Add support for ftrace at boot time --- lagarto-ox.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 18b024b..edbcd9b 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -62,6 +62,8 @@ DETECT_HUNG_TASK y WQ_WATCHDOG y WQ_CPU_INTENSIVE_REPORT y + TRACING y + BOOTTIME_TRACING y '' # Disable SMP so we don't have IPI + '' -- GitLab From 6531fd678c5c41ed56be70bbede8a2a49229113d Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 3 Jul 2024 20:39:17 +0200 Subject: [PATCH 036/310] WIP: Test M and S ext interrupt --- ox-plic.dts | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/ox-plic.dts b/ox-plic.dts index 8e8d30d..cae6f1c 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -145,12 +145,32 @@ PLIC: plic@40800000 { compatible = "riscv,plic0"; interrupt-controller; /* Receives interrupts */ + #address-cells = <0>; #interrupt-cells = <1>; /* Sends interrupts to HART interrupt controllers */ - interrupts-extended = <&HLIC0 2>; + + /* + * From: linux-6.6.1/arch/riscv/include/asm/csr.h + * + * Interrupt causes (minus the high bit) + * #define IRQ_S_SOFT 1 + * #define IRQ_VS_SOFT 2 + * #define IRQ_M_SOFT 3 + * #define IRQ_S_TIMER 5 + * #define IRQ_VS_TIMER 6 + * #define IRQ_M_TIMER 7 + * #define IRQ_S_EXT 9 + * #define IRQ_VS_EXT 10 + * #define IRQ_M_EXT 11 + * #define IRQ_S_GEXT 12 + * #define IRQ_PMU_OVF 13 + * #define IRQ_LOCAL_MAX (IRQ_PMU_OVF + 1) + * #define IRQ_LOCAL_MASK GENMASK((IRQ_LOCAL_MAX - 1), 0) + */ + interrupts-extended = <&HLIC0 9>, <&HLIC0 11>; reg = < 0x0 0x40800000 0x0 0x00400000>; - riscv,ndev = <0x3>; - riscv,max-priority = <0x7>; + riscv,ndev = <3>; + //riscv,max-priority = <0x7>; phandle = <0x3>; }; /* Core Local Interruptor: It directly connects to the timer and -- GitLab From 48263969621a95703a96a02011c1592da4d82d57 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 4 Jul 2024 11:39:35 +0200 Subject: [PATCH 037/310] Try enabling polling in the serial console --- lagarto-ox.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index edbcd9b..ed82f13 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -32,6 +32,7 @@ SERIAL_8250 y SERIAL_8250_CONSOLE y SERIAL_OF_PLATFORM y + CONSOLE_POLL y '' # Allows regions of persistent memory to be described in the device-tree. + '' -- GitLab From 471207c64a0fc68dd16d759b7798ef9f686b2131 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 4 Jul 2024 12:45:41 +0200 Subject: [PATCH 038/310] Allow access to all memory from userspace > If this option is disabled, you allow userspace (root) access to all > of memory, including kernel and userspace memory Currently is failing: ~ # cat /proc/iomem 40001000-400010ff : serial 60000000-7fffffff : Reserved 80000000-ffefffff : System RAM 80201000-81fa0b87 : Kernel image 80201000-80cb177f : Kernel code 81400000-819fffff : Kernel rodata 81c00000-81f18747 : Kernel data 81f19000-81fa0b87 : Kernel bss 100000000-1bfffffff : namespace0.0 ~ # devmem 0x40001000 devmem: mmap: Operation not permitted --- lagarto-ox.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index ed82f13..0e3db6a 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -65,6 +65,7 @@ WQ_CPU_INTENSIVE_REPORT y TRACING y BOOTTIME_TRACING y + STRICT_DEVMEM n '' # Disable SMP so we don't have IPI + '' -- GitLab From d5b5cc336360db3978d795504fe7d67f88eb25a3 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 4 Jul 2024 16:46:35 +0200 Subject: [PATCH 039/310] Change the order of interrupts-extended for PLIC In the SiFive DTS they are in this order, not sure it that may cause any difference. --- ox-plic.dts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ox-plic.dts b/ox-plic.dts index cae6f1c..af572d7 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -167,7 +167,7 @@ * #define IRQ_LOCAL_MAX (IRQ_PMU_OVF + 1) * #define IRQ_LOCAL_MASK GENMASK((IRQ_LOCAL_MAX - 1), 0) */ - interrupts-extended = <&HLIC0 9>, <&HLIC0 11>; + interrupts-extended = <&HLIC0 11>, <&HLIC0 9>; reg = < 0x0 0x40800000 0x0 0x00400000>; riscv,ndev = <3>; //riscv,max-priority = <0x7>; -- GitLab From fcf4977a65a7b71c5aaaa4932d16f8c98318035f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 4 Jul 2024 16:47:56 +0200 Subject: [PATCH 040/310] Extend the serial range from 0x100 to 0x1000 The AXI UART 16550 v2.0 from Xilinx only seem to require 32 bytes for the registes, but let's reduce the changes with the original DTS. --- ox-plic.dts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ox-plic.dts b/ox-plic.dts index af572d7..439e34b 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -88,7 +88,7 @@ ranges; SERIAL: serial@40001000 { compatible = "ns16550"; - reg = <0x0 0x40001000 0x0 0x100>; + reg = <0x0 0x40001000 0x0 0x1000>; interrupts = <1>; /* Output interrupt 1 */ interrupt-parent = <&PLIC>; reg-shift = <2>; -- GitLab From eb7679f6a23500d1bf4619d1af9a31491deb0b72 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 4 Jul 2024 17:22:43 +0200 Subject: [PATCH 041/310] Prepare device tree to accomodate another UART --- ox-plic.dts | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/ox-plic.dts b/ox-plic.dts index 439e34b..139ba51 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -4,6 +4,12 @@ #size-cells = <2>; /* 64 bits memory addresses */ compatible = "riscv,rv64i"; model = "Barcelona Supercomputing Center - Lagarto Ox (NixOS)"; + + aliases { + serial0 = &uart_console; // ttyS0 + serial1 = &uart_testing; // ttyS1 + }; + // chosen { // bootargs = "earlyprintk ignore_loglevel earlycon=sbi console=hvc0 root=/dev/pmem0p1 ro init=/bin/bash"; // }; @@ -86,14 +92,29 @@ #size-cells = <0x00000002>; compatible = "BSC,Lagarto-ox-soc", "simple-bus"; ranges; - SERIAL: serial@40001000 { + + /* For bitstream e97dd7b2-397f-11ef-abe0-bbd201a5a630 with two + * consoles */ + + /* The serial for the kernel console */ + uart_console: serial@40001000 { compatible = "ns16550"; reg = <0x0 0x40001000 0x0 0x1000>; + reg-shift = <2>; + /* No interrupts for this UART, use console=hvc0 */ + /* This clock is the SERIAL_CLK */ + clock-frequency = <25000000>; + current-speed = <115200>; + status = "okay"; + }; + /* The serial for interrupt tests */ + uart_testing: serial@40003000 { + compatible = "ns16550"; + reg = <0x0 0x40003000 0x0 0x1000>; interrupts = <1>; /* Output interrupt 1 */ interrupt-parent = <&PLIC>; reg-shift = <2>; - /* This clock is the SERIAL_CLK */ - clock-frequency = <50000000>; + clock-frequency = <25000000>; current-speed = <115200>; status = "okay"; }; -- GitLab From 9b37e2aed02a03b2513b1eeaf874ea49e61abb46 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 4 Jul 2024 17:27:43 +0200 Subject: [PATCH 042/310] Use the hvc0 console for the kernel --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 0e3db6a..24e647f 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -203,7 +203,7 @@ fdt set /memory@80000000 reg <0x0 0x80000000 0x0 0xc0000000> # Set kernel options - setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi boot.trace console=ttyS0,115200n8 init=${init}" + setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi boot.trace console=hvc0 init=${init}" EOF -- GitLab From d541462b4c64a19b4d414c44384f5019a42c0371 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 4 Jul 2024 17:28:15 +0200 Subject: [PATCH 043/310] Stop in the stage 1 debug shell --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 24e647f..83da2dc 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -203,7 +203,7 @@ fdt set /memory@80000000 reg <0x0 0x80000000 0x0 0xc0000000> # Set kernel options - setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi boot.trace console=hvc0 init=${init}" + setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi boot.trace console=hvc0 debug1 init=${init}" EOF -- GitLab From 7eeb16ecfb566a82c7cde8c4ed63a05284f4ce2a Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 4 Jul 2024 17:30:57 +0200 Subject: [PATCH 044/310] Reduce frequency of UART clock to 25 MHz --- ox-alveo-platform-plic.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ox-alveo-platform-plic.patch b/ox-alveo-platform-plic.patch index cd54047..5f13c3b 100644 --- a/ox-alveo-platform-plic.patch +++ b/ox-alveo-platform-plic.patch @@ -55,7 +55,7 @@ index 0000000..a359b34 + +#define OX_ALVEO_UART_BASE_ADDR 0x40000000 +#define OX_ALVEO_UART_OFFSET 0x1000 -+#define OX_ALVEO_UART_INPUT_FREQ 50000000 ++#define OX_ALVEO_UART_INPUT_FREQ 25000000 +#define OX_ALVEO_UART_BAUDRATE 115200 +#define OX_ALVEO_PLIC_ADDR 0x40800000 +#define OX_ALVEO_PLIC_NUM_SOURCES 3 -- GitLab From 05898c5f853866ffd7c2255969bf56a14e43e361 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 4 Jul 2024 17:57:26 +0200 Subject: [PATCH 045/310] Revert UART speed to 50MHz following vivado log > UART Name g_UART IntfLabel m_axi_uart SyncClk {Label CLK0 Freq 50000000 > Name clk_i} BaseAddr 0x0 Number 2 AxiAddrWidth 0 AxiDataWidth 0 AxiIdWidth > 0 AxiUserWidth 0 Mode normal IRQ uart_irq --- ox-alveo-platform-plic.patch | 2 +- ox-plic.dts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ox-alveo-platform-plic.patch b/ox-alveo-platform-plic.patch index 5f13c3b..cd54047 100644 --- a/ox-alveo-platform-plic.patch +++ b/ox-alveo-platform-plic.patch @@ -55,7 +55,7 @@ index 0000000..a359b34 + +#define OX_ALVEO_UART_BASE_ADDR 0x40000000 +#define OX_ALVEO_UART_OFFSET 0x1000 -+#define OX_ALVEO_UART_INPUT_FREQ 25000000 ++#define OX_ALVEO_UART_INPUT_FREQ 50000000 +#define OX_ALVEO_UART_BAUDRATE 115200 +#define OX_ALVEO_PLIC_ADDR 0x40800000 +#define OX_ALVEO_PLIC_NUM_SOURCES 3 diff --git a/ox-plic.dts b/ox-plic.dts index 139ba51..018c2e8 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -103,7 +103,7 @@ reg-shift = <2>; /* No interrupts for this UART, use console=hvc0 */ /* This clock is the SERIAL_CLK */ - clock-frequency = <25000000>; + clock-frequency = <50000000>; current-speed = <115200>; status = "okay"; }; @@ -114,7 +114,7 @@ interrupts = <1>; /* Output interrupt 1 */ interrupt-parent = <&PLIC>; reg-shift = <2>; - clock-frequency = <25000000>; + clock-frequency = <50000000>; current-speed = <115200>; status = "okay"; }; -- GitLab From 48688594bfb8568e9146dcd175c63e770adb73a2 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 5 Jul 2024 12:23:07 +0200 Subject: [PATCH 046/310] Add a heartbeat counter to check the kernel --- lagarto-ox.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 83da2dc..1eddbdc 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -79,6 +79,15 @@ # Avoid zstd as we don't have the tools in "cucu" machine compressor = "gzip"; kernelModules = [ ]; + + # Write a counter to the DMA region, so we can check the kernel is not + # dead. Monitor from the host with: + # while [ 1 ]; do xxd -s $((0x1bfff0000 - 0x60000000)) \ + # -l 4 /dev/qdma34000-MM-1; sleep 0.2; done + preDeviceCommands = '' + echo "Creating a heartbeat counter at 0x1bfff0000" + hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done & + ''; }; loader = { -- GitLab From 504c4aca3cc4be04acaf6fd429d4f60c99615c88 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 5 Jul 2024 12:56:33 +0200 Subject: [PATCH 047/310] Make the counter silent by spawning another shell --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 1eddbdc..0a96131 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -86,7 +86,7 @@ # -l 4 /dev/qdma34000-MM-1; sleep 0.2; done preDeviceCommands = '' echo "Creating a heartbeat counter at 0x1bfff0000" - hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done & + sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' & ''; }; -- GitLab From f509a0a51732bcbdd8fafd07db0a3d5553776256 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 5 Jul 2024 15:09:59 +0200 Subject: [PATCH 048/310] Enable ftrace at boot For now we only show init calls. --- lagarto-ox.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 0a96131..319cb5c 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -212,7 +212,8 @@ fdt set /memory@80000000 reg <0x0 0x80000000 0x0 0xc0000000> # Set kernel options - setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi boot.trace console=hvc0 debug1 init=${init}" + setenv debugargs "debug1 trace_options=sym-addr trace_event=initcall:* tp_printk trace_buf_size=1M" + setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi boot.trace console=hvc0 \''${debugargs} init=${init}" EOF -- GitLab From 934b67ac0b31fe7a349cb73aeb45e54bd67c5a48 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 5 Jul 2024 15:15:53 +0200 Subject: [PATCH 049/310] Trace sched events from stage1 Events from the kthread that dumps the buffer to the console are filtered, otherwise we make an infinite loop. --- lagarto-ox.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 319cb5c..2da2c5e 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -87,6 +87,16 @@ preDeviceCommands = '' echo "Creating a heartbeat counter at 0x1bfff0000" sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' & + '' + + + # Exclude the second pid, which is the kthread that will dump the trace to + # the console, otherwise we live lock the kernel. Then enable the sched + # events. + '' + echo "Exclude pid 2 from sched" + echo '(prev_pid != 2 && next_pid != 2)' > /sys/kernel/debug/tracing/events/sched/filter + echo "Enable all events from sched subsystem" + echo 1 > /sys/kernel/debug/tracing/events/sched/enable ''; }; -- GitLab From bc05dc267665553d1580693d578e9b2c2f7141cd Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 5 Jul 2024 15:24:43 +0200 Subject: [PATCH 050/310] Mount debugfs first --- lagarto-ox.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 2da2c5e..67cfb5a 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -93,6 +93,9 @@ # the console, otherwise we live lock the kernel. Then enable the sched # events. '' + echo "Mount debugfs" + mkdir -p /sys/kernel/debug/ + mount -t debugfs none /sys/kernel/debug/ echo "Exclude pid 2 from sched" echo '(prev_pid != 2 && next_pid != 2)' > /sys/kernel/debug/tracing/events/sched/filter echo "Enable all events from sched subsystem" -- GitLab From 32f4d117dbed958c00378d13848201b198939578 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 5 Jul 2024 15:37:50 +0200 Subject: [PATCH 051/310] Only enable sched_switch events As sched_stat_runtime is flooding the console. --- lagarto-ox.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 67cfb5a..dfd4f44 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -90,16 +90,16 @@ '' + # Exclude the second pid, which is the kthread that will dump the trace to - # the console, otherwise we live lock the kernel. Then enable the sched - # events. + # the console, otherwise we live lock the kernel. Then enable the + # sched_switch events. '' echo "Mount debugfs" mkdir -p /sys/kernel/debug/ mount -t debugfs none /sys/kernel/debug/ echo "Exclude pid 2 from sched" echo '(prev_pid != 2 && next_pid != 2)' > /sys/kernel/debug/tracing/events/sched/filter - echo "Enable all events from sched subsystem" - echo 1 > /sys/kernel/debug/tracing/events/sched/enable + echo "Enable sched_switch events" + echo 1 > /sys/kernel/debug/tracing/events/sched/sched_switch/enable ''; }; -- GitLab From 7e6fdff142d76e883168a7ec5297e8cb13965b4f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 5 Jul 2024 15:57:18 +0200 Subject: [PATCH 052/310] Disable proactive compation I see a lot of kcompactd0 entries in sched_switch log, so disable it for now. --- lagarto-ox.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index dfd4f44..f8f8a65 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -89,6 +89,11 @@ sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' & '' + + # Disable proactive compaction. May be better to disable CONFIG_COMPACTION. + '' + echo 0 > /proc/sys/vm/compaction_proactiveness + '' + + # Exclude the second pid, which is the kthread that will dump the trace to # the console, otherwise we live lock the kernel. Then enable the # sched_switch events. -- GitLab From b903eae7e5c0db350c9d0cb079db40a2b83d808e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 5 Jul 2024 16:23:09 +0200 Subject: [PATCH 053/310] Disable ftrace for now as it hangs the boot --- lagarto-ox.nix | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index f8f8a65..0aeb392 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -93,19 +93,21 @@ '' echo 0 > /proc/sys/vm/compaction_proactiveness '' - + - # Exclude the second pid, which is the kthread that will dump the trace to - # the console, otherwise we live lock the kernel. Then enable the - # sched_switch events. - '' - echo "Mount debugfs" - mkdir -p /sys/kernel/debug/ - mount -t debugfs none /sys/kernel/debug/ - echo "Exclude pid 2 from sched" - echo '(prev_pid != 2 && next_pid != 2)' > /sys/kernel/debug/tracing/events/sched/filter - echo "Enable sched_switch events" - echo 1 > /sys/kernel/debug/tracing/events/sched/sched_switch/enable - ''; +# FIXME: Disable sched_switch for now, as it still hangs the boot... +# + +# # Exclude the second pid, which is the kthread that will dump the trace to +# # the console, otherwise we live lock the kernel. Then enable the +# # sched_switch events. +# '' +# echo "Mount debugfs" +# mkdir -p /sys/kernel/debug/ +# mount -t debugfs none /sys/kernel/debug/ +# echo "Exclude pid 2 from sched" +# echo '(prev_pid != 2 && next_pid != 2)' > /sys/kernel/debug/tracing/events/sched/filter +# echo "Enable sched_switch events" +# echo 1 > /sys/kernel/debug/tracing/events/sched/sched_switch/enable +# '' + ; }; loader = { -- GitLab From b49471ba505266aea9afd381030f54673d2b73df Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 5 Jul 2024 16:30:29 +0200 Subject: [PATCH 054/310] Update nixpkgs to nixos-unstable --- flake.nix | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/flake.nix b/flake.nix index e6590f5..69f22a1 100644 --- a/flake.nix +++ b/flake.nix @@ -1,28 +1,13 @@ { - inputs.nixpkgs.url = "github:NixOS/nixpkgs/e4ad989506ec7d71f7302cc3067abd82730a4beb"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; inputs.bscpkgs.url = "git+https://git.sr.ht/~rodarima/bscpkgs?ref=riscv-benchmarks"; # When changing bscpkgs, take directly from disk to avoid having to push: #inputs.bscpkgs.url = "/home/Computational/rarias/bscpkgs"; - outputs = {self,...}@inputs: + outputs = {self, nixpkgs, ...}@inputs: let system = "x86_64-linux"; - remoteNixpkgsPatches = [ - { - meta.description = "sha256-ZCDQ7SpGhH8JvAwWzdcyrc68RFEWHxxAj0M2+AvEzIg="; - url = "https://github.com/NixOS/nixpkgs/pull/283460.diff"; - sha256 = "sha256-g6o4rqkOOYZ6OJTzv9kTPq9Zsu5Z1QXZmPLC3Q7Sq6w="; - } - ]; - originPkgs = inputs.nixpkgs.legacyPackages."x86_64-linux"; - nixpkgs = originPkgs.applyPatches { - name = "nixpkgs-patched"; - src = inputs.nixpkgs; - patches = map originPkgs.fetchpatch remoteNixpkgsPatches; - }; nixosSystem = import (nixpkgs + "/nixos/lib/eval-config.nix"); - - in { #overlay = import ./overlay.nix; nixosConfigurations = { -- GitLab From 254d1bd82b214193f081521d6b37cc62b749f3c3 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 5 Jul 2024 16:31:08 +0200 Subject: [PATCH 055/310] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/e4ad989506ec7d71f7302cc3067abd82730a4beb' (2023-11-19) → 'github:NixOS/nixpkgs/9f4128e00b0ae8ec65918efeba59db998750ead6' (2024-07-03) --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index cf69c3c..cf746de 100644 --- a/flake.lock +++ b/flake.lock @@ -34,17 +34,17 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1700390070, - "narHash": "sha256-de9KYi8rSJpqvBfNwscWdalIJXPo8NjdIZcEJum1mH0=", + "lastModified": 1720031269, + "narHash": "sha256-rwz8NJZV+387rnWpTYcXaRNvzUSnnF9aHONoJIYmiUQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e4ad989506ec7d71f7302cc3067abd82730a4beb", + "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6", "type": "github" }, "original": { "owner": "NixOS", + "ref": "nixos-unstable", "repo": "nixpkgs", - "rev": "e4ad989506ec7d71f7302cc3067abd82730a4beb", "type": "github" } }, -- GitLab From d52691ca338b9d03598f2346efe1dc5440a7e9c3 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 5 Jul 2024 16:53:07 +0200 Subject: [PATCH 056/310] Add journal to the repository --- JOURNAL.md | 649 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 649 insertions(+) create mode 100644 JOURNAL.md diff --git a/JOURNAL.md b/JOURNAL.md new file mode 100644 index 0000000..e563135 --- /dev/null +++ b/JOURNAL.md @@ -0,0 +1,649 @@ +## 2024-07-02 + +Aleix provided some notes on how to trace the kernel using tracepoints that are +printed to the console: + +``` +BOOT TIME TRACING + - compile with CONFIG_BOOTTIME_TRACING=y + - add to kernel parameters: + trace_options=sym-addr trace_event=initcall:* tp_printk trace_buf_size=1M ftrace=function ftrace_filter="vfs*" + trace_options=sym-addr trace_event=sched:* tp_printk trace_buf_size=1M + - tp_printk sends tracepoint (TRACE_EVENT) to printk buffer + - trace_event=[subsytem:event]; accepts + - *: + - :* + - *:* all + - many more options at https://www.kernel.org/doc/html/latest/trace/boottime-trace.html +``` + +Start with: + +``` +trace_options=sym-addr trace_event=sched:* tp_printk trace_buf_size=1M loglevel=7 +``` + +Can it be caused by the D extension? If I set the ISA to: + +``` +riscv,isa = "rv64imaf"; +``` + +It locks the stage1 script without providing any output. Let see trying again. +Locked again. + +I can disable the FPU in the kernel, and then it will act as a detector of any +floating point instruction. + +Then I just need to rebuild busybox without support for double instructions. + +Let see if I can build busybox with double instruction + +Nope, the u-boot is reporting the d extension is in the isa: + +> riscv,isa = "rv64imafd"; + +## 2024-07-03 + +I cannot switch to `gcc.arch = rv64ima` because rust fails to build. + +**Assumption**: The extensions F and D work well and don't cause a hang in the +CPU. + +Let's go back and try to get the initrd shell, so we can systematically hang it +in the `switch_root` + +**Observation**: The riscv-timer seems to be causing interrupts with IRQ 5: + +``` +[ 62.439060] irq_handler_entry: irq=5 name=riscv-timer +[ 62.444980] irq_handler_exit: irq=5 ret=handled +``` + +**Observation**: Rohan reports the serial startup routine being running *after*{{{ +the init begins. + +**Observation**: Only interrupts in timer, others are zero. + +With: + + commit 4c656bd8ddd2e41ccaa976ff8d6bd9209175a632 (HEAD -> lagarto-ox) + Author: Rodrigo Arias Mallo + Date: Wed Jul 3 13:21:04 2024 +0200 + + Add busybox patch to see debug lines + + The switch_root command seems to hang in the execv() syscall. + +I can see this: + + ~ # cat /proc/interrupts ; sleep 10 ; cat /proc/interrupts + CPU0 + 10: 42926 RISC-V INTC 5 Edge riscv-timer + IPI0: 0 Rescheduling interrupts + IPI1: 0 Function call interrupts + IPI2: 0 CPU stop interrupts + IPI3: 0 CPU stop (for crash dump) interrupts + IPI4: 0 IRQ work interrupts + IPI5: 0 Timer broadcast interrupts + CPU0 + 10: 46023 RISC-V INTC 5 Edge riscv-timer + IPI0: 0 Rescheduling interrupts + IPI1: 0 Function call interrupts + IPI2: 0 CPU stop interrupts + IPI3: 0 CPU stop (for crash dump) interrupts + IPI4: 0 IRQ work interrupts + IPI5: 0 Timer broadcast interrupts + +**Observation**: There is a timer configured in 0x40170000 but in the device +tree we only have one at `timer@40002000`. + + #define OX_ALVEO_TIMER_BASE 0x40170000 + #define ADDR_TIME_L 0x0u // 32 lower bits of the time register + #define ADDR_TIME_H 0x1u // 32 higher bits of the time register + #define ADDR_TIMECMP_L 0x2u // 32 lower bits of the time comparator + #define ADDR_TIMECMP_H 0x3u // 32 higher bits of the time comparator + + https://gitlab.bsc.es/hwdesign/bsc-linux/-/blob/d6d194bd30d9a8fe49c2a278ffb3c3ae7852e75d/bsc_tree/patches/ox_alveo/opensbi/0001-opensbi-ox_alveo-platform.patch#L63 + +**Observation**: When the serial console starts, the speed of the serial port +changes to 9600: + + [ 6.845400] io scheduler mq-deadline registered + [ 6.851500] io scheduler kyber registered + [ 17.644460] Serial: 8250/16550 driver, 4 ports, IRQ sharing disabled + [ 18.141160] printk: console [ttyS0] disabled + [ 18.229480] 40001000.serial: ttyS0 at MMIO 0x40001000 (irq = 11, base_baud = 3125000) is a 16550 + + *** baud: 230400 *** + + *** baud: 460800 *** + + *** baud: 500000 *** + + *** baud: 576000 *** + + *** baud: 500000 *** + + *** baud: 460800 *** + + *** baud: 230400 *** + + *** baud: 115200 *** + + *** baud: 57600 *** + + *** baud: 38400 *** + + *** baud: 19200 *** + faaa0?xx + +**Observation**: Trying to read from the serial console /dev/ttyS0 causes no +more messages in the console (or a hang). + +**Question**: Can we make a heartbeat for the kernel? The idea is to keep a +counter in some memory of the kernel so we can see it from the host being moved. + +**Question**: Can we disable the serial driver 8250 from loading? + + initcall_blacklist= + +I need to know the 8250 init function name: + + drivers/tty/serial/8250/8250_core.c:static int __init serial8250_init(void) + +So... + + initcall_blacklist=serial8250_init + +Yes, but that doesn't seem to do anything. It is hanging: + + + modprobe dm_mod + [ 627.473580] stage-1-init: [Thu Jan 1 00:10:26 UTC 1970] + echo init /nix/store/v6pi2mqfgshxdsbyxlvpm9nvawxrpijv-nixos-system-nixos-riscv-23.11pre-git/init + [ 628.249440] stage-1-init: [Thu Jan 1 00:10:27 UTC 1970] + set -- init /nix/store/v6pi2mqfgshxdsbyxlvpm9nvawxrpijv-nixos-system-nixos-riscv-23.11pre-git/init + [ 629.004840] stage-1-init: [Thu Jan 1 00:10:28 UTC 1970] + stage2Init=/nix/store/v6pi2mqfgshxdsbyxlvpm9nvawxrpijv-nixos-system-nixos-riscv-23.11pre-git/init + [ 629.733920] stage-1-init: [Thu Jan 1 00:10:29 UTC 1970] + echo /nix/store/snvvqpxmryw1szlllk0bxpm37p8vj8sw-extra-utils/bin/modprobe + + +**Question**: What happens if we remap the interruptions? + +- Move the serial from 0 to 1 +- Move the plic from 3 to 2 and remove 7 + +Now we have one context only: + + [ 0.000000] riscv-intc: 64 local interrupts mapped + [ 0.000000] plic: plic@40800000: mapped 3 interrupts with 0 handlers for 1 contexts. + [ 0.000000] riscv: providing IPIs using SBI IPI extension + +Rather than two: + + [ 0.000000] riscv-intc: 64 local interrupts mapped + [ 0.000000] plic: plic@40800000: mapped 3 interrupts with 0 handlers for 2 contexts. + [ 0.000000] riscv: providing IPIs using SBI IPI extension + +**Question**: What happens if we block the `sbi_ipi` driver? + + initcall_blacklist=sbi_ipi_init + +Nothing, it cannot be disabled it seems. I will remove SMP support so it won't +be compiled in. + +**Observation**: Searching for 'riscv,plic0' only matches irq-sifive-plic driver. + + hut% rg 'riscv,plic0' + Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml + 72: - const: riscv,plic0 + + drivers/irqchip/irq-sifive-plic.c + 572:IRQCHIP_DECLARE(riscv_plic0, "riscv,plic0", plic_init); /* for legacy systems */ + +So it looks that the only driver that setups the plic is the one used by SiFive. +Here is the doc: https://static.dev.sifive.com/U54-MC-RVCoreIP.pdf + +**Observation**: The number of handlers is 0, so there are no interruptions. + +It seems the number next to the phandle of the interrupts-extended attribute in +the plic follows a different convention of values. Using 9 and 11: + + plic: plic@40800000: mapped 3 interrupts with 1 handlers for 2 contexts. + +**Remark**: The key combination to run Magic SysRq using the HVC console is +Ctrl-O and then the SysRq key. It only works it the console is being actively +polled, otherwise it hangs. + +## 2024-07-04 + +**Observation**: I saw they changed this option in Cinco Ranch DTS for the +serial: + +> reg-shift = <0>; // regs are spaced on 8 bit boundary (modified from Xilinx UART16550 to be ns16550 compatible) + +Tested booting with debug1 and the ttyS0 console, and it goes extremely slow +(but still outputs at 115200) and then continues to fail to read keyboard input. + +**Question**: Let's try setting the console in poll mode. + +setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=uart,io,0x40001000,115200n8 boot.trace console=uart,io,0x40001000,115200n8 debug1 init=/nix/store/wavmnv6wjj8y10ha07wxd5f0sqacivj8-nixos-system-nixos-riscv-23.11pre-git/init" + + [ 5.909360] io scheduler mq-deadline registered + [ 5.914900] io scheduler kyber registered + [ 14.405980] Serial: 8250/16550 driver, 4 ports, IRQ sharing disabled + [ 14.856040] 40001000.serial: ttyS0 at MMIO 0x40001000 (irq = 2, base_baud = 3125000) is a 16550 + [ 15.059680] nd_pmem namespace0.0: unable to guarantee persistence of writes + ... + [ 40.250100] clk: Disabling unused clocks + [ 40.256220] Warning: unable to open an initial console. <---- SEE THIS + [ 40.618300] Freeing unused kernel image (initmem) memory: 5592K + [ 40.728300] Checked W+X mappings: passed, no W+X pages found + [ 40.735540] Run /init as init process + [ 40.740080] with arguments: + [ 40.743920] /init + [ 40.746660] with environment: + [ 40.750740] HOME=/ + [ 40.754020] TERM=linux + [ 46.569960] Kernel panic - not syncing: Attempted to kill init! exitcode=0x00000100 + [ 46.578100] CPU: 0 PID: 1 Comm: init Not tainted 6.6.1 #1-NixOS + [ 46.584600] Hardware name: Barcelona Supercomputing Center - Lagarto Ox (NixOS) (DT) + [ 46.592740] Call Trace: + [ 46.595640] [] dump_backtrace+0x38/0x48 + [ 46.601760] [] show_stack+0x50/0x68 + [ 46.607540] [] dump_stack_lvl+0x3c/0x5c + [ 46.613660] [] dump_stack+0x20/0x30 + [ 46.619400] [] panic+0x158/0x374 + [ 46.624900] [] do_exit+0x9e8/0x9f0 + [ 46.630580] [] do_group_exit+0x44/0xb0 + [ 46.636600] [] __wake_up_parent+0x0/0x40 + [ 46.642780] [] do_trap_ecall_u+0x14c/0x168 + [ 46.649140] [] ret_from_exception+0x0/0xac + [ 46.655500] Kernel Offset: 0x0 from 0xffffffff80000000 + [ 46.661160] ---[ end Kernel panic - not syncing: Attempted to kill init! exitcode=0x00000100 ]--- + +Also found: `no_console_suspend` + +**Observation**: There are messages of address space being assigned to +registers: + + Slave segment '/MEEP_uart_0/S_AXI/Reg' is being assigned into address space '/m_axi_uart0' at <0x0000_0000 [ 4K ]>. + Slave segment '/MEEP_uart_1/S_AXI/Reg' is being assigned into address space '/m_axi_uart1' at <0x0000_0000 [ 4K ]>. + +**Question**: What happens if I enable `CONFIG_CONSOLE_POLL`? + +With `console=ttyS0,115200n8 debug1` I cannot type. + +**Observation**: I can dump iomem memory with the tool devmem: + +But it seems I cannot dump the registers of the serial io mapped region: + + ~ # cat /proc/iomem + 40001000-400010ff : serial + 60000000-7fffffff : Reserved + 80000000-ffefffff : System RAM + 80201000-81fa0b87 : Kernel image + 80201000-80cb177f : Kernel code + 81400000-819fffff : Kernel rodata + 81c00000-81f18747 : Kernel data + 81f19000-81fa0b87 : Kernel bss + 100000000-1bfffffff : namespace0.0 + + ~ # devmem 0x40001000 + devmem: mmap: Operation not permitted + +It looks like the following options may be required to be disabled to allow +user-space tools read those regions. + + ~ # zcat /proc/config.gz | grep CONFIG_STRICT_DEVMEM + CONFIG_STRICT_DEVMEM=y + ~ # zcat /proc/config.gz | grep CONFIG_IO_STRICT_DEVMEM + CONFIG_IO_STRICT_DEVMEM=y + +Let's try disabling `CONFIG_STRICT_DEVMEM` and see if we can read the serial +registers. + +It works! + + ~ # cat /proc/iomem + 40001000-400010ff : serial + 60000000-7fffffff : Reserved + 80000000-ffefffff : System RAM + 80201000-81fa0b87 : Kernel image + 80201000-80cb159f : Kernel code + 81400000-819fffff : Kernel rodata + 81c00000-81f18707 : Kernel data + 81f19000-81fa0b87 : Kernel bss + 100000000-1bfffffff : namespace0.0 + ~ # devmem 0x40001000 + 0x0000006E + ~ # devmem 0x40001000 + 0x0000006C + ~ # devmem 0x40001000 + 0x00000072 + ~ # devmem 0x40001000 + 0x0000000D + +**Observation**: The interrupt register of the serial console is 0x0: + +Assuming the console registers follow AXI UART 16550, here is the IER: + +> 0x1004 IER R/W Interrupt Enable Register + +Which is zero: + +~ # devmem 0x40001004 +0x00000000 + +The line control register is 0x3: + +~ # devmem 0x4000100C +0x00000003 + +**Question**: Can I write to some memory address and see the result from the +host? + +For that I would need to find some address that is mapped to the DMA or to the +pmem. Xavi recommended `0x6000_0000` as it is uncached. + +It seems to have some content already: + + ~ # devmem 0x60000000 + 0x00000093 + ~ # devmem 0x60000004 + 0x00000113 + ~ # devmem 0x60000008 + 0x00000193 + ~ # devmem 0x60000010 + 0x00000293 + ~ # devmem 0x60000014 + 0x00000313 + ~ # devmem 0x60000018 + 0x00000393 + +Writing test seems to work: + + ~ # devmem 0x60000000 32 0x11223344 + ~ # devmem 0x60000000 + 0x11223344 + +So, technically we should be using the `/dev/qdma34000-MM-0` device, as we +already use the other one to map the memory. + + [bsc015557@fpgan02 nixos]$ for i in {0..16}; do addr=$(($i * 0x10000000)); \ + printf "addr 0x%09x: " $addr; dd if=/dev/qdma34000-MM-0 count=16 bs=1 skip=$addr 2>/dev/null | xxd; done + addr 0x000000000: 00000000: 4444 4444 4444 4444 4444 4444 4444 4444 DDDDDDDDDDDDDDDD + addr 0x010000000: 00000000: 0000 0000 0000 0000 0000 0000 0000 0000 ................ + addr 0x020000000: 00000000: 3333 3333 3333 3333 3333 3333 3333 3333 3333333333333333 + addr 0x030000000: 00000000: cccc cccc cccc cccc cccc cccc cccc cccc ................ + addr 0x040000000: 00000000: cccc cccc cccc cccc cccc cccc cccc cccc ................ + addr 0x050000000: 00000000: 0000 0000 0000 0000 0000 0000 0000 0000 ................ + addr 0x060000000: 00000000: 0000 0000 0000 0000 0000 0000 0000 0000 ................ + addr 0x070000000: 00000000: 0000 0000 0000 0000 0000 0000 0000 0000 ................ + addr 0x080000000: 00000000: 3333 3333 3333 3333 3333 3333 3333 3333 3333333333333333 + addr 0x090000000: 00000000: 0000 0000 0000 0000 0000 0000 0000 0000 ................ + addr 0x0a0000000: 00000000: 0000 0000 0000 0000 0000 0000 0000 0000 ................ + addr 0x0b0000000: 00000000: 2f2f 2f2f 2f2f 2f2f 2f2f 2f2f 2f2f 2f2f //////////////// + addr 0x0c0000000: 00000000: 6e6e 6e6e 6e6e 6e6e 6e6e 6e6e 6e6e 6e6e nnnnnnnnnnnnnnnn + addr 0x0d0000000: 00000000: 2020 2020 2020 2020 2020 2020 2020 2020 + addr 0x0e0000000: 00000000: 6c6c 6c6c 6c6c 6c6c 6c6c 6c6c 6c6c 6c6c llllllllllllllll + addr 0x0f0000000: 00000000: 6767 6767 6767 6767 6767 6767 6767 6767 gggggggggggggggg + addr 0x100000000: 00000000: 2424 2424 2424 2424 2424 2424 2424 2424 $$$$$$$$$$$$$$$$ + + [bsc015557@fpgan02 nixos]$ for i in {0..16}; do addr=$(($i * 0x10000000)); \ + printf "addr 0x%09x: " $addr; dd if=/dev/qdma34000-MM-1 count=16 bs=1 skip=$addr 2>/dev/null | xxd; done + addr 0x000000000: 00000000: 4444 4444 4444 4444 4444 4444 4444 4444 DDDDDDDDDDDDDDDD + addr 0x010000000: 00000000: 0000 0000 0000 0000 0000 0000 0000 0000 ................ + addr 0x020000000: 00000000: 3333 3333 3333 3333 3333 3333 3333 3333 3333333333333333 + addr 0x030000000: 00000000: cccc cccc cccc cccc cccc cccc cccc cccc ................ + addr 0x040000000: 00000000: cccc cccc cccc cccc cccc cccc cccc cccc ................ + addr 0x050000000: 00000000: 0000 0000 0000 0000 0000 0000 0000 0000 ................ + addr 0x060000000: 00000000: 0000 0000 0000 0000 0000 0000 0000 0000 ................ + addr 0x070000000: 00000000: 0000 0000 0000 0000 0000 0000 0000 0000 ................ + addr 0x080000000: 00000000: 3333 3333 3333 3333 3333 3333 3333 3333 3333333333333333 + addr 0x090000000: 00000000: 0000 0000 0000 0000 0000 0000 0000 0000 ................ + addr 0x0a0000000: 00000000: 0000 0000 0000 0000 0000 0000 0000 0000 ................ + addr 0x0b0000000: 00000000: 2f2f 2f2f 2f2f 2f2f 2f2f 2f2f 2f2f 2f2f //////////////// + addr 0x0c0000000: 00000000: 6e6e 6e6e 6e6e 6e6e 6e6e 6e6e 6e6e 6e6e nnnnnnnnnnnnnnnn + addr 0x0d0000000: 00000000: 2020 2020 2020 2020 2020 2020 2020 2020 + addr 0x0e0000000: 00000000: 6c6c 6c6c 6c6c 6c6c 6c6c 6c6c 6c6c 6c6c llllllllllllllll + addr 0x0f0000000: 00000000: 6767 6767 6767 6767 6767 6767 6767 6767 gggggggggggggggg + addr 0x100000000: 00000000: 2424 2424 2424 2424 2424 2424 2424 2424 $$$$$$$$$$$$$$$$ + +None of the two queues seem to have the value 0x11223344 at any multiple of `0x1000_0000`. + +Let's verify first that this method works. The kernel is loaded here: + + [bsc015557@fpgan02 nixos]$ printf '0x%x\n' $FPGACTL_KERNEL_ADDR + 0x24000000 + +So we should see the same values as the kernel file: + + [bsc015557@fpgan02 nixos]$ dd if=kernel.bin count=16 bs=1 2>/dev/null| xxd + 00000000: 6f00 400d 0000 0000 0000 2000 0000 0000 o.@....... ..... + +But we don't see the same: + + [bsc015557@fpgan02 nixos]$ dd if=/dev/qdma34000-MM-1 count=16 bs=1 skip=$FPGACTL_KERNEL_ADDR 2>/dev/null | xxd + 00000000: 9797 9797 9797 9797 9797 9797 9797 9797 ................ + [bsc015557@fpgan02 nixos]$ dd if=/dev/qdma34000-MM-0 count=16 bs=1 skip=$FPGACTL_KERNEL_ADDR 2>/dev/null | xxd + 00000000: 9797 9797 9797 9797 9797 9797 9797 9797 ................ +**Question**: Missing forward M to S via Mideleg? + +Can it be happening that he MEDELEG is not forwarding the interruptions to the +Supervisor (kernel)? + + Boot HART MIDELEG : 0x0000000000000222 + Boot HART MEDELEG : 0x000000000000b109 + +**Question**: Can we add a timer to the PLIC to test the interrupts? + +**Observation**: Here is the PLIC register dump: + + ~ # for i in `seq 0 16`; do addr=$((0x40600000 + $i)); printf '%08x: ' $addr; devmem $addr; done + 40600000: 0x00010002 + 40600001: 0x09000000 + 40600002: 0x00090000 + 40600003: 0x00000900 + 40600004: 0x00010009 + 40600005: 0x00000000 + 40600006: 0x00000000 + 40600007: 0x00000000 + 40600008: 0x00000000 + 40600009: 0x00000000 + 4060000a: 0x00000000 + 4060000b: 0x00000000 + 4060000c: 0x00000000 + 4060000d: 0x00000000 + 4060000e: 0x00000000 + 4060000f: 0x00000000 + 40600010: 0x00000000 + +**Question**: Can we boot with the new bitstream that includes the second UART? + +The interruptions are enabled for the UART 1, not the default UART 0. + + +**Observation**: I'm using 0x100 not 0x1000 in the serial range: + + reg = <0x0 0x40003000 0x0 0x100>; + reg = <0x0 0x40003000 0x0 0x1000>; + +Can this produce any problem? + +It doesn't seem to change anything, still unable to send any bytes. + +**Question**: Can we use virtio to mount a FS in the DMA shared memory? + +## 2024-07-05 + +**Observation**: The kernel continues working when the console hangs. + +Switching to 0x100000000 as 0x60000000 shows: + + ~ # devmem 0x6000000 + 0xBADCAB1E + +With the following loop: + + ~ # i=0; while [ 1 ]; do let i=$i+1; devmem 0x100000000 32 $i; done & + ~ # cat /dev/ttyS0 + (hangs) + +Shows the kernel works: + + [bsc015557@fpgan02 nixos]$ while [ 1 ]; do xxd -s $((0x100000000 - 0x60000000)) -l 4 /dev/qdma34000-MM-1; sleep 0.2; done + ... + a0000000: 6400 0000 d... + a0000000: 6500 0000 e... + a0000000: 6600 0000 f... + a0000000: 6700 0000 g... + a0000000: 6800 0000 h... + a0000000: 6900 0000 i... + +**Question**: Can we reproduce it with `switch_root`? + +For that I would have to ensure the process continues to operate, even if we +exit the console. Maybe I can make a double fork? + +I cannot use `0x1000_0000` as that is where the pmem will be. But I can try to +use an address in the end, as we are not filling the whole space. + + [0x1_0000_0000, 0x1_c000_0000) -> PMEM (3072 MiB) + +Maybe `0x1_bfff_0000`? Let's try first from the initrd shell. + + i=0; while [ 1 ]; do let i=$i+1; devmem 0x1bfff0000 32 $i; done & + +Then + + while [ 1 ]; do xxd -s $((0x1bfff0000 - 0x60000000)) -l 4 /dev/qdma34000-MM-1; sleep 0.2; done + +Yes, it seems to be working. Let's load the rootfs too. + +I added a loop in the stage1 script. + +**Question**: Can we see any clock in memory? This will allow us to check if the +AXI still works. + +**Observation**: The kernel stops updating the counter in the mount phase. + +Managed to reach the mount and hang there: + + [ 337.504740] stage-1-init: [Thu Jan 1 00:05:36 UTC 1970] + '[' -d + /dev/disk/by-label/NIXOS_SD ] + [ 338.284560] stage-1-init: [Thu Jan 1 00:05:37 UTC 1970] + mkdir -m 0755 -p + /mnt-root/ + [ 339.017420] stage-1-init: [Thu Jan 1 00:05:38 UTC 1970] + local 'n=0' + [ 339.752560] stage-1-init: [Thu Jan 1 00:05:39 UTC 1970] + true + [ 340.488960] stage-1-init: [Thu Jan 1 00:05:39 UTC 1970] + mount /mnt-root/ + +After almost 6 minutes, with 571 beats: + + $ xxd -s 5905514496 -l 4 /dev/qdma34000-MM-1 + 5fff0000: 3b02 0000 ;... + +It looks like the kernel is the one getting stuck *or* at least is unable to +propagate the heartbeat changes to the host. It would be nice to monitor a +hardware clock from the DMA region too, so we can discard problems in the AXI. + + + [ 177.009540] stage-1-init: [Thu Jan 1 00:02:56 UTC 1970] + udevadm settle + + kbd_mode -u -C /dev/console + kbd_mode: KDSKBMODE: Inappropriate ioctl for device + + printf '\033%%G' + + loadkmap + [ 266.301040] stage-1-init: [Thu Jan 1 00:04:25 UTC 1970] + kbd_mode -u -C /dev/console + +**Assumption**: The kernel hangs. + +If the kernel hangs, there must be an instruction or sequence of instructions +that causes it. First I need to determine what is being executed by the kernel. +For that I could use `ftrace` to see which program is running at the time it +hangs. + + trace_options=sym-addr trace_event=initcall:* tp_printk trace_buf_size=1M + + (prev_comm != 2 && next_comm != 2) + +So, we can just enable the `tp_printk` but not the tracer. Then in the initrd +script, I enable the function tracer and the filter. + +**Observation**: It takes a long time to init the pty: + +Interesting timing: + + [ 12.612620] initcall_start: func=pty_init+0x0/0x3f4 + [ 20.962640] initcall_finish: func=pty_init+0x0/0x3f4 ret=0 + +**Observation**: The kcompactd0 daemon is using the CPU: + + [ 290.394920] sched_switch: prev_comm=devmem prev_pid=129 prev_prio=120 prev_state=R ==> next_comm=init next_pid=69 next_prio=120 + [ 290.408160] sched_switch: prev_comm=init prev_pid=69 prev_prio=120 prev_state=R ==> next_comm=tee next_pid=68 next_prio=120 + [ 290.420720] sched_switch: prev_comm=tee prev_pid=68 prev_prio=120 prev_state=R+ ==> next_comm=ksoftirqd/0 next_pid=12 next_prio=120 + [ 290.433960] sched_switch: prev_comm=ksoftirqd/0 prev_pid=12 prev_prio=120 prev_state=R ==> next_comm=init next_pid=1 next_prio=120 + [ 290.447100] sched_switch: prev_comm=init prev_pid=1 prev_prio=120 prev_state=R ==> next_comm=kcompactd0 next_pid=22 next_prio=120 + [ 290.460180] sched_switch: prev_comm=kcompactd0 prev_pid=22 prev_prio=120 prev_state=R ==> next_comm=khvcd next_pid=31 next_prio=120 + [ 290.473400] sched_switch: prev_comm=khvcd prev_pid=31 prev_prio=120 prev_state=R ==> next_comm=kworker/u2:2 next_pid=19 next_prio=120 + [ 290.486960] sched_switch: prev_comm=kworker/u2:2 prev_pid=19 prev_prio=120 prev_state=R ==> next_comm=khungtaskd next_pid=18 next_prio=120 + [ 290.500800] sched_switch: prev_comm=khungtaskd prev_pid=18 prev_prio=120 prev_state=R ==> next_comm=kworker/0:1 next_pid=13 next_prio=120 + [ 290.514560] sched_switch: prev_comm=kworker/0:1 prev_pid=13 prev_prio=120 prev_state=R ==> next_comm=tee next_pid=68 next_prio=120 + [ 290.527720] sched_switch: prev_comm=tee prev_pid=68 prev_prio=120 prev_state=R+ ==> next_comm=init next_pid=69 next_prio=120 + [ 290.540360] sched_switch: prev_comm=init prev_pid=69 prev_prio=120 prev_state=R ==> next_comm=devmem next_pid=129 next_prio=120 + [ 290.553280] sched_switch: prev_comm=devmem prev_pid=129 prev_prio=120 prev_state=R ==> next_comm=ksoftirqd/0 next_pid=12 next_prio=120 + [ 290.566780] sched_switch: prev_comm=ksoftirqd/0 prev_pid=12 prev_prio=120 prev_state=R ==> next_comm=kcompactd0 next_pid=22 next_prio=120 + [ 290.580500] sched_switch: prev_comm=kcompactd0 prev_pid=22 prev_prio=120 prev_state=R ==> next_comm=init next_pid=1 next_prio=120 + [ 290.593740] sched_switch: prev_comm=init prev_pid=1 prev_prio=120 prev_state=R ==> next_comm=khvcd next_pid=31 next_prio=120 + [ 290.606340] sched_switch: prev_comm=khvcd prev_pid=31 prev_prio=120 prev_state=R ==> next_comm=kworker/u2:2 next_pid=19 next_prio=120 + [ 290.619780] sched_switch: prev_comm=kworker/u2:2 prev_pid=19 prev_prio=120 prev_state=R ==> next_comm=khungtaskd next_pid=18 next_prio=120 + [ 290.633620] sched_switch: prev_comm=khungtaskd prev_pid=18 prev_prio=120 prev_state=R ==> next_comm=tee next_pid=68 next_prio=120 + [ 290.646700] sched_switch: prev_comm=tee prev_pid=68 prev_prio=120 prev_state=R+ ==> next_comm=init next_pid=69 next_prio=120 + [ 290.659320] sched_switch: prev_comm=init prev_pid=69 prev_prio=120 prev_state=R ==> next_comm=kworker/0:1 next_pid=13 next_prio=120 + [ 290.672560] sched_switch: prev_comm=kworker/0:1 prev_pid=13 prev_prio=120 prev_state=R ==> next_comm=devmem next_pid=129 next_prio=120 + [ 290.686080] sched_switch: prev_comm=devmem prev_pid=129 prev_prio=120 prev_state=R ==> next_comm=ksoftirqd/0 next_pid=12 next_prio=120 + [ 290.699720] sched_switch: prev_comm=ksoftirqd/0 prev_pid=12 prev_prio=120 prev_state=R ==> next_comm=init next_pid=1 next_prio=120 + [ 290.712880] sched_switch: prev_comm=init prev_pid=1 prev_prio=120 prev_state=R ==> next_comm=khvcd next_pid=31 next_prio=120 + [ 290.725500] sched_switch: prev_comm=khvcd prev_pid=31 prev_prio=120 prev_state=R ==> next_comm=kcompactd0 next_pid=22 next_prio=120 + +**Question**: Can we reproduce this hang with 6.9.7? + -- GitLab From 540780e5081e31f398c60d9d5a618e06e18b92e5 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 5 Jul 2024 16:54:14 +0200 Subject: [PATCH 057/310] Remove rvb dependency for now to avoid clang --- JOURNAL.md | 22 ++++++++++++++++++++++ configuration.nix | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/JOURNAL.md b/JOURNAL.md index e563135..4221efd 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -647,3 +647,25 @@ Interesting timing: **Question**: Can we reproduce this hang with 6.9.7? +Disabling clang as it is failing to build: + + hut% nix develop '.#lagarto-ox' + error: builder for '/nix/store/x1nfa792pv28px70kvfakm3aalcfbdyw-clang-epi-479518d.drv' failed with exit code 2; + last 10 log lines: + > | ^~~~~~~~~~~~~~~ + > 2 errors generated. + > make[2]: *** [lib/Support/CMakeFiles/LLVMSupport.dir/build.make:1868: lib/Support/CMakeFiles/LLVMSupport.dir/Signals.cpp.o] Error 1 + > make[2]: *** Waiting for unfinished jobs.... + > [ 9%] Built target obj.clang-tblgen + > 1 warning generated. + > make[1]: *** [CMakeFiles/Makefile2:9468: lib/Support/CMakeFiles/LLVMSupport.dir/all] Error 2 + > make[1]: *** Waiting for unfinished jobs.... + > [ 9%] Built target obj.llvm-tblgen + > make: *** [Makefile:156: all] Error 2 + For full logs, run 'nix log /nix/store/x1nfa792pv28px70kvfakm3aalcfbdyw-clang-epi-479518d.drv'. + error: 1 dependencies of derivation '/nix/store/m54sxxyi3cg062djrcddcawp10z7r49l-riscv64-unknown-linux-gnu-clang-epi-wrapper-479518d.drv' failed to build + error: 1 dependencies of derivation '/nix/store/yl26dbqqj0snl807c0wjabg4dpbq5gvp-stdenv-linux.drv' failed to build + error: 1 dependencies of derivation '/nix/store/h6180fcl30kqy3apaqjsbkkik2p1spmr-rvb-riscv64-unknown-linux-gnu-da202d6.drv' failed to build + error: 1 dependencies of derivation '/nix/store/b13shgqj7128rdsdzzp4qicqbzl0wnfw-system-path.drv' failed to build + error: 1 dependencies of derivation '/nix/store/6qghlihqcyg6155309ldj5xm9m0v835i-nixos-system-nixos-riscv-24.11pre-git.drv' failed to build + error: 1 dependencies of derivation '/nix/store/l2x18cih29r1kn6vi8imwhkyk98yhw4i-nix-shell-riscv64-unknown-linux-gnu-env.drv' failed to build diff --git a/configuration.nix b/configuration.nix index 7337141..669fe60 100644 --- a/configuration.nix +++ b/configuration.nix @@ -36,5 +36,5 @@ }; #environment.systemPackages = with pkgs; [ vim gdb neofetch gcc bintools ]; - environment.systemPackages = with pkgs; [ rvb unalignedCheck ]; + #environment.systemPackages = with pkgs; [ rvb unalignedCheck ]; } -- GitLab From a7c460b0349f778cb8b7075aa842aacdd39bb33d Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 5 Jul 2024 17:05:28 +0200 Subject: [PATCH 058/310] Use headings to allow hrefs --- JOURNAL.md | 87 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 49 insertions(+), 38 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 4221efd..c52d45a 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -53,17 +53,17 @@ CPU. Let's go back and try to get the initrd shell, so we can systematically hang it in the `switch_root` -**Observation**: The riscv-timer seems to be causing interrupts with IRQ 5: +### OBSERVATION: The riscv-timer seems to be causing interrupts with IRQ 5: ``` [ 62.439060] irq_handler_entry: irq=5 name=riscv-timer [ 62.444980] irq_handler_exit: irq=5 ret=handled ``` -**Observation**: Rohan reports the serial startup routine being running *after*{{{ +### OBSERVATION: Rohan reports the serial startup routine being running *after* the init begins. -**Observation**: Only interrupts in timer, others are zero. +### OBSERVATION: Only interrupts in timer, others are zero. With: @@ -95,7 +95,7 @@ I can see this: IPI4: 0 IRQ work interrupts IPI5: 0 Timer broadcast interrupts -**Observation**: There is a timer configured in 0x40170000 but in the device +### OBSERVATION: There is a timer configured in 0x40170000 but in the device tree we only have one at `timer@40002000`. #define OX_ALVEO_TIMER_BASE 0x40170000 @@ -106,7 +106,7 @@ tree we only have one at `timer@40002000`. https://gitlab.bsc.es/hwdesign/bsc-linux/-/blob/d6d194bd30d9a8fe49c2a278ffb3c3ae7852e75d/bsc_tree/patches/ox_alveo/opensbi/0001-opensbi-ox_alveo-platform.patch#L63 -**Observation**: When the serial console starts, the speed of the serial port +### OBSERVATION: When the serial console starts, the speed of the serial port changes to 9600: [ 6.845400] io scheduler mq-deadline registered @@ -181,13 +181,14 @@ That was my mistake as I need to put the baud speed in the ttyS0, like this: console=ttyS0,115200n8 -**Observation**: Trying to read from the serial console /dev/ttyS0 causes no +### OBSERVATION: Trying to read from the serial console /dev/ttyS0 causes no more messages in the console (or a hang). -**Question**: Can we make a heartbeat for the kernel? The idea is to keep a -counter in some memory of the kernel so we can see it from the host being moved. +### QUESTION: Can we make a heartbeat for the kernel? +The idea is to keep a counter in some memory of the kernel so we can see it from +the host being moved. -**Question**: Can we disable the serial driver 8250 from loading? +### QUESTION: Can we disable the serial driver 8250 from loading? initcall_blacklist= @@ -208,7 +209,7 @@ Yes, but that doesn't seem to do anything. It is hanging: [ 629.733920] stage-1-init: [Thu Jan 1 00:10:29 UTC 1970] + echo /nix/store/snvvqpxmryw1szlllk0bxpm37p8vj8sw-extra-utils/bin/modprobe -**Question**: What happens if we remap the interruptions? +### QUESTION: What happens if we remap the interruptions? - Move the serial from 0 to 1 - Move the plic from 3 to 2 and remove 7 @@ -225,14 +226,14 @@ Rather than two: [ 0.000000] plic: plic@40800000: mapped 3 interrupts with 0 handlers for 2 contexts. [ 0.000000] riscv: providing IPIs using SBI IPI extension -**Question**: What happens if we block the `sbi_ipi` driver? +### QUESTION: What happens if we block the `sbi_ipi` driver? initcall_blacklist=sbi_ipi_init Nothing, it cannot be disabled it seems. I will remove SMP support so it won't be compiled in. -**Observation**: Searching for 'riscv,plic0' only matches irq-sifive-plic driver. +### OBSERVATION: Searching for 'riscv,plic0' only matches irq-sifive-plic driver. hut% rg 'riscv,plic0' Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml @@ -244,7 +245,7 @@ be compiled in. So it looks that the only driver that setups the plic is the one used by SiFive. Here is the doc: https://static.dev.sifive.com/U54-MC-RVCoreIP.pdf -**Observation**: The number of handlers is 0, so there are no interruptions. +### OBSERVATION: The number of handlers is 0, so there are no interruptions. It seems the number next to the phandle of the interrupts-extended attribute in the plic follows a different convention of values. Using 9 and 11: @@ -257,7 +258,7 @@ polled, otherwise it hangs. ## 2024-07-04 -**Observation**: I saw they changed this option in Cinco Ranch DTS for the +### OBSERVATION: I saw they changed this option in Cinco Ranch DTS for the serial: > reg-shift = <0>; // regs are spaced on 8 bit boundary (modified from Xilinx UART16550 to be ns16550 compatible) @@ -265,7 +266,7 @@ serial: Tested booting with debug1 and the ttyS0 console, and it goes extremely slow (but still outputs at 115200) and then continues to fail to read keyboard input. -**Question**: Let's try setting the console in poll mode. +### QUESTION: Let's try setting the console in poll mode. setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=uart,io,0x40001000,115200n8 boot.trace console=uart,io,0x40001000,115200n8 debug1 init=/nix/store/wavmnv6wjj8y10ha07wxd5f0sqacivj8-nixos-system-nixos-riscv-23.11pre-git/init" @@ -304,17 +305,17 @@ setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=uart,io,0x40001000, Also found: `no_console_suspend` -**Observation**: There are messages of address space being assigned to +### OBSERVATION: There are messages of address space being assigned to registers: Slave segment '/MEEP_uart_0/S_AXI/Reg' is being assigned into address space '/m_axi_uart0' at <0x0000_0000 [ 4K ]>. Slave segment '/MEEP_uart_1/S_AXI/Reg' is being assigned into address space '/m_axi_uart1' at <0x0000_0000 [ 4K ]>. -**Question**: What happens if I enable `CONFIG_CONSOLE_POLL`? +### QUESTION: What happens if I enable `CONFIG_CONSOLE_POLL`? With `console=ttyS0,115200n8 debug1` I cannot type. -**Observation**: I can dump iomem memory with the tool devmem: +### OBSERVATION: I can dump iomem memory with the tool devmem: But it seems I cannot dump the registers of the serial io mapped region: @@ -364,7 +365,7 @@ It works! ~ # devmem 0x40001000 0x0000000D -**Observation**: The interrupt register of the serial console is 0x0: +### OBSERVATION: The interrupt register of the serial console is 0x0: Assuming the console registers follow AXI UART 16550, here is the IER: @@ -380,8 +381,7 @@ The line control register is 0x3: ~ # devmem 0x4000100C 0x00000003 -**Question**: Can I write to some memory address and see the result from the -host? +### QUESTION: Can I write to some memory address and see the result from the host? For that I would need to find some address that is mapped to the DMA or to the pmem. Xavi recommended `0x6000_0000` as it is uncached. @@ -467,18 +467,22 @@ But we don't see the same: [bsc015557@fpgan02 nixos]$ dd if=/dev/qdma34000-MM-1 count=16 bs=1 skip=$FPGACTL_KERNEL_ADDR 2>/dev/null | xxd 00000000: 9797 9797 9797 9797 9797 9797 9797 9797 ................ [bsc015557@fpgan02 nixos]$ dd if=/dev/qdma34000-MM-0 count=16 bs=1 skip=$FPGACTL_KERNEL_ADDR 2>/dev/null | xxd - 00000000: 9797 9797 9797 9797 9797 9797 9797 9797 ................ -**Question**: Missing forward M to S via Mideleg? + 00000000: 9797 9797 9797 9797 9797 9797 9797 9797 ................ + + +### QUESTION: Missing forward M to S via Mideleg? Can it be happening that he MEDELEG is not forwarding the interruptions to the Supervisor (kernel)? Boot HART MIDELEG : 0x0000000000000222 Boot HART MEDELEG : 0x000000000000b109 + -**Question**: Can we add a timer to the PLIC to test the interrupts? +### QUESTION: Can we add a timer to the PLIC to test the interrupts? + -**Observation**: Here is the PLIC register dump: +### OBSERVATION: Here is the PLIC register dump: ~ # for i in `seq 0 16`; do addr=$((0x40600000 + $i)); printf '%08x: ' $addr; devmem $addr; done 40600000: 0x00010002 @@ -499,12 +503,12 @@ Supervisor (kernel)? 4060000f: 0x00000000 40600010: 0x00000000 -**Question**: Can we boot with the new bitstream that includes the second UART? +### QUESTION: Can we boot with the new bitstream that includes the second UART? The interruptions are enabled for the UART 1, not the default UART 0. -**Observation**: I'm using 0x100 not 0x1000 in the serial range: +### OBSERVATION: I'm using 0x100 not 0x1000 in the serial range: reg = <0x0 0x40003000 0x0 0x100>; reg = <0x0 0x40003000 0x0 0x1000>; @@ -512,12 +516,13 @@ The interruptions are enabled for the UART 1, not the default UART 0. Can this produce any problem? It doesn't seem to change anything, still unable to send any bytes. + -**Question**: Can we use virtio to mount a FS in the DMA shared memory? +### QUESTION: Can we use virtio to mount a FS in the DMA shared memory? ## 2024-07-05 -**Observation**: The kernel continues working when the console hangs. +### OBSERVATION: The kernel continues working when the console hangs. Switching to 0x100000000 as 0x60000000 shows: @@ -540,8 +545,9 @@ Shows the kernel works: a0000000: 6700 0000 g... a0000000: 6800 0000 h... a0000000: 6900 0000 i... + -**Question**: Can we reproduce it with `switch_root`? +### QUESTION: Can we reproduce it with `switch_root`? For that I would have to ensure the process continues to operate, even if we exit the console. Maybe I can make a double fork? @@ -563,10 +569,11 @@ Yes, it seems to be working. Let's load the rootfs too. I added a loop in the stage1 script. -**Question**: Can we see any clock in memory? This will allow us to check if the -AXI still works. +### QUESTION: Can we see any clock in memory? + +This will allow us to check if the AXI still works. -**Observation**: The kernel stops updating the counter in the mount phase. +### OBSERVATION: The kernel stops updating the counter in the mount phase. Managed to reach the mount and hang there: @@ -595,7 +602,7 @@ hardware clock from the DMA region too, so we can discard problems in the AXI. +### ASSUMPTION: The kernel hangs. If the kernel hangs, there must be an instruction or sequence of instructions that causes it. First I need to determine what is being executed by the kernel. @@ -607,16 +614,18 @@ hangs. (prev_comm != 2 && next_comm != 2) So, we can just enable the `tp_printk` but not the tracer. Then in the initrd -script, I enable the function tracer and the filter. +script, I enable the function tracer and the filter. -**Observation**: It takes a long time to init the pty: + +### OBSERVATION: It takes a long time to init the pty: Interesting timing: [ 12.612620] initcall_start: func=pty_init+0x0/0x3f4 [ 20.962640] initcall_finish: func=pty_init+0x0/0x3f4 ret=0 + -**Observation**: The kcompactd0 daemon is using the CPU: +### OBSERVATION: The kcompactd0 daemon is using the CPU: [ 290.394920] sched_switch: prev_comm=devmem prev_pid=129 prev_prio=120 prev_state=R ==> next_comm=init next_pid=69 next_prio=120 [ 290.408160] sched_switch: prev_comm=init prev_pid=69 prev_prio=120 prev_state=R ==> next_comm=tee next_pid=68 next_prio=120 @@ -644,8 +653,9 @@ Interesting timing: [ 290.699720] sched_switch: prev_comm=ksoftirqd/0 prev_pid=12 prev_prio=120 prev_state=R ==> next_comm=init next_pid=1 next_prio=120 [ 290.712880] sched_switch: prev_comm=init prev_pid=1 prev_prio=120 prev_state=R ==> next_comm=khvcd next_pid=31 next_prio=120 [ 290.725500] sched_switch: prev_comm=khvcd prev_pid=31 prev_prio=120 prev_state=R ==> next_comm=kcompactd0 next_pid=22 next_prio=120 + -**Question**: Can we reproduce this hang with 6.9.7? +### QUESTION: Can we reproduce this hang with 6.9.7? Disabling clang as it is failing to build: @@ -669,3 +679,4 @@ Disabling clang as it is failing to build: error: 1 dependencies of derivation '/nix/store/b13shgqj7128rdsdzzp4qicqbzl0wnfw-system-path.drv' failed to build error: 1 dependencies of derivation '/nix/store/6qghlihqcyg6155309ldj5xm9m0v835i-nixos-system-nixos-riscv-24.11pre-git.drv' failed to build error: 1 dependencies of derivation '/nix/store/l2x18cih29r1kn6vi8imwhkyk98yhw4i-nix-shell-riscv64-unknown-linux-gnu-env.drv' failed to build + -- GitLab From 6155c7e3f8d94e58294b7becf7110b03430f9d71 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 8 Jul 2024 08:48:56 +0200 Subject: [PATCH 059/310] Try to fill cache details --- JOURNAL.md | 36 ++++++++++++++++++++++++++++++++++++ ox-plic.dts | 20 ++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index c52d45a..3860946 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -679,4 +679,40 @@ Disabling clang as it is failing to build: error: 1 dependencies of derivation '/nix/store/b13shgqj7128rdsdzzp4qicqbzl0wnfw-system-path.drv' failed to build error: 1 dependencies of derivation '/nix/store/6qghlihqcyg6155309ldj5xm9m0v835i-nixos-system-nixos-riscv-24.11pre-git.drv' failed to build error: 1 dependencies of derivation '/nix/store/l2x18cih29r1kn6vi8imwhkyk98yhw4i-nix-shell-riscv64-unknown-linux-gnu-env.drv' failed to build + + +### QUESTION: Missing cache information may affect? + +Other CPUs report the cache details in the DT. For example this one +https://github.com/torvalds/linux/blob/master/arch/riscv/boot/dts/sifive/fu540-c000.dtsi#L45 + + cpu1: cpu@1 { + compatible = "sifive,u54-mc", "sifive,rocket0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <32>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <64>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <32>; + mmu-type = "riscv,sv39"; + reg = <1>; + riscv,isa = "rv64imafdc"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", "zicsr", + "zifencei", "zihpm"; + tlb-split; + next-level-cache = <&l2cache>; + cpu1_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + +We may want to add it to our DT to be sure that it has no effect. diff --git a/ox-plic.dts b/ox-plic.dts index 018c2e8..f66ce9a 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -27,6 +27,26 @@ riscv,isa = "rv64imafd"; mmu-type = "riscv,sv39"; tlb-split; + + // OpenPiton+Ariane Platform + // L1I Size / Assoc: 16 kB / 4 + // L1D Size / Assoc: 32 kB / 4 + // L15 Size / Assoc: 128 kB / 8 + // L2 Size / Assoc: 256 kB / 4 + // L15/L1D Cacheline size 64 + + i-cache-block-size = <64>; // Guess + i-cache-sets = <4>; + i-cache-size = <16384>; + i-tlb-sets = <1>; // Guess + i-tlb-size = <32>; // Guess + + d-cache-block-size = <64>; // Guess + d-cache-sets = <4>; + d-cache-size = <32768>; + d-tlb-sets = <1>; // Guess + d-tlb-size = <32>; // Guess + phandle = <0x00000004>; /* Hart-Level Interrupt Controller: Every interrupt is * ultimately routed through a hart's HLIC before it -- GitLab From ef7a100c3f1b1f702d3bb0e26df05df96e90fc98 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 8 Jul 2024 08:49:12 +0200 Subject: [PATCH 060/310] Disable secondary console --- ox-plic.dts | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/ox-plic.dts b/ox-plic.dts index f66ce9a..31cf650 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -7,7 +7,7 @@ aliases { serial0 = &uart_console; // ttyS0 - serial1 = &uart_testing; // ttyS1 +// serial1 = &uart_testing; // ttyS1 }; // chosen { @@ -120,24 +120,26 @@ uart_console: serial@40001000 { compatible = "ns16550"; reg = <0x0 0x40001000 0x0 0x1000>; - reg-shift = <2>; - /* No interrupts for this UART, use console=hvc0 */ - /* This clock is the SERIAL_CLK */ - clock-frequency = <50000000>; - current-speed = <115200>; - status = "okay"; - }; - /* The serial for interrupt tests */ - uart_testing: serial@40003000 { - compatible = "ns16550"; - reg = <0x0 0x40003000 0x0 0x1000>; interrupts = <1>; /* Output interrupt 1 */ interrupt-parent = <&PLIC>; reg-shift = <2>; + /* No interrupts for this UART, use console=hvc0 */ + /* This clock is the SERIAL_CLK */ clock-frequency = <50000000>; current-speed = <115200>; status = "okay"; }; +// /* The serial for interrupt tests */ +// uart_testing: serial@40003000 { +// compatible = "ns16550"; +// reg = <0x0 0x40003000 0x0 0x1000>; +// interrupts = <1>; /* Output interrupt 1 */ +// interrupt-parent = <&PLIC>; +// reg-shift = <2>; +// clock-frequency = <50000000>; +// current-speed = <115200>; +// status = "okay"; +// }; // ethernet0 { // xlnx,rxmem = <0x000005f2>; -- GitLab From aaca0bb2e6fd03e933df370e14e9166ec62335b5 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 8 Jul 2024 08:49:21 +0200 Subject: [PATCH 061/310] Reached stage2! --- JOURNAL.md | 90 +++++++- stage2.log | 668 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 756 insertions(+), 2 deletions(-) create mode 100644 stage2.log diff --git a/JOURNAL.md b/JOURNAL.md index 3860946..a6300fb 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -682,7 +682,7 @@ Disabling clang as it is failing to build: -### QUESTION: Missing cache information may affect? +### QUESTION: Missing cache information may affect? Other CPUs report the cache details in the DT. For example this one https://github.com/torvalds/linux/blob/master/arch/riscv/boot/dts/sifive/fu540-c000.dtsi#L45 @@ -715,4 +715,90 @@ https://github.com/torvalds/linux/blob/master/arch/riscv/boot/dts/sifive/fu540-c }; }; -We may want to add it to our DT to be sure that it has no effect. +We may want to add it to our DT to be sure that it has no effect. + + +### OBSERVATION: Arrived to stage 2! + + + kill -9 74 + + readlink /proc/75/exe + [ 374.961120] stage-1-init: [Thu Jan 1 00:06:14 UTC 1970] + test 0 -ge 8 + + '[' 75 -eq 1 ] + + kill -9 75 + + readlink /proc/102/exe + + '[' 102 -eq 1 ] + + kill -9 102 + + readlink /proc/137/exe + + continue + + readlink /proc/674/exe + + continue + + readlink /proc/675/exe + + continue + + test -n + + echo /sbin/modprobe + + '[' '!' -e /mnt-root//nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init ] + + mkdir -m 0755 -p /mnt-root/proc /mnt-root/sys /mnt-root/dev /mnt-root/run + + mount --move /proc /mnt-root/proc + + mount --move /sys /mnt-root/sys + + mount --move /dev /mnt-root/dev + + mount --move /run /mnt-root/run + + type -P switch_root + + exec env -i /nix/store/988axh0bq3wqp90gms4b4a0hkfwvkd3i-extra-utils/bin/switch_root /mnt-root /nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init + + <<< NixOS Stage 2 >>> + + [ 384.203680] EXT4-fs (pmem0p2): re-mounted 44444444-4444-4444-8888-888888888888 r/w. Quota mode: none. + [ 384.287600] booting system configuration /nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git + running activation script... + [ 388.163860] stage-2-init: running activation script... + [ 391.643500] random: perl: uninitialized urandom read (4 bytes read) + [ 391.884800] random: perl: uninitialized urandom read (4 bytes read) + [ 425.302000] random: perl: uninitialized urandom read (4 bytes read) + +But then it hangs. + +### QUESTION: Who sets the plic interrupts? + +Shouldn't OpenSBI read the DT and do some configuration in the plic while in +machine mode? + +### OBSERVATION: Semi-stack trace from CincoRanch + + hvc_remove? + console_unlock <-- only called from hvc_remove() + prb_read_valid + desc_read_finalized_seq + __memcpy (multiple times) + get_data + do_trap_break + report_bug + is_valid_bugaddr + copy_from_kernel_nofault + copy_from_kernel_nofault_allowed + find_bug + _printk + vprintk + vprintk_default + vprintk_emit + vprintk_store + sched_clock + vsnprintf + format_decode + __memcpy + printk_parse_prefix + prb_reserve + do_page_fault + fixup_exception + search_exception_tables + search_extable + cmp_ex_search (multiple times) + search_module_extables + __module_address + no_context.part.0 + die_kernel_fault <-- last frame(?) + +### QUESTION: Can we place a tracepoint in `hvc_remove`? + +If we are getting stuck in the same place, we should be able to see the +backtrace (assuming the console still works) just before we try to remove the +console device. diff --git a/stage2.log b/stage2.log new file mode 100644 index 0000000..1ad0652 --- /dev/null +++ b/stage2.log @@ -0,0 +1,668 @@ +OpenSBI v1.4 + ____ _____ ____ _____ + / __ \ / ____| _ \_ _| + | | | |_ __ ___ _ __ | (___ | |_) || | + | | | | '_ \ / _ \ '_ \ \___ \| _ < | | + | |__| | |_) | __/ | | |____) | |_) || |_ + \____/| .__/ \___|_| |_|_____/|____/_____| + | | + |_| + +Platform Name : ox (Rodrigo NixOS version) +Platform Features : medeleg +Platform HART Count : 1 +Platform IPI Device : --- +Platform Timer Device : axi_timer @ 50000000Hz +Platform Console Device : uart8250 +Platform HSM Device : --- +Platform PMU Device : --- +Platform Reboot Device : --- +Platform Shutdown Device : --- +Platform Suspend Device : --- +Platform CPPC Device : --- +Firmware Base : 0x80000000 +Firmware Size : 178 KB +Firmware RW Offset : 0x20000 +Firmware RW Size : 50 KB +Firmware Heap Offset : 0x24000 +Firmware Heap Size : 34 KB (total), 2 KB (reserved), 8 KB (used), 23 KB (free) +Firmware Scratch Size : 4096 B (total), 280 B (used), 3816 B (free) +Runtime SBI Version : 2.0 + +Domain0 Name : root +Domain0 Boot HART : 0 +Domain0 HARTs : 0* +Domain0 Region00 : 0x0000000040000000-0x0000000040000fff M: (I,R,W) S/U: (R,W) +Domain0 Region01 : 0x0000000080020000-0x000000008002ffff M: (R,W) S/U: () +Domain0 Region02 : 0x0000000080000000-0x000000008001ffff M: (R,X) S/U: () +Domain0 Region03 : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X) +Domain0 Next Address : 0x0000000080200000 +Domain0 Next Arg1 : 0x0000000080013000 +Domain0 Next Mode : S-mode +Domain0 SysReset : yes +Domain0 SysSuspend : yes + +Boot HART ID : 0 +Boot HART Domain : root +Boot HART Priv Version : v1.10 +Boot HART Base ISA : rv64imafdc +Boot HART ISA Extensions : zicntr,zihpm +Boot HART PMP Count : 0 +Boot HART PMP Granularity : 0 bits +Boot HART PMP Address Bits: 0 +Boot HART MHPM Info : 29 (0xfffffff8) +Boot HART MIDELEG : 0x0000000000000222 +Boot HART MEDELEG : 0x000000000000b109 + + +Core: 12 devices, 8 uclasses, devicetree: board +Loading Environment from nowhere... OK +In: serial,usbkbd +Out: serial,vidconsole +Err: serial,vidconsole +No working controllers found +Net: No ethernet found. +Working FDT set to 80013000 +Hit any key to stop autoboot: 0 + +Device 0: unknown device + +Device 1: unknown device +scanning bus for devices... + +Device 0: unknown device +starting USB... +No working controllers found +No ethernet found. +No ethernet found. +=> setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi boot.trace console=hvc0 ${debugargs} init=/nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init" +=> setenv ramdisk_size 12563951 +=> booti ${kernel_addr_r} ${ramdisk_addr_r}:${ramdisk_size} ${fdtcontroladdr} +Moving Image from 0x84000000 to 0x80200000, end=83044650 +## Flattened Device Tree blob at 80013000 + Booting using the fdt blob at 0x80013000 +Working FDT set to 80013000 + Using Device Tree in place at 0000000080013000, end 000000008001696f +Working FDT set to 80013000 + +Starting kernel ... + +[ 0.000000] Linux version 6.9.7 (nixbld@localhost) (riscv64-unknown-linux-gnu-gcc (GCC) 13.3.0, GNU ld (GNU Binutils) 2.41) #1-NixOS Thu Jun 27 11:52:32 UTC 2024 +[ 0.000000] Machine model: Barcelona Supercomputing Center - Lagarto Ox (NixOS) +[ 0.000000] SBI specification v2.0 detected +[ 0.000000] SBI implementation ID=0x1 Version=0x10004 +[ 0.000000] SBI TIME extension detected +[ 0.000000] SBI IPI extension detected +[ 0.000000] SBI RFENCE extension detected +[ 0.000000] SBI DBCN extension detected +[ 0.000000] earlycon: sbi0 at I/O port 0x0 (options '') +[ 0.000000] printk: legacy bootconsole [sbi0] enabled +[ 0.000000] Reserved memory: created DMA memory pool at 0x0000000060000000, size 256 MiB +[ 0.000000] OF: reserved mem: initialized node dma_pool@60000000, compatible id shared-dma-pool +[ 0.000000] OF: reserved mem: 0x0000000060000000..0x000000006fffffff (262144 KiB) map non-reusable dma_pool@60000000 +[ 0.000000] Reserved memory: created DMA memory pool at 0x0000000070000000, size 256 MiB +[ 0.000000] OF: reserved mem: initialized node dma_pool@70000000, compatible id shared-dma-pool +[ 0.000000] OF: reserved mem: 0x0000000070000000..0x000000007fffffff (262144 KiB) map non-reusable dma_pool@70000000 +[ 0.000000] cma: Reserved 16 MiB at 0x00000000fee00000 on node -1 +[ 0.000000] Zone ranges: +[ 0.000000] DMA32 [mem 0x0000000080000000-0x00000000ffefffff] +[ 0.000000] Normal empty +[ 0.000000] Movable zone start for each node +[ 0.000000] Early memory node ranges +[ 0.000000] node 0: [mem 0x0000000080000000-0x00000000ffefffff] +[ 0.000000] Initmem setup node 0 [mem 0x0000000080000000-0x00000000ffefffff] +[ 0.000000] On node 0, zone DMA32: 256 pages in unavailable ranges +[ 0.000000] Falling back to deprecated "riscv,isa" +[ 0.000000] riscv: base ISA extensions adfim +[ 0.000000] riscv: ELF capabilities adfim +[ 0.000000] pcpu-alloc: s0 r0 d131072 u131072 alloc=1*131072 +[ 0.000000] pcpu-alloc: [0] 0 +[ 0.000000] Kernel command line: root=/dev/ram0 loglevel=7 debug rw earlycon=sbi boot.trace console=hvc0 init=/nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init +[ 0.000000] Dentry cache hash table entries: 262144 (order: 9, 2097152 bytes, linear) +[ 0.000000] Inode-cache hash table entries: 131072 (order: 8, 1048576 bytes, linear) +[ 0.000000] Built 1 zonelists, mobility grouping on. Total pages: 515844 +[ 0.000000] mem auto-init: stack:all(zero), heap alloc:off, heap free:off +[ 0.000000] Virtual kernel memory layout: +[ 0.000000] fixmap : 0xffffffc6fea00000 - 0xffffffc6ff000000 (6144 kB) +[ 0.000000] pci io : 0xffffffc6ff000000 - 0xffffffc700000000 ( 16 MB) +[ 0.000000] vmemmap : 0xffffffc700000000 - 0xffffffc800000000 (4096 MB) +[ 0.000000] vmalloc : 0xffffffc800000000 - 0xffffffd800000000 ( 64 GB) +[ 0.000000] modules : 0xffffffff02e45000 - 0xffffffff80000000 (2001 MB) +[ 0.000000] lowmem : 0xffffffd800000000 - 0xffffffd87ff00000 (2047 MB) +[ 0.000000] kernel : 0xffffffff80000000 - 0xffffffffffffffff (2047 MB) +[ 0.000000] Memory: 1675400K/2096128K available (17075K kernel code, 9047K rwdata, 10240K rodata, 8737K init, 917K bss, 404344K reserved, 16384K cma-reserved) +[ 0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=1, Nodes=1 +[ 0.000000] ftrace: allocating 46961 entries in 184 pages +[ 0.000000] ftrace: allocated 184 pages with 4 groups +[ 0.000000] trace event string verifier disabled +[ 0.000000] RCU Tasks Rude: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1. +[ 0.000000] RCU Tasks Trace: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1. +[ 0.000000] NR_IRQS: 64, nr_irqs: 64, preallocated irqs: 0 +[ 0.000000] riscv-intc: 64 local interrupts mapped +[ 0.000000] clocksource: riscv_clocksource: mask: 0xffffffffffffffff max_cycles: 0x179dd7f66, max_idle_ns: 56421785867800 ns +[ 0.000020] sched_clock: 64 bits at 50kHz, resolution 20000ns, wraps every 70368744170000ns +[ 0.015140] kfence: initialized - using 2097152 bytes for 255 objects at 0x(____ptrval____)-0x(____ptrval____) +[ 0.037080] Console: colour dummy device 80x25 +[ 0.043380] Calibrating delay loop (skipped), value calculated using timer frequency.. 0.10 BogoMIPS (lpj=200) +[ 0.054600] pid_max: default: 32768 minimum: 301 +[ 0.206980] LSM: initializing lsm=capability,landlock,yama +[ 0.537640] landlock: Up and running. +[ 0.542180] Yama: becoming mindful. +[ 0.584380] Mount-cache hash table entries: 4096 (order: 3, 32768 bytes, linear) +[ 0.593160] Mountpoint-cache hash table entries: 4096 (order: 3, 32768 bytes, linear) +[ 0.885440] riscv: ELF compat mode unsupported +[ 0.885820] ASID allocator disabled (0 bits) +[ 0.992540] devtmpfs: initialized +[ 1.122280] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns +[ 1.133360] futex hash table entries: 256 (order: 1, 12288 bytes, linear) +[ 1.233120] pinctrl core: initialized pinctrl subsystem +[ 1.366300] NET: Registered PF_NETLINK/PF_ROUTE protocol family +[ 1.426680] DMA: preallocated 256 KiB GFP_KERNEL pool for atomic allocations +[ 1.441220] DMA: preallocated 256 KiB GFP_KERNEL|GFP_DMA32 pool for atomic allocations +[ 1.452360] audit: initializing netlink subsys (disabled) +[ 1.468360] audit: type=2000 audit(1.200:1): state=initialized audit_enabled=0 res=1 +[ 1.526400] thermal_sys: Registered thermal governor 'step_wise' +[ 1.527680] cpuidle: using governor ladder +[ 2.024460] cpu0: Ratio of byte access time to unaligned word access is 0.00, unaligned accesses are slow +[ 2.301780] HugeTLB: registered 1.00 GiB page size, pre-allocated 0 pages +[ 2.309580] HugeTLB: 16380 KiB vmemmap can be freed for a 1.00 GiB page +[ 2.317240] HugeTLB: registered 2.00 MiB page size, pre-allocated 0 pages +[ 2.324960] HugeTLB: 28 KiB vmemmap can be freed for a 2.00 MiB page +[ 2.807240] iommu: Default domain type: Translated +[ 2.813020] iommu: DMA domain TLB invalidation policy: strict mode +[ 2.938240] SCSI subsystem initialized +[ 2.966020] libata version 3.00 loaded. +[ 3.006820] usbcore: registered new interface driver usbfs +[ 3.028200] usbcore: registered new interface driver hub +[ 3.042520] usbcore: registered new device driver usb +[ 3.162860] Advanced Linux Sound Architecture Driver Initialized. +[ 3.331140] vgaarb: loaded +[ 3.374160] clocksource: Switched to clocksource riscv_clocksource +[ 18.084500] VFS: Disk quotas dquot_6.6.0 +[ 18.099980] VFS: Dquot-cache hash table entries: 512 (order 0, 4096 bytes) +[ 18.129800] netfs: FS-Cache loaded +[ 18.887880] NET: Registered PF_INET protocol family +[ 18.905140] IP idents hash table entries: 32768 (order: 6, 262144 bytes, linear) +[ 20.307580] tcp_listen_portaddr_hash hash table entries: 1024 (order: 3, 32768 bytes, linear) +[ 20.317920] Table-perturb hash table entries: 65536 (order: 6, 262144 bytes, linear) +[ 20.327840] TCP established hash table entries: 16384 (order: 5, 131072 bytes, linear) +[ 20.339640] TCP bind hash table entries: 16384 (order: 8, 1048576 bytes, linear) +[ 20.371220] TCP: Hash tables configured (established 16384 bind 16384) +[ 20.399500] MPTCP token hash table entries: 2048 (order: 4, 98304 bytes, linear) +[ 20.421380] UDP hash table entries: 1024 (order: 4, 98304 bytes, linear) +[ 20.432460] UDP-Lite hash table entries: 1024 (order: 4, 98304 bytes, linear) +[ 20.459500] NET: Registered PF_UNIX/PF_LOCAL protocol family +[ 20.506800] RPC: Registered named UNIX socket transport module. +[ 20.514160] RPC: Registered udp transport module. +[ 20.520000] RPC: Registered tcp transport module. +[ 20.525720] RPC: Registered tcp-with-tls transport module. +[ 20.532220] RPC: Registered tcp NFSv4.1 backchannel transport module. +[ 20.541120] NET: Registered PF_XDP protocol family +[ 20.547220] PCI: CLS 0 bytes, default 64 +[ 20.570800] Trying to unpack rootfs image as initramfs... +[ 20.733220] Initialise system trusted keyrings +[ 20.772720] workingset: timestamp_bits=46 max_order=19 bucket_order=0 +[ 22.193520] NFS: Registering the id_resolver key type +[ 22.220780] Key type id_resolver registered +[ 22.226940] Key type id_legacy registered +[ 22.259900] nfs4filelayout_init: NFSv4 File Layout Driver Registering... +[ 22.267740] nfs4flexfilelayout_init: NFSv4 Flexfile Layout Driver Registering... +[ 22.310900] 9p: Installing v9fs 9p2000 file system support +[ 30.612000] NET: Registered PF_ALG protocol family +[ 30.625120] Key type asymmetric registered +[ 30.635440] Asymmetric key parser 'x509' registered +[ 30.676100] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 244) +[ 30.712080] io scheduler mq-deadline registered +[ 30.717760] io scheduler kyber registered +[ 30.797740] riscv-plic 40800000.plic: mapped 3 interrupts with 1 handlers for 2 contexts. +[ 40.296340] printk: legacy console [hvc0] enabled +[ 40.296340] printk: legacy console [hvc0] enabled +[ 40.307600] printk: legacy bootconsole [sbi0] disabled +[ 40.307600] printk: legacy bootconsole [sbi0] disabled +[ 40.341220] Serial: 8250/16550 driver, 4 ports, IRQ sharing disabled +[ 40.697320] of_serial 40001000.serial: error -ENXIO: IRQ index 0 not found +[ 40.844360] 40001000.serial: ttyS0 at MMIO 0x40001000 (irq = 0, base_baud = 3125000) is a 16550 +[ 41.039860] 40003000.serial: ttyS1 at MMIO 0x40003000 (irq = 1, base_baud = 3125000) is a 16550 +[ 41.147500] SuperH (H)SCI(F) driver initialized +[ 42.464640] loop: module loaded +[ 42.621480] nd_pmem namespace0.0: unable to guarantee persistence of writes +[ 42.797320] pmem0: p1 p2 +[ 43.117840] usbcore: registered new interface driver uas +[ 43.151240] usbcore: registered new interface driver usb-storage +[ 43.187600] usbcore: registered new interface driver usbserial_generic +[ 43.198760] usbserial: USB Serial support registered for generic +[ 43.240820] mousedev: PS/2 mouse device common for all mice +[ 43.421760] sdhci: Secure Digital Host Controller Interface driver +[ 43.429680] sdhci: Copyright(c) Pierre Ossman +[ 43.461820] Synopsys Designware Multimedia Card Interface Driver +[ 43.497400] sdhci-pltfm: SDHCI platform and OF driver helper +[ 43.539140] hid: raw HID events driver (C) Jiri Kosina +[ 43.569620] usbcore: registered new interface driver usbhid +[ 43.576400] usbhid: USB HID core driver +[ 43.621260] riscv-pmu-sbi: SBI PMU extension is available +[ 43.633420] riscv-pmu-sbi: 16 firmware and 31 hardware counters +[ 43.640860] riscv-pmu-sbi: Perf sampling/filtering is not supported as sscof extension is not available +[ 43.792480] drop_monitor: Initializing network drop monitor service +[ 43.915420] NET: Registered PF_INET6 protocol family +[ 49.587120] Initramfs unpacking failed: invalid magic at start of compressed archive +[ 57.799900] Freeing initrd memory: 300428K +[ 57.863740] Segment Routing with IPv6 +[ 57.873220] In-situ OAM (IOAM) with IPv6 +[ 57.882320] sit: IPv6, IPv4 and MPLS over IPv4 tunneling driver +[ 57.964280] NET: Registered PF_PACKET protocol family +[ 57.993060] 9pnet: Installing 9P2000 support +[ 58.003340] Key type dns_resolver registered +[ 58.013600] start plist test +[ 58.094820] end plist test +[ 59.433600] registered taskstats version 1 +[ 59.447940] Loading compiled-in X.509 certificates +[ 62.396240] debug_vm_pgtable: [debug_vm_pgtable ]: Validating architecture page table helpers +[ 62.449040] Key type .fscrypt registered +[ 62.454140] Key type fscrypt-provisioning registered +[ 62.540060] hid_bpf: error while preloading HID BPF dispatcher: -22 +[ 62.552660] clk: Disabling unused clocks +[ 62.564820] PM: genpd: Disabling unused power domains +[ 62.571280] ALSA device list: +[ 62.575900] No soundcards found. +[ 63.179280] Freeing unused kernel image (initmem) memory: 8736K +[ 63.281280] Checked W+X mappings: passed, no W+X pages found +[ 63.289780] Run /init as init process +[ 63.295560] with arguments: +[ 63.299640] /init +[ 63.303080] with environment: +[ 63.307320] HOME=/ +[ 63.310820] TERM=linux + +<<< NixOS Stage 1 >>> + ++ IFS='=' ++ echo console hvc0 ++ set -- console hvc0 ++ params=hvc0 ++ IFS=, ++ echo hvc0 ++ set -- hvc0 ++ console=hvc0 ++ IFS='=' ++ echo init /nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init ++ set -- init /nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init ++ stage2Init=/nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init ++ echo /nix/store/988axh0bq3wqp90gms4b4a0hkfwvkd3i-extra-utils/bin/modprobe[ 71.111740] stage-1-init: [Thu Jan 1 00:01:10 UTC 1970] + IFS='=' + ++ basename dm_mod ++ info 'loading module dm_mod...' ++ '[[' -n 1 ]] ++ echo 'loading module dm_mod...' +loading module dm_mod... ++ modprobe dm_mod +[ 71.713020] stage-1-init: [Thu Jan 1 00:01:11 UTC 1970] + echo console hvc0 +[ 72.282000] stage-1-init: [Thu Jan 1 00:01:11 UTC 1970] + set -- console hvc0 +[ 72.828720] stage-1-init: [Thu Jan 1 00:01:12 UTC 1970] + params=hvc0 +[ 73.359520] stage-1-init: [Thu Jan 1 00:01:12 UTC 1970] + IFS=, +[ 73.889960] stage-1-init: [Thu Jan 1 00:01:13 UTC 1970] + echo hvc0 +[ 74.448740] stage-1-init: [Thu Jan 1 00:01:13 UTC 1970] + set -- hvc0 +[ 74.988080] stage-1-init: [Thu Jan 1 00:01:14 UTC 1970] + console=hvc0 +[ 75.523700] stage-1-init: [Thu Jan 1 00:01:14 UTC 1970] + IFS='=' +[ 76.152720] stage-1-init: [Thu Jan 1 00:01:15 UTC 1970] + echo init /nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init +[ 76.800900] stage-1-init: [Thu Jan 1 00:01:16 UTC 1970] + set -- init /nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init +[ 77.441300] stage-1-init: [Thu Jan 1 00:01:16 UTC 1970] + stage2Init=/nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init +[ 78.060720] stage-1-init: [Thu Jan 1 00:01:17 UTC 1970] + echo /nix/store/988axh0bq3wqp90gms4b4a0hkfwvkd3i-extra-utils/bin/modprobe +[ 78.613460] stage-1-init: [Thu Jan 1 00:01:17 UTC 1970] + basename dm_mod +[ 79.520040] stage-1-init: [Thu Jan 1 00:01:18 UTC 1970] + info 'loading module dm_mod...' +[ 79.958040] stage-1-init: [Thu Jan 1 00:01:19 UTC 1970] + '[[' -n 1 ]] +[ 80.571720] stage-1-init: [Thu Jan 1 00:01:19 UTC 1970] + echo 'loading module dm_mod...' +[ 80.607660] device-mapper: ioctl: 4.48.0-ioctl (2023-03-01) initialised: dm-devel@lists.linux.dev ++ echo 'Creating a heartbeat counter at 0x1bfff0000' +Creating a heartbeat counter at 0x1bfff0000 ++ echo 0 ++ info 'running udev...' ++ '[[' -n 1 ]] ++ echo 'running udev...' +running udev... ++ ln -sfn /proc/self/fd /dev/fd ++ sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' +[ 81.473120] stage-1-init: [Thu Jan 1 00:01:20 UTC 1970] loading module dm_mod... ++ ln -sfn /proc/self/fd/0 /dev/stdin +[ 82.329900] stage-1-init: [Thu Jan 1 00:01:21 UTC 1970] + modprobe dm_mod ++ ln -sfn /proc/self/fd/1 /dev/stdout +[ 83.264540] stage-1-init: [Thu Jan 1 00:01:22 UTC 1970] + echo 'Creating a heartbeat counter at 0x1bfff0000' ++ ln -sfn /proc/self/fd/2 /dev/stderr ++ mkdir -p /etc/systemd +[ 84.156700] stage-1-init: [Thu Jan 1 00:01:23 UTC 1970] Creating a heartbeat counter at 0x1bfff0000 ++ ln -sfn /nix/store/rwgnr0zbwm045ijjkhq9mmyqymffan17-link-units /etc/systemd/network +[ 84.989640] stage-1-init: [Thu Jan 1 00:01:24 UTC 1970] + echo 0 ++ mkdir -p /etc/udev +[ 85.867560] stage-1-init: [Thu Jan 1 00:01:25 UTC 1970] + info 'running udev...' ++ ln -sfn /nix/store/2lwmlx07crc2zys49kkwrfm3zhkj6zfr-udev-rules /etc/udev/rules.d +[ 86.717780] stage-1-init: [Thu Jan 1 00:01:26 UTC 1970] + '[[' -n 1 ]] ++ mkdir -p /dev/.mdadm +[ 87.589900] stage-1-init: [Thu Jan 1 00:01:26 UTC 1970] + echo 'running udev...' ++ systemd-udevd --daemon +[ 88.423760] stage-1-init: [Thu Jan 1 00:01:27 UTC 1970] running udev... +[ 89.309480] stage-1-init: [Thu Jan 1 00:01:28 UTC 1970] + ln -sfn /proc/self/fd /dev/fd +[ 90.287460] stage-1-init: [Thu Jan 1 00:01:29 UTC 1970] + sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' +Starting systemd-udevd version 255.6 +[ 91.248620] stage-1-init: [Thu Jan 1 00:01:30 UTC 1970] + ln -sfn /proc/self/fd/0 /dev/stdin ++ udevadm trigger '--action=add' +[ 92.431600] stage-1-init: [Thu Jan 1 00:01:31 UTC 1970] + ln -sfn /proc/self/fd/1 /dev/stdout +[ 93.623640] stage-1-init: [Thu Jan 1 00:01:32 UTC 1970] + ln -sfn /proc/self/fd/2 /dev/stderr +[ 94.768040] stage-1-init: [Thu Jan 1 00:01:34 UTC 1970] + mkdir -p /etc/systemd +[ 95.720180] stage-1-init: [Thu Jan 1 00:01:34 UTC 1970] + ln -sfn /nix/store/rwgnr0zbwm045ijjkhq9mmyqymffan17-link-units /etc/systemd/network +[ 96.587820] stage-1-init: [Thu Jan 1 00:01:35 UTC 1970] + mkdir -p /etc/udev +[ 97.519760] stage-1-init: [Thu Jan 1 00:01:36 UTC 1970] + ln -sfn /nix/store/2lwmlx07crc2zys49kkwrfm3zhkj6zfr-udev-rules /etc/udev/rules.d +[ 98.364160] stage-1-init: [Thu Jan 1 00:01:37 UTC 1970] + mkdir -p /dev/.mdadm +[ 99.194820] stage-1-init: [Thu Jan 1 00:01:38 UTC 1970] + systemd-udevd --daemon +[ 100.050720] stage-1-init: [Thu Jan 1 00:01:39 UTC 1970] Starting systemd-udevd version 255.6 +[ 100.907560] stage-1-init: [Thu Jan 1 00:01:40 UTC 1970] + udevadm trigger '--action=add' ++ udevadm settle +[ 204.792900] stage-1-init: [Thu Jan 1 00:03:24 UTC 1970] + udevadm settle ++ kbd_mode -u -C /dev/console +kbd_mode: KDSKBMODE: Inappropriate ioctl for device ++ printf '\033%%G' ++ loadkmap +[ 305.593520] stage-1-init: [Thu Jan 1 00:05:04 UTC 1970] + kbd_mode -u -C /dev/console +[ 306.507560] stage-1-init: [Thu Jan 1 00:05:05 UTC 1970] kbd_mode: KDSKBMODE: Inappropriate ioctl for device ++ info 'starting device mapper and LVM...' ++ '[[' -n 1 ]] ++ echo 'starting device mapper and LVM...' +starting device mapper and LVM... ++ lvm vgchange -ay +[ 307.384260] stage-1-init: [Thu Jan 1 00:05:06 UTC 1970] + printf '\033%%G' +[ 308.487820] stage-1-init: [Thu Jan 1 00:05:07 UTC 1970] + loadkmap +[ 309.589740] random: lvm: uninitialized urandom read (4 bytes read) +[ 309.676340] stage-1-init: [Thu Jan 1 00:05:08 UTC 1970] + info 'starting device mapper and LVM...' +[ 310.899120] stage-1-init: [Thu Jan 1 00:05:10 UTC 1970] + '[[' -n 1 ]] +[ 311.760340] stage-1-init: [Thu Jan 1 00:05:11 UTC 1970] + echo 'starting device mapper and LVM...' +[ 312.620080] stage-1-init: [Thu Jan 1 00:05:11 UTC 1970] starting device mapper and LVM... +[ 313.468280] stage-1-init: [Thu Jan 1 00:05:12 UTC 1970] + lvm vgchange -ay ++ test -n ++ test -e /sys/power/resume -a -e /sys/power/disk ++ '[' -n ] ++ mkdir -p /mnt-root +[ 317.268920] stage-1-init: [Thu Jan 1 00:05:16 UTC 1970] + test -n ++ exec ++ read -u 3 mountPoint ++ read -u 3 device ++ read -u 3 fsType ++ read -u 3 options ++ pseudoDevice= ++ test -z ++ waitDevice /dev/disk/by-label/NIXOS_SD ++ local 'device=/dev/disk/by-label/NIXOS_SD' ++ local IFS ++ '[' ext4 '=' bcachefs ] ++ test '!' -e /dev/disk/by-label/NIXOS_SD ++ udevadm settle +[ 318.217580] stage-1-init: [Thu Jan 1 00:05:17 UTC 1970] + test -e /sys/power/resume -a -e /sys/power/disk +[ 319.069060] stage-1-init: [Thu Jan 1 00:05:18 UTC 1970] + '[' -n ] +[ 319.916780] stage-1-init: [Thu Jan 1 00:05:19 UTC 1970] + mkdir -p /mnt-root +[ 320.769780] stage-1-init: [Thu Jan 1 00:05:20 UTC 1970] + exec ++ '[' -n ] ++ '[' / '=' / ] ++ '[' /dev/disk/by-label/NIXOS_SD '=' tmpfs ] ++ escapeFstab / ++ local 'original=/' ++ local 'escaped=/' ++ echo / +[ 321.672440] stage-1-init: [Thu Jan 1 00:05:20 UTC 1970] + read -u 3 mountPoint ++ escapeFstab x-initrd.mount ++ local 'original=x-initrd.mount' ++ local 'escaped=x-initrd.mount' ++ echo x-initrd.mount ++ mountFS /dev/disk/by-label/NIXOS_SD / x-initrd.mount ext4 ++ local 'device=/dev/disk/by-label/NIXOS_SD' ++ local 'mountPoint=/' ++ local 'options=x-initrd.mount' ++ local 'fsType=ext4' ++ '[' ext4 '=' auto ] ++ IFS=, ++ '[' x- '!=' x- ] ++ local 'optionsFiltered=' +[ 322.609040] stage-1-init: [Thu Jan 1 00:05:21 UTC 1970] + read -u 3 device ++ echo ++ sed -E 's#\<(lowerdir|upperdir|workdir)=#\1=/mnt-root#g' +[ 323.548540] stage-1-init: [Thu Jan 1 00:05:22 UTC 1970] + read -u 3 fsType ++ local 'optionsPrefixed=' ++ echo '/dev/disk/by-label/NIXOS_SD /mnt-root/ ext4 ' ++ checkFS /dev/disk/by-label/NIXOS_SD ext4 ++ local 'device=/dev/disk/by-label/NIXOS_SD' ++ local 'fsType=ext4' ++ '[' '!' -b /dev/disk/by-label/NIXOS_SD ] ++ '[' ext4 '=' iso9660 -o ext4 '=' udf ] ++ '[' ext4 '=' btrfs -o ext4 '=' zfs -o ext4 '=' bcachefs ] ++ '[' ext4 '=' apfs ] ++ '[' ext4 '=' nilfs2 ] ++ '[' ext4 '=' squashfs ] ++ '[' ext4 '=' erofs ] ++ '[' ext4 '=' auto ] ++ mount ++ grep -q '^/dev/disk/by-label/NIXOS_SD on ' +[ 324.688960] stage-1-init: [Thu Jan 1 00:05:23 UTC 1970] + read -u 3 options ++ test -z 1 -a '(' ext4 '=' ext3 -o ext4 '=' ext4 -o ext4 '=' reiserfs -o ext4 '=' xfs -o ext4 '=' jfs -o ext4 '=' f2fs ')' ++ echo 'checking /dev/disk/by-label/NIXOS_SD...' +checking /dev/disk/by-label/NIXOS_SD... ++ fsck -V -a /dev/disk/by-label/NIXOS_SD +[ 325.609200] stage-1-init: [Thu Jan 1 00:05:24 UTC 1970] + pseudoDevice= +fsck (busybox 1.36.1) +[fsck.ext4 (1) -- /mnt-root/] fsck.ext4 -a /dev/disk/by-label/NIXOS_SD +[ 326.476140] stage-1-init: [Thu Jan 1 00:05:25 UTC 1970] + test -z +NIXOS_SD: recovering journal +NIXOS_SD: clean, 52833/122160 files, 393133/491520 blocks +[ 327.380080] stage-1-init: [Thu Jan 1 00:05:26 UTC 1970] + waitDevice /dev/disk/by-label/NIXOS_SD ++ fsckResult=0 ++ test 2 '=' 0 ++ test 4 '=' 0 ++ test 0 -ge 8 ++ return 0 ++ '[' ext4 '=' overlay ] ++ info 'mounting /dev/disk/by-label/NIXOS_SD on /...' ++ '[[' -n 1 ]] ++ echo 'mounting /dev/disk/by-label/NIXOS_SD on /...' +mounting /dev/disk/by-label/NIXOS_SD on /... ++ makeMountPoint /dev/disk/by-label/NIXOS_SD / ++ local 'device=/dev/disk/by-label/NIXOS_SD' ++ local 'mountPoint=/' ++ local 'options=' ++ local 'IFS=,' ++ '[' -d /dev/disk/by-label/NIXOS_SD ] ++ mkdir -m 0755 -p /mnt-root/ +[ 328.351960] stage-1-init: [Thu Jan 1 00:05:27 UTC 1970] + local 'device=/dev/disk/by-label/NIXOS_SD' ++ local 'n=0' ++ true ++ mount /mnt-root/ +[ 329.204940] stage-1-init: [Thu Jan 1 00:05:28 UTC 1970] + local IFS +[ 329.524640] EXT4-fs (pmem0p2): mounted filesystem 44444444-4444-4444-8888-888888888888 r/w with ordered data mode. Quota mode: none. ++ break ++ mount /mnt-root/ -o remount, +[ 330.096240] stage-1-init: [Thu Jan 1 00:05:29 UTC 1970] + '[' ext4 '=' bcachefs ] +[ 330.423880] EXT4-fs (pmem0p2): re-mounted 44444444-4444-4444-8888-888888888888 r/w. Quota mode: none. ++ '[' / '==' / ] ++ '[' -f /mnt-root/etc/NIXOS_LUSTRATE ] ++ true ++ read -u 3 mountPoint ++ exec ++ '[' -e /mnt-root/iso ] ++ udevadm info --export '--export-prefix=ROOT_' '--device-id-of-file=/mnt-root' +[ 331.044220] stage-1-init: [Thu Jan 1 00:05:30 UTC 1970] + test '!' -e /dev/disk/by-label/NIXOS_SD +[ 331.904160] stage-1-init: [Thu Jan 1 00:05:31 UTC 1970] + udevadm settle +[ 332.755700] stage-1-init: [Thu Jan 1 00:05:32 UTC 1970] + '[' -n ] +[ 333.603600] stage-1-init: [Thu Jan 1 00:05:32 UTC 1970] + '[' / '=' / ] ++ eval 'ROOT_MAJOR=259' 'ROOT_MINOR=2' ++ ROOT_MAJOR=259 ROOT_MINOR=2 ++ '[' 259 -a 2 -a 259 '!=' 0 ] ++ mkdir -p /run/udev/rules.d +[ 334.527860] stage-1-init: [Thu Jan 1 00:05:33 UTC 1970] + '[' /dev/disk/by-label/NIXOS_SD '=' tmpfs ] ++ echo 'ACTION=="add|change", SUBSYSTEM=="block", ENV{MAJOR}=="259", ENV{MINOR}=="2", SYMLINK+="root"' ++ udevadm control --exit +[ 335.392500] stage-1-init: [Thu Jan 1 00:05:34 UTC 1970] + escapeFstab / +[ 336.237560] stage-1-init: [Thu Jan 1 00:05:35 UTC 1970] + local 'original=/' +[ 337.080820] stage-1-init: [Thu Jan 1 00:05:36 UTC 1970] + local 'escaped=/' ++ + evalexec + 'exec 8>&- 9>&-' ++ exec +[ 338.100000] stage-1-init: [Thu Jan 1 00:05:37 UTC 1970] + echo / ++ pgrep -v -f ^@ +[ 338.990840] stage-1-init: [Thu Jan 1 00:05:38 UTC 1970] + escapeFstab x-initrd.mount +[ 339.835920] stage-1-init: [Thu Jan 1 00:05:39 UTC 1970] + local 'original=x-initrd.mount' ++ readlink /proc/1/exe +[ 340.702820] stage-1-init: [Thu Jan 1 00:05:39 UTC 1970] + local 'escaped=x-initrd.mount' ++ '[' 1 -eq 1 ] ++ continue ++ readlink /proc/2/exe +[ 341.543740] stage-1-init: [Thu Jan 1 00:05:40 UTC 1970] + echo x-initrd.mount ++ continue ++ readlink /proc/3/exe +[ 342.457600] stage-1-init: [Thu Jan 1 00:05:41 UTC 1970] + mountFS /dev/disk/by-label/NIXOS_SD / x-initrd.mount ext4 ++ continue ++ readlink /proc/4/exe +[ 343.332000] stage-1-init: [Thu Jan 1 00:05:42 UTC 1970] + local 'device=/dev/disk/by-label/NIXOS_SD' ++ continue ++ readlink /proc/5/exe ++ [ 344.178760] stage-1-init: [Thu Jan 1 00:05:43 UTC 1970] + local 'mountPoint=/' +continue ++ readlink /proc/6/exe ++ continue ++ readlink /proc/7/exe +[ 345.050720] stage-1-init: [Thu Jan 1 00:05:44 UTC 1970] + local 'options=x-initrd.mount' ++ continue ++ readlink /proc/8/exe +[ 345.927260] stage-1-init: [Thu Jan 1 00:05:45 UTC 1970] + local 'fsType=ext4' ++ continue ++ readlink /proc/9/exe +[ 346.777360] stage-1-init: [Thu Jan 1 00:05:46 UTC 1970] + '[' ext4 '=' auto ] ++ continue ++ readlink /proc/10/exe +[ 347.588680] stage-1-init: [Thu Jan 1 00:05:46 UTC 1970] + IFS=, ++ continue ++ readlink /proc/11/exe +[ 348.448200] stage-1-init: [Thu Jan 1 00:05:47 UTC 1970] + '[' x- '!=' x- ] ++ continue ++ readlink /proc/12/exe +[ 349.316640] stage-1-init: [Thu Jan 1 00:05:48 UTC 1970] + local 'optionsFiltered=' ++ continue ++ readlink /proc/13/exe +[ 350.152600] stage-1-init: [Thu Jan 1 00:05:49 UTC 1970] + echo ++ continue ++ readlink /proc/14/exe +[ 351.035880] stage-1-init: [Thu Jan 1 00:05:50 UTC 1970] + sed -E 's#\<(lowerdir|upperdir|workdir)=#\1=/mnt-root#g' ++ continue ++ readlink /proc/15/exe +[ 351.897160] stage-1-init: [Thu Jan 1 00:05:51 UTC 1970] + local 'optionsPrefixed=' ++ continue ++ readlink /proc/16/exe +[ 352.813200] stage-1-init: [Thu Jan 1 00:05:52 UTC 1970] + echo '/dev/disk/by-label/NIXOS_SD /mnt-root/ ext4 ' ++ continue ++ readlink /proc/17/exe ++ continue ++ readlink /proc/18/exe[ 353.712780] stage-1-init: [Thu Jan 1 00:05:52 UTC 1970] + checkFS /dev/disk/by-label/NIXOS_SD ext4 + ++ continue ++ readlink /proc/19/exe +[ 354.601220] stage-1-init: [Thu Jan 1 00:05:53 UTC 1970] + local 'device=/dev/disk/by-label/NIXOS_SD' ++ continue ++ readlink /proc/20/exe +[ 355.468760] stage-1-init: [Thu Jan 1 00:05:54 UTC 1970] + local 'fsType=ext4' ++ continue ++ readlink /proc/21/exe +[ 356.367400] stage-1-init: [Thu Jan 1 00:05:55 UTC 1970] + '[' '!' -b /dev/disk/by-label/NIXOS_SD ] ++ continue ++ readlink /proc/22/exe +[ 357.265240] stage-1-init: [Thu Jan 1 00:05:56 UTC 1970] + '[' ext4 '=' iso9660 -o ext4 '=' udf ] ++ continue ++ readlink /proc/23/exe +[ 358.205080] stage-1-init: [Thu Jan 1 00:05:57 UTC 1970] + '[' ext4 '=' btrfs -o ext4 '=' zfs -o ext4 '=' bcachefs ] ++ continue ++ readlink /proc/24/exe +[ 359.079760] stage-1-init: [Thu Jan 1 00:05:58 UTC 1970] + '[' ext4 '=' apfs ] ++ continue ++ readlink /proc/25/exe +[ 359.919800] stage-1-init: [Thu Jan 1 00:05:59 UTC 1970] + '[' ext4 '=' nilfs2 ] ++ continue ++ readlink /proc/26/exe ++ continue +[ 360.787940] stage-1-init: [Thu Jan 1 00:06:00 UTC 1970] + '[' ext4 '=' squashfs ] ++ readlink /proc/27/exe ++ continue ++ readlink /proc/28/exe +[ 361.648260] stage-1-init: [Thu Jan 1 00:06:00 UTC 1970] + '[' ext4 '=' erofs ] ++ continue ++ readlink /proc/29/exe +[ 362.507320] stage-1-init: [Thu Jan 1 00:06:01 UTC 1970] + '[' ext4 '=' auto ] ++ continue ++ readlink /proc/30/exe +[ 363.316580] stage-1-init: [Thu Jan 1 00:06:02 UTC 1970] + mount ++ continue ++ readlink /proc/31/exe +[ 364.199960] stage-1-init: [Thu Jan 1 00:06:03 UTC 1970] + grep -q '^/dev/disk/by-label/NIXOS_SD on ' ++ continue ++ readlink /proc/32/exe +[ 365.232880] stage-1-init: [Thu Jan 1 00:06:04 UTC 1970] + test -z 1 -a '(' ext4 '=' ext3 -o ext4 '=' ext4 -o ext4 '=' reiserfs -o ext4 '=' xfs -o ext4 '=' jfs -o ext4 '=' f2fs ')' ++ continue ++ readlink /proc/33/exe +[ 366.162000] stage-1-init: [Thu Jan 1 00:06:05 UTC 1970] + echo 'checking /dev/disk/by-label/NIXOS_SD...' ++ continue ++ readlink /proc/34/exe +[ 367.042760] stage-1-init: [Thu Jan 1 00:06:06 UTC 1970] checking /dev/disk/by-label/NIXOS_SD... ++ continue ++ readlink /proc/35/exe ++ continue ++ readlink[ 367.939480] stage-1-init: [Thu Jan 1 00:06:07 UTC 1970] + fsck -V -a /dev/disk/by-label/NIXOS_SD + /proc/36/exe ++ continue ++ readlink /proc/37/exe +[ 368.785380] stage-1-init: [Thu Jan 1 00:06:08 UTC 1970] fsck (busybox 1.36.1) ++ continue ++ readlink /proc/38/exe +[ 369.728280] stage-1-init: [Thu Jan 1 00:06:09 UTC 1970] [fsck.ext4 (1) -- /mnt-root/] fsck.ext4 -a /dev/disk/by-label/NIXOS_SD ++ continue ++ readlink /proc/39/exe +[ 370.620960] stage-1-init: [Thu Jan 1 00:06:09 UTC 1970] NIXOS_SD: recovering journal ++ continue ++ readlink /proc/40/exe +[ 371.553240] stage-1-init: [Thu Jan 1 00:06:10 UTC 1970] NIXOS_SD: clean, 52833/122160 files, 393133/491520 blocks ++ continue ++ readlink /proc/47/exe +[ 372.419760] stage-1-init: [Thu Jan 1 00:06:11 UTC 1970] + fsckResult=0 ++ continue ++ readlink /proc/48/exe +[ 373.252460] stage-1-init: [Thu Jan 1 00:06:12 UTC 1970] + test 2 '=' 0 ++ continue ++ readlink /proc/74/exe +[ 374.083140] stage-1-init: [Thu Jan 1 00:06:13 UTC 1970] + test 4 '=' 0 ++ '[' 74 -eq 1 ] ++ kill -9 74 ++ readlink /proc/75/exe +[ 374.961120] stage-1-init: [Thu Jan 1 00:06:14 UTC 1970] + test 0 -ge 8 ++ '[' 75 -eq 1 ] ++ kill -9 75 ++ readlink /proc/102/exe ++ '[' 102 -eq 1 ] ++ kill -9 102 ++ readlink /proc/137/exe ++ continue ++ readlink /proc/674/exe ++ continue ++ readlink /proc/675/exe ++ continue ++ test -n ++ echo /sbin/modprobe ++ '[' '!' -e /mnt-root//nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init ] ++ mkdir -m 0755 -p /mnt-root/proc /mnt-root/sys /mnt-root/dev /mnt-root/run ++ mount --move /proc /mnt-root/proc ++ mount --move /sys /mnt-root/sys ++ mount --move /dev /mnt-root/dev ++ mount --move /run /mnt-root/run ++ type -P switch_root ++ exec env -i /nix/store/988axh0bq3wqp90gms4b4a0hkfwvkd3i-extra-utils/bin/switch_root /mnt-root /nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init + +<<< NixOS Stage 2 >>> + +[ 384.203680] EXT4-fs (pmem0p2): re-mounted 44444444-4444-4444-8888-888888888888 r/w. Quota mode: none. +[ 384.287600] booting system configuration /nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git +running activation script... +[ 388.163860] stage-2-init: running activation script... +[ 391.643500] random: perl: uninitialized urandom read (4 bytes read) +[ 391.884800] random: perl: uninitialized urandom read (4 bytes read) +[ 425.302000] random: perl: uninitialized urandom read (4 bytes read) + + -- GitLab From 87b4c91813f69f081690461affdf3157d7bb3e71 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 8 Jul 2024 08:55:50 +0200 Subject: [PATCH 062/310] Dump stack trace on hvc_remove --- lagarto-ox.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 0aeb392..37543d2 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -93,6 +93,17 @@ '' echo 0 > /proc/sys/vm/compaction_proactiveness '' + + + # Show stacktrace on calls to the hvc_remove function. + '' + echo "Mount debugfs" + mkdir -p /sys/kernel/debug/ + mount -t debugfs none /sys/kernel/debug/ + td=/sys/kernel/debug/tracing + echo hvc_remove > $td/set_ftrace_filter + echo function > $td/current_tracer + echo 1 > $td/options/func_stack_trace + '' # FIXME: Disable sched_switch for now, as it still hangs the boot... # + # # Exclude the second pid, which is the kthread that will dump the trace to -- GitLab From 931244a355d82c371af3f2419df376d1f6da2ec3 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 8 Jul 2024 09:22:15 +0200 Subject: [PATCH 063/310] Try to use openpiton based OpenSBI config The seem to be doing PLIC initialization based on the device tree, which may be relevant as that is where we are defining the interruptions. --- lagarto-ox.nix | 19 ++++++++++--------- opensbi-lagarto-ox.patch | 26 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 opensbi-lagarto-ox.patch diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 37543d2..71f0e49 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -255,16 +255,17 @@ ''; opensbi = prev.opensbi.overrideAttrs (old: rec { - version = "1.4"; - src = prev.fetchFromGitHub { - owner = "riscv-software-src"; - repo = "opensbi"; - rev = "v${version}"; - hash = "sha256-T8ZeAzjM9aeTXitjE7s+m+jjGGtDo2jK1qO5EuKiVLU="; - }; + #version = "1.4"; + #src = prev.fetchFromGitHub { + # owner = "riscv-software-src"; + # repo = "opensbi"; + # rev = "v${version}"; + # hash = "sha256-T8ZeAzjM9aeTXitjE7s+m+jjGGtDo2jK1qO5EuKiVLU="; + #}; #NIX_DEBUG=5; makeFlags = [ - "PLATFORM=fpga/ox_alveo" + "PLATFORM=fpga/openpiton" + #"PLATFORM=fpga/ox_alveo" #"CONFIG_SBI_ECALL_RFENCE=n" #"PLATFORM_RISCV_ISA=rv64imafd" # No compressed instructions #"PLATFORM_RISCV_ISA=rv64g" # No compressed instructions @@ -272,7 +273,7 @@ "FW_PAYLOAD_PATH=${final.uboot}/u-boot.bin" "FW_FDT_PATH=${final.ox-dtb}" ]; - patches = [ ./ox-alveo-platform-plic.patch ]; + patches = [ ./opensbi-lagarto-ox.patch ]; }); # opensbi = prev.opensbi.overrideAttrs (old: { # #NIX_DEBUG=5; diff --git a/opensbi-lagarto-ox.patch b/opensbi-lagarto-ox.patch new file mode 100644 index 0000000..d3dfce8 --- /dev/null +++ b/opensbi-lagarto-ox.patch @@ -0,0 +1,26 @@ +diff --git a/platform/fpga/openpiton/platform.c b/platform/fpga/openpiton/platform.c +index 2317a89..4a83ca9 100644 +--- a/platform/fpga/openpiton/platform.c ++++ b/platform/fpga/openpiton/platform.c +@@ -17,17 +17,17 @@ + #include + #include + +-#define OPENPITON_DEFAULT_UART_ADDR 0xfff0c2c000 +-#define OPENPITON_DEFAULT_UART_FREQ 60000000 ++#define OPENPITON_DEFAULT_UART_ADDR 0x40001000 ++#define OPENPITON_DEFAULT_UART_FREQ 50000000 + #define OPENPITON_DEFAULT_UART_BAUDRATE 115200 + #define OPENPITON_DEFAULT_UART_REG_SHIFT 0 + #define OPENPITON_DEFAULT_UART_REG_WIDTH 1 +-#define OPENPITON_DEFAULT_UART_REG_OFFSET 0 ++#define OPENPITON_DEFAULT_UART_REG_OFFSET 0x1000 + #define OPENPITON_DEFAULT_PLIC_ADDR 0xfff1100000 + #define OPENPITON_DEFAULT_PLIC_SIZE (0x200000 + \ + (OPENPITON_DEFAULT_HART_COUNT * 0x1000)) + #define OPENPITON_DEFAULT_PLIC_NUM_SOURCES 2 +-#define OPENPITON_DEFAULT_HART_COUNT 3 ++#define OPENPITON_DEFAULT_HART_COUNT 1 + #define OPENPITON_DEFAULT_CLINT_ADDR 0xfff1020000 + #define OPENPITON_DEFAULT_ACLINT_MTIMER_FREQ 1000000 + #define OPENPITON_DEFAULT_ACLINT_MSWI_ADDR \ -- GitLab From 1b4ab09c16c33feeb551404c4cd3c8d544ec92f5 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 8 Jul 2024 09:55:13 +0200 Subject: [PATCH 064/310] Select the second FPGA by default --- fpga/env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fpga/env.sh b/fpga/env.sh index 580e3bd..ac62ed5 100644 --- a/fpga/env.sh +++ b/fpga/env.sh @@ -29,7 +29,7 @@ function setup_meep() export PATH="$PATH:/apps/QDMA/meep-ionic/2022.1.4.4/linux-kernel/bin/" # Select the first FPGA in the node - local line=$(grep fpgan /etc/motd | head -1 | tr -d ' ') + local line=$(grep fpgan /etc/motd | sed -n 2p | tr -d ' ') export FPGACTL_PCIDEV=$(echo "$line" | awk -F'|' '{print $5}') export FPGACTL_SERIAL=$(echo "$line" | awk -F'|' '{print $4}') export FPGACTL_UART=$(echo "$line" | awk -F'|' '{print "/dev/"$7}') -- GitLab From 6721e1e22ca0aefad64a47f1882f2ee3cbd6bab7 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 8 Jul 2024 10:03:48 +0200 Subject: [PATCH 065/310] Revert "Try to use openpiton based OpenSBI config" This reverts commit 931244a355d82c371af3f2419df376d1f6da2ec3. --- lagarto-ox.nix | 19 +++++++++---------- opensbi-lagarto-ox.patch | 26 -------------------------- 2 files changed, 9 insertions(+), 36 deletions(-) delete mode 100644 opensbi-lagarto-ox.patch diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 71f0e49..37543d2 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -255,17 +255,16 @@ ''; opensbi = prev.opensbi.overrideAttrs (old: rec { - #version = "1.4"; - #src = prev.fetchFromGitHub { - # owner = "riscv-software-src"; - # repo = "opensbi"; - # rev = "v${version}"; - # hash = "sha256-T8ZeAzjM9aeTXitjE7s+m+jjGGtDo2jK1qO5EuKiVLU="; - #}; + version = "1.4"; + src = prev.fetchFromGitHub { + owner = "riscv-software-src"; + repo = "opensbi"; + rev = "v${version}"; + hash = "sha256-T8ZeAzjM9aeTXitjE7s+m+jjGGtDo2jK1qO5EuKiVLU="; + }; #NIX_DEBUG=5; makeFlags = [ - "PLATFORM=fpga/openpiton" - #"PLATFORM=fpga/ox_alveo" + "PLATFORM=fpga/ox_alveo" #"CONFIG_SBI_ECALL_RFENCE=n" #"PLATFORM_RISCV_ISA=rv64imafd" # No compressed instructions #"PLATFORM_RISCV_ISA=rv64g" # No compressed instructions @@ -273,7 +272,7 @@ "FW_PAYLOAD_PATH=${final.uboot}/u-boot.bin" "FW_FDT_PATH=${final.ox-dtb}" ]; - patches = [ ./opensbi-lagarto-ox.patch ]; + patches = [ ./ox-alveo-platform-plic.patch ]; }); # opensbi = prev.opensbi.overrideAttrs (old: { # #NIX_DEBUG=5; diff --git a/opensbi-lagarto-ox.patch b/opensbi-lagarto-ox.patch deleted file mode 100644 index d3dfce8..0000000 --- a/opensbi-lagarto-ox.patch +++ /dev/null @@ -1,26 +0,0 @@ -diff --git a/platform/fpga/openpiton/platform.c b/platform/fpga/openpiton/platform.c -index 2317a89..4a83ca9 100644 ---- a/platform/fpga/openpiton/platform.c -+++ b/platform/fpga/openpiton/platform.c -@@ -17,17 +17,17 @@ - #include - #include - --#define OPENPITON_DEFAULT_UART_ADDR 0xfff0c2c000 --#define OPENPITON_DEFAULT_UART_FREQ 60000000 -+#define OPENPITON_DEFAULT_UART_ADDR 0x40001000 -+#define OPENPITON_DEFAULT_UART_FREQ 50000000 - #define OPENPITON_DEFAULT_UART_BAUDRATE 115200 - #define OPENPITON_DEFAULT_UART_REG_SHIFT 0 - #define OPENPITON_DEFAULT_UART_REG_WIDTH 1 --#define OPENPITON_DEFAULT_UART_REG_OFFSET 0 -+#define OPENPITON_DEFAULT_UART_REG_OFFSET 0x1000 - #define OPENPITON_DEFAULT_PLIC_ADDR 0xfff1100000 - #define OPENPITON_DEFAULT_PLIC_SIZE (0x200000 + \ - (OPENPITON_DEFAULT_HART_COUNT * 0x1000)) - #define OPENPITON_DEFAULT_PLIC_NUM_SOURCES 2 --#define OPENPITON_DEFAULT_HART_COUNT 3 -+#define OPENPITON_DEFAULT_HART_COUNT 1 - #define OPENPITON_DEFAULT_CLINT_ADDR 0xfff1020000 - #define OPENPITON_DEFAULT_ACLINT_MTIMER_FREQ 1000000 - #define OPENPITON_DEFAULT_ACLINT_MSWI_ADDR \ -- GitLab From 4641e0d9a05892421c75e7ba465eb8921a733fb1 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 8 Jul 2024 10:41:50 +0200 Subject: [PATCH 066/310] Document hang missing hvc_remove trace point --- JOURNAL.md | 60 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index a6300fb..f12291c 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -43,7 +43,7 @@ Nope, the u-boot is reporting the d extension is in the isa: > riscv,isa = "rv64imafd"; -## 2024-07-03 +## 2024-07-03 I cannot switch to `gcc.arch = rv64ima` because rust fails to build. @@ -255,8 +255,8 @@ the plic follows a different convention of values. Using 9 and 11: **Remark**: The key combination to run Magic SysRq using the HVC console is Ctrl-O and then the SysRq key. It only works it the console is being actively polled, otherwise it hangs. - -## 2024-07-04 + +## 2024-07-04 ### OBSERVATION: I saw they changed this option in Cinco Ranch DTS for the serial: @@ -518,9 +518,11 @@ Can this produce any problem? It doesn't seem to change anything, still unable to send any bytes. -### QUESTION: Can we use virtio to mount a FS in the DMA shared memory? +### QUESTION: Can we use virtio to mount a FS in the DMA shared memory? -## 2024-07-05 + + +## 2024-07-05 ### OBSERVATION: The kernel continues working when the console hangs. @@ -568,11 +570,10 @@ Then Yes, it seems to be working. Let's load the rootfs too. I added a loop in the stage1 script. - -### QUESTION: Can we see any clock in memory? +### QUESTION: Can we see any clock in memory? This will allow us to check if the AXI still works. - + ### OBSERVATION: The kernel stops updating the counter in the mount phase. Managed to reach the mount and hang there: @@ -592,8 +593,10 @@ After almost 6 minutes, with 571 beats: It looks like the kernel is the one getting stuck *or* at least is unable to propagate the heartbeat changes to the host. It would be nice to monitor a -hardware clock from the DMA region too, so we can discard problems in the AXI. +hardware clock from the DMA region too, so we can discard problems in the AXI. + +### OBSERVATION: There is an ioctl failed for /dev/console [ 177.009540] stage-1-init: [Thu Jan 1 00:02:56 UTC 1970] + udevadm settle + kbd_mode -u -C /dev/console @@ -602,6 +605,7 @@ hardware clock from the DMA region too, so we can discard problems in the AXI. ### ASSUMPTION: The kernel hangs. If the kernel hangs, there must be an instruction or sequence of instructions @@ -681,7 +685,6 @@ Disabling clang as it is failing to build: error: 1 dependencies of derivation '/nix/store/l2x18cih29r1kn6vi8imwhkyk98yhw4i-nix-shell-riscv64-unknown-linux-gnu-env.drv' failed to build - ### QUESTION: Missing cache information may affect? Other CPUs report the cache details in the DT. For example this one @@ -716,9 +719,7 @@ https://github.com/torvalds/linux/blob/master/arch/riscv/boot/dts/sifive/fu540-c }; We may want to add it to our DT to be sure that it has no effect. - - -### OBSERVATION: Arrived to stage 2! +### OBSERVATION: Arrived to stage 2! + kill -9 74 + readlink /proc/75/exe @@ -756,13 +757,17 @@ We may want to add it to our DT to be sure that it has no effect. [ 425.302000] random: perl: uninitialized urandom read (4 bytes read) But then it hangs. + + +## 2024-07-08 -### QUESTION: Who sets the plic interrupts? +### QUESTION: Who sets the plic interrupts? -Shouldn't OpenSBI read the DT and do some configuration in the plic while in +Shouldn't OpenSBI read the DT and do some configuration in the PLIC while in machine mode? -### OBSERVATION: Semi-stack trace from CincoRanch + +### OBSERVATION: Semi-stack trace from CincoRanch hvc_remove? console_unlock <-- only called from hvc_remove() @@ -797,8 +802,29 @@ machine mode? no_context.part.0 die_kernel_fault <-- last frame(?) -### QUESTION: Can we place a tracepoint in `hvc_remove`? + +### QUESTION: Can we place a trace point in `hvc_remove`? If we are getting stuck in the same place, we should be able to see the backtrace (assuming the console still works) just before we try to remove the console device. + +Placed, but still unable to see anything in any hang. Here is a hang in the +Stage 2: + + <<< NixOS Stage 2 >>> + + [ 404.158340] EXT4-fs (pmem0p2): re-mounted 44444444-4444-4444-8888-888888888888 r/w. Quota mode: none. + [ 404.242500] booting system configuration /nix/store/0za1vqh5alk7mxqs59qxx8izmwmf21w6-nixos-system-nixos-riscv-24.11pre-git + running activation script... + [ 408.148380] stage-2-init: running activation script... + [ 411.612240] random: perl: uninitialized urandom read (4 bytes read) + [ 411.866440] random: perl: uninitialized urandom read (4 bytes read) + [ 447.588880] random: perl: uninitialized urandom read (4 bytes read) + +Still, it may be hang in a similar way, causing a loop of page faults just +while trying to printk to the console, which would explain why we don't see +anything and why the heartbeat stops. + + + -- GitLab From 4d246ad00e6b7e1d30d3ade3dcf0121aa96723d7 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 8 Jul 2024 10:44:51 +0200 Subject: [PATCH 067/310] Enable secondary serial console --- JOURNAL.md | 4 ++++ ox-plic.dts | 26 ++++++++++++-------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index f12291c..c0d3dc2 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -828,3 +828,7 @@ anything and why the heartbeat stops. +Fran has created another bitstream with two consoles enabled +(`ox_u55c_a234c132.bit`), let see if I manage to boot with it. + +First I will need to enable the consoles on the DTS. diff --git a/ox-plic.dts b/ox-plic.dts index 31cf650..f66ce9a 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -7,7 +7,7 @@ aliases { serial0 = &uart_console; // ttyS0 -// serial1 = &uart_testing; // ttyS1 + serial1 = &uart_testing; // ttyS1 }; // chosen { @@ -120,8 +120,6 @@ uart_console: serial@40001000 { compatible = "ns16550"; reg = <0x0 0x40001000 0x0 0x1000>; - interrupts = <1>; /* Output interrupt 1 */ - interrupt-parent = <&PLIC>; reg-shift = <2>; /* No interrupts for this UART, use console=hvc0 */ /* This clock is the SERIAL_CLK */ @@ -129,17 +127,17 @@ current-speed = <115200>; status = "okay"; }; -// /* The serial for interrupt tests */ -// uart_testing: serial@40003000 { -// compatible = "ns16550"; -// reg = <0x0 0x40003000 0x0 0x1000>; -// interrupts = <1>; /* Output interrupt 1 */ -// interrupt-parent = <&PLIC>; -// reg-shift = <2>; -// clock-frequency = <50000000>; -// current-speed = <115200>; -// status = "okay"; -// }; + /* The serial for interrupt tests */ + uart_testing: serial@40003000 { + compatible = "ns16550"; + reg = <0x0 0x40003000 0x0 0x1000>; + interrupts = <1>; /* Output interrupt 1 */ + interrupt-parent = <&PLIC>; + reg-shift = <2>; + clock-frequency = <50000000>; + current-speed = <115200>; + status = "okay"; + }; // ethernet0 { // xlnx,rxmem = <0x000005f2>; -- GitLab From 1f0ac646318af353e46d9405e8a011fda1d17fd9 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 8 Jul 2024 13:32:29 +0200 Subject: [PATCH 068/310] Add bitstream to Nix --- JOURNAL.md | 23 +++++++++++++++++++++++ flake.nix | 2 ++ fpga/upload.sh | 1 + lagarto-ox.nix | 3 +++ overlay.nix | 5 +++++ 5 files changed, 34 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index c0d3dc2..4f78f26 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -832,3 +832,26 @@ Fran has created another bitstream with two consoles enabled (`ox_u55c_a234c132.bit`), let see if I manage to boot with it. First I will need to enable the consoles on the DTS. + +It doesn't seem to produce any output in the UART. I cannot see OpenSBI while +loading it on each baud rate: + + Type [C-a] [C-h] to see available commands + Terminal ready + + *** baud: 57600 *** + + *** baud: 38400 *** + + *** baud: 19200 *** + + *** baud: 9600 *** + + [...] + + *** baud: 230400 *** + + *** baud: 460800 *** + +Let's keep the bitstream files in a repository, so I can carefully track them +with git too. diff --git a/flake.nix b/flake.nix index 69f22a1..e54b84c 100644 --- a/flake.nix +++ b/flake.nix @@ -107,6 +107,7 @@ INITRD = build.initialRamdisk; ROOTFS = build.sdImage; UBOOT_ENV = syspkgs.uboot-env; + BITSTREAM = syspkgs.bitstream; shellHook = '' echo "Here are the current system pieces:" echo " TOPLEVEL = $TOPLEVEL" @@ -115,6 +116,7 @@ echo " INITRD = $INITRD" echo " ROOTFS = $ROOTFS" echo " UBOOT_ENV = $UBOOT_ENV" + echo " BITSTREAM = $BITSTREAM" ''; }; diff --git a/fpga/upload.sh b/fpga/upload.sh index 783b0ef..e136f3f 100755 --- a/fpga/upload.sh +++ b/fpga/upload.sh @@ -21,6 +21,7 @@ if [ -n "$ROOTFS" ]; then else echo "Skipping rootfs" fi +rsync "$BITSTREAM" "$dst/bitstream.bit" rsync "$UBOOT_ENV" "$dst/uboot.env" echo "Now go to $dst and run ./boot.sh" diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 37543d2..74d9b4b 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -177,6 +177,9 @@ nativeBuildInputs = [ prev.buildPackages.dtc ]; } "dtc -O dtb -o $out $dtsFile"; + bitstream = "${final.bitstreams}/lagarto-3-ox/gold.bit"; + #bitstream = "${final.bitstreams}/lagarto-3-ox/ox_u55c_a234c132_two_uarts.bit"; + uboot = prev.ubootQemuRiscv64Smode.override { filesToInstall = [ "u-boot.bin" "u-boot-nodtb.bin" ]; #version = "2023.07.02-print-cpu-probe"; diff --git a/overlay.nix b/overlay.nix index 55d58ec..19ca0ec 100644 --- a/overlay.nix +++ b/overlay.nix @@ -36,4 +36,9 @@ final: prev: cp unalign_check $out/bin/ ''; }; + + bitstreams = builtins.fetchGit { + url = "git@bscpm03.bsc.es:rarias/bitstreams.git"; + rev = "57876cea158d8a4a63f7d35d715000092609d88c"; + }; } -- GitLab From 66ec07a0cbdbac7c537b68edaad2f33ff9e10cb8 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 8 Jul 2024 13:48:20 +0200 Subject: [PATCH 069/310] Switch to two uarts bitstream --- lagarto-ox.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 74d9b4b..2672017 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -177,8 +177,8 @@ nativeBuildInputs = [ prev.buildPackages.dtc ]; } "dtc -O dtb -o $out $dtsFile"; - bitstream = "${final.bitstreams}/lagarto-3-ox/gold.bit"; - #bitstream = "${final.bitstreams}/lagarto-3-ox/ox_u55c_a234c132_two_uarts.bit"; + #bitstream = "${final.bitstreams}/lagarto-3-ox/gold.bit"; + bitstream = "${final.bitstreams}/lagarto-3-ox/ox_u55c_a234c132_two_uarts.bit"; uboot = prev.ubootQemuRiscv64Smode.override { filesToInstall = [ "u-boot.bin" "u-boot-nodtb.bin" ]; -- GitLab From 5f90528b51654405fa54e92eb548f6ed3d8c24bc Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 8 Jul 2024 17:37:46 +0200 Subject: [PATCH 070/310] Add bootrom support --- JOURNAL.md | 7 +++++++ flake.nix | 2 ++ fpga/boot.sh | 2 +- fpga/env.sh | 5 ++++- fpga/fpgactl | 9 +++++++-- fpga/upload.sh | 1 + lagarto-ox.nix | 2 ++ overlay.nix | 2 +- 8 files changed, 25 insertions(+), 5 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 4f78f26..eeb79f3 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -855,3 +855,10 @@ loading it on each baud rate: Let's keep the bitstream files in a repository, so I can carefully track them with git too. + +### OBSERVATION: The new bitstream requires a bootrom to start + +I added it to the bitstream repository, as it is a binary blob too. Now I need +to update the load addresses: + +https://gitlab.bsc.es/hwdesign/fpga/integration-lab/fpga-tools/-/blob/6a63bcea6d1d59df3c7d62311aa4935efd54d3a3/boot_riscv/boot_sa.sh#L36-40 diff --git a/flake.nix b/flake.nix index e54b84c..ed621bf 100644 --- a/flake.nix +++ b/flake.nix @@ -108,6 +108,7 @@ ROOTFS = build.sdImage; UBOOT_ENV = syspkgs.uboot-env; BITSTREAM = syspkgs.bitstream; + BOOTROM = syspkgs.bootrom; shellHook = '' echo "Here are the current system pieces:" echo " TOPLEVEL = $TOPLEVEL" @@ -117,6 +118,7 @@ echo " ROOTFS = $ROOTFS" echo " UBOOT_ENV = $UBOOT_ENV" echo " BITSTREAM = $BITSTREAM" + echo " BOOTROM = $BOOTROM" ''; }; diff --git a/fpga/boot.sh b/fpga/boot.sh index 2de91b4..41a6813 100755 --- a/fpga/boot.sh +++ b/fpga/boot.sh @@ -7,6 +7,6 @@ source ./env.sh #bitstream="-w system-acme_ea-4h2v.bit" -./fpgactl $bitstream -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img +./fpgactl $bitstream -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img -R bootrom.bin picocom -b 115200 /dev/ttyUSB2 diff --git a/fpga/env.sh b/fpga/env.sh index ac62ed5..3768b63 100644 --- a/fpga/env.sh +++ b/fpga/env.sh @@ -37,11 +37,14 @@ function setup_meep() # Setup mappings # Delta between where we load in the dma device and RAM - local delta_addr=-0x60000000 + local delta_addr=-0x5ffe0000 + + # See https://gitlab.bsc.es/hwdesign/fpga/integration-lab/fpga-tools/-/blob/6a63bcea6d1d59df3c7d62311aa4935efd54d3a3/boot_riscv/boot_sa.sh#L36-40 export FPGACTL_BOOTLOADER_ADDR=$((0x80000000+$delta_addr)) export FPGACTL_KERNEL_ADDR=$((0x84000000+$delta_addr)) export FPGACTL_INITRD_ADDR=$((0x8c300000+$delta_addr)) export FPGACTL_ROOTFS_ADDR=$((0x100000000+$delta_addr)) + export FPGACTL_BOOTROM_ADDR=$((0x00000100)) } hostname=$(hostname) diff --git a/fpga/fpgactl b/fpga/fpgactl index 78b9c1b..6a2540e 100755 --- a/fpga/fpgactl +++ b/fpga/fpgactl @@ -370,6 +370,7 @@ bootloader= kernel= initrd= rootfs= +bootrom= resetcpu= verbose= pcidev= @@ -383,6 +384,7 @@ bootloader_addr="${FPGACTL_BOOTLOADER_ADDR:-0x80000000}" kernel_addr="${FPGACTL_KERNEL_ADDR:-0x84000000}" initrd_addr="${FPGACTL_INITRD_ADDR:-0x8c300000}" rootfs_addr="${FPGACTL_ROOTFS_ADDR:-0x140000000}" +bootrom_addr="${FPGACTL_BOOTROM_ADDR:-0x60000100}" hostname="${hostname:-$(hostname)}" echo "hostname=$hostname" @@ -390,7 +392,7 @@ echo "hostname=$hostname" function usage() { echo "" >&2 - echo "Usage: $0 [-p pcidev] [-v] [-w bitstream] [-j serial] [-b bootloader] [-k kernel] [-i initrd]" >&2 + echo "Usage: $0 [-p pcidev] [-v] [-w bitstream] [-j serial] [-b bootloader] [-k kernel] [-i initrd] [-R bootroom] " >&2 echo "" >&2 echo "First writes the bitstream if given. Then loads the rest of files" >&2 echo "into memory and restarts the CPU." >&2 @@ -405,13 +407,14 @@ function usage() echo " -k kernel Load the kernel file in $kernel_addr" >&2 echo " -i initrd Load the initrd file in $initrd_addr" >&2 echo " -r rootfs Load the rootfs file in $rootfs_addr" >&2 + echo " -R bootrom Load the bootrom file in $bootrom_addr" >&2 echo " -m model CPU model: Either 'hun' or 'ox' (default ox)" >&2 echo " -v Be verbose" >&2 echo "" >&2 exit 1 } -while getopts "hvw:b:k:i:r:p:j:m:" opt; do +while getopts "hvw:b:k:i:r:p:j:m:R:" opt; do case "${opt}" in v) verbose=1 ;; w) bitstream="${OPTARG}" ;; @@ -419,6 +422,7 @@ while getopts "hvw:b:k:i:r:p:j:m:" opt; do k) kernel="${OPTARG}"; resetcpu=1 ;; i) initrd="${OPTARG}"; resetcpu=1 ;; r) rootfs="${OPTARG}"; resetcpu=1 ;; + R) bootrom="${OPTARG}"; resetcpu=1 ;; p) pcidev="${OPTARG}" ;; j) jtagserial="${OPTARG}" ;; m) model="${OPTARG}" ;; @@ -441,6 +445,7 @@ test "$bootloader" && load_file_in_memory "$bootloader" $bootloader_addr test "$kernel" && load_file_in_memory "$kernel" $kernel_addr test "$initrd" && load_file_in_memory "$initrd" $initrd_addr test "$rootfs" && load_file_in_memory "$rootfs" $rootfs_addr +test "$bootrom" && load_file_in_memory "$bootrom" $bootrom_addr test "$resetcpu" && do_cpu_release exit 0 diff --git a/fpga/upload.sh b/fpga/upload.sh index e136f3f..67ee05b 100755 --- a/fpga/upload.sh +++ b/fpga/upload.sh @@ -22,6 +22,7 @@ else echo "Skipping rootfs" fi rsync "$BITSTREAM" "$dst/bitstream.bit" +rsync "$BOOTROM" "$dst/bootrom.bin" rsync "$UBOOT_ENV" "$dst/uboot.env" echo "Now go to $dst and run ./boot.sh" diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 2672017..f222110 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -180,6 +180,8 @@ #bitstream = "${final.bitstreams}/lagarto-3-ox/gold.bit"; bitstream = "${final.bitstreams}/lagarto-3-ox/ox_u55c_a234c132_two_uarts.bit"; + bootrom = "${final.bitstreams}/lagarto-3-ox/large_bootrom.bin"; + uboot = prev.ubootQemuRiscv64Smode.override { filesToInstall = [ "u-boot.bin" "u-boot-nodtb.bin" ]; #version = "2023.07.02-print-cpu-probe"; diff --git a/overlay.nix b/overlay.nix index 19ca0ec..0ad4189 100644 --- a/overlay.nix +++ b/overlay.nix @@ -39,6 +39,6 @@ final: prev: bitstreams = builtins.fetchGit { url = "git@bscpm03.bsc.es:rarias/bitstreams.git"; - rev = "57876cea158d8a4a63f7d35d715000092609d88c"; + rev = "ad901b0c21ffbdb310ff1dfb269f169f6ac6bde6"; }; } -- GitLab From bef5a6eac59259c5437676a06f966a083b73ef2d Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 8 Jul 2024 18:27:29 +0200 Subject: [PATCH 071/310] Disable CONFIG_BUG It may be causing a loop on WARN_ON_ONCE() and page fults trying to write to the console. --- JOURNAL.md | 23 +++++++++++++++++++++++ lagarto-ox.nix | 1 + 2 files changed, 24 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index eeb79f3..351dd6c 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -862,3 +862,26 @@ I added it to the bitstream repository, as it is a binary blob too. Now I need to update the load addresses: https://gitlab.bsc.es/hwdesign/fpga/integration-lab/fpga-tools/-/blob/6a63bcea6d1d59df3c7d62311aa4935efd54d3a3/boot_riscv/boot_sa.sh#L36-40 + +Continues to hang just after those perl messages: + + + mount --move /proc /mnt-root/proc + + mount --move /sys /mnt-root/sys + + mount --move /dev /mnt-root/dev + + mount --move /run /mnt-root/run + + type -P switch_root + + exec env -i /nix/store/988axh0bq3wqp90gms4b4a0hkfwvkd3i-extra-utils/bin/switch_root /mnt-root /nix/store/0za1vqh5alk7mxqs59qxx8izmwmf21w6-nixos-system-nixos-riscv-24.11pre-git/init + + <<< NixOS Stage 2 >>> + + [ 541.559320] EXT4-fs (pmem0p2): re-mounted 44444444-4444-4444-8888-888888888888 r/w. Quota mode: none. + [ 541.641280] booting system configuration /nix/store/0za1vqh5alk7mxqs59qxx8izmwmf21w6-nixos-system-nixos-riscv-24.11pre-git + running activation script... + [ 545.569700] stage-2-init: running activation script... + [ 549.019380] random: perl: uninitialized urandom read (4 bytes read) + [ 549.274940] random: perl: uninitialized urandom read (4 bytes read) + +### QUESTION: What happens if we disable `CONFIG_BUG` + +May be a long shot, but if we are experiencing the same page fault loop as in +cincoranch we may as well try. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index f222110..9cc4aa6 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -66,6 +66,7 @@ TRACING y BOOTTIME_TRACING y STRICT_DEVMEM n + BUG n '' # Disable SMP so we don't have IPI + '' -- GitLab From 5b34b3b97b4f140571ff16e2ceb0a33ea660f2c4 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 8 Jul 2024 19:02:01 +0200 Subject: [PATCH 072/310] Add csrtool to view and change CSR registers --- JOURNAL.md | 5 +++++ csrtool.c | 34 ++++++++++++++++++++++++++++++++++ lagarto-ox.nix | 6 ++++++ overlay.nix | 16 ++++++++++++++++ 4 files changed, 61 insertions(+) create mode 100644 csrtool.c diff --git a/JOURNAL.md b/JOURNAL.md index 351dd6c..ae07f53 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -885,3 +885,8 @@ Continues to hang just after those perl messages: May be a long shot, but if we are experiencing the same page fault loop as in cincoranch we may as well try. + +### QUESTION: Maybe we can try without out-of-order? + +I made a small tool in C to view and change the CSR register that controls the +in-order/out-of-order. Maybe we can try with the "in-order" setting. diff --git a/csrtool.c b/csrtool.c new file mode 100644 index 0000000..1ae6cee --- /dev/null +++ b/csrtool.c @@ -0,0 +1,34 @@ +#include +#include +#include + +int main(int argc, char *argv[]) +{ + /* Print */ + if (argc > 1) { + // Wait for all memory operations to finish + __asm__ volatile ("fence"); + + if (strcmp(argv[1], "mem-in-order") == 0) { + __asm__ volatile ("fence"); + __asm__ volatile ("csrwi 0x801, 2"); + } else if (strcmp(argv[1], "all-in-order") == 0) { + __asm__ volatile ("fence"); + __asm__ volatile ("csrwi 0x801, 7"); + } else if (strcmp(argv[1], "all-out-of-order") == 0) { + __asm__ volatile ("fence"); + __asm__ volatile ("csrwi 0x801, 0"); + } else { + fprintf(stderr, "unknown '%s', use: mem-in-order, all-in-order or all-out-of-order\n", argv[1]); + exit(1); + } + } + + // Wait for all memory operations to finish + __asm__ volatile ("fence"); + unsigned result; + asm("csrr %0, 0x801" : "=r"(result) : ); + printf("CSR 0x801 = %xu\n", result); + + return 0; +} diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 9cc4aa6..35e53bf 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -81,6 +81,12 @@ compressor = "gzip"; kernelModules = [ ]; + # Add the csrtool to the initrd so we can change the + # in-order/out-of-order. + extraUtilsCommands = '' + cp -a ${pkgs.csrtool}/bin/csrtool $out/bin + ''; + # Write a counter to the DMA region, so we can check the kernel is not # dead. Monitor from the host with: # while [ 1 ]; do xxd -s $((0x1bfff0000 - 0x60000000)) \ diff --git a/overlay.nix b/overlay.nix index 0ad4189..1614a9a 100644 --- a/overlay.nix +++ b/overlay.nix @@ -37,6 +37,22 @@ final: prev: ''; }; + csrtool = prev.pkgsStatic.stdenv.mkDerivation { + name = "csrtool"; + src = ./csrtool.c; + unpackPhase = '' + cp ${./csrtool.c} csrtool.c + ''; + dontConfigure = true; + buildPhase = '' + $CC -static csrtool.c -o csrtool + ''; + installPhase = '' + mkdir -p $out/bin + cp csrtool $out/bin/ + ''; + }; + bitstreams = builtins.fetchGit { url = "git@bscpm03.bsc.es:rarias/bitstreams.git"; rev = "ad901b0c21ffbdb310ff1dfb269f169f6ac6bde6"; -- GitLab From dd6082e805c4e04266b15b15cc9d26545e7d2aad Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 9 Jul 2024 12:26:00 +0200 Subject: [PATCH 073/310] Add results of changing the CSR It seems to arrive to systemd with all-in-order, but hangs there. --- JOURNAL.md | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index ae07f53..6080c78 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -886,7 +886,83 @@ Continues to hang just after those perl messages: May be a long shot, but if we are experiencing the same page fault loop as in cincoranch we may as well try. +## 2024-07-09 + ### QUESTION: Maybe we can try without out-of-order? I made a small tool in C to view and change the CSR register that controls the in-order/out-of-order. Maybe we can try with the "in-order" setting. + +We arrive to execute `systemd`: + + + Starting interactive shell... + + setsid /nix/store/xm3mpj9aldz5r4s5yb7p08jdjv98hj4w-extra-utils/bin/ash -c 'exec /nix/store/xm3mpj9aldz5r4s5yb7p08jdjv98hj4w-extra-utils/bin/ash < /dev/hvc0 >/dev/hvc0 2>/dev/hvc0' + [ 90.077300] stage-1-init: [Thu Jan 1 00:01:27 UTC 1970] + '[' -n 1 -a i '=' f ] + [ 90.639760] stage-1-init: [Thu Jan 1 00:01:28 UTC 1970] + '[' -n 1 -a i '=' i ] + ~ # [ 90.967260] stage-1-init: [Thu Jan 1 00:01:28 UTC 1970] + echo 'Starting interactive shell...' + [ 91.234980] stage-1-init: [Thu Jan 1 00:01:28 UTC 1970] Starting interactive shell... + [ 91.569580] stage-1-init: [Thu Jan 1 00:01:29 UTC 1970] + setsid /nix/store/xm3mpj9aldz5r4s5yb7p08jdjv98hj4w-extra-utils/bin/ash -c 'exec /nix/store/xm3mpj9aldz5r4s5yb7p08jdjv98hj4w-extr + a-utils/bin/ash < /dev/hvc0 >/dev/hvc0 2>/dev/hvc0' + which csrtool + /nix/store/xm3mpj9aldz5r4s5yb7p08jdjv98hj4w-extra-utils/bin/csrtool + ~ # csrtool + CSR 0x801 = 0u + ~ # csrtool o + unknown 'o', use: mem-in-order, all-in-order or all-out-of-order + ~ # csrtool all-in-order + CSR 0x801 = 7u + ~ # csrtool + CSR 0x801 = 7u + ~ # + + IFS='=' + + echo init /nix/store/xmagm60y90pfh3yvqanvmaswa0m3cb0a-nixos-system-nixos-riscv-24.11pre-git/init + + set -- init /nix/store/xmagm60y90pfh3yvqanvmaswa0m3cb0a-nixos-system-nixos-riscv-24.11pre-git/init + + stage2Init=/nix/store/xmagm60y90pfh3yvqanvmaswa0m3cb0a-nixos-system-nixos-riscv-24.11pre-git/init + + echo /nix/store/xm3mpj9aldz5r4s5yb7p08jdjv98hj4w-extra-utils/bin/modprobe + + basename dm_mod + [...] + + echo /sbin/modprobe + + '[' '!' -e /mnt-root//nix/store/xmagm60y90pfh3yvqanvmaswa0m3cb0a-nixos-system-nixos-riscv-24.11pre-git/init ] + + mkdir -m 0755 -p /mnt-root/proc /mnt-root/sys /mnt-root/dev /mnt-root/run + + mount --move /proc /mnt-root/proc + + mount --move /sys /mnt-root/sys + + mount --move /dev /mnt-root/dev + + mount --move /run /mnt-root/run + + type -P switch_root + + exec env -i /nix/store/xm3mpj9aldz5r4s5yb7p08jdjv98hj4w-extra-utils/bin/switch_root /mnt-root /nix/store/xmagm60y90pfh3yvqanvmaswa0m3cb0a-nixos-system-nixos-riscv-24.11pre-git/init + + <<< NixOS Stage 2 >>> + + [ 967.703320] EXT4-fs (pmem0p2): re-mounted 44444444-4444-4444-8888-888888888888 r/w. Quota mode: none. + [ 967.928020] booting system configuration /nix/store/xmagm60y90pfh3yvqanvmaswa0m3cb0a-nixos-system-nixos-riscv-24.11pre-git + running activation script... + [ 977.608980] stage-2-init: running activation script... + bbbbbbbbbbbbbbbbbbbbsetting up /etc... + [ 1084.376420] stage-2-init: setting up /etc... + starting systemd... + +Not sure if it is a good reproducer, as it is taking around 15 minutes to hang +in a very large piece of software, while when we have the out-of-order enabled, +we can hang in half of the time in some script. + +Either way, we need to see a backtrace of where it is hanging to understand why +it does. We may also enable a stage-2 heartbeat to be sure that it is hanging +the kernel and not only the console. + +Another idea is to arrive at a proper bash shell, where we can have debugging +tools, which may allow us to go slowly until we catch the bug. + +### OBSERVATION: Setting memory in-order only causes a hang + +Tested with: + + $ csrtool mem-in-order + +And it hangs just after exiting the tool. + +### QUESTION: Can we see the printk buffer from the host? + +If the problem that we are observing is somehow related to the recursive +segfault of the kernel in Cincoranch, we may be able to see the printk ring +buffer by directly poking at the memory from the host. -- GitLab From 82630f3eefa1855adc82dc1de775c348351a3e50 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 9 Jul 2024 12:35:53 +0200 Subject: [PATCH 074/310] Enable CONFIG_BUG again It doesn't seem to make any difference, and it would be good in case it catches a bug. --- lagarto-ox.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 35e53bf..4a750a1 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -66,7 +66,6 @@ TRACING y BOOTTIME_TRACING y STRICT_DEVMEM n - BUG n '' # Disable SMP so we don't have IPI + '' -- GitLab From f617efdcac628c4bd8f4fc51b0475095b3dfec76 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 9 Jul 2024 15:15:35 +0200 Subject: [PATCH 075/310] Add memtool program to test the memory --- JOURNAL.md | 5 +++ lagarto-ox.nix | 3 +- memtool.c | 119 +++++++++++++++++++++++++++++++++++++++++++++++++ overlay.nix | 16 +++++++ 4 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 memtool.c diff --git a/JOURNAL.md b/JOURNAL.md index 6080c78..54e6c2f 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -966,3 +966,8 @@ And it hangs just after exiting the tool. If the problem that we are observing is somehow related to the recursive segfault of the kernel in Cincoranch, we may be able to see the printk ring buffer by directly poking at the memory from the host. + +### QUESTION: Can we crash the CPU by exercising the memory? + +I did a small tool `memtool` that performs allocations and +deallocations. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 4a750a1..4b3be5d 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -81,9 +81,10 @@ kernelModules = [ ]; # Add the csrtool to the initrd so we can change the - # in-order/out-of-order. + # in-order/out-of-order, and memtool to stress the memory. extraUtilsCommands = '' cp -a ${pkgs.csrtool}/bin/csrtool $out/bin + cp -a ${pkgs.memtool}/bin/memtool $out/bin ''; # Write a counter to the DMA region, so we can check the kernel is not diff --git a/memtool.c b/memtool.c new file mode 100644 index 0000000..b5b6f2a --- /dev/null +++ b/memtool.c @@ -0,0 +1,119 @@ +/* Copyright (c) 2024 Barcelona Supercomputing Center (BSC) + * SPDX-License-Identifier: MIT + * Author: Rodrigo Arias Mallo */ + +/* This is just a small tool to exercise the memory which attempts to + * stress the virtual memory, in a crude attempt to reproduce the hangs + * that we were observing while booting NixOS. */ + +#include +#include +#include + +#define MAX_SIZE (1024L * 1024L) + +struct block { + struct block *next; + size_t size; + uint32_t data[]; +}; + +struct block *front = NULL; +struct block *tail = NULL; +long nblocks = 0; +long nbytes = 0; + +static int +allocate(void) +{ + long n = (long) rand() % MAX_SIZE; + /* Constraint the size */ + n = n % MAX_SIZE; + + size_t size = sizeof(struct block) + n * sizeof(uint32_t); + + struct block *b = malloc(size); + + /* No mem */ + if (b == NULL) + return -1; + + b->size = size; + b->next = NULL; + + /* Populate the block with some data */ + for (long i = 0; i < n; i++) + b->data[i] = rand(); + + /* Add it to the chain */ + if (tail) + tail->next = b; + + tail = b; + + /* And to the front if it is the first */ + if (!front) + front = b; + + nblocks++; + nbytes += size; + + return 0; +} + +static int +deallocate(void) +{ + /* May run out of blocks */ + if (!front) + return -1; + + struct block *b = front; + + front = b->next; + + /* Last block */ + if (tail == b) + tail = NULL; + + nblocks--; + nbytes -= b->size; + + free(b); + + return 0; +} + +static void +torture(void) +{ + srand(123); + + for (long iter = 0; ; iter++) { + int p = rand() % 100; + int is_alloc = (p > 10); + int ret = 0; + char c; + if (is_alloc) { + if (allocate() == 0) + c = 'A'; + else + c = '-'; + } else { + if (deallocate() == 0) + c = 'D'; + else + c = '-'; + } + + printf("iter %ld, nblocks %ld, nbytes %.1fM (%c)\n", + iter, nblocks, (double) nbytes / (1024. * 1024.), + c); + } +} + +int main(int argc, char *argv[]) +{ + torture(); + return 0; +} diff --git a/overlay.nix b/overlay.nix index 1614a9a..5813639 100644 --- a/overlay.nix +++ b/overlay.nix @@ -53,6 +53,22 @@ final: prev: ''; }; + memtool = prev.pkgsStatic.stdenv.mkDerivation { + name = "memtool"; + src = ./memtool.c; + unpackPhase = '' + cp ${./memtool.c} memtool.c + ''; + dontConfigure = true; + buildPhase = '' + $CC -static memtool.c -o memtool + ''; + installPhase = '' + mkdir -p $out/bin + cp memtool $out/bin/ + ''; + }; + bitstreams = builtins.fetchGit { url = "git@bscpm03.bsc.es:rarias/bitstreams.git"; rev = "ad901b0c21ffbdb310ff1dfb269f169f6ac6bde6"; -- GitLab From 5a88ed36b41efc90a4e5d9877faaddc55ee2df01 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 9 Jul 2024 15:38:52 +0200 Subject: [PATCH 076/310] Managed to reproduce the crash consistently! --- JOURNAL.md | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index 54e6c2f..c5653f5 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -971,3 +971,70 @@ buffer by directly poking at the memory from the host. I did a small tool `memtool` that performs allocations and deallocations. + +Good news, the tool has hang the console (potentially the kernel too). + + ~ # which memtool + /nix/store/amj11aclwx62d4mnvkhdgj19kq5gjb9y-extra-utils/bin/memtool + ~ # memtool + iter 0, nblocks 1, nbytes 0.1M (A) + iter 1, nblocks 2, nbytes 3.5M (A) + iter 2, nblocks 3, nbytes 3.8M (A) + iter 3, nblocks 4, nbytes 5.7M (A) + iter 4, nblocks 5, nbytes 6.8M (A) + iter 5, nblocks 6, nbytes 10.7M (A) + iter 6, nblocks 5, nbytes 10.6M (D) + iter 7, nblocks 6, nbytes 13.9M (A) + iter 8, nblocks 7, nbytes 16.4M (A) + iter 9, nblocks 8, nbytes 19.8M (A) + iter 10, nblocks 9, nbytes 21.2M (A) + iter 11, nblocks 10, nbytes 24.3M (A) + iter 12, nblocks 11, nbytes 27.4M (A) + iter 13, nblocks 12, nbytes 28.3M (A) + iter 14, nblocks 13, nbytes 31.6M (A) + iter 15, nblocks 12, nbytes 28.2M (D) + iter 16, nblocks 13, nbytes 29.2M (A) + iter 17, nblocks 14, nbytes 30.8M (A) + iter 18, nblocks 15, nbytes 32.6M (A) + iter 19, nblocks 16, nbytes 32.8M (A) + iter 20, nblocks 17, nbytes 36.8M (A) + iter 21, nblocks 18, nbytes 39.6M (A) + iter 22, nblocks 19, nbytes 41.1M (A) + iter 23, nblocks 20, nbytes 44.1M (A) + iter 24, nblocks 21, nbytes 46.9M (A) + iter 25, nblocks 20, nbytes 46.5M (D) + iter 26, nblocks 21, nbytes 50.2M (A) + iter 27, nblocks 22, nbytes 53.8M (A) + +Let's see if we can reproduce it again in the same position. + + ~ # memtool + iter 0, nblocks 1, nbytes 0.1M (A) + iter 1, nblocks 2, nbytes 3.5M (A) + iter 2, nblocks 3, nbytes 3.8M (A) + iter 3, nblocks 4, nbytes 5.7M (A) + iter 4, nblocks 5, nbytes 6.8M (A) + iter 5, nblocks 6, nbytes 10.7M (A) + iter 6, nblocks 5, nbytes 10.6M (D) + iter 7, nblocks 6, nbytes 13.9M (A) + iter 8, nblocks 7, nbytes 16.4M (A) + iter 9, nblocks 8, nbytes 19.8M (A) + iter 10, nblocks 9, nbytes 21.2M (A) + iter 11, nblocks 10, nbytes 24.3M (A) + iter 12, nblocks 11, nbytes 27.4M (A) + iter 13, nblocks 12, nbytes 28.3M (A) + iter 14, nblocks 13, nbytes 31.6M (A) + iter 15, nblocks 12, nbytes 28.2M (D) + iter 16, nblocks 13, nbytes 29.2M (A) + iter 17, nblocks 14, nbytes 30.8M (A) + iter 18, nblocks 15, nbytes 32.6M (A) + iter 19, nblocks 16, nbytes 32.8M (A) + iter 20, nblocks 17, nbytes 36.8M (A) + iter 21, nblocks 18, nbytes 39.6M (A) + iter 22, nblocks 19, nbytes 41.1M (A) + iter 23, nblocks 20, nbytes 44.1M (A) + iter 24, nblocks 21, nbytes 46.9M (A) + iter 25, nblocks 20, nbytes 46.5M (D) + iter 26, nblocks 21, nbytes 50.2M (A) + iter 27, nblocks 22, nbytes 53.8M (A) + -- GitLab From d488c0b3b784dccf543ab8b4de0ad97d78113557 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 9 Jul 2024 17:06:50 +0200 Subject: [PATCH 077/310] Always run memtool at the start --- JOURNAL.md | 2 ++ lagarto-ox.nix | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index c5653f5..4733b14 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1038,3 +1038,5 @@ Let's see if we can reproduce it again in the same position. iter 26, nblocks 21, nbytes 50.2M (A) iter 27, nblocks 22, nbytes 53.8M (A) +Let's make it automatic, so we only need to boot and confirm that it +hangs. Just in case we can make it not hang by a miracle. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 4b3be5d..175649a 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -96,6 +96,11 @@ sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' & '' + + # Run our memtool to hang the kernel here. + '' + memtool + '' + + # Disable proactive compaction. May be better to disable CONFIG_COMPACTION. '' echo 0 > /proc/sys/vm/compaction_proactiveness -- GitLab From b8f1ca672f3d3038fc0d2f927a3e935a7d69f471 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 9 Jul 2024 17:25:48 +0200 Subject: [PATCH 078/310] Run the memtool as first program --- lagarto-ox.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 175649a..1a1e4e6 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -91,16 +91,17 @@ # dead. Monitor from the host with: # while [ 1 ]; do xxd -s $((0x1bfff0000 - 0x60000000)) \ # -l 4 /dev/qdma34000-MM-1; sleep 0.2; done - preDeviceCommands = '' - echo "Creating a heartbeat counter at 0x1bfff0000" - sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' & - '' - + + preDeviceCommands = # Run our memtool to hang the kernel here. '' memtool '' + + '' + echo "Creating a heartbeat counter at 0x1bfff0000" + sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' & + '' + + # Disable proactive compaction. May be better to disable CONFIG_COMPACTION. '' echo 0 > /proc/sys/vm/compaction_proactiveness -- GitLab From 4707a8b1434c92884e1eea57ef67ee14e3f253df Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 9 Jul 2024 17:27:07 +0200 Subject: [PATCH 079/310] Simplify boot parameters and document options --- lagarto-ox.nix | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 1a1e4e6..bc494f5 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -260,9 +260,18 @@ # Reduce memory to 3 GiB [0x80000000, 0x140000000) fdt set /memory@80000000 reg <0x0 0x80000000 0x0 0xc0000000> - # Set kernel options - setenv debugargs "debug1 trace_options=sym-addr trace_event=initcall:* tp_printk trace_buf_size=1M" - setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi boot.trace console=hvc0 \''${debugargs} init=${init}" + # Set kernel bootcmd options. + # NixOS interesting options: + # debug1 enable debug shell in stage 1 + # boot.trace enable set -x in stage 1 + # boot.tracedebug enable set -x in stage 2 + # Ftrace interesting options: + # trace_event=initcall:* trace the init function of drivers + # trace_options=sym-addr display function address + # tp_printk write ftrace events to console + # trace_buf_size=1M set ftrace buffer to 1M + # + setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 init=${init}" EOF -- GitLab From c6726cce28d46eef38e4228acbde066fe8420f78 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 9 Jul 2024 17:58:12 +0200 Subject: [PATCH 080/310] Reduce blocksize in memtool to 64K Let's see if we can hang a bit later on. --- lagarto-ox.nix | 5 +++-- memtool.c | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index bc494f5..ab96e84 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -92,9 +92,10 @@ # while [ 1 ]; do xxd -s $((0x1bfff0000 - 0x60000000)) \ # -l 4 /dev/qdma34000-MM-1; sleep 0.2; done preDeviceCommands = - # Run our memtool to hang the kernel here. + # Run our memtool to hang the kernel here. Set the maximum block + # size to 64K. '' - memtool + memtool $((64 * 1024)) '' + '' diff --git a/memtool.c b/memtool.c index b5b6f2a..7dd347a 100644 --- a/memtool.c +++ b/memtool.c @@ -22,13 +22,14 @@ struct block *front = NULL; struct block *tail = NULL; long nblocks = 0; long nbytes = 0; +long maxsize = MAX_SIZE; static int allocate(void) { - long n = (long) rand() % MAX_SIZE; - /* Constraint the size */ - n = n % MAX_SIZE; + /* Constraint the number of elements based on the maxsize */ + long maxn = maxsize / sizeof(uint32_t); + long n = (long) rand() % maxn; size_t size = sizeof(struct block) + n * sizeof(uint32_t); @@ -106,14 +107,20 @@ torture(void) c = '-'; } - printf("iter %ld, nblocks %ld, nbytes %.1fM (%c)\n", - iter, nblocks, (double) nbytes / (1024. * 1024.), + printf("iter=%ld nblocks=%ld allocated=%ldK (%c)\n", + iter, nblocks, nbytes / 1024, c); } } int main(int argc, char *argv[]) { + if (argc > 1) + maxsize = atol(argv[1]); + + printf("memtool v1.0.0 maxsize=%ldK\n", maxsize / 1024); + torture(); + return 0; } -- GitLab From 9ed8f812c8e5c767563bcf0c78024412cfb00566 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 9 Jul 2024 18:21:33 +0200 Subject: [PATCH 081/310] Make our own init script The problem with the Stage 1 script is that is spawning another tee process to forward the output to the console, but this is not required. So let's only keep the memtool running by creating our own init script. --- lagarto-ox.nix | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index ab96e84..6b51268 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -80,6 +80,20 @@ compressor = "gzip"; kernelModules = [ ]; + # Custom init script + extraFiles."/init2".source = pkgs.writeScript "init2" '' + #!${config.system.build.extraUtils}/bin/ash + + set -x + + export PATH=${config.system.build.extraUtils}/bin + + memtool $((64 * 1024)) + + # Unlikely to reach this point + exec /init + ''; + # Add the csrtool to the initrd so we can change the # in-order/out-of-order, and memtool to stress the memory. extraUtilsCommands = '' @@ -91,14 +105,7 @@ # dead. Monitor from the host with: # while [ 1 ]; do xxd -s $((0x1bfff0000 - 0x60000000)) \ # -l 4 /dev/qdma34000-MM-1; sleep 0.2; done - preDeviceCommands = - # Run our memtool to hang the kernel here. Set the maximum block - # size to 64K. - '' - memtool $((64 * 1024)) - '' - + - '' + preDeviceCommands = '' echo "Creating a heartbeat counter at 0x1bfff0000" sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' & '' @@ -272,7 +279,7 @@ # tp_printk write ftrace events to console # trace_buf_size=1M set ftrace buffer to 1M # - setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 init=${init}" + setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 rdinit=/init2 init=${init}" EOF -- GitLab From 813e7cae2fe687608e06ad5fe954a6efbfd31d30 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 9 Jul 2024 20:58:55 +0200 Subject: [PATCH 082/310] Increase memtool max block size to 512 K --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 6b51268..58248ad 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -88,7 +88,7 @@ export PATH=${config.system.build.extraUtils}/bin - memtool $((64 * 1024)) + memtool $((512 * 1024)) # Unlikely to reach this point exec /init -- GitLab From 6b3af5b18806b092066908a2208861205031f2e2 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 9 Jul 2024 21:11:44 +0200 Subject: [PATCH 083/310] Reduce RAM to 1 GiB --- JOURNAL.md | 25 +++++++++++++++++++++++++ ox-plic.dts | 6 +++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 4733b14..b18a591 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1040,3 +1040,28 @@ Let's see if we can reproduce it again in the same position. Let's make it automatic, so we only need to boot and confirm that it hangs. Just in case we can make it not hang by a miracle. + + +With blocks of up to 64K we hang in the ~50 M region. + + iter=2042 nblocks=1577 allocated=50458K (A) + iter=2043 nblocks=1578 allocated=50489K (A) + iter=2044 nblocks=1579 allocated=50550K (A) + iter=2045 nblocks=1580 allocated=50605K (A) + +With blocks of maxsize=512K it also hangs around ~57M. + + memtool v1.0.0 maxsize=512K + iter=0 nblocks=1 allocated=88K (A) + iter=1 nblocks=2 allocated=464K (A) + ... + iter=275 nblocks=218 allocated=56674K (A) + iter=276 nblocks=219 allocated=56787K (A) + iter=277 nblocks=220 allocated=57252K (A) + iter=278 nblocks=221 allocated=57493K (A) + iter=279 nblocks=222 allocated=57581K (A) + iter=280 nblocks=221 allocated=57416K (D) + iter=281 nblocks=222 allocated=57521K (A) + +Maybe there is a problem in the memory segment? Can we reduce it to 1 GiB only +and see if it has any effect? diff --git a/ox-plic.dts b/ox-plic.dts index f66ce9a..3bc0e47 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -70,14 +70,14 @@ * * [0x0_6000_0000, 0x0_7000_0000) -> DMA pool (256 MiB) * [0x0_7000_0000, 0x0_8000_0000) -> DMA pool (256 MiB) - * [0x0_8000_0000, 0x0_fff0_0000) -> RAM memory (2047 MiB) - * [0x0_fff0_0000, 0x1_0000_0000) -> Empty (1 MiB) + * [0x0_8000_0000, 0x0_fff0_0000) -> RAM memory (1024 MiB) + * [0x0_fff0_0000, 0x1_0000_0000) -> Empty (1024 MiB) * [0x1_0000_0000, 0x1_c000_0000) -> PMEM (3072 MiB) * [0x1_c000_0000, 0x2_8000_0000) -> Empty (3072 MiB) */ memory@80000000 { device_type = "memory"; - reg = <0x0 0x80000000 0x0 0x7ff00000>; + reg = <0x0 0x80000000 0x0 0x40000000>; }; reserved-memory { #address-cells = <2>; /* Starting address and size */ -- GitLab From 6c8d6354bc8ff52998a5370319a6a42104d932fe Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 9 Jul 2024 22:12:55 +0200 Subject: [PATCH 084/310] Add debug in memtool to see where it hangs --- JOURNAL.md | 7 +++++++ memtool.c | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index b18a591..0630708 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1065,3 +1065,10 @@ With blocks of maxsize=512K it also hangs around ~57M. Maybe there is a problem in the memory segment? Can we reduce it to 1 GiB only and see if it has any effect? + + iter=289 nblocks=228 allocated=58636K (A) + iter=290 nblocks=227 allocated=58412K (D) + iter=291 nblocks=228 allocated=58480K (A) + iter=292 nblocks=229 allocated=58599K (A) + +Has changed, but not much. diff --git a/memtool.c b/memtool.c index 7dd347a..b718e63 100644 --- a/memtool.c +++ b/memtool.c @@ -33,6 +33,8 @@ allocate(void) size_t size = sizeof(struct block) + n * sizeof(uint32_t); + printf("allocating...\n"); + struct block *b = malloc(size); /* No mem */ @@ -43,6 +45,7 @@ allocate(void) b->next = NULL; /* Populate the block with some data */ + printf("filling...\n"); for (long i = 0; i < n; i++) b->data[i] = rand(); @@ -80,6 +83,7 @@ deallocate(void) nblocks--; nbytes -= b->size; + printf("deallocating...\n"); free(b); return 0; -- GitLab From 91080535f882716105d80bf73bf92ab63b11c6c9 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 10 Jul 2024 11:01:32 +0200 Subject: [PATCH 085/310] Add fill mode in memtool --- JOURNAL.md | 19 +++++++ lagarto-ox.nix | 3 +- memtool.c | 144 ++++++++++++++++++++++++++++++++++++++----------- 3 files changed, 135 insertions(+), 31 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 0630708..c2b819a 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1072,3 +1072,22 @@ and see if it has any effect? iter=292 nblocks=229 allocated=58599K (A) Has changed, but not much. + + aaaiter=291 nblocks=228 allocated=58480K (A) + allocating... + filling... + aaaaaaaaiter=292 nblocks=229 allocated=58599K (A) + allocating... + aafilling... + +It seems to be getting stuck in the filling phase. Can we trace it down with +ftrace? It should be generating page faults. + +### 2024-07-10 + +So, if we manage to crash in the filling phase, we can further pinpoint the +issue and remove any effect of `malloc()`. It would be only related to a page +fault and the MMU at this point. + +Let's make a much simpler program that only allocates once a buffer of N bytes +and then begins filling it, printing the progress in the output. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 58248ad..faff021 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -88,7 +88,8 @@ export PATH=${config.system.build.extraUtils}/bin - memtool $((512 * 1024)) + #memtool chain $((512 * 1024)) + memtool fill $((512 * 1024 * 1024)) # Unlikely to reach this point exec /init diff --git a/memtool.c b/memtool.c index b718e63..f961c71 100644 --- a/memtool.c +++ b/memtool.c @@ -6,9 +6,14 @@ * stress the virtual memory, in a crude attempt to reproduce the hangs * that we were observing while booting NixOS. */ +/* Changelog: + * v0.0.1 (2024-07-10): Start version with "chain" and "fill" tests. + */ + #include #include #include +#include #define MAX_SIZE (1024L * 1024L) @@ -18,17 +23,19 @@ struct block { uint32_t data[]; }; -struct block *front = NULL; -struct block *tail = NULL; -long nblocks = 0; -long nbytes = 0; -long maxsize = MAX_SIZE; +struct chain { + struct block *front; + struct block *tail; + long maxsize; + long nbytes; + long nblocks; +}; static int -allocate(void) +allocate(struct chain *chain) { /* Constraint the number of elements based on the maxsize */ - long maxn = maxsize / sizeof(uint32_t); + long maxn = chain->maxsize / sizeof(uint32_t); long n = (long) rand() % maxn; size_t size = sizeof(struct block) + n * sizeof(uint32_t); @@ -50,38 +57,38 @@ allocate(void) b->data[i] = rand(); /* Add it to the chain */ - if (tail) - tail->next = b; + if (chain->tail) + chain->tail->next = b; - tail = b; + chain->tail = b; /* And to the front if it is the first */ - if (!front) - front = b; + if (!chain->front) + chain->front = b; - nblocks++; - nbytes += size; + chain->nblocks++; + chain->nbytes += size; return 0; } static int -deallocate(void) +deallocate(struct chain *chain) { /* May run out of blocks */ - if (!front) + if (!chain->front) return -1; - struct block *b = front; + struct block *b = chain->front; - front = b->next; + chain->front = b->next; /* Last block */ - if (tail == b) - tail = NULL; + if (chain->tail == b) + chain->tail = NULL; - nblocks--; - nbytes -= b->size; + chain->nblocks--; + chain->nbytes -= b->size; printf("deallocating...\n"); free(b); @@ -90,8 +97,18 @@ deallocate(void) } static void -torture(void) +do_chain(int argc, char *argv[]) { + struct chain chain = {0}; + + /* Default 1 MiB */ + chain.maxsize = 1024L * 1024L; + + if (argc > 0) + chain.maxsize = atol(argv[0]); + + printf("mode chain: maxsize=%ldK\n", chain.maxsize / 1024); + srand(123); for (long iter = 0; ; iter++) { @@ -100,31 +117,98 @@ torture(void) int ret = 0; char c; if (is_alloc) { - if (allocate() == 0) + if (allocate(&chain) == 0) c = 'A'; else c = '-'; } else { - if (deallocate() == 0) + if (deallocate(&chain) == 0) c = 'D'; else c = '-'; } printf("iter=%ld nblocks=%ld allocated=%ldK (%c)\n", - iter, nblocks, nbytes / 1024, + iter, chain.nblocks, chain.nbytes / 1024, c); } } +static void +do_fill(int argc, char *argv[]) +{ + /* Default: 256 MiB */ + long nbytes = 256L * 1024L * 1024L; + + if (argc > 0) + nbytes = atol(argv[0]); + + long n = nbytes / sizeof(int); + + printf("mode fill: nbytes=%ldM, n=%ld\n", + nbytes / (1024L * 1024L), n); + + int *buf = malloc(nbytes); + + if (!buf) { + perror("malloc failed"); + exit(1); + } + + for (long i = 0; i < n; i++) { + buf[i] = i; + if ((i % (1024L * 1024L)) == 0) + printf("written=%ldK, addr=%p OK\n", + i * sizeof(int) / 1024L, + &buf[i]); + } + + free(buf); + + printf("fill test OK\n"); +} + +static void +usage(void) +{ + printf( +"Usage: memtool [...]\n" +"\n" +"Available commands:\n" +" chain []\n" +" Creates a chain of blocks of random size, each up to maxsize\n" +" or 1MiB if not given. Blocks are freed with 10% probability\n" +" starting from the oldest.\n" +"\n" +" fill []\n" +" Allocates a vector of the given size (or 256 MiB if not given)\n" +" and initializes it with a increasing value per element.\n" +"\n"); + + exit(1); +} + int main(int argc, char *argv[]) { - if (argc > 1) - maxsize = atol(argv[1]); + printf("memtool v0.0.1 - Rodrigo Arias Mallo \n"); + + if (argc < 2) + usage(); + + /* Skip program name */ + argc--; argv++; + + const char *mode = argv[0]; - printf("memtool v1.0.0 maxsize=%ldK\n", maxsize / 1024); + /* Skip mode */ + argc--; argv++; - torture(); + if (strcmp(mode, "chain") == 0) + do_chain(argc, argv); + else if (strcmp(mode, "fill") == 0) + do_fill(argc, argv); + else + usage(); return 0; } -- GitLab From 6a63798308dc0281b8250d5aedd48239e7bfbc55 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 10 Jul 2024 11:13:05 +0200 Subject: [PATCH 086/310] The memtool fill mode also gets stuck --- JOURNAL.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index c2b819a..08a69fd 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1091,3 +1091,25 @@ fault and the MMU at this point. Let's make a much simpler program that only allocates once a buffer of N bytes and then begins filling it, printing the progress in the output. + +### OBSERVATION: Writing to a vector also stops around 58 MiB + + + memtool fill 536870912 + memtool v0.0.1 - Rodrigo Arias Mallo + mode fill: nbytes=512M, n=134217728 + written=0K, addr=0x3f9b800020 OK + written=4096K, addr=0x3f9bc00020 OK + written=8192K, addr=0x3f9c000020 OK + written=12288K, addr=0x3f9c400020 OK + written=16384K, addr=0x3f9c800020 OK + written=20480K, addr=0x3f9cc00020 OK + written=24576K, addr=0x3f9d000020 OK + written=28672K, addr=0x3f9d400020 OK + written=32768K, addr=0x3f9d800020 OK + written=36864K, addr=0x3f9dc00020 OK + written=40960K, addr=0x3f9e000020 OK + written=45056K, addr=0x3f9e400020 OK + written=49152K, addr=0x3f9e800020 OK + written=53248K, addr=0x3f9ec00020 OK + written=57344K, addr=0x3f9f000020 OK + -- GitLab From 93936cbe145f8480959b231a164370f5914f21ed Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 10 Jul 2024 11:34:40 +0200 Subject: [PATCH 087/310] Trace function graph after a page fault --- JOURNAL.md | 1 + lagarto-ox.nix | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index 08a69fd..e61aab2 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1113,3 +1113,4 @@ and then begins filling it, printing the progress in the output. written=53248K, addr=0x3f9ec00020 OK written=57344K, addr=0x3f9f000020 OK +Let's see if we can trace the page fault. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index faff021..5954a7a 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -88,6 +88,14 @@ export PATH=${config.system.build.extraUtils}/bin + mkdir -p /sys/kernel/debug/ + mount -t debugfs none /sys/kernel/debug/ + td=/sys/kernel/debug/tracing + echo nop > $td/current_tracer + echo 100 > $td/max_graph_depth + echo do_page_fault > $td/set_graph_function + echo function_graph > $td/current_tracer + #memtool chain $((512 * 1024)) memtool fill $((512 * 1024 * 1024)) -- GitLab From 9ee8fb06dcbd47802fa964fd919de95b870cf0ce Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 10 Jul 2024 12:21:24 +0200 Subject: [PATCH 088/310] Try again the fill test but with all-in-order --- JOURNAL.md | 19 +++++++++++++++++++ lagarto-ox.nix | 9 +-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index e61aab2..7648b5c 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1114,3 +1114,22 @@ and then begins filling it, printing the progress in the output. written=57344K, addr=0x3f9f000020 OK Let's see if we can trace the page fault. + +In today's meeting, Jonnatan suggests test the memtool program with all-in-order +configuration. I should also try to reproduce this hang with a "production" +bitstream (from master). + +Let's do the quick CSR test first, and then we go back to the ftrace testing, +which will take more time. + +Here are the commands I was testing, but nothing comes out of the console, even +after booting with the `tp_printk trace_buf_size=1M` boot options: + + mkdir -p /sys/kernel/debug/ + mount -t debugfs none /sys/kernel/debug/ + td=/sys/kernel/debug/tracing + echo nop > $td/current_tracer + echo 100 > $td/max_graph_depth + echo do_page_fault > $td/set_graph_function + echo function_graph > $td/current_tracer + diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 5954a7a..86e1ae2 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -88,15 +88,8 @@ export PATH=${config.system.build.extraUtils}/bin - mkdir -p /sys/kernel/debug/ - mount -t debugfs none /sys/kernel/debug/ - td=/sys/kernel/debug/tracing - echo nop > $td/current_tracer - echo 100 > $td/max_graph_depth - echo do_page_fault > $td/set_graph_function - echo function_graph > $td/current_tracer + csrtool all-in-order - #memtool chain $((512 * 1024)) memtool fill $((512 * 1024 * 1024)) # Unlikely to reach this point -- GitLab From b562c9650f58e02f9834f560ea89bb95c5c4579e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 10 Jul 2024 12:30:16 +0200 Subject: [PATCH 089/310] Same hang with all-in-order configuration --- JOURNAL.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index 7648b5c..56dd628 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1133,3 +1133,34 @@ after booting with the `tp_printk trace_buf_size=1M` boot options: echo do_page_fault > $td/set_graph_function echo function_graph > $td/current_tracer +### OBSERVATION: Using all-in-order causes the hang in the same place + +After setting the CSR 0x801 register to 0x7, the all-in-order configuration, the +memtool fill tests continues to hang in the same position: + + + csrtool all-in-order + CSR 0x801 = 7u + + memtool fill 536870912 + memtool v0.0.1 - Rodrigo Arias Mallo + mode fill: nbytes=512M, n=134217728 + written=0K, addr=0x3f8d600020 OK + written=4096K, addr=0x3f8da00020 OK + written=8192K, addr=0x3f8de00020 OK + written=12288K, addr=0x3f8e200020 OK + written=16384K, addr=0x3f8e600020 OK + written=20480K, addr=0x3f8ea00020 OK + written=24576K, addr=0x3f8ee00020 OK + written=28672K, addr=0x3f8f200020 OK + written=32768K, addr=0x3f8f600020 OK + written=36864K, addr=0x3f8fa00020 OK + written=40960K, addr=0x3f8fe00020 OK + written=45056K, addr=0x3f90200020 OK + written=49152K, addr=0x3f90600020 OK + written=53248K, addr=0x3f90a00020 OK + written=57344K, addr=0x3f90e00020 OK + +It doesn't seem to have any observable effect with this test, other than going +more slow. + +Interesting [article](https://wiki.osdev.org/RISC-V_Bare_Bones) on how to write +a simple bootrom that outputs some ASCII text into the console. -- GitLab From 4c2e21033bed7ae942e8b1a30a70b209031ce576 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 10 Jul 2024 13:02:43 +0200 Subject: [PATCH 090/310] Try tracing handle_page_fault --- JOURNAL.md | 6 ++++++ lagarto-ox.nix | 12 ++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 56dd628..e81a9e6 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1119,6 +1119,9 @@ In today's meeting, Jonnatan suggests test the memtool program with all-in-order configuration. I should also try to reproduce this hang with a "production" bitstream (from master). +Also, Xavi reports that the memtool chain test continued to run until the memory +was exhausted using an old bistream. + Let's do the quick CSR test first, and then we go back to the ftrace testing, which will take more time. @@ -1164,3 +1167,6 @@ more slow. Interesting [article](https://wiki.osdev.org/RISC-V_Bare_Bones) on how to write a simple bootrom that outputs some ASCII text into the console. + +Regarding the `do_page_fault` filter, it seems that is no available. I need to +pick one from `available_filter_functions` instead. Let's try `handle_page_fault`. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 86e1ae2..de55287 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -88,7 +88,15 @@ export PATH=${config.system.build.extraUtils}/bin - csrtool all-in-order + mkdir -p /tracing + mount -t tracefs nodev /tracing + + cd /tracing + echo nop > current_tracer + echo 100 > max_graph_depth + echo handle_page_fault > set_graph_function + echo function_graph > current_tracer + cd / memtool fill $((512 * 1024 * 1024)) @@ -281,7 +289,7 @@ # tp_printk write ftrace events to console # trace_buf_size=1M set ftrace buffer to 1M # - setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 rdinit=/init2 init=${init}" + setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 tp_printk trace_buf_size=1M rdinit=/init2 init=${init}" EOF -- GitLab From b7d02f867a99cc0cd0283a9fc6e5ffaff865c9f8 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 10 Jul 2024 14:19:49 +0200 Subject: [PATCH 091/310] Disable support for huge pages --- JOURNAL.md | 6 ++++++ lagarto-ox.nix | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index e81a9e6..d103ff3 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1170,3 +1170,9 @@ a simple bootrom that outputs some ASCII text into the console. Regarding the `do_page_fault` filter, it seems that is no available. I need to pick one from `available_filter_functions` instead. Let's try `handle_page_fault`. + +It still doesn't seem to appear in the console. It is visible with a SysRq +request Ctrl+O then 'z'. + +Maybe I can try disabling the huge pages, just to discard that it may be +related to it: `CONFIG_HUGETLBFS`. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index de55287..336f2fb 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -71,6 +71,11 @@ + '' SMP n '' + # Disable huge pages + + '' + HUGETLBFS n + HUGETLB_PAGE n + '' ; } ]; -- GitLab From 31f85577994a325317c634daa5fa77c007247959 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 10 Jul 2024 14:20:39 +0200 Subject: [PATCH 092/310] Open interactive shell on start --- lagarto-ox.nix | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 336f2fb..292986d 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -93,15 +93,18 @@ export PATH=${config.system.build.extraUtils}/bin - mkdir -p /tracing - mount -t tracefs nodev /tracing - - cd /tracing - echo nop > current_tracer - echo 100 > max_graph_depth - echo handle_page_fault > set_graph_function - echo function_graph > current_tracer - cd / + #mkdir -p /tracing + #mount -t tracefs nodev /tracing + + #cd /tracing + #echo nop > current_tracer + #echo 100 > max_graph_depth + #echo handle_page_fault > set_graph_function + #echo function_graph > current_tracer + #cd / + + # Open an interactive shell + ash memtool fill $((512 * 1024 * 1024)) -- GitLab From 28efecdbc7c9ad92a98ed9dc724a8ef5f3666f97 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 10 Jul 2024 17:32:58 +0200 Subject: [PATCH 093/310] Enable U-Boot mtest command --- JOURNAL.md | 30 ++++++++++++++++++++++++++++++ lagarto-ox.nix | 1 + 2 files changed, 31 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index d103ff3..84245af 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1176,3 +1176,33 @@ request Ctrl+O then 'z'. Maybe I can try disabling the huge pages, just to discard that it may be related to it: `CONFIG_HUGETLBFS`. + +Same hang with huge pages disabled, but a bit further ~70 MB. + +### OBSERVATION: The Linux memtest fails in the first round + + [ 0.000000] Linux version 6.9.7 (nixbld@localhost) (riscv64-unknown-linux-gnu-gcc (GCC) 13.3.0, GNU ld (GNU Binutils) 2.41) #1-NixOS Thu Jun 27 11:52:32 UTC 2024 + [ 0.000000] Machine model: Barcelona Supercomputing Center - Lagarto Ox (NixOS) + [ 0.000000] SBI specification v2.0 detected + [ 0.000000] SBI implementation ID=0x1 Version=0x10004 + [ 0.000000] SBI TIME extension detected + [ 0.000000] SBI IPI extension detected + [ 0.000000] SBI RFENCE extension detected + [ 0.000000] SBI DBCN extension detected + [ 0.000000] earlycon: sbi0 at I/O port 0x0 (options '') + [ 0.000000] printk: legacy bootconsole [sbi0] enabled + [ 0.000000] Reserved memory: created DMA memory pool at 0x0000000060000000, size 256 MiB + [ 0.000000] OF: reserved mem: initialized node dma_pool@60000000, compatible id shared-dma-pool + [ 0.000000] OF: reserved mem: 0x0000000060000000..0x000000006fffffff (262144 KiB) map non-reusable dma_pool@60000000 + [ 0.000000] Reserved memory: created DMA memory pool at 0x0000000070000000, size 256 MiB + [ 0.000000] OF: reserved mem: initialized node dma_pool@70000000, compatible id shared-dma-pool + [ 0.000000] OF: reserved mem: 0x0000000070000000..0x000000007fffffff (262144 KiB) map non-reusable dma_pool@70000000 + [ 0.000000] cma: Reserved 16 MiB at 0x00000000bf000000 on node -1 + [ 0.000000] early_memtest: # of tests: 3 + [ 0.000000] 0x0000000080000000 - 0x0000000080013000 pattern 5555555555555555 + +This is not suggesting that the problem is not in the virtual memory, but with +the actual physical memory. + +I will try a similar test with uboot with the `mtest` command, but requires +enabling it first. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 292986d..f9208ca 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -253,6 +253,7 @@ CONFIG_DEBUG_SBI_CONSOLE=y CONFIG_SMP=n CONFIG_TRACE_EARLY=y + CONFIG_CMD_MEMTEST=y '' # # Enable debug logs # + -- GitLab From b7dba89d63193a4412791715347dad176f71f944 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 10 Jul 2024 18:04:11 +0200 Subject: [PATCH 094/310] Reduce RAM to 768 MiB to avoid a memory problem --- JOURNAL.md | 23 +++++++++++++++++++++++ lagarto-ox.nix | 25 +++++++++++++------------ ox-plic.dts | 7 ++++--- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 84245af..1c4e949 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1206,3 +1206,26 @@ the actual physical memory. I will try a similar test with uboot with the `mtest` command, but requires enabling it first. + +### OBSERVATION: Memory in the 0xb0000000..0xc0000000 range is bad + +Reproduced from U-Boot: + + => mtest 0x80000000 0x90000000 0 2 + Testing 80000000 ... 90000000: + Pattern FFFFFFFFFFFFFFFF Writing... Reading...Iteration: 2 + Tested 2 iteration(s) with 0 errors. + => mtest 0x90000000 0xa0000000 0 2 + Testing 90000000 ... a0000000: + Pattern FFFFFFFFFFFFFFFF Writing... Reading...Iteration: 2 + Tested 2 iteration(s) with 0 errors. + => mtest 0xa0000000 0xb0000000 0 2 + Testing a0000000 ... b0000000: + Pattern FFFFFFFFFFFFFFFF Writing... Reading...Iteration: 2 + Tested 2 iteration(s) with 0 errors. + => mtest 0xb0000000 0xc0000000 0 2 + Testing b0000000 ... c0000000: + Pattern 0000000000000000 Writing... + +Let's see if we can fix the boot hang by reducing the memory enough to avoid +this bad region. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index f9208ca..568d349 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -132,17 +132,17 @@ '' echo 0 > /proc/sys/vm/compaction_proactiveness '' - + - # Show stacktrace on calls to the hvc_remove function. - '' - echo "Mount debugfs" - mkdir -p /sys/kernel/debug/ - mount -t debugfs none /sys/kernel/debug/ - td=/sys/kernel/debug/tracing - echo hvc_remove > $td/set_ftrace_filter - echo function > $td/current_tracer - echo 1 > $td/options/func_stack_trace - '' +# + +# # Show stacktrace on calls to the hvc_remove function. +# '' +# echo "Mount debugfs" +# mkdir -p /sys/kernel/debug/ +# mount -t debugfs none /sys/kernel/debug/ +# td=/sys/kernel/debug/tracing +# echo hvc_remove > $td/set_ftrace_filter +# echo function > $td/current_tracer +# echo 1 > $td/options/func_stack_trace +# '' # FIXME: Disable sched_switch for now, as it still hangs the boot... # + # # Exclude the second pid, which is the kthread that will dump the trace to @@ -288,6 +288,7 @@ fdt set /memory@80000000 reg <0x0 0x80000000 0x0 0xc0000000> # Set kernel bootcmd options. + # rdinit=/init2 boot custom init script # NixOS interesting options: # debug1 enable debug shell in stage 1 # boot.trace enable set -x in stage 1 @@ -298,7 +299,7 @@ # tp_printk write ftrace events to console # trace_buf_size=1M set ftrace buffer to 1M # - setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 tp_printk trace_buf_size=1M rdinit=/init2 init=${init}" + setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 init=${init}" EOF diff --git a/ox-plic.dts b/ox-plic.dts index 3bc0e47..e7e9249 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -70,14 +70,15 @@ * * [0x0_6000_0000, 0x0_7000_0000) -> DMA pool (256 MiB) * [0x0_7000_0000, 0x0_8000_0000) -> DMA pool (256 MiB) - * [0x0_8000_0000, 0x0_fff0_0000) -> RAM memory (1024 MiB) - * [0x0_fff0_0000, 0x1_0000_0000) -> Empty (1024 MiB) + * [0x0_8000_0000, 0x0_b000_0000) -> RAM memory (768 MiB) + * [0x0_b000_0000, 0x0_c000_0000) -> Broken? (256 MiB) + * [0x0_c000_0000, 0x1_0000_0000) -> Empty (1024 MiB) * [0x1_0000_0000, 0x1_c000_0000) -> PMEM (3072 MiB) * [0x1_c000_0000, 0x2_8000_0000) -> Empty (3072 MiB) */ memory@80000000 { device_type = "memory"; - reg = <0x0 0x80000000 0x0 0x40000000>; + reg = <0x0 0x80000000 0x0 0x30000000>; }; reserved-memory { #address-cells = <2>; /* Starting address and size */ -- GitLab From 131713e7fcc7235af32fc7ce619c4e07b1a06c6c Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 11 Jul 2024 15:36:52 +0200 Subject: [PATCH 095/310] Add more experiments with mtest --- JOURNAL.md | 162 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index 1c4e949..5574148 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1229,3 +1229,165 @@ Reproduced from U-Boot: Let's see if we can fix the boot hang by reducing the memory enough to avoid this bad region. + +## 2024-07-11 + +### OBSERVATION: U-Boot mtest hangs in the last 256 MiB + +After reducing the size of the RAM segment, I run again the mtest, but this time +it fails in the last 256 MiB block. + +I assume that U-Boot moves itself to the last part of the memory, and them mtest +overwrites the U-Boot code, causing a hang. + +So, I simply changed the FDT from U-Boot to skip the first 2M: + + fdt set /memory@80000000 reg <0x0 0x80200000 0x0 0x40000000> + +And then I enabled the memtest in the kernel boot parameters: + + => fdt set /memory@80000000 reg <0x0 0x80200000 0x0 0x30000000> + => setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 boot.trace boot.tracedebug init=/nix/store/zxbq93zfg8ijkyq5cq5sb4742rczqfck-nixos-system-nixos-riscv-24.11pre-git/init" + => setenv ramdisk_size 12611657 + => setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 boot.trace boot.tracedebug memtest=3 init=/nix/store/zxbq93zfg8ijkyq5cq5sb4742rczqfck-nixos-system-nixos-risc=> 4.11pre-git/init" + => booti ${kernel_addr_r} ${ramdisk_addr_r}:${ramdisk_size} ${fdtcontroladdr} + Moving Image from 0x84000000 to 0x80200000, end=8303c4d0 + ## Flattened Device Tree blob at 80013000 + Booting using the fdt blob at 0x80013000 + Working FDT set to 80013000 + Using Device Tree in place at 0000000080013000, end 000000008001696f + Working FDT set to 80013000 + + Starting kernel ... + + [ 0.000000] Linux version 6.9.7 (nixbld@localhost) (riscv64-unknown-linux-gnu-gcc (GCC) 13.3.0, GNU ld (GNU Binutils) 2.41) #1-NixOS Thu Jun 27 11:52:32 UTC 2024 + [ 0.000000] Machine model: Barcelona Supercomputing Center - Lagarto Ox (NixOS) + [ 0.000000] SBI specification v2.0 detected + [ 0.000000] SBI implementation ID=0x1 Version=0x10004 + [ 0.000000] SBI TIME extension detected + [ 0.000000] SBI IPI extension detected + [ 0.000000] SBI RFENCE extension detected + [ 0.000000] SBI DBCN extension detected + [ 0.000000] earlycon: sbi0 at I/O port 0x0 (options '') + [ 0.000000] printk: legacy bootconsole [sbi0] enabled + [ 0.000000] Reserved memory: created DMA memory pool at 0x0000000060000000, size 256 MiB + [ 0.000000] OF: reserved mem: initialized node dma_pool@60000000, compatible id shared-dma-pool + [ 0.000000] OF: reserved mem: 0x0000000060000000..0x000000006fffffff (262144 KiB) map non-reusable dma_pool@60000000 + [ 0.000000] Reserved memory: created DMA memory pool at 0x0000000070000000, size 256 MiB + [ 0.000000] OF: reserved mem: initialized node dma_pool@70000000, compatible id shared-dma-pool + [ 0.000000] OF: reserved mem: 0x0000000070000000..0x000000007fffffff (262144 KiB) map non-reusable dma_pool@70000000 + [ 0.000000] cma: Reserved 16 MiB at 0x00000000af000000 on node -1 + [ 0.000000] early_memtest: # of tests: 3 + [ 0.000000] 0x0000000083200000 - 0x000000008c300000 pattern 5555555555555555 + [ 0.000000] 0x000000009e912000 - 0x00000000aeff9308 pattern 5555555555555555 + [ 0.000000] 0x00000000aeff9337 - 0x00000000aeff9338 pattern 5555555555555555 + [ 0.000000] 0x00000000aeff9367 - 0x00000000aeff9368 pattern 5555555555555555 + [ 0.000000] 0x00000000aeffcffc - 0x00000000aeffd000 pattern 5555555555555555 + [ 0.000000] 0x0000000083200000 - 0x000000008c300000 pattern ffffffffffffffff + [ 0.000000] 0x000000009e912000 - 0x00000000aeff9308 pattern ffffffffffffffff + [ 0.000000] 0x00000000aeff9337 - 0x00000000aeff9338 pattern ffffffffffffffff + [ 0.000000] 0x00000000aeff9367 - 0x00000000aeff9368 pattern ffffffffffffffff + [ 0.000000] 0x00000000aeffcffc - 0x00000000aeffd000 pattern ffffffffffffffff + [ 0.000000] 0x0000000083200000 - 0x000000008c300000 pattern 0000000000000000 + [ 0.000000] 0x000000009e912000 - 0x00000000aeff9308 pattern 0000000000000000 + [ 0.000000] 0x00000000aeff9337 - 0x00000000aeff9338 pattern 0000000000000000 + [ 0.000000] 0x00000000aeff9367 - 0x00000000aeff9368 pattern 0000000000000000 + [ 0.000000] 0x00000000aeffcffc - 0x00000000aeffd000 pattern 0000000000000000 + [ 0.000000] Zone ranges: + [ 0.000000] DMA32 [mem 0x0000000080200000-0x00000000b01fffff] + [ 0.000000] Normal empty + [ 0.000000] Movable zone start for each node + [ 0.000000] Early memory node ranges + [ 0.000000] node 0: [mem 0x0000000080200000-0x00000000b01fffff] + +That seems to pass the memtest fine, however the boot process hangs in different +stages. + +### OBSERVATION: Cannot open /dev/ttyS0 + + ~ # setserial -g /dev/ttyS1 -a + /dev/ttyS1, Line 1, UART: 16550, Port: 0x0000, IRQ: 1 + Baud_base: 3125000, close_delay: 50, divisor: 0 + closing_wait: 3000 + Flags: spd_normal + + ~ # setserial -g /dev/ttyS0 -a + (hangs) + +This page seems to have good resources on the serial console: + + https://tldp.org/HOWTO/Serial-HOWTO-16.html + +It seems that there are some differences in the way the serial port is handled +regarding 16550 and 16550A. + +I can write to the UART console from U-Boot by directly writing in the +0x40001000 address (A = 0x41): + + => help mw + mw - memory write (fill) + + Usage: + mw [.b, .w, .l, .q] address value [count] + => mw 0x40001000 0x41 + A=> mw 0x40001000 0x42 + B=> mw 0x40001000 0x43 + C=> + +### OBSERVATION: I can type with the ttyS0 8250 driver + +Tried to boot too, but hangs: + + => fdt set /memory@80000000 reg <0x0 0x80200000 0x0 0x30000000> + => setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=ttyS0,115200n8 boot.trace boot.tracedebug init=/nix/store/zxbq93zfg8ijkyq5cq5sb4742rczqfck-nixos- + system-nixos-riscv-24.11pre-git/init" + => setenv ramdisk_size 12611657 + => booti ${kernel_addr_r} ${ramdisk_addr_r}:${ramdisk_size} ${fdtcontroladdr} + ... + [ 30.740740] riscv-plic 40800000.plic: mapped 3 interrupts with 1 handlers for 2 contexts. + [ 40.048300] Serial: 8250/16550 driver, 4 ports, IRQ sharing disabled + [ 40.420000] of_serial 40001000.serial: error -ENXIO: IRQ index 0 not found + [ 40.496720] printk: legacy console [ttyS0] disabled + [ 40.558860] 40001000.serial: ttyS0 at MMIO 0x40001000 (irq = 0, base_baud = 3125000) is a 16550 + [ 40.583940] printk: legacy console [ttyS0] enabled + [ 40.583940] printk: legacy console [ttyS0] enabled + [ 40.595760] printk: legacy bootconsole [sbi0] disabled + [ 40.595760] printk: legacy bootconsole [sbi0] disabled + [ 40.820380] 40003000.serial: ttyS1 at MMIO 0x40003000 (irq = 1, base_baud = 3125000) is a 16550 + + ... + + <<< NixOS Stage 2 >>> + + [ 394.678980] EXT4-fs (pmem0p2): re-mounted 44444444-4444-4444-8888-888888888888 r/w. Quota mode: none. + [ 394.764620] booting system configuration /nix/store/zxbq93zfg8ijkyq5cq5sb4742rczqfck-nixos-system-nixos-riscv-24.11pre-git + runnin[ 398.543300] stage-2-init: running activation script... + g activation script... + +So, if we observe a hang when writing to a bad memory segment, can there be a +problem in the place we are placing the pmem? Maybe we can test it with u-boot +first. + +Another note, the serial device 16550 doesn't seem to use a FIFO, but the +16550A does. + +We may want to switch to the A variant, as it seems to be supported by U-boot and +the kernel: + + https://github.com/u-boot/u-boot/blob/master/drivers/serial/ns16550.c#L607-L619 + https://github.com/torvalds/linux/blob/v6.9/drivers/tty/serial/8250/8250_of.c#L285 + +And defines a FIFO size of 16 bytes: + + https://github.com/torvalds/linux/blob/v6.9/drivers/tty/serial/8250/8250_port.c#L74-L81 + +Still, we would have to wait for a bitstream that can forward the interrupts +from the host to the serial console to test it. + +### OBSERVATION: The memory for the pmem seems to be ok + + => mtest 0x100000000 0x1c0000000 0 2 + Testing 100000000 ... 1c0000000: + Pattern FFFFFFFFFFFFFFFF Writing... Reading...Iteration: 2 + Tested 2 iteration(s) with 0 errors. + -- GitLab From 1a15ddc0e75902e210b0db6903cf54bc55d31bf9 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 11 Jul 2024 15:37:41 +0200 Subject: [PATCH 096/310] Restore support for huge pages --- JOURNAL.md | 2 ++ lagarto-ox.nix | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 5574148..8458bac 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1391,3 +1391,5 @@ from the host to the serial console to test it. Pattern FFFFFFFFFFFFFFFF Writing... Reading...Iteration: 2 Tested 2 iteration(s) with 0 errors. +It may be worth enabling huge pages again, as there didn't seem to have any +effect. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 568d349..104f85c 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -71,11 +71,6 @@ + '' SMP n '' - # Disable huge pages - + '' - HUGETLBFS n - HUGETLB_PAGE n - '' ; } ]; -- GitLab From 5ddea498cadde8d858cfb84a7734d489320d3a31 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 11 Jul 2024 15:47:29 +0200 Subject: [PATCH 097/310] Add reserved segment from 0x80000000 --- ox-plic.dts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ox-plic.dts b/ox-plic.dts index e7e9249..833bf4d 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -84,6 +84,10 @@ #address-cells = <2>; /* Starting address and size */ #size-cells = <2>; /* 64 bits memory addresses */ ranges; + reserved@80000000 { + reg = <0x0 0x80000000 0x0 0x04000000>; + no-map; + }; eth_pool: dma_pool@60000000 { reg = <0x0 0x60000000 0x0 0x10000000>; compatible = "shared-dma-pool"; -- GitLab From 9f6f8bcddcc68bec294fb4d119c954fc6d8c1814 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 11 Jul 2024 16:00:43 +0200 Subject: [PATCH 098/310] Don't use DTB in U-Boot, read it from OpenSBI --- JOURNAL.md | 18 ++++++++++++++++++ lagarto-ox.nix | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 8458bac..cf7bce2 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1393,3 +1393,21 @@ from the host to the serial console to test it. It may be worth enabling huge pages again, as there didn't seem to have any effect. + +### OBSERVATION: There are extra regions mapped by OpenSBI + +In the OpenSBI 1.2 test from buildroot: + + Domain0 Region00 : 0x0000000080000000-0x000000008003ffff () + Domain0 Region01 : 0x0000000000000000-0xffffffffffffffff (R,W,X) + Domain0 Next Address : 0x0000000080200000 + Domain0 Next Arg1 : 0x0000000080010000 + +But with OpenSBI 1.4: + + Domain0 Region00 : 0x0000000040000000-0x0000000040000fff M: (I,R,W) S/U: (R,W) + Domain0 Region01 : 0x0000000080020000-0x000000008002ffff M: (R,W) S/U: () + Domain0 Region02 : 0x0000000080000000-0x000000008001ffff M: (R,X) S/U: () + Domain0 Region03 : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X) + Domain0 Next Address : 0x0000000080200000 + Domain0 Next Arg1 : 0x0000000080013000 diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 104f85c..a3f75df 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -217,7 +217,7 @@ bootrom = "${final.bitstreams}/lagarto-3-ox/large_bootrom.bin"; uboot = prev.ubootQemuRiscv64Smode.override { - filesToInstall = [ "u-boot.bin" "u-boot-nodtb.bin" ]; + filesToInstall = [ "u-boot-nodtb.bin" ]; #version = "2023.07.02-print-cpu-probe"; #src = builtins.fetchGit { # url = "file:///home/Computational/rarias/riscv/u-boot"; @@ -319,7 +319,7 @@ #"PLATFORM_RISCV_ISA=rv64imafd" # No compressed instructions #"PLATFORM_RISCV_ISA=rv64g" # No compressed instructions #"PLATFORM_RISCV_ABI=lp64d" - "FW_PAYLOAD_PATH=${final.uboot}/u-boot.bin" + "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin" "FW_FDT_PATH=${final.ox-dtb}" ]; patches = [ ./ox-alveo-platform-plic.patch ]; -- GitLab From c23d33a51c2e818bde314657ef17a3f808d6cff3 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 09:34:04 +0200 Subject: [PATCH 099/310] Update to OpenSBI 1.5 --- JOURNAL.md | 8 ++++++++ lagarto-ox.nix | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index cf7bce2..8401ece 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1411,3 +1411,11 @@ But with OpenSBI 1.4: Domain0 Region03 : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X) Domain0 Next Address : 0x0000000080200000 Domain0 Next Arg1 : 0x0000000080013000 + +## 2024-07-12 + +I find it strange that if we are writing to the 0x80000000 region from the +kernel and OpenSBI has mapped a segment for Machine mode only, there is no error +message about it in the console. + +Let's update to OpenSBI 1.5 for now, and see if we get any more information. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index a3f75df..d03a206 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -305,12 +305,12 @@ ''; opensbi = prev.opensbi.overrideAttrs (old: rec { - version = "1.4"; + version = "1.5"; src = prev.fetchFromGitHub { owner = "riscv-software-src"; repo = "opensbi"; rev = "v${version}"; - hash = "sha256-T8ZeAzjM9aeTXitjE7s+m+jjGGtDo2jK1qO5EuKiVLU="; + hash = "sha256-vK14P97FcaVz4GDr/0055Z6s/k7BPKPQGZ/MQxbOWu0="; }; #NIX_DEBUG=5; makeFlags = [ -- GitLab From 40def167f19634d31e834208f0cf605886ea2645 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 10:20:32 +0200 Subject: [PATCH 100/310] Update journal --- JOURNAL.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index 8401ece..47890ca 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1419,3 +1419,11 @@ kernel and OpenSBI has mapped a segment for Machine mode only, there is no error message about it in the console. Let's update to OpenSBI 1.5 for now, and see if we get any more information. + +It seems to be booting fine. + +So, I observed the `udevadm settle` to take a lot of time, maybe I can get some +clues by enabling the debug mode. + +If a trap is being issued, we may be able to modify OpenSBI to print some +information to the serial console so we can understand what is going on. -- GitLab From 8c2b28cf21d5e6c35dd8d848ac8507285a006f63 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 10:50:28 +0200 Subject: [PATCH 101/310] Enable systemd debug log and all-in-order --- JOURNAL.md | 8 ++++++++ lagarto-ox.nix | 50 +++++++++++++++++++++++--------------------------- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 47890ca..cb36ee4 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1427,3 +1427,11 @@ clues by enabling the debug mode. If a trap is being issued, we may be able to modify OpenSBI to print some information to the serial console so we can understand what is going on. + +In the meanwhile, let's try a simple test. We can enable debug output of systemd +with the following boot options: + + systemd.log_level=debug systemd.log_target=console + +We will also need to run `csrtool all-in-order` to arrive to systemd. + diff --git a/lagarto-ox.nix b/lagarto-ox.nix index d03a206..ab0ab8f 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -81,31 +81,24 @@ kernelModules = [ ]; # Custom init script - extraFiles."/init2".source = pkgs.writeScript "init2" '' - #!${config.system.build.extraUtils}/bin/ash - - set -x - - export PATH=${config.system.build.extraUtils}/bin - - #mkdir -p /tracing - #mount -t tracefs nodev /tracing - - #cd /tracing - #echo nop > current_tracer - #echo 100 > max_graph_depth - #echo handle_page_fault > set_graph_function - #echo function_graph > current_tracer - #cd / - - # Open an interactive shell - ash - - memtool fill $((512 * 1024 * 1024)) - - # Unlikely to reach this point - exec /init - ''; + extraFiles = { + "/shell".source = pkgs.writeScript "shell" '' + #!${config.system.build.extraUtils}/bin/ash + + set -x + + export PATH=${config.system.build.extraUtils}/bin + ash + ''; + + "/preinit".source = pkgs.writeScript "preinit" '' + #!${config.system.build.extraUtils}/bin/ash + set -x + export PATH=${config.system.build.extraUtils}/bin + csrtool all-in-order + exec /init + ''; + }; # Add the csrtool to the initrd so we can change the # in-order/out-of-order, and memtool to stress the memory. @@ -283,7 +276,10 @@ fdt set /memory@80000000 reg <0x0 0x80000000 0x0 0xc0000000> # Set kernel bootcmd options. - # rdinit=/init2 boot custom init script + # rdinit=/preinit boot custom preinit script + # Systemd options + # systemd.log_level=debug + # systemd.log_target=console # NixOS interesting options: # debug1 enable debug shell in stage 1 # boot.trace enable set -x in stage 1 @@ -294,7 +290,7 @@ # tp_printk write ftrace events to console # trace_buf_size=1M set ftrace buffer to 1M # - setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 init=${init}" + setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 systemd.log_level=debug systemd.log_target=console rdinit=/preinit init=${init}" EOF -- GitLab From dc8a8c2ce9c56d3e3eac44b202497f6537e6a103 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 12:13:06 +0200 Subject: [PATCH 102/310] Build OpenSBI with the generic platform --- JOURNAL.md | 149 +++++++++++++++++++++++++++++++++++++++++++++++++ fpga/upload.sh | 2 +- lagarto-ox.nix | 4 +- 3 files changed, 152 insertions(+), 3 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index cb36ee4..8f4d367 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1435,3 +1435,152 @@ with the following boot options: We will also need to run `csrtool all-in-order` to arrive to systemd. +### OBSERVATION: Hangs in `switch_root` again. + +Technically, we cannot discard the hypothesis than only the console has crashed, +as when we switch to the stage 2 we don't have the heartbeat counter. We may as +well run it again before we arrive to systemd just to verify that userland +crashed. + +On the other hand, I don't understand why we hang in such a way when we try to +write to the `0x8000_0000` area from the kernel memtest. I've been reading the +OpenSBI source code and they seem to have a trap handler that can emit verbose +information to the console when a problem with the trap is detected. I would +expect to see some error being dumped to the console in that case. + +From the OpenSBI information, this line: + + Domain0 Region02 : 0x0000000080000000-0x000000008001ffff M: (R,X) S/U: () + +Suggests that it registers a region with no write permission at `0x8000_0000`, +so it should fail right away from the kernel side. However, this is not reported +anywhere in the console. + +As we have an easy way to trigger this situation, maybe we can use it as a test +to modify OpenSBI to report that problem to the console and verify that it is +working. With that information, we could rule out that a similar problem is +happening when we try to run systemd. Maybe we could also try to debug other +traps. + +Another observation is that the memtest lines we see on the console are printed +*before* the actual test begins: + + pr_info(" %pa - %pa pattern %016llx\n", + &this_start, &this_end, cpu_to_be64(pattern)); + memtest(pattern, this_start, this_end - this_start); + +So when this line is shown: + + [ 0.000000] early_memtest: # of tests: 3 + [ 0.000000] 0x0000000080000000 - 0x0000000080013000 pattern 5555555555555555 + +We an infer that the problem is located in that region, which agrees with the +hypothesis that is related with the OpenSBI regions. + +This is the output I get with OpenSBI 1.5: + + OpenSBI v1.5 + ____ _____ ____ _____ + / __ \ / ____| _ \_ _| + | | | |_ __ ___ _ __ | (___ | |_) || | + | | | | '_ \ / _ \ '_ \ \___ \| _ < | | + | |__| | |_) | __/ | | |____) | |_) || |_ + \____/| .__/ \___|_| |_|_____/|____/_____| + | | + |_| + + Platform Name : ox (Rodrigo NixOS version) + Platform Features : medeleg + Platform HART Count : 1 + Platform IPI Device : --- + Platform Timer Device : axi_timer @ 50000000Hz + Platform Console Device : uart8250 + Platform HSM Device : --- + Platform PMU Device : --- + Platform Reboot Device : --- + Platform Shutdown Device : --- + Platform Suspend Device : --- + Platform CPPC Device : --- + Firmware Base : 0x80000000 + Firmware Size : 310 KB + Firmware RW Offset : 0x40000 + Firmware RW Size : 54 KB + Firmware Heap Offset : 0x45000 + Firmware Heap Size : 34 KB (total), 2 KB (reserved), 11 KB (used), 20 KB (free) + Firmware Scratch Size : 4096 B (total), 368 B (used), 3728 B (free) + Runtime SBI Version : 2.0 + + Domain0 Name : root + Domain0 Boot HART : 0 + Domain0 HARTs : 0* + Domain0 Region00 : 0x0000000040000000-0x0000000040000fff M: (I,R,W) S/U: (R,W) + Domain0 Region01 : 0x0000000080040000-0x000000008004ffff M: (R,W) S/U: () + Domain0 Region02 : 0x0000000080000000-0x000000008003ffff M: (R,X) S/U: () + Domain0 Region03 : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X) + Domain0 Next Address : 0x0000000080200000 + Domain0 Next Arg1 : 0x0000000080017000 + Domain0 Next Mode : S-mode + Domain0 SysReset : yes + Domain0 SysSuspend : yes + + Boot HART ID : 0 + Boot HART Domain : root + Boot HART Priv Version : v1.10 + Boot HART Base ISA : rv64imafdc + Boot HART ISA Extensions : zicntr,zihpm,sdtrig + Boot HART PMP Count : 0 + Boot HART PMP Granularity : 0 bits + Boot HART PMP Address Bits: 0 + Boot HART MHPM Info : 29 (0xfffffff8) + Boot HART Debug Triggers : 0 triggers + Boot HART MIDELEG : 0x0000000000000222 + Boot HART MEDELEG : 0x000000000000b109 + + + Core: 12 devices, 8 uclasses, devicetree: board + Loading Environment from nowhere... OK + In: serial,usbkbd + Out: serial,vidconsole + Err: serial,vidconsole + No working controllers found + Net: No ethernet found. + Working FDT set to 80017000 + Hit any key to stop autoboot: 0 + + Device 0: unknown device + + Device 1: unknown device + scanning bus for devices... + + Device 0: unknown device + starting USB... + No working controllers found + No ethernet found. + No ethernet found. + => + +Where now the regions are slightly off: + + Domain0 Region00 : 0x0000000040000000-0x0000000040000fff M: (I,R,W) S/U: (R,W) + Domain0 Region01 : 0x0000000080040000-0x000000008004ffff M: (R,W) S/U: () + Domain0 Region02 : 0x0000000080000000-0x000000008003ffff M: (R,X) S/U: () + Domain0 Region03 : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X) + +I would assume that the region 1 is where OpenSBI places its own data, and +region 2 is where it places its own code. Then, in region 0 there is the serial +area. + +Interestingly, I can read and write to the 0x80000000 - 0x81000000 from u-boot +without problems: + + => mtest 0x80000000 0x81000000 0 4 + Testing 80000000 ... 81000000: + Pattern FFFFFFFFFFFFFFFF Writing... Reading...Iteration: 4 + Tested 4 iteration(s) with 0 errors. + +So I suspect that it disables those regions before jumping into U-Boot. + +What I don't understand is why the MMIO region 0 is starting at 0x40000000 while +the UART port should be mapped in 0x40001000 as per the device tree. Maybe we +could try with the generic configuration of OpenSBI and see if it can load the +plic and the serial ports properly directly from the device tree. diff --git a/fpga/upload.sh b/fpga/upload.sh index 67ee05b..0c8807e 100755 --- a/fpga/upload.sh +++ b/fpga/upload.sh @@ -13,7 +13,7 @@ fi rsync -a fpga/fpgactl "$dst" #rsync -a fpga/boot.sh "$dst" rsync -a fpga/env.sh "$dst" -rsync $OPENSBI/share/opensbi/*/fpga/*/firmware/fw_payload.bin "$dst/opensbi.bin" +rsync $(find "$OPENSBI" -name fw_payload.bin) "$dst/opensbi.bin" rsync "$KERNEL/Image" "$dst/kernel.bin" rsync "$INITRD/initrd" "$dst/initrd.bin" if [ -n "$ROOTFS" ]; then diff --git a/lagarto-ox.nix b/lagarto-ox.nix index ab0ab8f..d88f845 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -310,7 +310,7 @@ }; #NIX_DEBUG=5; makeFlags = [ - "PLATFORM=fpga/ox_alveo" + "PLATFORM=generic" #"CONFIG_SBI_ECALL_RFENCE=n" #"PLATFORM_RISCV_ISA=rv64imafd" # No compressed instructions #"PLATFORM_RISCV_ISA=rv64g" # No compressed instructions @@ -318,7 +318,7 @@ "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin" "FW_FDT_PATH=${final.ox-dtb}" ]; - patches = [ ./ox-alveo-platform-plic.patch ]; + #patches = [ ./ox-alveo-platform-plic.patch ]; }); # opensbi = prev.opensbi.overrideAttrs (old: { # #NIX_DEBUG=5; -- GitLab From 6604911264988f4c77540b4a4167b1e5808d8c19 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 12:38:34 +0200 Subject: [PATCH 103/310] Remove reg-names property from clint --- JOURNAL.md | 26 ++++++++++++++++++++++++++ ox-plic.dts | 3 +-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 8f4d367..fe1b16d 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1584,3 +1584,29 @@ What I don't understand is why the MMIO region 0 is starting at 0x40000000 while the UART port should be mapped in 0x40001000 as per the device tree. Maybe we could try with the generic configuration of OpenSBI and see if it can load the plic and the serial ports properly directly from the device tree. + +Nice, with OpenSBI 1.5 I can see the console errors when trying the generic +configuration: + + OpenSBI v1.5 + ____ _____ ____ _____ + / __ \ / ____| _ \_ _| + | | | |_ __ ___ _ __ | (___ | |_) || | + | | | | '_ \ / _ \ '_ \ \___ \| _ < | | + | |__| | |_) | __/ | | |____) | |_) || |_ + \____/| .__/ \___|_| |_|_____/|____/_____| + | | + |_| + + init_coldboot: timer init failed (error -3) + +Seems to be failing in `sbi_timer_init()` with `cold_boot = true`. + +And the -3 error seems to be: + + #define SBI_ERR_INVALID_PARAM -3 + +I assume it is calling `fdt_timer_init()`. + +Let's try removing the `reg-names` property, as it seems to cause it to enter a +different branch, but "control" is never matched there. diff --git a/ox-plic.dts b/ox-plic.dts index 833bf4d..35cf1e2 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -228,9 +228,8 @@ * property is described in * Documentation/devicetree/bindings/riscv/cpus.yaml */ - timer@40002000 { + clint: clint@40002000 { reg = <0x0 0x40002000 0x0 0x000c0000>; - reg-names = "control"; interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; /*<&CPU0 0x3>, <&CPU0 0x7>,*/ /*<&onic_pool 0x3>, <&onic_pool 0x7>,*/ -- GitLab From 83df8edd28c800eadf28faa4ed2fcbc743b6112d Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 12:44:21 +0200 Subject: [PATCH 104/310] Switch to fpga/openpiton in OpenSBI --- JOURNAL.md | 4 ++++ lagarto-ox.nix | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/JOURNAL.md b/JOURNAL.md index fe1b16d..42a37b3 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1610,3 +1610,7 @@ I assume it is calling `fdt_timer_init()`. Let's try removing the `reg-names` property, as it seems to cause it to enter a different branch, but "control" is never matched there. + +Still failing, the problem must be somewhere else. + +Let's try with openpiton configuration instead. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index d88f845..640140e 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -310,7 +310,7 @@ }; #NIX_DEBUG=5; makeFlags = [ - "PLATFORM=generic" + "PLATFORM=fpga/openpiton" #"CONFIG_SBI_ECALL_RFENCE=n" #"PLATFORM_RISCV_ISA=rv64imafd" # No compressed instructions #"PLATFORM_RISCV_ISA=rv64g" # No compressed instructions -- GitLab From 722f185525a65943232b862addd9717252c1dc8a Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 13:19:10 +0200 Subject: [PATCH 105/310] Add timer debug patch for OpenSBI --- JOURNAL.md | 9 ++++ lagarto-ox.nix | 7 ++- opensbi-timer-debug.patch | 100 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 opensbi-timer-debug.patch diff --git a/JOURNAL.md b/JOURNAL.md index 42a37b3..a8c1ddd 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1614,3 +1614,12 @@ different branch, but "control" is never matched there. Still failing, the problem must be somewhere else. Let's try with openpiton configuration instead. + +Doesn't even start the UART: + + GGGGGGGGG + +These G's must be coming from the bootrom. + +So let's go back to the generic platform and place some `printf()` calls to +determine where it is failing. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 640140e..c9ab49c 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -310,7 +310,7 @@ }; #NIX_DEBUG=5; makeFlags = [ - "PLATFORM=fpga/openpiton" + "PLATFORM=generic" #"CONFIG_SBI_ECALL_RFENCE=n" #"PLATFORM_RISCV_ISA=rv64imafd" # No compressed instructions #"PLATFORM_RISCV_ISA=rv64g" # No compressed instructions @@ -318,7 +318,10 @@ "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin" "FW_FDT_PATH=${final.ox-dtb}" ]; - #patches = [ ./ox-alveo-platform-plic.patch ]; + patches = [ + ./opensbi-timer-debug.patch + #./ox-alveo-platform-plic.patch + ]; }); # opensbi = prev.opensbi.overrideAttrs (old: { # #NIX_DEBUG=5; diff --git a/opensbi-timer-debug.patch b/opensbi-timer-debug.patch new file mode 100644 index 0000000..93e2df1 --- /dev/null +++ b/opensbi-timer-debug.patch @@ -0,0 +1,100 @@ +diff --git a/lib/sbi/sbi_timer.c b/lib/sbi/sbi_timer.c +index 7b618de..65e42b0 100644 +--- a/lib/sbi/sbi_timer.c ++++ b/lib/sbi/sbi_timer.c +@@ -183,13 +183,17 @@ int sbi_timer_init(struct sbi_scratch *scratch, bool cold_boot) + u64 *time_delta; + const struct sbi_platform *plat = sbi_platform_ptr(scratch); + ++ sbi_printf("sbi_timer_init: begins\n"); ++ + if (cold_boot) { + time_delta_off = sbi_scratch_alloc_offset(sizeof(*time_delta)); + if (!time_delta_off) + return SBI_ENOMEM; + +- if (sbi_hart_has_extension(scratch, SBI_HART_EXT_ZICNTR)) ++ if (sbi_hart_has_extension(scratch, SBI_HART_EXT_ZICNTR)) { ++ sbi_printf("sbi_timer_init: got Zicntr extension\n"); + get_time_val = get_ticks; ++ } + } else { + if (!time_delta_off) + return SBI_ENOMEM; +@@ -198,7 +202,10 @@ int sbi_timer_init(struct sbi_scratch *scratch, bool cold_boot) + time_delta = sbi_scratch_offset_ptr(scratch, time_delta_off); + *time_delta = 0; + +- return sbi_platform_timer_init(plat, cold_boot); ++ int rc = sbi_platform_timer_init(plat, cold_boot); ++ if (rc) ++ sbi_printf("sbi_platform_timer: sbi_platform_timer_init failed (%d)\n", rc); ++ return rc; + } + + void sbi_timer_exit(struct sbi_scratch *scratch) +diff --git a/lib/utils/timer/fdt_timer.c b/lib/utils/timer/fdt_timer.c +index f468730..db20526 100644 +--- a/lib/utils/timer/fdt_timer.c ++++ b/lib/utils/timer/fdt_timer.c +@@ -7,6 +7,7 @@ + * Anup Patel + */ + ++#include + #include + #include + #include +@@ -39,19 +40,26 @@ static int fdt_timer_cold_init(void) + void *fdt = fdt_get_address(); + + for (pos = 0; pos < fdt_timer_drivers_size; pos++) { ++ sbi_printf("fdt_timer_cold_init: pos = %d\n", pos); + drv = fdt_timer_drivers[pos]; + + noff = -1; + while ((noff = fdt_find_match(fdt, noff, + drv->match_table, &match)) >= 0) { ++ ++ sbi_printf("fdt_timer_cold_init: got match, name = %s\n", match->compatible); + if (!fdt_node_is_enabled(fdt, noff)) + continue; + ++ sbi_printf("fdt_timer_cold_init: enabled\n"); ++ + /* drv->cold_init must not be NULL */ + if (drv->cold_init == NULL) + return SBI_EFAIL; + + rc = drv->cold_init(fdt, noff, match); ++ sbi_printf("fdt_timer_cold_init: drc->cold_init = %d\n", rc); ++ + if (rc == SBI_ENODEV) + continue; + if (rc) +@@ -69,6 +77,7 @@ static int fdt_timer_cold_init(void) + * We can't fail here since systems with Sstc might not provide + * mtimer/clint DT node in the device tree. + */ ++ sbi_printf("fdt_timer_cold_init: returns 0\n"); + return 0; + } + +@@ -78,9 +87,15 @@ int fdt_timer_init(bool cold_boot) + + if (cold_boot) { + rc = fdt_timer_cold_init(); +- if (rc) ++ if (rc) { ++ sbi_printf("fdt_timer_init: fdt_timer_cold_init failed (%d)\n", rc); + return rc; ++ } + } + +- return fdt_timer_warm_init(); ++ rc = fdt_timer_warm_init(); ++ if (rc) ++ sbi_printf("fdt_timer_init: fdt_timer_warm_init failed (%d)\n", rc); ++ ++ return rc; + } -- GitLab From 6576a92ba57266dcf551854ef665c8250e6d1a07 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 13:47:01 +0200 Subject: [PATCH 106/310] Debug timer_mtimer_cold_init() in OpenSBI --- JOURNAL.md | 51 +++++++++++++++++++++++++++++++++++ opensbi-timer-debug.patch | 57 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index a8c1ddd..0c888da 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1623,3 +1623,54 @@ These G's must be coming from the bootrom. So let's go back to the generic platform and place some `printf()` calls to determine where it is failing. + + OpenSBI v1.5 + ____ _____ ____ _____ + / __ \ / ____| _ \_ _| + | | | |_ __ ___ _ __ | (___ | |_) || | + | | | | '_ \ / _ \ '_ \ \___ \| _ < | | + | |__| | |_) | __/ | | |____) | |_) || |_ + \____/| .__/ \___|_| |_|_____/|____/_____| + | | + |_| + + sbi_timer_init: begins + sbi_timer_init: got Zicntr extension + fdt_timer_cold_init: pos = 0 + fdt_timer_cold_init: got match, name = riscv,clint0 + fdt_timer_cold_init: enabled + fdt_timer_cold_init: drc->cold_init = -3 + fdt_timer_init: fdt_timer_cold_init failed (-3) + sbi_platform_timer: sbi_platform_timer_init failed (-3) + init_coldboot: timer init failed (error -3) + +Okay, now we can see where it failed. I wonder why aren't these messages enabled +by default. I'll guess this is the `timer_mtimer_cold_init()` function, so let's +add some more instrumentation there. + +It seems that SiFive timer has a very long weird offset: + + if (is_clint) { /* SiFive CLINT */ + /* Set CLINT addresses */ + mt->mtimecmp_addr = addr[0] + ACLINT_DEFAULT_MTIMECMP_OFFSET; + mt->mtimecmp_size = ACLINT_DEFAULT_MTIMECMP_SIZE; + if (!quirks->clint_without_mtime) { + mt->mtime_addr = addr[0] + ACLINT_DEFAULT_MTIME_OFFSET; + mt->mtime_size = size[0] - mt->mtimecmp_size; + /* Adjust MTIMER address and size for CLINT device */ + mt->mtime_addr += quirks->clint_mtime_offset; + mt->mtime_size -= quirks->clint_mtime_offset; + } else { + mt->mtime_addr = mt->mtime_size = 0; + } + mt->mtimecmp_addr += quirks->clint_mtime_offset; + } else { /* RISC-V ACLINT MTIMER */ + /* Set ACLINT MTIMER addresses */ + mt->mtime_addr = addr[0]; + mt->mtime_size = size[0]; + mt->mtimecmp_addr = addr[1]; + mt->mtimecmp_size = size[1]; + } + +We may want to use the ACLINT timer instead. Let's first see where the addresses +lay in memory, and then use that to verify we change it to 0x0 and 0x8. diff --git a/opensbi-timer-debug.patch b/opensbi-timer-debug.patch index 93e2df1..f2155f2 100644 --- a/opensbi-timer-debug.patch +++ b/opensbi-timer-debug.patch @@ -98,3 +98,60 @@ index f468730..db20526 100644 + + return rc; } +diff --git a/lib/utils/timer/fdt_timer_mtimer.c b/lib/utils/timer/fdt_timer_mtimer.c +index 9e27e3a..fba48ca 100644 +--- a/lib/utils/timer/fdt_timer_mtimer.c ++++ b/lib/utils/timer/fdt_timer_mtimer.c +@@ -8,6 +8,7 @@ + */ + + #include ++#include + #include + #include + #include +@@ -33,6 +34,7 @@ static struct aclint_mtimer_data *mt_reference = NULL; + static int timer_mtimer_cold_init(void *fdt, int nodeoff, + const struct fdt_match *match) + { ++ + int rc; + unsigned long addr[2], size[2]; + struct timer_mtimer_node *mtn, *n; +@@ -40,6 +42,8 @@ static int timer_mtimer_cold_init(void *fdt, int nodeoff, + const struct timer_mtimer_quirks *quirks = match->data; + bool is_clint = quirks && quirks->is_clint; + ++ sbi_printf("timer_mtimer_cold_init: begins, is_clint = %d\n", (int) is_clint); ++ + mtn = sbi_zalloc(sizeof(*mtn)); + if (!mtn) + return SBI_ENOMEM; +@@ -49,6 +53,7 @@ static int timer_mtimer_cold_init(void *fdt, int nodeoff, + &addr[0], &size[0], &addr[1], &size[1], + &mt->first_hartid, &mt->hart_count); + if (rc) { ++ sbi_printf("timer_mtimer_cold_init: fdt_parse_aclint_node failed (%d)\n", rc); + sbi_free(mtn); + return rc; + } +@@ -57,6 +62,7 @@ static int timer_mtimer_cold_init(void *fdt, int nodeoff, + + rc = fdt_parse_timebase_frequency(fdt, &mt->mtime_freq); + if (rc) { ++ sbi_printf("timer_mtimer_cold_init: fdt_parse_timebase_frequency failed (%d)\n", rc); + sbi_free(mtn); + return rc; + } +@@ -83,6 +89,11 @@ static int timer_mtimer_cold_init(void *fdt, int nodeoff, + mt->mtimecmp_size = size[1]; + } + ++ sbi_printf("timer_mtimer_cold_init: mtime_addr = 0x%08lx\n", mt->mtime_addr); ++ sbi_printf("timer_mtimer_cold_init: mtime_size = 0x%08lx\n", mt->mtime_size); ++ sbi_printf("timer_mtimer_cold_init: mtime_addr = 0x%08lx\n", mt->mtimecmp_addr); ++ sbi_printf("timer_mtimer_cold_init: mtime_addr = 0x%08lx\n", mt->mtimecmp_size); ++ + /* Apply additional quirks */ + if (quirks) { + mt->has_64bit_mmio = quirks->has_64bit_mmio; -- GitLab From 6a57dd004fe09bbd86f1bb3c4da7dd341e00fd02 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 13:55:08 +0200 Subject: [PATCH 107/310] Switch to "riscv,aclint-mtimer" timer --- JOURNAL.md | 38 ++++++++++++++++++++++++++++++++++++++ opensbi-timer-debug.patch | 6 +++--- ox-plic.dts | 4 +++- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 0c888da..9a5b9df 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1674,3 +1674,41 @@ It seems that SiFive timer has a very long weird offset: We may want to use the ACLINT timer instead. Let's first see where the addresses lay in memory, and then use that to verify we change it to 0x0 and 0x8. + + GGGGGGGGG + OpenSBI v1.5 + ____ _____ ____ _____ + / __ \ / ____| _ \_ _| + | | | |_ __ ___ _ __ | (___ | |_) || | + | | | | '_ \ / _ \ '_ \ \___ \| _ < | | + | |__| | |_) | __/ | | |____) | |_) || |_ + \____/| .__/ \___|_| |_|_____/|____/_____| + | | + |_| + + sbi_timer_init: begins + sbi_timer_init: got Zicntr extension + fdt_timer_cold_init: pos = 0 + fdt_timer_cold_init: got match, name = riscv,clint0 + fdt_timer_cold_init: enabled + timer_mtimer_cold_init: begins, is_clint = 1 + timer_mtimer_cold_init: mtime_addr = 0x4000dff8 + timer_mtimer_cold_init: mtime_size = 0x000b4008 + timer_mtimer_cold_init: mtime_addr = 0x40006000 + timer_mtimer_cold_init: mtime_addr = 0x00007ff8 + fdt_timer_cold_init: drc->cold_init = -3 + fdt_timer_init: fdt_timer_cold_init failed (-3) + sbi_platform_timer: sbi_platform_timer_init failed (-3) + init_coldboot: timer init failed (error -3) + +Yeah, those addresses are not what we want. Based on the device tree, the clint +must be at 0x40002000, so they should be 40002000 and 40002008. Also I made some +typos in the printf command, this is the patch: + + + sbi_printf("timer_mtimer_cold_init: mtime_addr = 0x%08lx\n", mt->mtime_addr); + + sbi_printf("timer_mtimer_cold_init: mtime_size = 0x%08lx\n", mt->mtime_size); + + sbi_printf("timer_mtimer_cold_init: mtime_addr = 0x%08lx\n", mt->mtimecmp_addr); + + sbi_printf("timer_mtimer_cold_init: mtime_addr = 0x%08lx\n", mt->mtimecmp_size); + +So, let's fix the patch and switch to the "riscv,aclint-mtimer" timer, which +doesn't set any quirk or weird offset. diff --git a/opensbi-timer-debug.patch b/opensbi-timer-debug.patch index f2155f2..292e46c 100644 --- a/opensbi-timer-debug.patch +++ b/opensbi-timer-debug.patch @@ -99,7 +99,7 @@ index f468730..db20526 100644 + return rc; } diff --git a/lib/utils/timer/fdt_timer_mtimer.c b/lib/utils/timer/fdt_timer_mtimer.c -index 9e27e3a..fba48ca 100644 +index 9e27e3a..cef2ee6 100644 --- a/lib/utils/timer/fdt_timer_mtimer.c +++ b/lib/utils/timer/fdt_timer_mtimer.c @@ -8,6 +8,7 @@ @@ -149,8 +149,8 @@ index 9e27e3a..fba48ca 100644 + sbi_printf("timer_mtimer_cold_init: mtime_addr = 0x%08lx\n", mt->mtime_addr); + sbi_printf("timer_mtimer_cold_init: mtime_size = 0x%08lx\n", mt->mtime_size); -+ sbi_printf("timer_mtimer_cold_init: mtime_addr = 0x%08lx\n", mt->mtimecmp_addr); -+ sbi_printf("timer_mtimer_cold_init: mtime_addr = 0x%08lx\n", mt->mtimecmp_size); ++ sbi_printf("timer_mtimer_cold_init: mtimecmp_addr = 0x%08lx\n", mt->mtimecmp_addr); ++ sbi_printf("timer_mtimer_cold_init: mtimecmp_size = 0x%08lx\n", mt->mtimecmp_size); + /* Apply additional quirks */ if (quirks) { diff --git a/ox-plic.dts b/ox-plic.dts index 35cf1e2..1959cfb 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -234,7 +234,9 @@ /*<&CPU0 0x3>, <&CPU0 0x7>,*/ /*<&onic_pool 0x3>, <&onic_pool 0x7>,*/ /*<&SERIAL 0x3>, <&SERIAL 0x7>*/ - compatible = "riscv,clint0"; + + /* Allows using the "generic" platform in OpenSBI. */ + compatible = "riscv,aclint-mtimer"; }; /* Guesswork: There must be a timer at 0x40170000 as it is -- GitLab From 8b2821f2f239cf52f814816ebf93a4220c1a2eb5 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 14:14:25 +0200 Subject: [PATCH 108/310] Try setting the mtimecmp address --- JOURNAL.md | 40 ++++++++++++++++++++++++++++++++++++++++ ox-plic.dts | 3 ++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/JOURNAL.md b/JOURNAL.md index 9a5b9df..f53252c 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1712,3 +1712,43 @@ typos in the printf command, this is the patch: So, let's fix the patch and switch to the "riscv,aclint-mtimer" timer, which doesn't set any quirk or weird offset. + + GGGGGGGGG + OpenSBI v1.5 + ____ _____ ____ _____ + / __ \ / ____| _ \_ _| + | | | |_ __ ___ _ __ | (___ | |_) || | + | | | | '_ \ / _ \ '_ \ \___ \| _ < | | + | |__| | |_) | __/ | | |____) | |_) || |_ + \____/| .__/ \___|_| |_|_____/|____/_____| + | | + |_| + + sbi_timer_init: begins + sbi_timer_init: got Zicntr extension + fdt_timer_cold_init: pos = 0 + fdt_timer_cold_init: got match, name = riscv,aclint-mtimer + fdt_timer_cold_init: enabled + timer_mtimer_cold_init: begins, is_clint = 0 + timer_mtimer_cold_init: mtime_addr = 0x40002000 + timer_mtimer_cold_init: mtime_size = 0x000c0000 + timer_mtimer_cold_init: mtimecmp_addr = 0x00000000 + timer_mtimer_cold_init: mtimecmp_size = 0x00000000 + fdt_timer_cold_init: drc->cold_init = -3 + fdt_timer_init: fdt_timer_cold_init failed (-3) + sbi_platform_timer: sbi_platform_timer_init failed (-3) + init_coldboot: timer init failed (error -3) + +So, now we have better addresses, but this is not what I would expect. The regs +property is being used to set the mtime address and size, while I would expect +it to set the `mtimecmp_addr` too, but it seems it is expecting two pairs of +address and size in the regs: + + reg = <0x0 0x40002000 0x0 0x000c0000>; + +So, first let's add the mtimecmp after the mtime registers and see if that +clears the -3 error. + +I'll leave it at 0x40002000, but I suspect the address might be 0x40170000: + + #define OX_ALVEO_TIMER_BASE 0x40170000 diff --git a/ox-plic.dts b/ox-plic.dts index 1959cfb..1499c37 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -229,7 +229,8 @@ * Documentation/devicetree/bindings/riscv/cpus.yaml */ clint: clint@40002000 { - reg = <0x0 0x40002000 0x0 0x000c0000>; + /* MTIME and MTIMECMP address and size pairs */ + reg = <0x0 0x40002000 0x0 0x8>, <0x0 0x40002008 0x0 0x8>; interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; /*<&CPU0 0x3>, <&CPU0 0x7>,*/ /*<&onic_pool 0x3>, <&onic_pool 0x7>,*/ -- GitLab From 983dc939ab652b3fc61837d0c228c663c31d6def Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 14:43:09 +0200 Subject: [PATCH 109/310] Move the FDT to 0x80100000 --- JOURNAL.md | 118 +++++++++++++++++++++++++++++++++++++++++++++++++ lagarto-ox.nix | 4 ++ 2 files changed, 122 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index f53252c..642d562 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1752,3 +1752,121 @@ clears the -3 error. I'll leave it at 0x40002000, but I suspect the address might be 0x40170000: #define OX_ALVEO_TIMER_BASE 0x40170000 + +After booting again, now it seems to work. But I'm not sure if that +may cause more problems down the line. + + GGGGGGGGG + OpenSBI v1.5 + ____ _____ ____ _____ + / __ \ / ____| _ \_ _| + | | | |_ __ ___ _ __ | (___ | |_) || | + | | | | '_ \ / _ \ '_ \ \___ \| _ < | | + | |__| | |_) | __/ | | |____) | |_) || |_ + \____/| .__/ \___|_| |_|_____/|____/_____| + | | + |_| + + sbi_timer_init: begins + sbi_timer_init: got Zicntr extension + fdt_timer_cold_init: pos = 0 + fdt_timer_cold_init: got match, name = riscv,aclint-mtimer + fdt_timer_cold_init: enabled + timer_mtimer_cold_init: begins, is_clint = 0 + timer_mtimer_cold_init: mtime_addr = 0x40002000 + timer_mtimer_cold_init: mtime_size = 0x00000008 + timer_mtimer_cold_init: mtimecmp_addr = 0x40002008 + timer_mtimer_cold_init: mtimecmp_size = 0x00000008 + fdt_timer_cold_init: drc->cold_init = 0 + fdt_timer_cold_init: pos = 1 + fdt_timer_cold_init: returns 0 + Platform Name : Barcelona Supercomputing Center - Lagarto Ox (NixOS) + Platform Features : medeleg + Platform HART Count : 1 + Platform IPI Device : --- + Platform Timer Device : aclint-mtimer @ 50000Hz + Platform Console Device : uart8250 + Platform HSM Device : --- + Platform PMU Device : --- + Platform Reboot Device : --- + Platform Shutdown Device : --- + Platform Suspend Device : --- + Platform CPPC Device : --- + Firmware Base : 0x80000000 + Firmware Size : 327 KB + Firmware RW Offset : 0x40000 + Firmware RW Size : 71 KB + Firmware Heap Offset : 0x49000 + Firmware Heap Size : 35 KB (total), 2 KB (reserved), 11 KB (used), 21 KB (free) + Firmware Scratch Size : 4096 B (total), 408 B (used), 3688 B (free) + Runtime SBI Version : 2.0 + + Domain0 Name : root + Domain0 Boot HART : 0 + Domain0 HARTs : 0* + Domain0 Region00 : 0x0000000040002000-0x000000004000200f M: (I,R,W) S/U: () + Domain0 Region01 : 0x0000000040001000-0x0000000040001fff M: (I,R,W) S/U: (R,W) + Domain0 Region02 : 0x0000000080040000-0x000000008005ffff M: (R,W) S/U: () + Domain0 Region03 : 0x0000000080000000-0x000000008003ffff M: (R,X) S/U: () + Domain0 Region04 : 0x0000000040800000-0x0000000040bfffff M: (I,R,W) S/U: (R,W) + Domain0 Region05 : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X) + Domain0 Next Address : 0x0000000080200000 + Domain0 Next Arg1 : 0x0000000082200000 + Domain0 Next Mode : S-mode + Domain0 SysReset : yes + Domain0 SysSuspend : yes + + Boot HART ID : 0 + Boot HART Domain : root + Boot HART Priv Version : v1.10 + Boot HART Base ISA : rv64imafdc + Boot HART ISA Extensions : zicntr,zihpm,sdtrig + Boot HART PMP Count : 0 + Boot HART PMP Granularity : 0 bits + Boot HART PMP Address Bits: 0 + Boot HART MHPM Info : 29 (0xfffffff8) + Boot HART Debug Triggers : 0 triggers + Boot HART MIDELEG : 0x0000000000000222 + Boot HART MEDELEG : 0x000000000000b109 + + + Core: 12 devices, 8 uclasses, devicetree: board + Loading Environment from nowhere... OK + In: serial,usbkbd + Out: serial,vidconsole + Err: serial,vidconsole + No working controllers found + Net: No ethernet found. + Working FDT set to 82200000 + Hit any key to stop autoboot: 0 + + Device 0: unknown device + + Device 1: unknown device + scanning bus for devices... + + Device 0: unknown device + starting USB... + No working controllers found + No ethernet found. + No ethernet found. + +However, now I cannot boot the kernel, as it is overwritting the FDT: + + => printenv fdtcontroladdr + fdtcontroladdr=82200000 + => fdt addr ${fdtcontroladdr } + Working fdt: 82200000 + => setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0" + => setenv ramdisk_size 12614846 + => #booti ${kernel_addr_r} ${ramdisk_addr_r}:${ramdisk_size} ${fdtcontroladdr} + => printenv kernel_addr_r + kernel_addr_r=0x84000000 + => printenv ramdisk_addr_r + ramdisk_addr_r=0x8c300000 + => booti ${kernel_addr_r} ${ramdisk_addr_r}:${ramdisk_size} ${fdtcontroladdr} + Moving Image from 0x84000000 to 0x80200000, end=83044650 + ERROR: Did not find a cmdline Flattened Device Tree + Could not find a valid device tree + +Let's move the FDT to 0x80100000. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index c9ab49c..2cecee9 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -317,6 +317,10 @@ #"PLATFORM_RISCV_ABI=lp64d" "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin" "FW_FDT_PATH=${final.ox-dtb}" + + # Ensure it doesn't overlap from the 0x80200000 where the kernel will be + # placed. + "FW_PAYLOAD_FDT_ADDR=0x80100000" ]; patches = [ ./opensbi-timer-debug.patch -- GitLab From 97c0dd685924a09d9a816c2468edc719067d00d8 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 14:50:59 +0200 Subject: [PATCH 110/310] Remove reserved region hack --- JOURNAL.md | 200 ++++++++++++++++++++++++++++++++++++++++++++++++++++ ox-plic.dts | 4 -- 2 files changed, 200 insertions(+), 4 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 642d562..ae48e9e 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1870,3 +1870,203 @@ However, now I cannot boot the kernel, as it is overwritting the FDT: Could not find a valid device tree Let's move the FDT to 0x80100000. + +Now I can load the kernel, but it gets stuck in the middle of the boot: + + GGGGGGGGG + OpenSBI v1.5 + ____ _____ ____ _____ + / __ \ / ____| _ \_ _| + | | | |_ __ ___ _ __ | (___ | |_) || | + | | | | '_ \ / _ \ '_ \ \___ \| _ < | | + | |__| | |_) | __/ | | |____) | |_) || |_ + \____/| .__/ \___|_| |_|_____/|____/_____| + | | + |_| + + sbi_timer_init: begins + sbi_timer_init: got Zicntr extension + fdt_timer_cold_init: pos = 0 + fdt_timer_cold_init: got match, name = riscv,aclint-mtimer + fdt_timer_cold_init: enabled + timer_mtimer_cold_init: begins, is_clint = 0 + timer_mtimer_cold_init: mtime_addr = 0x40002000 + timer_mtimer_cold_init: mtime_size = 0x00000008 + timer_mtimer_cold_init: mtimecmp_addr = 0x40002008 + timer_mtimer_cold_init: mtimecmp_size = 0x00000008 + fdt_timer_cold_init: drc->cold_init = 0 + fdt_timer_cold_init: pos = 1 + fdt_timer_cold_init: returns 0 + Platform Name : Barcelona Supercomputing Center - Lagarto Ox (NixOS) + Platform Features : medeleg + Platform HART Count : 1 + Platform IPI Device : --- + Platform Timer Device : aclint-mtimer @ 50000Hz + Platform Console Device : uart8250 + Platform HSM Device : --- + Platform PMU Device : --- + Platform Reboot Device : --- + Platform Shutdown Device : --- + Platform Suspend Device : --- + Platform CPPC Device : --- + Firmware Base : 0x80000000 + Firmware Size : 327 KB + Firmware RW Offset : 0x40000 + Firmware RW Size : 71 KB + Firmware Heap Offset : 0x49000 + Firmware Heap Size : 35 KB (total), 2 KB (reserved), 11 KB (used), 21 KB (free) + Firmware Scratch Size : 4096 B (total), 408 B (used), 3688 B (free) + Runtime SBI Version : 2.0 + + Domain0 Name : root + Domain0 Boot HART : 0 + Domain0 HARTs : 0* + Domain0 Region00 : 0x0000000040002000-0x000000004000200f M: (I,R,W) S/U: () + Domain0 Region01 : 0x0000000040001000-0x0000000040001fff M: (I,R,W) S/U: (R,W) + Domain0 Region02 : 0x0000000080040000-0x000000008005ffff M: (R,W) S/U: () + Domain0 Region03 : 0x0000000080000000-0x000000008003ffff M: (R,X) S/U: () + Domain0 Region04 : 0x0000000040800000-0x0000000040bfffff M: (I,R,W) S/U: (R,W) + Domain0 Region05 : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X) + Domain0 Next Address : 0x0000000080200000 + Domain0 Next Arg1 : 0x0000000080100000 + Domain0 Next Mode : S-mode + Domain0 SysReset : yes + Domain0 SysSuspend : yes + + Boot HART ID : 0 + Boot HART Domain : root + Boot HART Priv Version : v1.10 + Boot HART Base ISA : rv64imafdc + Boot HART ISA Extensions : zicntr,zihpm,sdtrig + Boot HART PMP Count : 0 + Boot HART PMP Granularity : 0 bits + Boot HART PMP Address Bits: 0 + Boot HART MHPM Info : 29 (0xfffffff8) + Boot HART Debug Triggers : 0 triggers + Boot HART MIDELEG : 0x0000000000000222 + Boot HART MEDELEG : 0x000000000000b109 + + + Core: 12 devices, 8 uclasses, devicetree: board + Loading Environment from nowhere... OK + In: serial,usbkbd + Out: serial,vidconsole + Err: serial,vidconsole + No working controllers found + Net: No ethernet found. + Working FDT set to 80100000 + Hit any key to stop autoboot: 0 + + Device 0: unknown device + + Device 1: unknown device + scanning bus for devices... + + Device 0: unknown device + starting USB... + No working controllers found + No ethernet found. + No ethernet found. + => setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0" + => setenv ramdisk_size 12614846 + => booti ${kernel_addr_r} ${ramdisk_addr_r}:${ramdisk_size} ${fdtcontroladdr} + Moving Image from 0x84000000 to 0x80200000, end=83044650 + ## Flattened Device Tree blob at 80100000 + Booting using the fdt blob at 0x80100000 + Working FDT set to 80100000 + ERROR: reserving fdt memory region failed (addr=80000000 size=4000000 flags=4) + Using Device Tree in place at 0000000080100000, end 0000000080103dd0 + Working FDT set to 80100000 + + Starting kernel ... + + [ 0.000000] Linux version 6.9.7 (nixbld@localhost) (riscv64-unknown-linux-gnu-gcc (GCC) 13.3.0, GNU ld (GNU Binutils) 2.41) #1-NixOS Thu Jun 27 11:52:32 UTC 2024 + [ 0.000000] Machine model: Barcelona Supercomputing Center - Lagarto Ox (NixOS) + [ 0.000000] SBI specification v2.0 detected + [ 0.000000] SBI implementation ID=0x1 Version=0x10005 + [ 0.000000] SBI TIME extension detected + [ 0.000000] SBI IPI extension detected + [ 0.000000] SBI RFENCE extension detected + [ 0.000000] SBI DBCN extension detected + [ 0.000000] earlycon: sbi0 at I/O port 0x0 (options '') + [ 0.000000] printk: legacy bootconsole [sbi0] enabled + [ 0.000000] OF: reserved mem: Reserved memory: failed to reserve memory for node 'reserved@80000000': base 0x0000000080000000, size 64 MiB + [ 0.000000] OF: reserved mem: OVERLAP DETECTED! + [ 0.000000] mmode_resv1@80000000 (0x0000000080000000--0x0000000080040000) overlaps with reserved@80000000 (0x0000000080000000--0x0000000084000000) + [ 0.000000] OF: reserved mem: OVERLAP DETECTED! + [ 0.000000] reserved@80000000 (0x0000000080000000--0x0000000084000000) overlaps with mmode_resv0@80040000 (0x0000000080040000--0x0000000080060000) + [ 0.000000] Reserved memory: created DMA memory pool at 0x0000000060000000, size 256 MiB + [ 0.000000] OF: reserved mem: initialized node dma_pool@60000000, compatible id shared-dma-pool + [ 0.000000] OF: reserved mem: 0x0000000060000000..0x000000006fffffff (262144 KiB) map non-reusable dma_pool@60000000 + [ 0.000000] Reserved memory: created DMA memory pool at 0x0000000070000000, size 256 MiB + [ 0.000000] OF: reserved mem: initialized node dma_pool@70000000, compatible id shared-dma-pool + [ 0.000000] OF: reserved mem: 0x0000000070000000..0x000000007fffffff (262144 KiB) map non-reusable dma_pool@70000000 + [ 0.000000] OF: reserved mem: 0x0000000080000000..0x000000008003ffff (256 KiB) nomap non-reusable mmode_resv1@80000000 + [ 0.000000] OF: reserved mem: 0x0000000080000000..0x0000000083ffffff (65536 KiB) nomap non-reusable reserved@80000000 + [ 0.000000] OF: reserved mem: 0x0000000080040000..0x000000008005ffff (128 KiB) nomap non-reusable mmode_resv0@80040000 + [ 0.000000] cma: Reserved 16 MiB at 0x00000000af000000 on node -1 + [ 0.000000] Zone ranges: + [ 0.000000] DMA32 [mem 0x0000000080000000-0x00000000afffffff] + [ 0.000000] Normal empty + [ 0.000000] Movable zone start for each node + [ 0.000000] Early memory node ranges + [ 0.000000] node 0: [mem 0x0000000080000000-0x000000008005ffff] + [ 0.000000] node 0: [mem 0x0000000080060000-0x00000000afffffff] + [ 0.000000] Initmem setup node 0 [mem 0x0000000080000000-0x00000000afffffff] + [ 0.000000] Falling back to deprecated "riscv,isa" + [ 0.000000] riscv: base ISA extensions adfim + [ 0.000000] riscv: ELF capabilities adfim + [ 0.000000] pcpu-alloc: s0 r0 d131072 u131072 alloc=1*131072 + [ 0.000000] pcpu-alloc: [0] 0 + [ 0.000000] Kernel command line: root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 + [ 0.000000] Dentry cache hash table entries: 131072 (order: 8, 1048576 bytes, linear) + [ 0.000000] Inode-cache hash table entries: 65536 (order: 7, 524288 bytes, linear) + [ 0.000000] Built 1 zonelists, mobility grouping on. Total pages: 193536 + [ 0.000000] mem auto-init: stack:all(zero), heap alloc:off, heap free:off + [ 0.000000] Virtual kernel memory layout: + [ 0.000000] fixmap : 0xffffffc6fea00000 - 0xffffffc6ff000000 (6144 kB) + [ 0.000000] pci io : 0xffffffc6ff000000 - 0xffffffc700000000 ( 16 MB) + [ 0.000000] vmemmap : 0xffffffc700000000 - 0xffffffc800000000 (4096 MB) + [ 0.000000] vmalloc : 0xffffffc800000000 - 0xffffffd800000000 ( 64 GB) + [ 0.000000] modules : 0xffffffff02e45000 - 0xffffffff80000000 (2001 MB) + [ 0.000000] lowmem : 0xffffffd800000000 - 0xffffffd830000000 ( 768 MB) + [ 0.000000] kernel : 0xffffffff80000000 - 0xffffffffffffffff (2047 MB) + [ 0.000000] Memory: 386632K/786432K available (17075K kernel code, 9047K rwdata, 10240K rodata, 8737K init, 917K bss, 383416K reserved, 16384K cma-reserved) + [ 0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=1, Nodes=1 + [ 0.000000] ftrace: allocating 46961 entries in 184 pages + [ 0.000000] ftrace: allocated 184 pages with 4 groups + [ 0.000000] trace event string verifier disabled + [ 0.000000] RCU Tasks Rude: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1. + [ 0.000000] RCU Tasks Trace: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1. + [ 0.000000] NR_IRQS: 64, nr_irqs: 64, preallocated irqs: 0 + [ 0.000000] riscv-intc: 64 local interrupts mapped + [ 0.000000] clocksource: riscv_clocksource: mask: 0xffffffffffffffff max_cycles: 0x179dd7f66, max_idle_ns: 56421785867800 ns + [ 0.000020] sched_clock: 64 bits at 50kHz, resolution 20000ns, wraps every 70368744170000ns + [ 0.015020] kfence: initialized - using 2097152 bytes for 255 objects at 0x(____ptrval____)-0x(____ptrval____) + [ 0.035900] Console: colour dummy device 80x25 + [ 0.041720] Calibrating delay loop (skipped), value calculated using timer frequency.. 0.10 BogoMIPS (lpj=200) + [ 0.052480] pid_max: default: 32768 minimum: 301 + [ 0.187380] LSM: initializing lsm=capability,landlock,yama + [ 0.479640] landlock: Up and running. + [ 0.483700] Yama: becoming mindful. + [ 0.521600] Mount-cache hash table entries: 2048 (order: 2, 16384 bytes, linear) + [ 0.529780] Mountpoint-cache hash table entries: 2048 (order: 2, 16384 bytes, linear) + [ 0.784220] riscv: ELF compat mode unsupported + [ 0.784600] ASID allocator disabled (0 bits) + [ 0.867620] devtmpfs: initialized + [ 0.986340] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns + [ 0.996900] futex hash table entries: 256 (order: 1, 12288 bytes, linear) + [ 1.084560] pinctrl core: initialized pinctrl subsystem + [ 1.195460] NET: Registered PF_NETLINK/PF_ROUTE protocol family + [ 1.244960] DMA: preallocated 128 KiB GFP_KERNEL pool for atomic allocations + [ 1.255960] DMA: preallocated 128 KiB GFP_KERNEL|GFP_DMA32 pool for atomic allocations + [ 1.266500] audit: initializing netlink subsys (disabled) + [ 1.317920] thermal_sys: Registered thermal governor 'step_wise' + [ 1.319800] cpuidle: using governor ladder + +This is very interesting, because all these lockups show the same symptoms and +they may be related with the kernel trying to access an area of memory that it +shouldn't. + +First, let's remove the reserved region, as now OpenSBI properly forwards the +regions to the kernel. diff --git a/ox-plic.dts b/ox-plic.dts index 1499c37..18ae4a5 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -84,10 +84,6 @@ #address-cells = <2>; /* Starting address and size */ #size-cells = <2>; /* 64 bits memory addresses */ ranges; - reserved@80000000 { - reg = <0x0 0x80000000 0x0 0x04000000>; - no-map; - }; eth_pool: dma_pool@60000000 { reg = <0x0 0x60000000 0x0 0x10000000>; compatible = "shared-dma-pool"; -- GitLab From 8819c091eb1a9cdd58c206e66e2b9e1d06ff35d2 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 14:57:43 +0200 Subject: [PATCH 111/310] Remove clint from device tree for now --- JOURNAL.md | 23 +++++++++++++++++++++++ ox-plic.dts | 22 +++++++++++----------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index ae48e9e..c72e264 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -2070,3 +2070,26 @@ shouldn't. First, let's remove the reserved region, as now OpenSBI properly forwards the regions to the kernel. + +It continues to hang, but at least now we don't have overlap of memory regions: + + [ 0.000000] Reserved memory: created DMA memory pool at 0x0000000060000000, size 256 MiB + [ 0.000000] OF: reserved mem: initialized node dma_pool@60000000, compatible id shared-dma-pool + [ 0.000000] OF: reserved mem: 0x0000000060000000..0x000000006fffffff (262144 KiB) map non-reusable dma_pool@60000000 + [ 0.000000] Reserved memory: created DMA memory pool at 0x0000000070000000, size 256 MiB + [ 0.000000] OF: reserved mem: initialized node dma_pool@70000000, compatible id shared-dma-pool + [ 0.000000] OF: reserved mem: 0x0000000070000000..0x000000007fffffff (262144 KiB) map non-reusable dma_pool@70000000 + [ 0.000000] OF: reserved mem: 0x0000000080000000..0x000000008003ffff (256 KiB) nomap non-reusable mmode_resv1@80000000 + [ 0.000000] OF: reserved mem: 0x0000000080040000..0x000000008005ffff (128 KiB) nomap non-reusable mmode_resv0@80040000 + [ 0.000000] cma: Reserved 16 MiB at 0x00000000af000000 on node -1 + [ 0.000000] Zone ranges: + [ 0.000000] DMA32 [mem 0x0000000080000000-0x00000000afffffff] + [ 0.000000] Normal empty + [ 0.000000] Movable zone start for each node + [ 0.000000] Early memory node ranges + [ 0.000000] node 0: [mem 0x0000000080000000-0x000000008005ffff] + [ 0.000000] node 0: [mem 0x0000000080060000-0x00000000afffffff] + [ 0.000000] Initmem setup node 0 [mem 0x0000000080000000-0x00000000afffffff] + +Let's remove the clint from the device tree for now, just to see if it is +affecting. diff --git a/ox-plic.dts b/ox-plic.dts index 18ae4a5..11f95b5 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -224,17 +224,17 @@ * property is described in * Documentation/devicetree/bindings/riscv/cpus.yaml */ - clint: clint@40002000 { - /* MTIME and MTIMECMP address and size pairs */ - reg = <0x0 0x40002000 0x0 0x8>, <0x0 0x40002008 0x0 0x8>; - interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; - /*<&CPU0 0x3>, <&CPU0 0x7>,*/ - /*<&onic_pool 0x3>, <&onic_pool 0x7>,*/ - /*<&SERIAL 0x3>, <&SERIAL 0x7>*/ - - /* Allows using the "generic" platform in OpenSBI. */ - compatible = "riscv,aclint-mtimer"; - }; +// clint: clint@40002000 { +// /* MTIME and MTIMECMP address and size pairs */ +// reg = <0x0 0x40002000 0x0 0x8>, <0x0 0x40002008 0x0 0x8>; +// interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; +// /*<&CPU0 0x3>, <&CPU0 0x7>,*/ +// /*<&onic_pool 0x3>, <&onic_pool 0x7>,*/ +// /*<&SERIAL 0x3>, <&SERIAL 0x7>*/ +// +// /* Allows using the "generic" platform in OpenSBI. */ +// compatible = "riscv,aclint-mtimer"; +// }; /* Guesswork: There must be a timer at 0x40170000 as it is * initialized in OpenSBI. It seems to drive the console. */ -- GitLab From 71c81f8dcd4a21698134c4180c918b23f4023497 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 16:09:36 +0200 Subject: [PATCH 112/310] Add debug in OpenSBI trap handler --- JOURNAL.md | 111 ++++++++++++++++++++++++++++++++++++++ opensbi-timer-debug.patch | 32 +++++++++++ 2 files changed, 143 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index c72e264..377cd98 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -2093,3 +2093,114 @@ It continues to hang, but at least now we don't have overlap of memory regions: Let's remove the clint from the device tree for now, just to see if it is affecting. + +Disabling the clint doesn't seem to cause any effect. In fact, the kernel is +still able to get a clock as shown in the kernel messages. + +I enabled the ftrace for initcall and I can see that is getting stuck in the +unaligned check: + + Domain0 Name : root + Domain0 Boot HART : 0 + Domain0 HARTs : 0* + Domain0 Region00 : 0x0000000040001000-0x0000000040001fff M: (I,R,W) S/U: (R,W) + Domain0 Region01 : 0x0000000080040000-0x000000008005ffff M: (R,W) S/U: () + Domain0 Region02 : 0x0000000080000000-0x000000008003ffff M: (R,X) S/U: () + Domain0 Region03 : 0x0000000040800000-0x0000000040bfffff M: (I,R,W) S/U: (R,W) + Domain0 Region04 : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X) + Domain0 Next Address : 0x0000000080200000 + Domain0 Next Arg1 : 0x0000000080100000 + Domain0 Next Mode : S-mode + Domain0 SysReset : yes + Domain0 SysSuspend : yes + ... + => setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 trace_event=initcall:* trace_options=sym-addr tp_printk trace_buf_size=1M" + => setenv ramdisk_size 12614846 + => booti ${kernel_addr_r} ${ramdisk_addr_r}:${ramdisk_size} ${fdtcontroladdr} + ... + [ 0.000000] printk: legacy bootconsole [sbi0] enabled + [ 0.000000] Reserved memory: created DMA memory pool at 0x0000000060000000, size 256 MiB + [ 0.000000] OF: reserved mem: initialized node dma_pool@60000000, compatible id shared-dma-pool + [ 0.000000] OF: reserved mem: 0x0000000060000000..0x000000006fffffff (262144 KiB) map non-reusable dma_pool@60000000 + [ 0.000000] Reserved memory: created DMA memory pool at 0x0000000070000000, size 256 MiB + [ 0.000000] OF: reserved mem: initialized node dma_pool@70000000, compatible id shared-dma-pool + [ 0.000000] OF: reserved mem: 0x0000000070000000..0x000000007fffffff (262144 KiB) map non-reusable dma_pool@70000000 + [ 0.000000] OF: reserved mem: 0x0000000080000000..0x000000008003ffff (256 KiB) nomap non-reusable mmode_resv1@80000000 + [ 0.000000] OF: reserved mem: 0x0000000080040000..0x000000008005ffff (128 KiB) nomap non-reusable mmode_resv0@80040000 + [ 0.000000] cma: Reserved 16 MiB at 0x00000000af000000 on node -1 + [ 0.000000] Zone ranges: + [ 0.000000] DMA32 [mem 0x0000000080000000-0x00000000afffffff] + [ 0.000000] Normal empty + [ 0.000000] Movable zone start for each node + [ 0.000000] Early memory node ranges + [ 0.000000] node 0: [mem 0x0000000080000000-0x000000008005ffff] + [ 0.000000] node 0: [mem 0x0000000080060000-0x00000000afffffff] + [ 0.000000] Initmem setup node 0 [mem 0x0000000080000000-0x00000000afffffff] + ... + [ 2.736300] initcall_finish: func=init_ladder+0x0/0x40 ret=0 + [ 2.742640] initcall_start: func=init_menu+0x0/0x38 + [ 2.748200] initcall_finish: func=init_menu+0x0/0x38 ret=0 + [ 2.754400] initcall_start: func=rpmsg_init+0x0/0xc0 + [ 2.763400] initcall_finish: func=rpmsg_init+0x0/0xc0 ret=0 + [ 2.769840] initcall_start: func=rpmsg_chrdev_init+0x0/0xc0 + [ 2.777660] initcall_finish: func=rpmsg_chrdev_init+0x0/0xc0 ret=0 + [ 2.784540] initcall_start: func=rpmsg_ctrldev_init+0x0/0xbc + [ 2.792460] initcall_finish: func=rpmsg_ctrldev_init+0x0/0xbc ret=0 + [ 2.799420] initcall_start: func=rpmsg_ns_init+0x0/0x6c + [ 2.807460] initcall_finish: func=rpmsg_ns_init+0x0/0x6c ret=0 + [ 2.814120] initcall_start: func=kobject_uevent_init+0x0/0x30 + [ 2.821120] initcall_finish: func=kobject_uevent_init+0x0/0x30 ret=0 + [ 2.830980] initcall_level: level=arch + [ 2.835460] initcall_start: func=riscv_cpuinfo_init+0x0/0x78 + [ 2.841960] initcall_finish: func=riscv_cpuinfo_init+0x0/0x78 ret=0 + [ 2.848920] initcall_start: func=vdso_init+0x0/0x4c + [ 2.855560] initcall_finish: func=vdso_init+0x0/0x4c ret=0 + [ 2.861900] initcall_start: func=check_unaligned_access_all_cpus+0x0/0x1d4 + +This makes me wonder if what has just happened is that somehow we have +overwritten the OpenSBI trap for unaligned access and we are now executing some +garbage code in machine mode. + +Based on the OpenSBI regions, I would assume here is where the traps should be: + + Domain0 Region02 : 0x0000000080000000-0x000000008003ffff M: (R,X) S/U: () + +It should be feasible to read the memory from the host where those traps are +placed. + +Here are the FDT reserved areas that U-Boot sees before switching to the kernel: + + => fdt print /reserved-memory + reserved-memory { + #address-cells = <0x00000002>; + #size-cells = <0x00000002>; + ranges; + mmode_resv1@80000000 { + reg = <0x00000000 0x80000000 0x00000000 0x00040000>; + no-map; + }; + mmode_resv0@80040000 { + reg = <0x00000000 0x80040000 0x00000000 0x00020000>; + no-map; + }; + dma_pool@60000000 { + reg = <0x00000000 0x60000000 0x00000000 0x10000000>; + compatible = "shared-dma-pool"; + }; + dma_pool@70000000 { + reg = <0x00000000 0x70000000 0x00000000 0x10000000>; + compatible = "shared-dma-pool"; + }; + }; + +Which match with the ones the kernel parse: + + [ 0.000000] OF: reserved mem: 0x0000000080000000..0x000000008003ffff (256 KiB) nomap non-reusable mmode_resv1@80000000 + [ 0.000000] OF: reserved mem: 0x0000000080040000..0x000000008005ffff (128 KiB) nomap non-reusable mmode_resv0@80040000 + +And with the OpenSBI regions (in reverse order): + + Domain0 Region01 : 0x0000000080040000-0x000000008005ffff M: (R,W) S/U: () + Domain0 Region02 : 0x0000000080000000-0x000000008003ffff M: (R,X) S/U: () + +Let's add some instrumentation in the code that handles the traps in OpenSBI. diff --git a/opensbi-timer-debug.patch b/opensbi-timer-debug.patch index 292e46c..8e3fc97 100644 --- a/opensbi-timer-debug.patch +++ b/opensbi-timer-debug.patch @@ -33,6 +33,38 @@ index 7b618de..65e42b0 100644 } void sbi_timer_exit(struct sbi_scratch *scratch) +diff --git a/lib/sbi/sbi_trap.c b/lib/sbi/sbi_trap.c +index b4f3a17..3c05a5b 100644 +--- a/lib/sbi/sbi_trap.c ++++ b/lib/sbi/sbi_trap.c +@@ -306,15 +306,18 @@ struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) + + switch (mcause) { + case CAUSE_ILLEGAL_INSTRUCTION: ++ sbi_printf("OpenSBI: illegall instruction\n"); + rc = sbi_illegal_insn_handler(tcntx); + msg = "illegal instruction handler failed"; + break; + case CAUSE_MISALIGNED_LOAD: ++ sbi_printf("OpenSBI: misaligned load\n"); + sbi_pmu_ctr_incr_fw(SBI_PMU_FW_MISALIGNED_LOAD); + rc = sbi_misaligned_load_handler(tcntx); + msg = "misaligned load handler failed"; + break; + case CAUSE_MISALIGNED_STORE: ++ sbi_printf("OpenSBI: misaligned store\n"); + sbi_pmu_ctr_incr_fw(SBI_PMU_FW_MISALIGNED_STORE); + rc = sbi_misaligned_store_handler(tcntx); + msg = "misaligned store handler failed"; +@@ -344,6 +347,8 @@ struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) + trap_done: + if (rc) + sbi_trap_error(msg, rc, tcntx); ++ else ++ sbi_printf("OpenSBI: good trap\n"); + + if (((regs->mstatus & MSTATUS_MPP) >> MSTATUS_MPP_SHIFT) != PRV_M) + sbi_sse_process_pending_events(regs); diff --git a/lib/utils/timer/fdt_timer.c b/lib/utils/timer/fdt_timer.c index f468730..db20526 100644 --- a/lib/utils/timer/fdt_timer.c -- GitLab From c6e2db8c2d6bb9ae1803d4536b7793e20e0a8af0 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 16:32:55 +0200 Subject: [PATCH 113/310] Add extra debug messages --- JOURNAL.md | 16 ++++++++++++++++ opensbi-timer-debug.patch | 30 +++++++++++++++++++++++------- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 377cd98..560d526 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -2204,3 +2204,19 @@ And with the OpenSBI regions (in reverse order): Domain0 Region02 : 0x0000000080000000-0x000000008003ffff M: (R,X) S/U: () Let's add some instrumentation in the code that handles the traps in OpenSBI. + +Memory regions: + + 0x8000_0000 to 0x8003_ffff : OpenSBI code + 0x8004_0000 to 0x8005_ffff : OpenSBI data + 0x8010_0000 to 0x801._.... : FDT + 0x8020_0000 to 0x8020_.... : U-Boot (later kernel) + + 0x8020_1000 to 0x81fa_0b87 : Kernel image + 80201000-80cb177f : Kernel code + 81400000-819fffff : Kernel rodata + 81c00000-81f18747 : Kernel data + 81f19000-81fa0b87 : Kernel bss + + 0x8400_0000 to 0x84.._.... M: (R,W) S/U: () Linux kernel + diff --git a/opensbi-timer-debug.patch b/opensbi-timer-debug.patch index 8e3fc97..735f55e 100644 --- a/opensbi-timer-debug.patch +++ b/opensbi-timer-debug.patch @@ -34,34 +34,50 @@ index 7b618de..65e42b0 100644 void sbi_timer_exit(struct sbi_scratch *scratch) diff --git a/lib/sbi/sbi_trap.c b/lib/sbi/sbi_trap.c -index b4f3a17..3c05a5b 100644 +index b4f3a17..b60315f 100644 --- a/lib/sbi/sbi_trap.c +++ b/lib/sbi/sbi_trap.c -@@ -306,15 +306,18 @@ struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) +@@ -283,6 +283,7 @@ static int sbi_trap_aia_irq(void) + */ + struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) + { ++ sbi_printf("!"); + int rc = SBI_ENOTSUPP; + const char *msg = "trap handler failed"; + struct sbi_scratch *scratch = sbi_scratch_thishart_ptr(); +@@ -295,6 +296,7 @@ struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) + sbi_trap_set_context(scratch, tcntx); + + if (mcause & MCAUSE_IRQ_MASK) { ++ sbi_printf("OpenSBI: local interrupt"); + if (sbi_hart_has_extension(sbi_scratch_thishart_ptr(), + SBI_HART_EXT_SMAIA)) + rc = sbi_trap_aia_irq(); +@@ -306,15 +308,18 @@ struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) switch (mcause) { case CAUSE_ILLEGAL_INSTRUCTION: -+ sbi_printf("OpenSBI: illegall instruction\n"); ++ sbi_printf("OpenSBI: illegal instruction"); rc = sbi_illegal_insn_handler(tcntx); msg = "illegal instruction handler failed"; break; case CAUSE_MISALIGNED_LOAD: -+ sbi_printf("OpenSBI: misaligned load\n"); ++ sbi_printf("OpenSBI: misaligned load"); sbi_pmu_ctr_incr_fw(SBI_PMU_FW_MISALIGNED_LOAD); rc = sbi_misaligned_load_handler(tcntx); msg = "misaligned load handler failed"; break; case CAUSE_MISALIGNED_STORE: -+ sbi_printf("OpenSBI: misaligned store\n"); ++ sbi_printf("OpenSBI: misaligned store"); sbi_pmu_ctr_incr_fw(SBI_PMU_FW_MISALIGNED_STORE); rc = sbi_misaligned_store_handler(tcntx); msg = "misaligned store handler failed"; -@@ -344,6 +347,8 @@ struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) +@@ -344,6 +349,8 @@ struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) trap_done: if (rc) sbi_trap_error(msg, rc, tcntx); + else -+ sbi_printf("OpenSBI: good trap\n"); ++ sbi_printf("$\n"); if (((regs->mstatus & MSTATUS_MPP) >> MSTATUS_MPP_SHIFT) != PRV_M) sbi_sse_process_pending_events(regs); -- GitLab From 7d9f375e4a7631b4cdd4bc9243a832aaf2b80869 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 17:38:53 +0200 Subject: [PATCH 114/310] Remove newline in OpenSBI trap debug line --- JOURNAL.md | 107 +++++++++++++++++++++++++++++++++++++- opensbi-timer-debug.patch | 4 +- 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 560d526..99a6e6f 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -1851,7 +1851,7 @@ may cause more problems down the line. No ethernet found. No ethernet found. -However, now I cannot boot the kernel, as it is overwritting the FDT: +However, now I cannot boot the kernel, as it is overwriting the FDT: => printenv fdtcontroladdr fdtcontroladdr=82200000 @@ -2097,6 +2097,8 @@ affecting. Disabling the clint doesn't seem to cause any effect. In fact, the kernel is still able to get a clock as shown in the kernel messages. +### OBSERVATION: The kernel hangs after the unaligned check begins + I enabled the ftrace for initcall and I can see that is getting stuck in the unaligned check: @@ -2220,3 +2222,106 @@ Memory regions: 0x8400_0000 to 0x84.._.... M: (R,W) S/U: () Linux kernel +Okay, so we enter the unaligned access check: + + ![ 2.947680] initcall_start: func=check_unaligned_access_all_cpus+0x0/0x1d4 + $ + !OpenSBI: misaligned load$ + !OpenSBI: misaligned load$ + !OpenSBI: misaligned load$ + !OpenSBI: misaligned load$ + !OpenSBI: misaligned load$ + !OpenSBI: misaligned load$ + !OpenSBI: misaligned load$ + !OpenSBI: misaligned load$ + !OpenSBI: misaligned store$ + !OpenSBI: misaligned store$ + !OpenSBI: misaligned store$ + !OpenSBI: misaligned store$ + !OpenSBI: misaligned store$ + !OpenSBI: misaligned store$ + !OpenSBI: misaligned store$ + !OpenSBI: misaligned store$ + ... + !OpenSBI: misaligned load$ + !OpenSBI: misaligned load$ + !OpenSBI: misaligned load$ + !OpenSBI: misaligned load$ + !OpenSBI: misaligned load$ + !OpenSBI: misaligned load$ + !OpenSBI: misaligned load$ + !OpenSBI: misaligned load$ + !OpenSBI: misaligned store$ + !OpenSBI: misaligned store$ + !OpenSBI: misaligned store$ + !OpenSBI: misaligned store$ + !OpenSBI: misaligned store$ + !OpenSBI: misaligned store$ + !OpenSBI: misaligned store$ + !OpenSBI: misaligned store$ + +But we never arrive to anywhere else. + +Here are the current options that match ALIGN: + + hut% grep ALIGN /nix/store/c9jr35xnh2ffzjvkq8nvzj9i2siz1n4s-linux-config-riscv64-unknown-linux-gnu-6.9.7 + CONFIG_SYSCTL_ARCH_UNALIGN_ALLOW=y + CONFIG_RISCV_MISALIGNED=y + CONFIG_RISCV_PROBE_UNALIGNED_ACCESS=y + # CONFIG_RISCV_EMULATED_UNALIGNED_ACCESS is not set + # CONFIG_RISCV_SLOW_UNALIGNED_ACCESS is not set + # CONFIG_RISCV_EFFICIENT_UNALIGNED_ACCESS is not set + CONFIG_HAVE_64BIT_ALIGNED_ACCESS=y + CONFIG_FUNCTION_ALIGNMENT=0 + CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC=y + CONFIG_CMA_ALIGNMENT=8 + # CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B is not set + +We may want to set these two: + + # CONFIG_RISCV_EMULATED_UNALIGNED_ACCESS is not set + # CONFIG_RISCV_SLOW_UNALIGNED_ACCESS is not set + +And disable + + CONFIG_RISCV_PROBE_UNALIGNED_ACCESS=y + +So we don't perform the probing. However, this may bite later, so probably is a +better idea to debug it now. I could bound where it is failing as it doesn't +seem to be outside the speed check function. + + +Those 8 pairs of load and store calls seem to match this assembly function in +arch/riscv/kernel/copy-unaligned.S: + + /* void __riscv_copy_bytes_unaligned(void *, const void *, size_t) */ + /* Performs a memcpy without aligning buffers, using only byte accesses. */ + /* Note: The size is truncated to a multiple of 8 */ + SYM_FUNC_START(__riscv_copy_bytes_unaligned) + andi a4, a2, ~(8-1) + beqz a4, 2f + add a3, a1, a4 + 1: + lb a4, 0(a1) + lb a5, 1(a1) + lb a6, 2(a1) + lb a7, 3(a1) + lb t0, 4(a1) + lb t1, 5(a1) + lb t2, 6(a1) + lb t3, 7(a1) + sb a4, 0(a0) + sb a5, 1(a0) + sb a6, 2(a0) + sb a7, 3(a0) + sb t0, 4(a0) + sb t1, 5(a0) + sb t2, 6(a0) + sb t3, 7(a0) + addi a0, a0, 8 + addi a1, a1, 8 + bltu a1, a3, 1b + + 2: + ret + SYM_FUNC_END(__riscv_copy_bytes_unaligned) diff --git a/opensbi-timer-debug.patch b/opensbi-timer-debug.patch index 735f55e..33471af 100644 --- a/opensbi-timer-debug.patch +++ b/opensbi-timer-debug.patch @@ -34,7 +34,7 @@ index 7b618de..65e42b0 100644 void sbi_timer_exit(struct sbi_scratch *scratch) diff --git a/lib/sbi/sbi_trap.c b/lib/sbi/sbi_trap.c -index b4f3a17..b60315f 100644 +index b4f3a17..742eea6 100644 --- a/lib/sbi/sbi_trap.c +++ b/lib/sbi/sbi_trap.c @@ -283,6 +283,7 @@ static int sbi_trap_aia_irq(void) @@ -77,7 +77,7 @@ index b4f3a17..b60315f 100644 if (rc) sbi_trap_error(msg, rc, tcntx); + else -+ sbi_printf("$\n"); ++ sbi_printf("$"); if (((regs->mstatus & MSTATUS_MPP) >> MSTATUS_MPP_SHIFT) != PRV_M) sbi_sse_process_pending_events(regs); -- GitLab From 384069130b065938a9d097c59617a5aa9292e845 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 17:45:57 +0200 Subject: [PATCH 115/310] Reduce OpenSBI trap debug message --- opensbi-timer-debug.patch | 42 ++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/opensbi-timer-debug.patch b/opensbi-timer-debug.patch index 33471af..0f4b635 100644 --- a/opensbi-timer-debug.patch +++ b/opensbi-timer-debug.patch @@ -34,14 +34,14 @@ index 7b618de..65e42b0 100644 void sbi_timer_exit(struct sbi_scratch *scratch) diff --git a/lib/sbi/sbi_trap.c b/lib/sbi/sbi_trap.c -index b4f3a17..742eea6 100644 +index b4f3a17..cde2073 100644 --- a/lib/sbi/sbi_trap.c +++ b/lib/sbi/sbi_trap.c @@ -283,6 +283,7 @@ static int sbi_trap_aia_irq(void) */ struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) { -+ sbi_printf("!"); ++ sbi_printf("<"); int rc = SBI_ENOTSUPP; const char *msg = "trap handler failed"; struct sbi_scratch *scratch = sbi_scratch_thishart_ptr(); @@ -49,35 +49,59 @@ index b4f3a17..742eea6 100644 sbi_trap_set_context(scratch, tcntx); if (mcause & MCAUSE_IRQ_MASK) { -+ sbi_printf("OpenSBI: local interrupt"); ++ sbi_printf("i"); if (sbi_hart_has_extension(sbi_scratch_thishart_ptr(), SBI_HART_EXT_SMAIA)) rc = sbi_trap_aia_irq(); -@@ -306,15 +308,18 @@ struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) +@@ -306,35 +308,42 @@ struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) switch (mcause) { case CAUSE_ILLEGAL_INSTRUCTION: -+ sbi_printf("OpenSBI: illegal instruction"); ++ sbi_printf("I"); rc = sbi_illegal_insn_handler(tcntx); msg = "illegal instruction handler failed"; break; case CAUSE_MISALIGNED_LOAD: -+ sbi_printf("OpenSBI: misaligned load"); ++ sbi_printf("L"); sbi_pmu_ctr_incr_fw(SBI_PMU_FW_MISALIGNED_LOAD); rc = sbi_misaligned_load_handler(tcntx); msg = "misaligned load handler failed"; break; case CAUSE_MISALIGNED_STORE: -+ sbi_printf("OpenSBI: misaligned store"); ++ sbi_printf("S"); sbi_pmu_ctr_incr_fw(SBI_PMU_FW_MISALIGNED_STORE); rc = sbi_misaligned_store_handler(tcntx); msg = "misaligned store handler failed"; -@@ -344,6 +349,8 @@ struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) + break; + case CAUSE_SUPERVISOR_ECALL: + case CAUSE_MACHINE_ECALL: ++ sbi_printf("E"); + rc = sbi_ecall_handler(tcntx); + msg = "ecall handler failed"; + break; + case CAUSE_LOAD_ACCESS: ++ sbi_printf("l"); + sbi_pmu_ctr_incr_fw(SBI_PMU_FW_ACCESS_LOAD); + rc = sbi_load_access_handler(tcntx); + msg = "load fault handler failed"; + break; + case CAUSE_STORE_ACCESS: ++ sbi_printf("s"); + sbi_pmu_ctr_incr_fw(SBI_PMU_FW_ACCESS_STORE); + rc = sbi_store_access_handler(tcntx); + msg = "store fault handler failed"; + break; + default: ++ sbi_printf("R"); + /* If the trap came from S or U mode, redirect it there */ + msg = "trap redirect failed"; + rc = sbi_trap_redirect(regs, trap); +@@ -344,6 +353,8 @@ struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx) trap_done: if (rc) sbi_trap_error(msg, rc, tcntx); + else -+ sbi_printf("$"); ++ sbi_printf(">"); if (((regs->mstatus & MSTATUS_MPP) >> MSTATUS_MPP_SHIFT) != PRV_M) sbi_sse_process_pending_events(regs); -- GitLab From 9a7b230b5b6cb37f73952eb1fcccae0b40781b61 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 18:07:25 +0200 Subject: [PATCH 116/310] Rollback to OpenSBI 1.4 --- JOURNAL.md | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++ lagarto-ox.nix | 8 ++++-- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 99a6e6f..6ed6ce6 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -2325,3 +2325,77 @@ arch/riscv/kernel/copy-unaligned.S: 2: ret SYM_FUNC_END(__riscv_copy_bytes_unaligned) + +Booted again with simpler output: + + + > + + +So here is the simplified output using the format `` where X is one character +that differentiates the type of trap. + +Based on the code that performs the check: + + /* + * For a fixed amount of time, repeatedly try the function, and take + * the best time in cycles as the measurement. + */ + while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { + start_cycles = get_cycles64(); + /* Ensure the CSR read can't reorder WRT to the copy. */ + mb(); + __riscv_copy_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); + /* Ensure the copy ends before the end time is snapped. */ + mb(); + end_cycles = get_cycles64(); + if ((end_cycles - start_cycles) < word_cycles) + word_cycles = end_cycles - start_cycles; + } + + byte_cycles = -1ULL; + __riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); + start_jiffies = jiffies; + while ((now = jiffies) == start_jiffies) + cpu_relax(); + + while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { + start_cycles = get_cycles64(); + mb(); + __riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); + mb(); + end_cycles = get_cycles64(); + if ((end_cycles - start_cycles) < byte_cycles) + byte_cycles = end_cycles - start_cycles; + } + + preempt_enable(); + + /* Don't divide by zero. */ + if (!word_cycles || !byte_cycles) { + pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned access speed\n", + cpu); + + return 0; + } + + if (word_cycles < byte_cycles) + speed = RISCV_HWPROBE_MISALIGNED_FAST; + + ratio = div_u64((byte_cycles * 100), word_cycles); + pr_info("cpu%d: Ratio of byte access time to unaligned word access is %d.%02d, unaligned accesses are %s\n", + cpu, + ratio / 100, + ratio % 100, + (speed == RISCV_HWPROBE_MISALIGNED_FAST) ? "fast" : "slow"); + +The fact that the print line with the "Ratio of byte access..." doesn't appear, +puts the hang place at some point in between the check and the `pr_info()` call. + +If this is a problem on the OpenSBI side, we can bisect the code to find out +where the problem was introduced. But first, I would have to try OpenSBI 1.4 and +ensure we can reproduce it. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 2cecee9..b03be3d 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -301,12 +301,14 @@ ''; opensbi = prev.opensbi.overrideAttrs (old: rec { - version = "1.5"; + version = "1.4"; + #version = "1.5"; src = prev.fetchFromGitHub { owner = "riscv-software-src"; repo = "opensbi"; rev = "v${version}"; - hash = "sha256-vK14P97FcaVz4GDr/0055Z6s/k7BPKPQGZ/MQxbOWu0="; + hash = "sha256-T8ZeAzjM9aeTXitjE7s+m+jjGGtDo2jK1qO5EuKiVLU="; #1.4 + #hash = "sha256-vK14P97FcaVz4GDr/0055Z6s/k7BPKPQGZ/MQxbOWu0="; #1.5 }; #NIX_DEBUG=5; makeFlags = [ @@ -323,7 +325,7 @@ "FW_PAYLOAD_FDT_ADDR=0x80100000" ]; patches = [ - ./opensbi-timer-debug.patch + #./opensbi-timer-debug.patch #./ox-alveo-platform-plic.patch ]; }); -- GitLab From bed3c7ba5c18365993ede8dc178dac69dee24267 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 18:19:58 +0200 Subject: [PATCH 117/310] Disable PLIC from DT --- JOURNAL.md | 38 ++++++++++++++++++++++++++++++ ox-plic.dts | 66 ++++++++++++++++++++++++++--------------------------- 2 files changed, 71 insertions(+), 33 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 6ed6ce6..12fc4b4 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -2399,3 +2399,41 @@ puts the hang place at some point in between the check and the `pr_info()` call. If this is a problem on the OpenSBI side, we can bisect the code to find out where the problem was introduced. But first, I would have to try OpenSBI 1.4 and ensure we can reproduce it. + +Okay so with OpenSBI 1.4 we have a hang in the same place. + +Lets compare the domain regions: + +With OpenSBI 1.4 `fpga/alveo_ox`: + + Domain0 Region00 : 0x0000000040000000-0x0000000040000fff M: (I,R,W) S/U: (R,W) + Domain0 Region01 : 0x0000000080040000-0x000000008004ffff M: (R,W) S/U: () + Domain0 Region02 : 0x0000000080000000-0x000000008003ffff M: (R,X) S/U: () + Domain0 Region03 : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X) + Domain0 Next Address : 0x0000000080200000 + Domain0 Next Arg1 : 0x0000000080017000 + +With OpenSBI 1.4 `generic`: + + Domain0 Region00 : 0x0000000040001000-0x0000000040001fff M: (I,R,W) S/U: (R,W) + Domain0 Region01 : 0x0000000080040000-0x000000008005ffff M: (R,W) S/U: () + Domain0 Region02 : 0x0000000080000000-0x000000008003ffff M: (R,X) S/U: () + Domain0 Region03 : 0x0000000040800000-0x0000000040bfffff M: (I,R,W) S/U: (R,W) + Domain0 Region04 : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X) + Domain0 Next Address : 0x0000000080200000 + Domain0 Next Arg1 : 0x0000000080100000 + +With OpenSBI 1.5 `generic`: + + Domain0 Region00 : 0x0000000040001000-0x0000000040001fff M: (I,R,W) S/U: (R,W) + Domain0 Region01 : 0x0000000080040000-0x000000008005ffff M: (R,W) S/U: () + Domain0 Region02 : 0x0000000080000000-0x000000008003ffff M: (R,X) S/U: () + Domain0 Region03 : 0x0000000040800000-0x0000000040bfffff M: (I,R,W) S/U: (R,W) + Domain0 Region04 : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X) + Domain0 Next Address : 0x0000000080200000 + Domain0 Next Arg1 : 0x0000000080100000 + +So we have several changes. + +First, the PLIC has a new memory map. Let's comment it out in the device tree, +and see what happens. diff --git a/ox-plic.dts b/ox-plic.dts index 11f95b5..0f41ba0 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -132,8 +132,8 @@ uart_testing: serial@40003000 { compatible = "ns16550"; reg = <0x0 0x40003000 0x0 0x1000>; - interrupts = <1>; /* Output interrupt 1 */ - interrupt-parent = <&PLIC>; +// interrupts = <1>; /* Output interrupt 1 */ +// interrupt-parent = <&PLIC>; reg-shift = <2>; clock-frequency = <50000000>; current-speed = <115200>; @@ -184,37 +184,37 @@ /* Platform-Level Interrupt Controller: Delivers interrupts to * HARTs. */ - PLIC: plic@40800000 { - compatible = "riscv,plic0"; - interrupt-controller; /* Receives interrupts */ - #address-cells = <0>; - #interrupt-cells = <1>; - /* Sends interrupts to HART interrupt controllers */ - - /* - * From: linux-6.6.1/arch/riscv/include/asm/csr.h - * - * Interrupt causes (minus the high bit) - * #define IRQ_S_SOFT 1 - * #define IRQ_VS_SOFT 2 - * #define IRQ_M_SOFT 3 - * #define IRQ_S_TIMER 5 - * #define IRQ_VS_TIMER 6 - * #define IRQ_M_TIMER 7 - * #define IRQ_S_EXT 9 - * #define IRQ_VS_EXT 10 - * #define IRQ_M_EXT 11 - * #define IRQ_S_GEXT 12 - * #define IRQ_PMU_OVF 13 - * #define IRQ_LOCAL_MAX (IRQ_PMU_OVF + 1) - * #define IRQ_LOCAL_MASK GENMASK((IRQ_LOCAL_MAX - 1), 0) - */ - interrupts-extended = <&HLIC0 11>, <&HLIC0 9>; - reg = < 0x0 0x40800000 0x0 0x00400000>; - riscv,ndev = <3>; - //riscv,max-priority = <0x7>; - phandle = <0x3>; - }; +// PLIC: plic@40800000 { +// compatible = "riscv,plic0"; +// interrupt-controller; /* Receives interrupts */ +// #address-cells = <0>; +// #interrupt-cells = <1>; +// /* Sends interrupts to HART interrupt controllers */ +// +// /* +// * From: linux-6.6.1/arch/riscv/include/asm/csr.h +// * +// * Interrupt causes (minus the high bit) +// * #define IRQ_S_SOFT 1 +// * #define IRQ_VS_SOFT 2 +// * #define IRQ_M_SOFT 3 +// * #define IRQ_S_TIMER 5 +// * #define IRQ_VS_TIMER 6 +// * #define IRQ_M_TIMER 7 +// * #define IRQ_S_EXT 9 +// * #define IRQ_VS_EXT 10 +// * #define IRQ_M_EXT 11 +// * #define IRQ_S_GEXT 12 +// * #define IRQ_PMU_OVF 13 +// * #define IRQ_LOCAL_MAX (IRQ_PMU_OVF + 1) +// * #define IRQ_LOCAL_MASK GENMASK((IRQ_LOCAL_MAX - 1), 0) +// */ +// interrupts-extended = <&HLIC0 11>, <&HLIC0 9>; +// reg = < 0x0 0x40800000 0x0 0x00400000>; +// riscv,ndev = <3>; +// //riscv,max-priority = <0x7>; +// phandle = <0x3>; +// }; /* Core Local Interruptor: It directly connects to the timer and * inter-processor interrupt lines of various HARTs (or CPUs) so * RISC-V per-HART (or per-CPU) local interrupt controller is -- GitLab From 444bb635f6bd239ca05322c841539726d151451f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 18:25:14 +0200 Subject: [PATCH 118/310] Move serial to 0x4000_0000 --- JOURNAL.md | 13 +++++++++++++ ox-plic.dts | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 12fc4b4..4f0a2d1 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -2437,3 +2437,16 @@ So we have several changes. First, the PLIC has a new memory map. Let's comment it out in the device tree, and see what happens. + +Hangs in the same place, but now we don't have the 0x40800000 region with +OpenSBI 1.4 generic: + + Domain0 Region00 : 0x0000000040001000-0x0000000040001fff M: (I,R,W) S/U: (R,W) + Domain0 Region01 : 0x0000000080040000-0x000000008005ffff M: (R,W) S/U: () + Domain0 Region02 : 0x0000000080000000-0x000000008003ffff M: (R,X) S/U: () + Domain0 Region03 : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X) + Domain0 Next Address : 0x0000000080200000 + Domain0 Next Arg1 : 0x0000000080100000 + +Let's try to move the serial region to 0x40000000, although that seems to be +wrong. diff --git a/ox-plic.dts b/ox-plic.dts index 0f41ba0..7d33acf 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -118,9 +118,9 @@ * consoles */ /* The serial for the kernel console */ - uart_console: serial@40001000 { + uart_console: serial@40000000 { compatible = "ns16550"; - reg = <0x0 0x40001000 0x0 0x1000>; + reg = <0x0 0x40000000 0x0 0x1000>; reg-shift = <2>; /* No interrupts for this UART, use console=hvc0 */ /* This clock is the SERIAL_CLK */ -- GitLab From d3779d0f95b1346030d619dd0a6c4788fe9144fe Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 18:28:43 +0200 Subject: [PATCH 119/310] Revert serial address and move FDT --- JOURNAL.md | 5 +++++ lagarto-ox.nix | 2 +- ox-plic.dts | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 4f0a2d1..395aac8 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -2450,3 +2450,8 @@ OpenSBI 1.4 generic: Let's try to move the serial region to 0x40000000, although that seems to be wrong. + +Yep, that causes OpenSBI to not emit any message in the console, so let's put it +back in 0x40001000. + +Next, we may want to place the FDT in the previous location, at 0x80017000. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index b03be3d..e25a3c3 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -322,7 +322,7 @@ # Ensure it doesn't overlap from the 0x80200000 where the kernel will be # placed. - "FW_PAYLOAD_FDT_ADDR=0x80100000" + "FW_PAYLOAD_FDT_ADDR=0x80017000" ]; patches = [ #./opensbi-timer-debug.patch diff --git a/ox-plic.dts b/ox-plic.dts index 7d33acf..0f41ba0 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -118,9 +118,9 @@ * consoles */ /* The serial for the kernel console */ - uart_console: serial@40000000 { + uart_console: serial@40001000 { compatible = "ns16550"; - reg = <0x0 0x40000000 0x0 0x1000>; + reg = <0x0 0x40001000 0x0 0x1000>; reg-shift = <2>; /* No interrupts for this UART, use console=hvc0 */ /* This clock is the SERIAL_CLK */ -- GitLab From 80c93613d2147ffdbc103caa88cd0d361b2800f0 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 18:40:39 +0200 Subject: [PATCH 120/310] Try FDT address 0x80013000 --- JOURNAL.md | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++ lagarto-ox.nix | 2 +- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/JOURNAL.md b/JOURNAL.md index 395aac8..99576df 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -2455,3 +2455,61 @@ Yep, that causes OpenSBI to not emit any message in the console, so let's put it back in 0x40001000. Next, we may want to place the FDT in the previous location, at 0x80017000. + + => setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 trace_event=initcall:* trace_options=sym-addr tp_printk trace_buf_size=1M" + => setenv ramdisk_size 12614846 + => booti ${kernel_addr_r} ${ramdisk_addr_r}:${ramdisk_size} ${fdtcontroladdr} + +Hmm, continues to hang in the same point: + + [ 2.852200] initcall_start: func=check_unaligned_access_all_cpus+0x0/0x1d4 + +With this regions: + + OpenSBI v1.4 + ____ _____ ____ _____ + / __ \ / ____| _ \_ _| + | | | |_ __ ___ _ __ | (___ | |_) || | + | | | | '_ \ / _ \ '_ \ \___ \| _ < | | + | |__| | |_) | __/ | | |____) | |_) || |_ + \____/| .__/ \___|_| |_|_____/|____/_____| + | | + |_| + + Platform Name : Barcelona Supercomputing Center - Lagarto Ox (NixOS) + Platform Features : medeleg + Platform HART Count : 1 + Platform IPI Device : --- + Platform Timer Device : --- @ 0Hz + Platform Console Device : uart8250 + Platform HSM Device : --- + Platform PMU Device : --- + Platform Reboot Device : --- + Platform Shutdown Device : --- + Platform Suspend Device : --- + Platform CPPC Device : --- + Firmware Base : 0x80000000 + Firmware Size : 323 KB + Firmware RW Offset : 0x40000 + Firmware RW Size : 67 KB + Firmware Heap Offset : 0x48000 + Firmware Heap Size : 35 KB (total), 2 KB (reserved), 8 KB (used), 24 KB (free) + Firmware Scratch Size : 4096 B (total), 288 B (used), 3808 B (free) + Runtime SBI Version : 2.0 + + Domain0 Name : root + Domain0 Boot HART : 0 + Domain0 HARTs : 0* + Domain0 Region00 : 0x0000000040001000-0x0000000040001fff M: (I,R,W) S/U: (R,W) + Domain0 Region01 : 0x0000000080040000-0x000000008005ffff M: (R,W) S/U: () + Domain0 Region02 : 0x0000000080000000-0x000000008003ffff M: (R,X) S/U: () + Domain0 Region03 : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X) + Domain0 Next Address : 0x0000000080200000 + Domain0 Next Arg1 : 0x0000000080017000 + Domain0 Next Mode : S-mode + Domain0 SysReset : yes + Domain0 SysSuspend : yes + +I will also try 0x80013000 as address as I saw it being used before, but I don't +think it is the problem. It must be related with how the initialization is now +different. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index e25a3c3..faf6624 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -322,7 +322,7 @@ # Ensure it doesn't overlap from the 0x80200000 where the kernel will be # placed. - "FW_PAYLOAD_FDT_ADDR=0x80017000" + "FW_PAYLOAD_FDT_ADDR=0x80013000" ]; patches = [ #./opensbi-timer-debug.patch -- GitLab From fc7bfddd64360e7b991a30412227eac665cd8563 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 18:47:48 +0200 Subject: [PATCH 121/310] Switch to openpiton configuration --- JOURNAL.md | 35 +++++++++++++++++++++++++++++++++++ lagarto-ox.nix | 4 ++-- ox-plic.dts | 24 ++++++++++++------------ 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 99576df..4e69a0f 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -2513,3 +2513,38 @@ With this regions: I will also try 0x80013000 as address as I saw it being used before, but I don't think it is the problem. It must be related with how the initialization is now different. + + OpenSBI v1.4 + ____ _____ ____ _____ + / __ \ / ____| _ \_ _| + | | | |_ __ ___ _ __ | (___ | |_) || | + | | | | '_ \ / _ \ '_ \ \___ \| _ < | | + | |__| | |_) | __/ | | |____) | |_) || |_ + \____/| .__/ \___|_| |_|_____/|____/_____| + | | + |_| + + sbi_trap_error: hart0: illegal instruction handler failed (error -2) + sbi_trap_error: hart0: mcause=0x0000000000000002 mtval=0x0000000000000000 + sbi_trap_error: hart0: mepc=0x00000000800131a4 mstatus=0x8000000a00007800 + sbi_trap_error: hart0: ra=0x0000000080013bb0 sp=0x0000000080046e70 + sbi_trap_error: hart0: gp=0x0000000000000000 tp=0x0000000080047000 + sbi_trap_error: hart0: s0=0x0000000080046eb0 s1=0x0000000000000118 + sbi_trap_error: hart0: a0=0x0000000080013000 a1=0x0000000080046e74 + sbi_trap_error: hart0: a2=0x00000000800133c0 a3=0x0000000000000600 + sbi_trap_error: hart0: a4=0x000000000000062c a5=0x0000000000000284 + sbi_trap_error: hart0: a6=0x0000000000000000 a7=0x000000000000002c + sbi_trap_error: hart0: s2=0x0000000080013000 s3=0x0000000000000000 + sbi_trap_error: hart0: s4=0x0000000080047000 s5=0x0000000000000000 + sbi_trap_error: hart0: s6=0x0000000000000000 s7=0x0000000000000001 + sbi_trap_error: hart0: s8=0x0000000000002000 s9=0x00000000800436f0 + sbi_trap_error: hart0: s10=0x0000000000000000 s11=0x0000000000000000 + sbi_trap_error: hart0: t0=0x0000000080009796 t1=0x0000000000000268 + sbi_trap_error: hart0: t2=0x0000000000001000 t3=0x00000000000000d7 + sbi_trap_error: hart0: t4=0x000000007fffffff t5=0x0000000000000001 + sbi_trap_error: hart0: t6=0x0000000000000004 + +With 0x80013000 we seem to enter OpenSBI code region. + +Let's try with openpiton again, without the FDT address. We should disable the +secondary console from the DT too. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index faf6624..2dcfe9d 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -312,7 +312,7 @@ }; #NIX_DEBUG=5; makeFlags = [ - "PLATFORM=generic" + "PLATFORM=fpga/openpiton" #"CONFIG_SBI_ECALL_RFENCE=n" #"PLATFORM_RISCV_ISA=rv64imafd" # No compressed instructions #"PLATFORM_RISCV_ISA=rv64g" # No compressed instructions @@ -322,7 +322,7 @@ # Ensure it doesn't overlap from the 0x80200000 where the kernel will be # placed. - "FW_PAYLOAD_FDT_ADDR=0x80013000" + #"FW_PAYLOAD_FDT_ADDR=0x80013000" ]; patches = [ #./opensbi-timer-debug.patch diff --git a/ox-plic.dts b/ox-plic.dts index 0f41ba0..6e21697 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -7,7 +7,7 @@ aliases { serial0 = &uart_console; // ttyS0 - serial1 = &uart_testing; // ttyS1 +// serial1 = &uart_testing; // ttyS1 }; // chosen { @@ -128,17 +128,17 @@ current-speed = <115200>; status = "okay"; }; - /* The serial for interrupt tests */ - uart_testing: serial@40003000 { - compatible = "ns16550"; - reg = <0x0 0x40003000 0x0 0x1000>; -// interrupts = <1>; /* Output interrupt 1 */ -// interrupt-parent = <&PLIC>; - reg-shift = <2>; - clock-frequency = <50000000>; - current-speed = <115200>; - status = "okay"; - }; +// /* The serial for interrupt tests */ +// uart_testing: serial@40003000 { +// compatible = "ns16550"; +// reg = <0x0 0x40003000 0x0 0x1000>; +//// interrupts = <1>; /* Output interrupt 1 */ +//// interrupt-parent = <&PLIC>; +// reg-shift = <2>; +// clock-frequency = <50000000>; +// current-speed = <115200>; +// status = "okay"; +// }; // ethernet0 { // xlnx,rxmem = <0x000005f2>; -- GitLab From c38edfe737aa04e529aa7fc9099c708c2e659e8a Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 18:51:17 +0200 Subject: [PATCH 122/310] Enable PLIC --- JOURNAL.md | 2 ++ ox-plic.dts | 62 ++++++++++++++++++++++++++--------------------------- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 4e69a0f..ffab2fc 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -2548,3 +2548,5 @@ With 0x80013000 we seem to enter OpenSBI code region. Let's try with openpiton again, without the FDT address. We should disable the secondary console from the DT too. + +No output, lets enable the PLIC again in the DT. diff --git a/ox-plic.dts b/ox-plic.dts index 6e21697..e0191f4 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -184,37 +184,37 @@ /* Platform-Level Interrupt Controller: Delivers interrupts to * HARTs. */ -// PLIC: plic@40800000 { -// compatible = "riscv,plic0"; -// interrupt-controller; /* Receives interrupts */ -// #address-cells = <0>; -// #interrupt-cells = <1>; -// /* Sends interrupts to HART interrupt controllers */ -// -// /* -// * From: linux-6.6.1/arch/riscv/include/asm/csr.h -// * -// * Interrupt causes (minus the high bit) -// * #define IRQ_S_SOFT 1 -// * #define IRQ_VS_SOFT 2 -// * #define IRQ_M_SOFT 3 -// * #define IRQ_S_TIMER 5 -// * #define IRQ_VS_TIMER 6 -// * #define IRQ_M_TIMER 7 -// * #define IRQ_S_EXT 9 -// * #define IRQ_VS_EXT 10 -// * #define IRQ_M_EXT 11 -// * #define IRQ_S_GEXT 12 -// * #define IRQ_PMU_OVF 13 -// * #define IRQ_LOCAL_MAX (IRQ_PMU_OVF + 1) -// * #define IRQ_LOCAL_MASK GENMASK((IRQ_LOCAL_MAX - 1), 0) -// */ -// interrupts-extended = <&HLIC0 11>, <&HLIC0 9>; -// reg = < 0x0 0x40800000 0x0 0x00400000>; -// riscv,ndev = <3>; -// //riscv,max-priority = <0x7>; -// phandle = <0x3>; -// }; + PLIC: plic@40800000 { + compatible = "riscv,plic0"; + interrupt-controller; /* Receives interrupts */ + #address-cells = <0>; + #interrupt-cells = <1>; + /* Sends interrupts to HART interrupt controllers */ + + /* + * From: linux-6.6.1/arch/riscv/include/asm/csr.h + * + * Interrupt causes (minus the high bit) + * #define IRQ_S_SOFT 1 + * #define IRQ_VS_SOFT 2 + * #define IRQ_M_SOFT 3 + * #define IRQ_S_TIMER 5 + * #define IRQ_VS_TIMER 6 + * #define IRQ_M_TIMER 7 + * #define IRQ_S_EXT 9 + * #define IRQ_VS_EXT 10 + * #define IRQ_M_EXT 11 + * #define IRQ_S_GEXT 12 + * #define IRQ_PMU_OVF 13 + * #define IRQ_LOCAL_MAX (IRQ_PMU_OVF + 1) + * #define IRQ_LOCAL_MASK GENMASK((IRQ_LOCAL_MAX - 1), 0) + */ + interrupts-extended = <&HLIC0 11>, <&HLIC0 9>; + reg = < 0x0 0x40800000 0x0 0x00400000>; + riscv,ndev = <3>; + //riscv,max-priority = <0x7>; + phandle = <0x3>; + }; /* Core Local Interruptor: It directly connects to the timer and * inter-processor interrupt lines of various HARTs (or CPUs) so * RISC-V per-HART (or per-CPU) local interrupt controller is -- GitLab From 8725d0453388c16acd54133bfba605bf58300916 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 18:53:16 +0200 Subject: [PATCH 123/310] Switch OpenSBI to generic --- JOURNAL.md | 4 +++- lagarto-ox.nix | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index ffab2fc..723e04d 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -2549,4 +2549,6 @@ With 0x80013000 we seem to enter OpenSBI code region. Let's try with openpiton again, without the FDT address. We should disable the secondary console from the DT too. -No output, lets enable the PLIC again in the DT. +No output, lets enable the PLIC again in the DT. Same. + +Probably we have a better chance to fix it in the generic platform. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 2dcfe9d..07eaf79 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -312,7 +312,7 @@ }; #NIX_DEBUG=5; makeFlags = [ - "PLATFORM=fpga/openpiton" + "PLATFORM=generic" #"CONFIG_SBI_ECALL_RFENCE=n" #"PLATFORM_RISCV_ISA=rv64imafd" # No compressed instructions #"PLATFORM_RISCV_ISA=rv64g" # No compressed instructions -- GitLab From 9fac87a00e1fe74bb0aeafefab5dbe6be6e56908 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 12 Jul 2024 18:59:48 +0200 Subject: [PATCH 124/310] Move FDT to 0xc0000000 --- JOURNAL.md | 2 ++ lagarto-ox.nix | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/JOURNAL.md b/JOURNAL.md index 723e04d..fe4a9d8 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -2552,3 +2552,5 @@ secondary console from the DT too. No output, lets enable the PLIC again in the DT. Same. Probably we have a better chance to fix it in the generic platform. + +Let's use a very far away address for the FDT just in case: `0x0_c000_0000` diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 07eaf79..f026011 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -322,7 +322,7 @@ # Ensure it doesn't overlap from the 0x80200000 where the kernel will be # placed. - #"FW_PAYLOAD_FDT_ADDR=0x80013000" + "FW_PAYLOAD_FDT_ADDR=0xc0000000" ]; patches = [ #./opensbi-timer-debug.patch -- GitLab From eee26f2b4dd2785f9ea4728ae4ce76fa1bea8764 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 15 Jul 2024 11:02:27 +0200 Subject: [PATCH 125/310] More experiments with OpenSBI 1.5 with debug --- JOURNAL.md | 35 +++++++++++++++++++++++++++++++++++ lagarto-ox.nix | 10 +++++----- ox-plic.dts | 22 +++++++++++----------- 3 files changed, 51 insertions(+), 16 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index fe4a9d8..333b33a 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -2554,3 +2554,38 @@ No output, lets enable the PLIC again in the DT. Same. Probably we have a better chance to fix it in the generic platform. Let's use a very far away address for the FDT just in case: `0x0_c000_0000` + +This one doesn't work: + + => setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 ftrace=function ftrace_filter=* tp_printk" + +This one yes: + + => setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 trace_event=*:* tp_printk trace_buf_size=1M" + => setenv ramdisk_size 12614846 + => booti ${kernel_addr_r} ${ramdisk_addr_r}:${ramdisk_size} ${fdtcontroladdr} + + [ 164.849640] initcall_finish: func=vdso_init+0x0/0x4c ret=0 + [ 164.849960] console: initcall_finish: func=vdso_init+0x0/0x4c ret=0 + [ 164.862820] initcall_start: func=check_unaligned_access_all_cpus+0x0/0x1d4 + [ 164.863140] console: initcall_start: func=check_unaligned_access_all_cpus+0x0/0x1d4 + [ 164.878860] kmalloc: call_site=check_unaligned_access_all_cpus+0xa8/0x1d4 ptr=(____ptrval____) bytes_req=8 bytes_alloc=8 gfp_flags=GFP_KERNEL|__GFP_ZERO node=-1 accounted=false + [ 164.879200] console: kmalloc: call_site=check_unaligned_access_all_cpus+0xa8/0x1d4 ptr=(____ptrval____) bytes_req=8 bytes_alloc=8 gfp_flags=GFP_KERNEL|__GFP_ZERO node=-1 accounted=false + [ 164.912380] mm_page_alloc_zone_locked: page=(____ptrval____) pfn=0x83d28 order=2 migratetype=0 percpu_refill=1 + [ 164.912680] console: mm_page_alloc_zone_locked: page=(____ptrval____) pfn=0x83d28 order=2 migratetype=0 percpu_refill=1 + [ 164.934340] mm_page_alloc_zone_locked: page=(____ptrval____) pfn=0x83d2c order=2 migratetype=0 percpu_refill=1 + [ 164.934660] console: mm_page_alloc_zone_locked: page=(____ptrval____) pfn=0x83d2c order=2 migratetype=0 percpu_refill=1 + [ 164.956300] mm_page_alloc_zone_locked: page=(____ptrval____) pfn=0x83d30 order=2 migratetype=0 percpu_refill=1 + [ 164.956620] console: mm_page_alloc_zone_locked: page=(____ptrval____) pfn=0x83d30 order=2 migratetype=0 percpu_refill=1 + [ 164.978280] mm_page_alloc_zone_locked: page=(____ptrval____) pfn=0x83d34 order=2 migratetype=0 percpu_refill=1 + [ 164.978600] console: mm_page_alloc_zone_locked: page=(____ptrval____) pfn=0x83d34 order=2 migratetype=0 percpu_refill=1 + [ 165.000260] mm_page_alloc_zone_locked: page=(____ptrval____) pfn=0x83d38 order=2 migratetype=0 percpu_refill=1 + [ 165.000560] console: mm_page_alloc_zone_locked: page=(____ptrval____) pfn=0x83d38 order=2 migratetype=0 percpu_refill=1 + [ 165.022200] mm_page_alloc_zone_locked: page=(____ptrval____) pfn=0x83d3c order=2 migratetype=0 percpu_refill=1 + [ 165.022540] console: mm_page_alloc_zone_locked: page=(____ptrval____) pfn=0x83d3c order=2 migratetype=0 percpu_refill=1 + [ 165.044200] mm_page_alloc_zone_locked: page=(____ptrval____) pfn=0x83d40 order=2 migratetype=0 percpu_refill=1 + [ 165.044500] console: mm_page_alloc_zone_locked: page=(____ptrval____) pfn=0x83d40 order=2 migratetype=0 percpu_refill=1 + [ 165.066220] mm_page_alloc: page=(____ptrval____) pfn=0x83d28 order=2 migratetype=0 gfp_flags=GFP_KERNEL + [ 165.066540] console: mm_page_alloc: page=(____ptrval____) pfn=0x83d28 order=2 migratetype=0 gfp_flags=GFP_KERNEL + + diff --git a/lagarto-ox.nix b/lagarto-ox.nix index f026011..c389923 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -301,14 +301,14 @@ ''; opensbi = prev.opensbi.overrideAttrs (old: rec { - version = "1.4"; - #version = "1.5"; + #version = "1.4"; + version = "1.5"; src = prev.fetchFromGitHub { owner = "riscv-software-src"; repo = "opensbi"; rev = "v${version}"; - hash = "sha256-T8ZeAzjM9aeTXitjE7s+m+jjGGtDo2jK1qO5EuKiVLU="; #1.4 - #hash = "sha256-vK14P97FcaVz4GDr/0055Z6s/k7BPKPQGZ/MQxbOWu0="; #1.5 + #hash = "sha256-T8ZeAzjM9aeTXitjE7s+m+jjGGtDo2jK1qO5EuKiVLU="; #1.4 + hash = "sha256-vK14P97FcaVz4GDr/0055Z6s/k7BPKPQGZ/MQxbOWu0="; #1.5 }; #NIX_DEBUG=5; makeFlags = [ @@ -325,7 +325,7 @@ "FW_PAYLOAD_FDT_ADDR=0xc0000000" ]; patches = [ - #./opensbi-timer-debug.patch + ./opensbi-timer-debug.patch #./ox-alveo-platform-plic.patch ]; }); diff --git a/ox-plic.dts b/ox-plic.dts index e0191f4..a5fdff6 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -224,17 +224,17 @@ * property is described in * Documentation/devicetree/bindings/riscv/cpus.yaml */ -// clint: clint@40002000 { -// /* MTIME and MTIMECMP address and size pairs */ -// reg = <0x0 0x40002000 0x0 0x8>, <0x0 0x40002008 0x0 0x8>; -// interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; -// /*<&CPU0 0x3>, <&CPU0 0x7>,*/ -// /*<&onic_pool 0x3>, <&onic_pool 0x7>,*/ -// /*<&SERIAL 0x3>, <&SERIAL 0x7>*/ -// -// /* Allows using the "generic" platform in OpenSBI. */ -// compatible = "riscv,aclint-mtimer"; -// }; + clint: clint@40002000 { + /* MTIME and MTIMECMP address and size pairs */ + reg = <0x0 0x40002000 0x0 0x8>, <0x0 0x40002008 0x0 0x8>; + interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; + /*<&CPU0 0x3>, <&CPU0 0x7>,*/ + /*<&onic_pool 0x3>, <&onic_pool 0x7>,*/ + /*<&SERIAL 0x3>, <&SERIAL 0x7>*/ + + /* Allows using the "generic" platform in OpenSBI. */ + compatible = "riscv,aclint-mtimer"; + }; /* Guesswork: There must be a timer at 0x40170000 as it is * initialized in OpenSBI. It seems to drive the console. */ -- GitLab From cd7eb7179f37d81d227b01ccb5913c16ad9a6c01 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 1 Aug 2024 19:29:04 +0200 Subject: [PATCH 126/310] Update journal with PLIC experiments --- JOURNAL.md | 665 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 665 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index 333b33a..154f370 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -2588,4 +2588,669 @@ This one yes: [ 165.066220] mm_page_alloc: page=(____ptrval____) pfn=0x83d28 order=2 migratetype=0 gfp_flags=GFP_KERNEL [ 165.066540] console: mm_page_alloc: page=(____ptrval____) pfn=0x83d28 order=2 migratetype=0 gfp_flags=GFP_KERNEL +## 2024-08-01 +Now that we have a new bitstream with a CLINT connected to a PLIC input, we may +be able to generate an interrupt. + +Here is the comment where I gather the pieces: + +---8<--- + +From https://gitlab.bsc.es/hwdesign/rtl/core-tile/sa-fpga/ I can see that the +auxiliary timer [is in fact another +CLINT](https://gitlab.bsc.es/hwdesign/rtl/core-tile/sa-fpga/-/blob/10ba8b2a11ef105d7cda065e13838a3d28f3c951/fpga_core_bridge/rtl/fpga_core_bridge.sv#L685). + +I don't have access to the [hlib +repository](https://gitlab.bsc.es/hwdesign/hlib.git) (@jmendoza can I get access +to it?) to see the CLINT definition, but based on [this +CLINT](https://github.com/openhwgroup/cva6/blob/master/corev_apu/clint/clint.sv) +and [this one](https://github.com/pulp-platform/clint/blob/master/src/clint.sv) +I can estimate some of the previous information: + +> - The information on which port number of the PLIC the timer is connected to. + +https://gitlab.bsc.es/hwdesign/rtl/core-tile/sa-fpga/-/blob/main/fpga_core_bridge/rtl/fpga_core_bridge.sv#L1114 + +``` + plic #( + .PARAMETER_BITWIDTH (7), + .NUM_TARGETS (1), + .NUM_SOURCES (4) + ) plic_inst ( + .clk_i (clk_i), + .rstn_i (reset), + .irq_sources_i ({plic_timer_eirq,eth_irq,uart1_irq}), + .eip_targets_o (irq), +``` + +If I read it from right to left starting at 1, it should be **at 4**, as the +`eth_irq` has two "wires". + + +> - The memory address of the timer and the mapped registers, so I can see it +> increasing its value. I think the `aux_timer` you had in the past would be +> fine. + +https://gitlab.bsc.es/hwdesign/rtl/core-tile/sa-fpga/-/blob/main/fpga_core_bridge/rtl/local_includes/defines.svh#L33-36 + +``` +//Size: 64KB +`define AUX_TIMER_XBAR_ID 2 +`define AUX_TIMER_BASE_ADDR 64'h0000_0000_4001_0000 // Need to be this space because we use a clint as aux timer +`define AUX_TIMER_END_ADDR 64'h0000_0000_4001_FFFF +``` + +> - The specific operations I need to do in machine mode to configure the timer +> to fire at 1 Hz (probably setting two registers). + +Based on the source of the CLINT, **only one interrupt will be generated** after +setting the mtimecmp register to something larger than the mtime register. Then +I suspect I would have to make the interrupt run some code to rearm it again by +modifying the mtimecmp register to some value in the future: + +``` +// ----------------------------- +// IRQ Generation +// ----------------------------- +// The mtime register has a 64-bit precision on all RV32, RV64, and RV128 systems. Platforms provide a 64-bit +// memory-mapped machine-mode timer compare register (mtimecmp), which causes a timer interrupt to be posted when the +// mtime register contains a value greater than or equal (mtime >= mtimecmp) to the value in the mtimecmp register. +// The interrupt remains posted until it is cleared by writing the mtimecmp register. The interrupt will only be taken +// if interrupts are enabled and the MTIE bit is set in the mie register. +always_comb begin : irq_gen + // check that the mtime cmp register is set to a meaningful value + for (int unsigned i = 0; i < NR_CORES; i++) begin + if (mtime_q >= mtimecmp_q[i]) begin + timer_irq_o[i] = 1'b1; + end else begin + timer_irq_o[i] = 1'b0; + end + end +end +``` + +I could ensure that an interrupt has been fired by reading the mtime and +mtimecmp values, and checking that mtime > mtimecmp. + +Now I only need to find a bitstream that has been generated with +https://gitlab.bsc.es/hwdesign/rtl/core-tile/sa-fpga/-/commit/10ba8b2a11ef105d7cda065e13838a3d28f3c951. + + +This may work: + +https://gitlab.bsc.es/hwdesign/fpga/integration-lab/fpga-shell/-/jobs/968583/raw + +> Submodule path 'sa-fpga': checked out '12b77cb50cf1c416f107d4c7ab1c52d7b5e59056' + +Which is based on fpga-shell https://gitlab.bsc.es/hwdesign/fpga/integration-lab/fpga-shell/-/commit/01265d197f256bce2c7e82d21c7f4bf5dcb44e68 + +Here is the bitstream job: https://gitlab.bsc.es/hwdesign/fpga/integration-lab/fpga-shell/-/jobs/968585 + +And the bitstream: [artifacts.zip](/uploads/d8240a779cd485771b9e3d0147e342d1/artifacts.zip) + +And full log: [job.log](/uploads/a4215e4d039065b77f7a2d2b1403e475/job.log) + +The memory map would need a bit of adjustment in the device tree, but to play with the timer in machine mode not much is needed. + +I think I have all the pieces now. + +---8<--- + +I will try with the last bitstream that I already had compiled, as I will have +to rebuild the required packages in nix. + +To compute the memory position of the registers: + + `define AUX_TIMER_XBAR_ID 2 + `define AUX_TIMER_BASE_ADDR 64'h0000_0000_4001_0000 // Need to be this space because we use a clint as aux timer + `define AUX_TIMER_END_ADDR 64'h0000_0000_4001_FFFF + + localparam logic [15:0] MSIP_BASE = 16'h0; + localparam logic [15:0] MTIMECMP_BASE = 16'h4000; + localparam logic [15:0] MTIME_BASE = 16'hbff8; + +So, the base address 0x40010000 and the first MTIME at 0xbff8 would give us a +timer at 0x4001bff8. + +Here it is: + + => md 0x4001bff8 1 + 4001bff8: 006e65b8 .en. + => md 0x4001bff8 1 + 4001bff8: 006e9a26 &.n. + => md 0x4001bff8 1 + 4001bff8: 006ebae1 ..n. + => md 0x4001bff8 1 + 4001bff8: 006eda45 E.n. + => md 0x4001bff8 1 + 4001bff8: 006ef9d4 ..n. + => md 0x4001bff8 1 + 4001bff8: 006f1abb ..o. + +Now, the MTIMECMP should be at 0x40014000, which should be 0. + + => md 0x40014000 1 + 40014000: 00000000 .... + +Good. + +Now, I suspect the MSIP is not used, so it should be 0 at 0x40010000 too: + + => md 0x40010000 1 + 40010000: 00000000 .... + +Nice. + +Just for testing, let's see if I can make the timer cause any change in the MSIP +register by setting the MTIMECMP to a value: + + => mw 0x40014000 0x01700000 # Write the MTIMECMP + => md 0x40014000 1 + 40014000: 01700000 ..p. + => md 0x4001bff8 1 + 4001bff8: 016da81a ..m. + => md 0x40010000 1 + 40010000: 00000000 .... + => md 0x4001bff8 1 + 4001bff8: 016f947c |.o. + => md 0x4001bff8 1 + 4001bff8: 016fff96 ..o. + => md 0x4001bff8 1 + 4001bff8: 01704367 gCp. # Now we passed it + => md 0x40010000 1 + 40010000: 00000000 .... # But MSIP is still 0 + +As expected, nothing happens. We cannot monitor the interrupt line from the +timer itself. + +Now, let see if we can inspect the state of the PLIC. + +From the `plic_interface` I can see where are the memory addresses of the +registers exposed. + +The PLIC is mapped here: + + //Size: 4MB + `define PLIC_XBAR_ID 5 + `define PLIC_BASE_ADDR 64'h0000_0000_4080_0000 + `define PLIC_END_ADDR 64'h0000_0000_40BF_FFFF + +There are several ways in which the interrupts are not forwarded to the +destination, and several destinations. The PLIC specification is a good resource +to understand it: + + https://github.com/riscv/riscv-plic-spec + +This is important: + +> The interrupt gateways are responsible for converting global interrupt signals +> into a common interrupt request format, and for controlling the flow of +> interrupt requests to the PLIC core. At most one interrupt request per +> interrupt source can be pending in the PLIC core at any time, indicated by +> setting the source’s IP bit. The gateway only forwards a new interrupt request +> to the PLIC core after receiving notification that the interrupt handler +> servicing the previous interrupt request from the same source has completed. + +So, there cannot be any pending interrupt, otherwise no more interrupts will be +sent to the core. + +Assuming the PLIC uses the standard memory layout, we should find: + + base + 0x000000: Reserved (interrupt source 0 does not exist) + base + 0x000004: Interrupt source 1 priority + base + 0x000008: Interrupt source 2 priority + +Which they should begin at 0x40800000. + + => md 0x40800000 8 + 40800000: 00000000 00000000 00000000 00000000 ................ + 40800010: 00000000 00000000 00000000 00000000 ................ + +All the priorities are set to 0. + +Let's see the pending interrupts: + + base + 0x000FFC: Interrupt source 1023 priority + base + 0x001000: Interrupt Pending bit 0-31 + base + 0x00107C: Interrupt Pending bit 992-1023 + +They should be at 0x40801000: + + => md 0x40801000 8 + 40801000: 00000010 00000000 00000000 00000000 ................ + 40801010: 00000000 00000000 00000000 00000000 ................ + +Whoa, look at that. + + 4321 + 0x00000010 = 10000 + | | + | int 0 (reserved) + int 4 = timer + +We got the interrupt 4 pending in context 0! + +Other context don't seem to see anything: + + => md 0x40801080 1 + 40801080: 00000000 .... + => md 0x40801100 1 + 40801100: 00000000 .... + => md 0x40801180 1 + 40801180: 00000000 .... + => md 0x40801200 1 + 40801200: 00000000 .... + => md 0x40801280 1 + 40801280: 00000000 .... + => md 0x40801300 1 + 40801300: 00000000 .... + => md 0x40801380 1 + 40801380: 00000000 .... + +So, as the priority is 0, this means it is ignored: + +> If PLIC supports Interrupt Priorities, then each PLIC interrupt source can be +> assigned a priority by writing to its 32-bit memory-mapped priority register. +> A priority value of 0 is reserved to mean "never interrupt" and effectively +> disables the interrupt. Priority 1 is the lowest active priority while the +> maximum level of priority depends on PLIC implementation. Ties between global +> interrupts of the same priority are broken by the Interrupt ID; interrupts +> with the lowest ID have the highest effective priority. + +Let's claim the interrupt, by just performing a read from 0x40a00004: + + => md 0x40801000 1 + 40801000: 00000010 .... + => md 0x40a00004 1 + 40a00004: 00000000 .... + => md 0x40801000 1 + 40801000: 00000010 .... + +So, it continues to be pending. + +We have to write the completed interrupt, by writing the number 4 to the same +register: + + => mw 0x40a00004 4 + => md 0x40801000 1 + 40801000: 00000010 .... + +Still not cleared. + +Let's try making the MTIMECMP value much higher than MTIME: + + => md 0x40014000 1 + 40014000: 01700000 ..p. + => md 0x4001bff8 1 + 4001bff8: 03a4584b KX.. + => mw 0x40014000 0xaaaaaaaa + => md 0x40014000 1 + 40014000: aaaaaaaa .... + => md 0x4001bff8 1 + 4001bff8: 03abc84d M... + +So... the ID that must be written to the completion register is not the +interrupt number, but the value read from the claim register, which is 0. + + => mw 0x40a00004 0 + => md 0x40801000 1 + 40801000: 00000010 .... + +Still, nothing. + +All interrupts are disabled: + + => md 0x40802000 4 + 40802000: 00000000 00000000 00000000 00000000 ................ + +Let's try enabling the interrupt 4, by writting: + + => mw 0x40802000 0x10 + => md 0x40802000 1 + 40802000: 00000010 .... + => md 0x40801000 1 + 40801000: 00000010 .... + +Now, let's set the priority to something else than 0. + +First, lets make sure that the context 0 threshold priority is set to 0, so we +allow all interrupts: + + 0x200000: Priority threshold for context 0 + + => md 0x40a00000 1 + 40a00000: 00000007 .... + +Oh, so we are only receiving interrupts with priority 7 or higher. But our +interrupt has priority 0! + + => md 0x40800004 1 + 40800004: 00000000 .... + +Let's make the threshold 0 and our interrupt have priority 1. + + => mw 0x40a00000 0 + => mw 0x40800004 1 + => md 0x40800004 1 + 40800004: 00000001 .... + => md 0x40a00000 + 40a00000: 00000000 .... + +Not let's see again the interrupt state: + + => md 0x40801000 1 + 40801000: 00000010 .... + +Still on. + +Let's read the claim register again. + + => md 0x40a00004 + 40a00004: 00000000 .... + +Still 0, let's try to complete it: + + => mw 0x40a00004 0 + => md 0x40801000 1 + 40801000: 00000010 .... + +Nope, still pending. + +What, what the hell. The threshold value has changed to 1: + + => md 0x40800004 1 + 40800004: 00000001 .... + => md 0x40a00000 1 + 40a00000: 00000001 .... <-- this was 0 + +Let's configure the interruption priority to something bigger than 1. + +Wait, I put the priority in the wrong source: + + 0x000000: Reserved (interrupt source 0 does not exist) + 0x000004: Interrupt source 1 priority + 0x000008: Interrupt source 2 priority + +Our timer should be the source 4, so 12 or 0xc: + + => md 0x4080000c 1 + 4080000c: 00000000 .... + +(This is wrong, should be 0x40800010, see below) + +Let's make it have priority 0xd: + + => mw 0x4080000c 0xd + => md 0x4080000c 1 + 4080000c: 0000000d .... + +Something weird is going on with the priority register? + + => md 0x40a00000 1 + 40a00000: 00000000 .... + => md 0x40a00000 1 + 40a00000: 0000000d .... + => md 0x40a00000 1 + 40a00000: 0000000d .... + => md 0x40a00000 1 + 40a00000: 0000000d .... + => md 0x40a00000 1 + 40a00000: 0000000d .... + +Let's see the claim register, which should be in the next word: + + => md 0x40a00004 1 + 40a00004: 00000004 .... + +Yes! Now I can see the claim register with a proper ID. Let's complete this +interrupt by writing the 4 back to that register: + + => mw 0x40a00004 4 + => md 0x40801000 1 + 40801000: 00000000 .... + +Perfect! It properly caused the pending interrupt to disappear. + +Let's try now setting the MTIMECMP to something smaller than the MTIME, so it +causes an interrupt. With a value 0 should always work, but lets choose a non +zero value: + + => md 0x40014000 + 40014000: aaaaaaaa .... + => mw 0x40014000 00aaaaaa + => md 0x40014000 + 40014000: 00aaaaaa .... + => md 0x4001bff8 + 4001bff8: 06211a0c ..!. + => md 0x40801000 1 + 40801000: 00000010 .... + +Perfect! It causes the interrupt to appear as pending. + +So, using the context 0, we can properly see the interrupt pending, claim it and +complete it. But the context 0 is not used in OpenSBI, only the 9 and 11: + +From `include/sbi/riscv_encoding.h`: + + #define IRQ_S_SOFT 1 + #define IRQ_VS_SOFT 2 + #define IRQ_M_SOFT 3 + #define IRQ_S_TIMER 5 + #define IRQ_VS_TIMER 6 + #define IRQ_M_TIMER 7 + #define IRQ_S_EXT 9 + #define IRQ_VS_EXT 10 + #define IRQ_M_EXT 11 + #define IRQ_S_GEXT 12 + #define IRQ_PMU_OVF 13 + +And from `lib/utils/irqchip/fdt_irqchip_plic.c`: + + static int irqchip_plic_update_hartid_table(void *fdt, int nodeoff, + struct plic_data *pd) + { + const fdt32_t *val; + u32 phandle, hwirq, hartid; + struct sbi_scratch *scratch; + int i, err, count, cpu_offset, cpu_intc_offset; + + val = fdt_getprop(fdt, nodeoff, "interrupts-extended", &count); + if (!val || count < sizeof(fdt32_t)) + return SBI_EINVAL; + count = count / sizeof(fdt32_t); + + for (i = 0; i < count; i += 2) { + phandle = fdt32_to_cpu(val[i]); + hwirq = fdt32_to_cpu(val[i + 1]); + + cpu_intc_offset = fdt_node_offset_by_phandle(fdt, phandle); + if (cpu_intc_offset < 0) + continue; + + cpu_offset = fdt_parent_offset(fdt, cpu_intc_offset); + if (cpu_offset < 0) + continue; + + err = fdt_parse_hart_id(fdt, cpu_offset, &hartid); + if (err) + continue; + + scratch = sbi_hartid_to_scratch(hartid); + if (!scratch) + continue; + + plic_set_hart_data_ptr(scratch, pd); + switch (hwirq) { + case IRQ_M_EXT: + plic_set_hart_mcontext(scratch, i / 2); + break; + case IRQ_S_EXT: + plic_set_hart_scontext(scratch, i / 2); + break; + } + } + + return 0; + } + +So, lets try to do the same, but with the context 11 for machine mode +`IRQ_M_EXT`. + +Let's compute the address of the input source for context 11: + + base + 0x002000: Enable bits for sources 0-31 on context 0 + base + 0x002004: Enable bits for sources 32-63 on context 0 + ... + base + 0x00207C: Enable bits for sources 992-1023 on context 0 + base + 0x002080: Enable bits for sources 0-31 on context 1 + base + 0x002084: Enable bits for sources 32-63 on context 1 + ... + base + 0x0020FC: Enable bits for sources 992-1023 on context 1 + base + 0x002100: Enable bits for sources 0-31 on context 2 + base + 0x002104: Enable bits for sources 32-63 on context 2 + ... + base + 0x00217C: Enable bits for sources 992-1023 on context 2 + ... + base + 0x1F1F80: Enable bits for sources 0-31 on context 15871 + base + 0x1F1F84: Enable bits for sources 32-63 on context 15871 + base + 0x1F1FFC: Enable bits for sources 992-1023 on context 15871 + ... + +It should be: + + >>> hex(0x40800000 + 0x2000 + (11 * 0x80)) + '0x40802580' + +They are all disabled: + + => md 0x40802580 + 40802580: 00000000 .... + +So, let's enable the source 4 by writing 0x10 + + => mw 0x40802580 0x10 + => md 0x40801000 1 + 40801000: 00000010 .... + +Now, let's check the context 11 priority threshold: + + 0x200000: Priority threshold for context 0 + 0x201000: Priority threshold for context 1 + 0x202000: Priority threshold for context 2 + 0x203000: Priority threshold for context 3 + +The priority threshold for context 11 should be at: + + >>> hex(0x40800000 + 0x200000 + (11 * 0x1000)) + '0x40a0b000' + + => md 0x40a0b000 + 40a0b000: 00000000 .... + +It has value 0, so all interrupts with non-zero priority should pass: + +> For example, a threshold value of zero permits all interrupts with non-zero +> priority. + +Let's see the priority of source 4 in context 11: + + 0x000000: Reserved (interrupt source 0 does not exist) + 0x000004: Interrupt source 1 priority + 0x000008: Interrupt source 2 priority + ... + 0x000FFC: Interrupt source 1023 priority + +The address should be at: + + >>> hex(0x40800000 + (4 * 0x4)) + + => md 0x40800010 + 40800010: 00000000 .... + +It has priority 0, so it would never work. Let's make it priority 1: + + => mw 0x40800010 1 + => md 0x40800010 1 + 40800010: 00000001 + +Let's check the pending interrupts: + + => md 0x40801000 1 + 40801000: 00000010 .... + +It is still pending, so let's clear it my setting the MTIMECMP to a large value. + + => md 0x40014000 + 40014000: 00aaaaaa .... + => mw 0x40014000 0xaaaaaaaa + => md 0x40014000 + 40014000: aaaaaaaa .... + => md 0x4001bff8 + 4001bff8: 0e8e6066 f`.. + => md 0x4001bff8 + 4001bff8: 0e8ea4c9 .... + => md 0x4001bff8 + 4001bff8: 0e8ece24 $... + +Now, let's claim and complete it for the context 0 which was already enabled +from the test before. + + => md 0x40a00004 1 + 40a00004: 00000004 .... + => mw 0x40a00004 4 + => md 0x40801000 1 + 40801000: 00000000 .... + +Perfect, now it is not pending anymore. + +Now, the context 0 is still enabled, so the interruptions may be sent there +instead of context 11. So let's disable the context 0 first. + + => mw 0x40802000 0 + => md 0x40802000 1 + 40802000: 00000000 .... + +Now let's fire the MTIMECMP and see if OpenSBI sees a machine trap. + + => md 0x40014000 1 + 40014000: aaaaaaaa .... + => mw 0x40014000 00aaaaaa + => md 0x40014000 1 + 40014000: 00aaaaaa .... + +Nothing happened. + +The interrupt is pending: + + => md 0x40801000 1 + 40801000: 00000010 .... + +The claim on context 0 returns 0, so not interrupt there which is expected: + + => md 0x40a00004 1 + 40a00004: 00000000 .... + +Let's compute the claim register on context 11: + + 0x200004: Interrupt Claim Process for context 0 + 0x201004: Interrupt Claim Process for context 1 + 0x202004: Interrupt Claim Process for context 2 + 0x203004: Interrupt Claim Process for context 3 + ... + + >>> hex(0x40800000 + 0x200004 + (11 * 0x1000)) + '0x40a0b004' + + => md 0x40a0b004 1 + 40a0b004: 00000000 .... + +Hmm, there is no claim ID. + +So, I checked again, and I cannot enable the interrupt on context 11: + + => md 0x40802580 1 + 40802580: 00000000 .... + => mw 0x40802580 0x10 + => md 0x40802580 1 + 40802580: 00000000 .... -- GitLab From 038d2e71566bf42c8ae3b8230209c062b8d15e1d Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 1 Aug 2024 20:22:43 +0200 Subject: [PATCH 127/310] Try opensbi without supervisor ext delegation --- lagarto-ox.nix | 1 + opensbi-dont-delegate.patch | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 opensbi-dont-delegate.patch diff --git a/lagarto-ox.nix b/lagarto-ox.nix index c389923..e757532 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -326,6 +326,7 @@ ]; patches = [ ./opensbi-timer-debug.patch + ./opensbi-dont-delegate.patch #./ox-alveo-platform-plic.patch ]; }); diff --git a/opensbi-dont-delegate.patch b/opensbi-dont-delegate.patch new file mode 100644 index 0000000..5d7388a --- /dev/null +++ b/opensbi-dont-delegate.patch @@ -0,0 +1,13 @@ +diff --git a/lib/sbi/sbi_hart.c b/lib/sbi/sbi_hart.c +index c366701..c5b5249 100644 +--- a/lib/sbi/sbi_hart.c ++++ b/lib/sbi/sbi_hart.c +@@ -199,7 +199,7 @@ static int delegate_traps(struct sbi_scratch *scratch) + return 0; + + /* Send M-mode interrupts and most exceptions to S-mode */ +- interrupts = MIP_SSIP | MIP_STIP | MIP_SEIP; ++ interrupts = MIP_SSIP | MIP_STIP; + interrupts |= sbi_pmu_irq_bit(); + + exceptions = (1U << CAUSE_MISALIGNED_FETCH) | (1U << CAUSE_BREAKPOINT) | -- GitLab From e67965cc0ae269e94d26455b86ce1de1f8521f54 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 2 Aug 2024 10:33:31 +0200 Subject: [PATCH 128/310] Always enable external machine mode interrupts --- JOURNAL.md | 49 ++++++++++++++++++++++++++++++++++--- opensbi-dont-delegate.patch | 24 ++++++++++++++++++ 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 154f370..6c00508 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -2595,7 +2595,7 @@ be able to generate an interrupt. Here is the comment where I gather the pieces: ----8<--- +---8<---{{{ From https://gitlab.bsc.es/hwdesign/rtl/core-tile/sa-fpga/ I can see that the auxiliary timer [is in fact another @@ -2695,7 +2695,7 @@ The memory map would need a bit of adjustment in the device tree, but to play wi I think I have all the pieces now. ----8<--- +---8<---}}} I will try with the last bitstream that I already had compiled, as I will have to rebuild the required packages in nix. @@ -3037,7 +3037,7 @@ From `include/sbi/riscv_encoding.h`: #define IRQ_VS_SOFT 2 #define IRQ_M_SOFT 3 #define IRQ_S_TIMER 5 - #define IRQ_VS_TIMER 6 + #define IRQ_VS_TIMER 6 #define IRQ_M_TIMER 7 #define IRQ_S_EXT 9 #define IRQ_VS_EXT 10 @@ -3254,3 +3254,46 @@ So, I checked again, and I cannot enable the interrupt on context 11: => mw 0x40802580 0x10 => md 0x40802580 1 40802580: 00000000 .... + + +Note, the first value is 0 and must be claimed: + + => md 0x40801000 1 + 40801000: 00000010 .... + => md 0x40802000 1 + 40802000: 00000010 + => md 0x40a00004 1 + 40a00004: 00000000 .... + => mw 0x40a00004 1 + => mw 0x40a00004 4 + => md 0x40a00004 1 + 40a00004: 00000004 .... + => mw 0x40a00004 4 + => md 0x40a00004 1 + 40a00004: 00000004 .... + => mw 0x40a00004 4 + => md 0x40a00004 1 + 40a00004: 00000004 .... + +## 2024-08-02 + +I see that the MIE sets the machine mode external interrupt enable in this way: + + int sbi_irqchip_init(struct sbi_scratch *scratch, bool cold_boot) + { + int rc; + const struct sbi_platform *plat = sbi_platform_ptr(scratch); + + rc = sbi_platform_irqchip_init(plat, cold_boot); + if (rc) + return rc; + + if (ext_irqfn != default_irqfn) + csr_set(CSR_MIE, MIP_MEIP); + + return 0; + } + +Only if the external interrupt function is not the default one. But for the +PLIC, it looks like the default one is being used. So let's enable the machine +mode interrupts unconditionally. diff --git a/opensbi-dont-delegate.patch b/opensbi-dont-delegate.patch index 5d7388a..343ca93 100644 --- a/opensbi-dont-delegate.patch +++ b/opensbi-dont-delegate.patch @@ -11,3 +11,27 @@ index c366701..c5b5249 100644 interrupts |= sbi_pmu_irq_bit(); exceptions = (1U << CAUSE_MISALIGNED_FETCH) | (1U << CAUSE_BREAKPOINT) | +diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c +index 0ae604a..dd4592a 100644 +--- a/lib/sbi/sbi_irqchip.c ++++ b/lib/sbi/sbi_irqchip.c +@@ -37,8 +37,7 @@ int sbi_irqchip_init(struct sbi_scratch *scratch, bool cold_boot) + if (rc) + return rc; + +- if (ext_irqfn != default_irqfn) +- csr_set(CSR_MIE, MIP_MEIP); ++ csr_set(CSR_MIE, MIP_MEIP); + + return 0; + } +@@ -47,8 +46,7 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) + { + const struct sbi_platform *plat = sbi_platform_ptr(scratch); + +- if (ext_irqfn != default_irqfn) +- csr_clear(CSR_MIE, MIP_MEIP); ++ csr_clear(CSR_MIE, MIP_MEIP); + + sbi_platform_irqchip_exit(plat); + } -- GitLab From 995b1e3848e7600abdaf1827a51f14a984933dbc Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 2 Aug 2024 12:00:24 +0200 Subject: [PATCH 129/310] Delegate external interrupts to U-Boot --- JOURNAL.md | 41 ++++++++++++++++++++++++++++++++ lagarto-ox.nix | 7 ++++-- uboot-debug-ext-interrupts.patch | 31 ++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 uboot-debug-ext-interrupts.patch diff --git a/JOURNAL.md b/JOURNAL.md index 6c00508..88b4d27 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -3297,3 +3297,44 @@ I see that the MIE sets the machine mode external interrupt enable in this way: Only if the external interrupt function is not the default one. But for the PLIC, it looks like the default one is being used. So let's enable the machine mode interrupts unconditionally. + +Let's try to cause an interruption. I would need to list all the steps. + + mw 0x40802000 0x10 # Enable interrupt for source 4 (timer) + mw 0x40800010 0xff # Make source 4 priority large + md 0x40a00004 1 # Show which value should be claimed + mw 0x40a00004 0 # Claim 0 + +Nice, I can see the trap: + + Boot HART MIDELEG : 0x0000000000000022 + Boot HART MEDELEG : 0x000000000000b109 + ... + => mw 0x40802000 0x10 # Enable interrupt for source 4 (timer) + => mw 0x40800010 0xff # Make source 4 priority large + => md 0x40a00004 1 # Show which value should be claimed + 40a00004: 00000000 .... + => mw 0x40a00004 0 # Claim 0 (weird) + Date: Fri, 2 Aug 2024 13:18:18 +0200 Subject: [PATCH 130/310] Add exception sregs command to U-Boot --- JOURNAL.md | 31 +++++++++++++++++++++++++++++-- lagarto-ox.nix | 2 ++ uboot-sregs.patch | 45 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 uboot-sregs.patch diff --git a/JOURNAL.md b/JOURNAL.md index 88b4d27..23e2cb1 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -3300,10 +3300,15 @@ mode interrupts unconditionally. Let's try to cause an interruption. I would need to list all the steps. + mw 0x40014000 0xffffffff # Disable clock interrupt + md 0x40801000 1 # Show pending interrupts (should be 0x10) mw 0x40802000 0x10 # Enable interrupt for source 4 (timer) mw 0x40800010 0xff # Make source 4 priority large - md 0x40a00004 1 # Show which value should be claimed - mw 0x40a00004 0 # Claim 0 + md 0x40a00004 1 # Claim interrupt (should read 4) + mw 0x40a00004 4 # Complete 4 + md 0x40801000 1 # Show pending interrupts (should be 0x00) + mw 0x40014000 0x00000000 # Enable clock interrupt (should cause one) + md 0x40801000 1 # Show pending interrupts (should be 0x10) Nice, I can see the trap: @@ -3338,3 +3343,25 @@ Nice, I can see the trap: sbi_trap_error: hart0: trap0: t6=0x00000000aeed3aa0 Now let's try delegating it to u-boot, and see if I can print some information. + +I don't see any message: + + Boot HART MIDELEG : 0x0000000000000222 + Boot HART MEDELEG : 0x000000000000b109 + ... + => mw 0x40014000 0xffffffff # Disable clock interrupt + => md 0x40801000 1 # Show pending interrupts (should be 0x10) + 40801000: 00000010 .... + => mw 0x40802000 0x10 # Enable interrupt for source 4 (timer) + => mw 0x40800010 0xff # Make source 4 priority large + => md 0x40a00004 1 # Claim interrupt (should read 4) + 40a00004: 00000004 .... + => mw 0x40a00004 4 # Complete 4 + => md 0x40801000 1 # Show pending interrupts (should be 0x00) + 40801000: 00000000 .... + => mw 0x40014000 0x00000000 # Enable clock interrupt (should cause one) + => md 0x40801000 1 # Show pending interrupts (should be 0x10) + 40801000: 00000010 + +I made a small subcommand U-Boot "exception sregs" to be able to dump the +supervisor registers, to check they have the proper values. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index dcdb300..fabab92 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -219,6 +219,7 @@ patches = [ #./u-boot-debug.patch ./uboot-debug-ext-interrupts.patch + ./uboot-sregs.patch ]; # # CONFIG_SERIAL_PRESENT=n @@ -245,6 +246,7 @@ CONFIG_SMP=n CONFIG_TRACE_EARLY=y CONFIG_CMD_MEMTEST=y + CONFIG_CMD_EXCEPTION=y '' # # Enable debug logs # + diff --git a/uboot-sregs.patch b/uboot-sregs.patch new file mode 100644 index 0000000..ad707e1 --- /dev/null +++ b/uboot-sregs.patch @@ -0,0 +1,45 @@ +diff --git a/cmd/riscv/exception.c b/cmd/riscv/exception.c +index f38f454a0b..87877ab235 100644 +--- a/cmd/riscv/exception.c ++++ b/cmd/riscv/exception.c +@@ -56,6 +56,23 @@ static int do_undefined(struct cmd_tbl *cmdtp, int flag, int argc, + return CMD_RET_FAILURE; + } + ++static int do_sregs(struct cmd_tbl *cmdtp, int flag, int argc, ++ char *const argv[]) ++{ ++ ulong stvec, sie, sstatus; ++ ++ asm volatile ("fence"); ++ asm volatile ("csrr %0, stvec" : "=r"(stvec) : ); ++ asm volatile ("csrr %0, sie" : "=r"(sie) : ); ++ asm volatile ("csrr %0, sstatus" : "=r"(sstatus) : ); ++ ++ printf("stvec : 0x%08lx\n", stvec); ++ printf("sie : 0x%08lx\n", sie); ++ printf("sstatus : 0x%08lx\n", sstatus); ++ ++ return CMD_RET_SUCCESS; ++} ++ + static struct cmd_tbl cmd_sub[] = { + U_BOOT_CMD_MKENT(compressed, CONFIG_SYS_MAXARGS, 1, do_compressed, + "", ""), +@@ -67,6 +84,8 @@ static struct cmd_tbl cmd_sub[] = { + "", ""), + U_BOOT_CMD_MKENT(undefined, CONFIG_SYS_MAXARGS, 1, do_undefined, + "", ""), ++ U_BOOT_CMD_MKENT(sregs, CONFIG_SYS_MAXARGS, 1, do_sregs, ++ "", ""), + }; + + static char exception_help_text[] = +@@ -77,6 +96,7 @@ static char exception_help_text[] = + " ialign16 - 16 bit aligned instruction\n" + " undefined - illegal instruction\n" + " unaligned - load address misaligned\n" ++ " sregs - print supervisor registers\n" + ; + + #include -- GitLab From 33fb07481a1318a3a895f488943f394d6d90c00b Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 2 Aug 2024 13:43:56 +0200 Subject: [PATCH 131/310] Add exception enable U-Boot command --- lagarto-ox.nix | 2 +- ...regs.patch => uboot-exception-extras.patch | 36 ++++++++++++++----- 2 files changed, 29 insertions(+), 9 deletions(-) rename uboot-sregs.patch => uboot-exception-extras.patch (52%) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index fabab92..13a0155 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -219,7 +219,7 @@ patches = [ #./u-boot-debug.patch ./uboot-debug-ext-interrupts.patch - ./uboot-sregs.patch + ./uboot-exception-extras.patch ]; # # CONFIG_SERIAL_PRESENT=n diff --git a/uboot-sregs.patch b/uboot-exception-extras.patch similarity index 52% rename from uboot-sregs.patch rename to uboot-exception-extras.patch index ad707e1..b2511ba 100644 --- a/uboot-sregs.patch +++ b/uboot-exception-extras.patch @@ -1,24 +1,41 @@ diff --git a/cmd/riscv/exception.c b/cmd/riscv/exception.c -index f38f454a0b..87877ab235 100644 +index f38f454a0b..9de4effe47 100644 --- a/cmd/riscv/exception.c +++ b/cmd/riscv/exception.c -@@ -56,6 +56,23 @@ static int do_undefined(struct cmd_tbl *cmdtp, int flag, int argc, +@@ -56,6 +56,40 @@ static int do_undefined(struct cmd_tbl *cmdtp, int flag, int argc, return CMD_RET_FAILURE; } +static int do_sregs(struct cmd_tbl *cmdtp, int flag, int argc, + char *const argv[]) +{ -+ ulong stvec, sie, sstatus; ++ ulong stvec, sie, sip, sstatus; + + asm volatile ("fence"); + asm volatile ("csrr %0, stvec" : "=r"(stvec) : ); + asm volatile ("csrr %0, sie" : "=r"(sie) : ); ++ asm volatile ("csrr %0, sip" : "=r"(sip) : ); + asm volatile ("csrr %0, sstatus" : "=r"(sstatus) : ); + -+ printf("stvec : 0x%08lx\n", stvec); -+ printf("sie : 0x%08lx\n", sie); -+ printf("sstatus : 0x%08lx\n", sstatus); ++ printf("stvec : 0x%016lx\n", stvec); ++ printf("sie : 0x%016lx\n", sie); ++ printf("sip : 0x%016lx\n", sip); ++ printf("sstatus : 0x%016lx\n", sstatus); ++ ++ return CMD_RET_SUCCESS; ++} ++ ++static int do_enable(struct cmd_tbl *cmdtp, int flag, int argc, ++ char *const argv[]) ++{ ++ ulong which = SIE_SSIE | SIE_SEIE | SIE_STIE; ++ ++ asm volatile ( ++ "csrsi sstatus, 2\n" /* Enable SIE */ ++ "csrs sie, %0\n" /* Enable selected interrupts */ ++ : /* no output */ ++ : "r" (which) ++ ); + + return CMD_RET_SUCCESS; +} @@ -26,20 +43,23 @@ index f38f454a0b..87877ab235 100644 static struct cmd_tbl cmd_sub[] = { U_BOOT_CMD_MKENT(compressed, CONFIG_SYS_MAXARGS, 1, do_compressed, "", ""), -@@ -67,6 +84,8 @@ static struct cmd_tbl cmd_sub[] = { +@@ -67,6 +101,10 @@ static struct cmd_tbl cmd_sub[] = { "", ""), U_BOOT_CMD_MKENT(undefined, CONFIG_SYS_MAXARGS, 1, do_undefined, "", ""), + U_BOOT_CMD_MKENT(sregs, CONFIG_SYS_MAXARGS, 1, do_sregs, ++ "", ""), ++ U_BOOT_CMD_MKENT(enable, CONFIG_SYS_MAXARGS, 1, do_enable, + "", ""), }; static char exception_help_text[] = -@@ -77,6 +96,7 @@ static char exception_help_text[] = +@@ -77,6 +115,8 @@ static char exception_help_text[] = " ialign16 - 16 bit aligned instruction\n" " undefined - illegal instruction\n" " unaligned - load address misaligned\n" + " sregs - print supervisor registers\n" ++ " enable - enable supervisor interrupts\n" ; #include -- GitLab From 98f794e52d21564811dff70268faccf794636306 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 2 Aug 2024 15:27:20 +0200 Subject: [PATCH 132/310] Enable timer command in U-Boot --- JOURNAL.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ lagarto-ox.nix | 1 + 2 files changed, 46 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index 23e2cb1..fb84783 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -3365,3 +3365,48 @@ I don't see any message: I made a small subcommand U-Boot "exception sregs" to be able to dump the supervisor registers, to check they have the proper values. + + Boot HART MIDELEG : 0x0000000000000222 + Boot HART MEDELEG : 0x000000000000b109 + >>Core: 11 devices, 8 uclasses, devicetree: board + Loading Environment from nowhere... OK + In: serial,usbkbd + Out: serial,vidconsole + Err: serial,vidconsole + No working controllers found + Net: No ethernet found. + Working FDT set to c0000000 + Hit any key to stop autoboot: 0 + => exception sregs + stvec : 0x00000000af6f4400 + sie : 0x0000000000000000 + sip : 0x0000000000000000 + sstatus : 0x8000000200006000 + => exception enable + => exception sregs + stvec : 0x00000000af6f4400 + sie : 0x0000000000000222 + sip : 0x0000000000000000 + sstatus : 0x8000000200006002 + => mw 0x40014000 0xffffffff # Disable clock interrupt + => md 0x40801000 1 # Show pending interrupts (should be 0x10) + 40801000: 00000010 .... + => mw 0x40802000 0x10 # Enable interrupt for source 4 (timer) + => mw 0x40800010 0xff # Make source 4 priority large + => md 0x40a00004 1 # Claim interrupt (should read 4) + 40a00004: 00000004 .... + => mw 0x40a00004 4 # Complete 4 + => md 0x40801000 1 # Show pending interrupts (should be 0x00) + 40801000: 00000000 .... + => mw 0x40014000 0x00000000 # Enable clock interrupt (should cause one) + => md 0x40801000 1 # Show pending interrupts (should be 0x10) + 40801000: 00000010 .... + => exception sregs + stvec : 0x00000000af6f4400 + sie : 0x0000000000000222 + sip : 0x0000000000000200 + sstatus : 0x8000000200006002 + +Now I can see the external interrupt in supervisor arriving to the SIP. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 13a0155..636cc8e 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -247,6 +247,7 @@ CONFIG_TRACE_EARLY=y CONFIG_CMD_MEMTEST=y CONFIG_CMD_EXCEPTION=y + CONFIG_CMD_TIMER=y '' # # Enable debug logs # + -- GitLab From 91d3e9b163c15f5ac3b7a362cb0a4906f77ed233 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 2 Aug 2024 16:11:52 +0200 Subject: [PATCH 133/310] Set stvec to zero --- JOURNAL.md | 50 ++++++++++++++++++++++++++++++++++++ uboot-exception-extras.patch | 9 ++++--- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index fb84783..eb92795 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -3410,3 +3410,53 @@ supervisor registers, to check they have the proper values. sstatus : 0x8000000200006002 Now I can see the external interrupt in supervisor arriving to the SIP. + +Let's try to cause an interrupt with the normal CLINT: + + `define CLINT_XBAR_ID 3 + `define CLINT_BASE_ADDR 64'h0000_0000_4010_0000 + `define CLINT_END_ADDR 64'h0000_0000_4010_FFFF + + `define AUX_TIMER_XBAR_ID 2 + `define AUX_TIMER_BASE_ADDR 64'h0000_0000_4001_0000 // Need to be this space because we use a clint as aux timer + `define AUX_TIMER_END_ADDR 64'h0000_0000_4001_FFFF + + => exception sregs + stvec : 0x00000000af6f4400 + sie : 0x0000000000000000 + sip : 0x0000000000000000 + sstatus : 0x8000000200006000 + => exception enable + => exception sregs + stvec : 0x00000000af6f4400 + sie : 0x0000000000000222 + sip : 0x0000000000000000 + sstatus : 0x8000000200006002 + => md 0x4010bff8 # Show normal CLINT mtime value + 4010bff8: 00c8159b .... + => md 0x4010bff8 # Show normal CLINT mtime value + 4010bff8: 00c84453 SD.. + => md 0x4010bff8 # Show normal CLINT mtime value + 4010bff8: 00c865b5 .e.. + => md 0x40104000 # Show normal CLINT mtimecmp value + 40104000: 00000000 .... + => mw 0x40104000 aaaaaaaa # Disable interrupt for CLINT + => md 0x40104000 # Show normal CLINT mtimecmp value + 40104000: aaaaaaaa .... + => exception sregs + stvec : 0x00000000af6f4400 + sie : 0x0000000000000222 + sip : 0x0000000000000000 + sstatus : 0x8000000200006002 + => mw 0x40104000 0 # Enable interrupt for CLINT + => exception sregs + stvec : 0x00000000af6f4400 + sie : 0x0000000000000222 + sip : 0x0000000000000000 <-- nothing here + sstatus : 0x8000000200006002 + => md 0x40104000 # Show normal CLINT mtimecmp value + 40104000: 00000000 + +No interrupts seem to arrive at the SIP register. + +Let's set the stvec to zero, so it causes a machine exception. diff --git a/uboot-exception-extras.patch b/uboot-exception-extras.patch index b2511ba..7f35966 100644 --- a/uboot-exception-extras.patch +++ b/uboot-exception-extras.patch @@ -1,8 +1,8 @@ diff --git a/cmd/riscv/exception.c b/cmd/riscv/exception.c -index f38f454a0b..9de4effe47 100644 +index f38f454a0b..9bc554b0aa 100644 --- a/cmd/riscv/exception.c +++ b/cmd/riscv/exception.c -@@ -56,6 +56,40 @@ static int do_undefined(struct cmd_tbl *cmdtp, int flag, int argc, +@@ -56,6 +56,41 @@ static int do_undefined(struct cmd_tbl *cmdtp, int flag, int argc, return CMD_RET_FAILURE; } @@ -33,6 +33,7 @@ index f38f454a0b..9de4effe47 100644 + asm volatile ( + "csrsi sstatus, 2\n" /* Enable SIE */ + "csrs sie, %0\n" /* Enable selected interrupts */ ++ "csrwi stvec, 0\n" /* Redirect trap handler to NULL */ + : /* no output */ + : "r" (which) + ); @@ -43,7 +44,7 @@ index f38f454a0b..9de4effe47 100644 static struct cmd_tbl cmd_sub[] = { U_BOOT_CMD_MKENT(compressed, CONFIG_SYS_MAXARGS, 1, do_compressed, "", ""), -@@ -67,6 +101,10 @@ static struct cmd_tbl cmd_sub[] = { +@@ -67,6 +102,10 @@ static struct cmd_tbl cmd_sub[] = { "", ""), U_BOOT_CMD_MKENT(undefined, CONFIG_SYS_MAXARGS, 1, do_undefined, "", ""), @@ -54,7 +55,7 @@ index f38f454a0b..9de4effe47 100644 }; static char exception_help_text[] = -@@ -77,6 +115,8 @@ static char exception_help_text[] = +@@ -77,6 +116,8 @@ static char exception_help_text[] = " ialign16 - 16 bit aligned instruction\n" " undefined - illegal instruction\n" " unaligned - load address misaligned\n" -- GitLab From 19dbb857c8631a538b81026be2a8bf6c5819543a Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Sat, 3 Aug 2024 14:55:35 +0200 Subject: [PATCH 134/310] Enable MEIP in OpenSBI --- lagarto-ox.nix | 1 + opensbi-enable-meip.patch | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 opensbi-enable-meip.patch diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 636cc8e..7863535 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -332,6 +332,7 @@ ]; patches = [ ./opensbi-timer-debug.patch + ./opensbi-enable-meip.patch #./opensbi-dont-delegate.patch #./ox-alveo-platform-plic.patch ]; diff --git a/opensbi-enable-meip.patch b/opensbi-enable-meip.patch new file mode 100644 index 0000000..9121efc --- /dev/null +++ b/opensbi-enable-meip.patch @@ -0,0 +1,24 @@ +diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c +index 0ae604a..dd4592a 100644 +--- a/lib/sbi/sbi_irqchip.c ++++ b/lib/sbi/sbi_irqchip.c +@@ -37,8 +37,7 @@ int sbi_irqchip_init(struct sbi_scratch *scratch, bool cold_boot) + if (rc) + return rc; + +- if (ext_irqfn != default_irqfn) +- csr_set(CSR_MIE, MIP_MEIP); ++ csr_set(CSR_MIE, MIP_MEIP); + + return 0; + } +@@ -47,8 +46,7 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) + { + const struct sbi_platform *plat = sbi_platform_ptr(scratch); + +- if (ext_irqfn != default_irqfn) +- csr_clear(CSR_MIE, MIP_MEIP); ++ csr_clear(CSR_MIE, MIP_MEIP); + + sbi_platform_irqchip_exit(plat); + } -- GitLab From 62bd702929f6ac9e80f602ba24a3723aaaf7f29d Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 06:24:42 +0200 Subject: [PATCH 135/310] Restore stvec to its original value --- JOURNAL.md | 71 ++++++++++++++++++++++++++++++++++++ uboot-exception-extras.patch | 9 ++--- 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index eb92795..25681be 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -3460,3 +3460,74 @@ Let's try to cause an interrupt with the normal CLINT: No interrupts seem to arrive at the SIP register. Let's set the stvec to zero, so it causes a machine exception. + +# 2024-08-21 + +I tried with the new bitstream (ox_u55c_46619ef4.bit) setting the stvec to zero +and I can see the OpenSBI handler stopping, probably due to the jump to zero +address: + + Boot HART MIDELEG : 0x0000000000000222 + Boot HART MEDELEG : 0x000000000000b109 + >>Core: 11 devices, 8 uclasses, devicetree: board + Loading Environment from nowhere... OK + In: serial,usbkbd + Out: serial,vidconsole + Err: serial,vidconsole + No working controllers found + Net: No ethernet found. + Working FDT set to c0000000 + Hit any key to stop autoboot: 0 + + Device 0: unknown device + + Device 1: unknown device + scanning bus for devices... + + Device 0: unknown device + starting USB... + No working controllers found + No ethernet found. + No ethernet found. + => exception sregs + stvec : 0x00000000af6f4400 + sie : 0x0000000000000000 + sip : 0x0000000000000000 + sstatus : 0x8000000200006000 + => exception enable + => exception sregs + stvec : 0x0000000000000000 + sie : 0x0000000000000222 + sip : 0x0000000000000000 + sstatus : 0x8000000200006002 + => mw 0x40014000 0xffffffff # Disable clock interrupt + => md 0x40801000 1 # Show pending interrupts (should be 0x10) + 40801000: 00000010 .... + => mw 0x40802000 0x10 # Enable interrupt for source 4 (timer) + => mw 0x40800010 0xff # Make source 4 priority large + Date: Wed, 21 Aug 2024 07:13:45 +0200 Subject: [PATCH 136/310] Dump machine registers in OpenSBI --- JOURNAL.md | 56 ++++++++++++++++++++++++++++++++++++++++ lagarto-ox.nix | 1 + opensbi-dump-mregs.patch | 19 ++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 opensbi-dump-mregs.patch diff --git a/JOURNAL.md b/JOURNAL.md index 25681be..2a8f981 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -3531,3 +3531,59 @@ address: Let's revert the stvec to its original value and try again and see if we can see the delegation properly being handed to the supervisor handler. + +It continues to do the same trap in machine mode, rather than handing the +interrupt over to supervisor. + + Boot HART MIDELEG : 0x0000000000000222 + Boot HART MEDELEG : 0x000000000000b109 + >>Core: 11 devices, 8 uclasses, devicetree: board + Loading Environment from nowhere... OK + In: serial,usbkbd + Out: serial,vidconsole + Err: serial,vidconsole + No working controllers found + Net: No ethernet found. + Working FDT set to c0000000 + Hit any key to stop autoboot: 0 + => exception sregs + stvec : 0x00000000af6f4400 + sie : 0x0000000000000000 + sip : 0x0000000000000000 + sstatus : 0x8000000200006000 + => exception enable + => exception sregs + stvec : 0x00000000af6f4400 <-------- now stvec is left as-is + sie : 0x0000000000000222 + sip : 0x0000000000000000 + sstatus : 0x8000000200006002 + => mw 0x40014000 0xffffffff # Disable clock interrupt + => md 0x40801000 1 # Show pending interrupts (should be 0x10) + 40801000: 00000010 .... + => mw 0x40802000 0x10 # Enable interrupt for source 4 (timer) + => mw 0x40800010 0xff # Make source 4 priority large + Date: Wed, 21 Aug 2024 07:56:17 +0200 Subject: [PATCH 137/310] Disable MEIE bit in MIE register --- JOURNAL.md | 2 ++ lagarto-ox.nix | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/JOURNAL.md b/JOURNAL.md index 2a8f981..c73db79 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -3587,3 +3587,5 @@ interrupt over to supervisor. sbi_trap_error: hart0: trap0: t6=0x00000000aeed2aa0 Let's dump the machine registers too. + +Let's also disable the MEIE bit in MIE register. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index e9c6986..af77342 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -332,7 +332,7 @@ ]; patches = [ ./opensbi-timer-debug.patch - ./opensbi-enable-meip.patch + #./opensbi-enable-meip.patch ./opensbi-dump-mregs.patch #./opensbi-dont-delegate.patch #./ox-alveo-platform-plic.patch -- GitLab From 60631ec37f034210226c0017b45da4e9e0c6b2bb Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 08:00:31 +0200 Subject: [PATCH 138/310] Enable SEIE in MIE directly from machine mode --- lagarto-ox.nix | 1 + opensbi-enable-seip.patch | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 opensbi-enable-seip.patch diff --git a/lagarto-ox.nix b/lagarto-ox.nix index af77342..dd8ee93 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -333,6 +333,7 @@ patches = [ ./opensbi-timer-debug.patch #./opensbi-enable-meip.patch + ./opensbi-enable-seip.patch ./opensbi-dump-mregs.patch #./opensbi-dont-delegate.patch #./ox-alveo-platform-plic.patch diff --git a/opensbi-enable-seip.patch b/opensbi-enable-seip.patch new file mode 100644 index 0000000..b1bf8b0 --- /dev/null +++ b/opensbi-enable-seip.patch @@ -0,0 +1,24 @@ +diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c +index 0ae604a..dd4592a 100644 +--- a/lib/sbi/sbi_irqchip.c ++++ b/lib/sbi/sbi_irqchip.c +@@ -37,8 +37,7 @@ int sbi_irqchip_init(struct sbi_scratch *scratch, bool cold_boot) + if (rc) + return rc; + +- if (ext_irqfn != default_irqfn) +- csr_set(CSR_MIE, MIP_MEIP); ++ csr_set(CSR_MIE, MIP_SEIP); + + return 0; + } +@@ -47,8 +46,7 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) + { + const struct sbi_platform *plat = sbi_platform_ptr(scratch); + +- if (ext_irqfn != default_irqfn) +- csr_clear(CSR_MIE, MIP_MEIP); ++ csr_clear(CSR_MIE, MIP_SEIP); + + sbi_platform_irqchip_exit(plat); + } -- GitLab From a858b3548e2e783c553b26c9d1e67b6aa1e11741 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 08:08:13 +0200 Subject: [PATCH 139/310] Enable SIE in MSTATUS too --- opensbi-enable-seip.patch | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/opensbi-enable-seip.patch b/opensbi-enable-seip.patch index b1bf8b0..778ff9e 100644 --- a/opensbi-enable-seip.patch +++ b/opensbi-enable-seip.patch @@ -1,24 +1,26 @@ diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c -index 0ae604a..dd4592a 100644 +index 0ae604a..94832c8 100644 --- a/lib/sbi/sbi_irqchip.c +++ b/lib/sbi/sbi_irqchip.c -@@ -37,8 +37,7 @@ int sbi_irqchip_init(struct sbi_scratch *scratch, bool cold_boot) +@@ -37,8 +37,8 @@ int sbi_irqchip_init(struct sbi_scratch *scratch, bool cold_boot) if (rc) return rc; - if (ext_irqfn != default_irqfn) - csr_set(CSR_MIE, MIP_MEIP); + csr_set(CSR_MIE, MIP_SEIP); ++ csr_set(CSR_MSTATUS, MSTATUS_SIE); return 0; } -@@ -47,8 +46,7 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) +@@ -47,8 +47,8 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) { const struct sbi_platform *plat = sbi_platform_ptr(scratch); - if (ext_irqfn != default_irqfn) - csr_clear(CSR_MIE, MIP_MEIP); + csr_clear(CSR_MIE, MIP_SEIP); ++ csr_clear(CSR_MSTATUS, MSTATUS_SIE); sbi_platform_irqchip_exit(plat); } -- GitLab From 298595ba3c95282d14b8ec044c0436c6f0300a0c Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 08:16:14 +0200 Subject: [PATCH 140/310] Enable also MEIE and MIE --- opensbi-enable-seip.patch | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opensbi-enable-seip.patch b/opensbi-enable-seip.patch index 778ff9e..d1861e5 100644 --- a/opensbi-enable-seip.patch +++ b/opensbi-enable-seip.patch @@ -8,8 +8,8 @@ index 0ae604a..94832c8 100644 - if (ext_irqfn != default_irqfn) - csr_set(CSR_MIE, MIP_MEIP); -+ csr_set(CSR_MIE, MIP_SEIP); -+ csr_set(CSR_MSTATUS, MSTATUS_SIE); ++ csr_set(CSR_MIE, MIP_MEIP | MIP_SEIP); ++ csr_set(CSR_MSTATUS, MSTATUS_MIE | MSTATUS_SIE); return 0; } @@ -19,8 +19,8 @@ index 0ae604a..94832c8 100644 - if (ext_irqfn != default_irqfn) - csr_clear(CSR_MIE, MIP_MEIP); -+ csr_clear(CSR_MIE, MIP_SEIP); -+ csr_clear(CSR_MSTATUS, MSTATUS_SIE); ++ csr_clear(CSR_MIE, MIP_MEIP | MIP_SEIP); ++ csr_clear(CSR_MSTATUS, MSTATUS_MIE | MSTATUS_SIE); sbi_platform_irqchip_exit(plat); } -- GitLab From b7287bd4df14d07480414df479a27ec944e654ce Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 09:22:43 +0200 Subject: [PATCH 141/310] Port supervisor PLIC test to OpenSBI --- lagarto-ox.nix | 3 +- opensbi-test-plic.patch | 182 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 opensbi-test-plic.patch diff --git a/lagarto-ox.nix b/lagarto-ox.nix index dd8ee93..d55b1f6 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -333,7 +333,8 @@ patches = [ ./opensbi-timer-debug.patch #./opensbi-enable-meip.patch - ./opensbi-enable-seip.patch + #./opensbi-enable-seip.patch + ./opensbi-test-plic.patch ./opensbi-dump-mregs.patch #./opensbi-dont-delegate.patch #./ox-alveo-platform-plic.patch diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch new file mode 100644 index 0000000..49c2950 --- /dev/null +++ b/opensbi-test-plic.patch @@ -0,0 +1,182 @@ +diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c +index 0ae604a..fc0c286 100644 +--- a/lib/sbi/sbi_irqchip.c ++++ b/lib/sbi/sbi_irqchip.c +@@ -9,6 +9,9 @@ + + #include + #include ++#include ++ ++static void do_plic_test(void); + + static int default_irqfn(void) + { +@@ -37,8 +40,10 @@ int sbi_irqchip_init(struct sbi_scratch *scratch, bool cold_boot) + if (rc) + return rc; + +- if (ext_irqfn != default_irqfn) +- csr_set(CSR_MIE, MIP_MEIP); ++ //csr_set(CSR_MIE, MIP_SEIP); ++ //csr_set(CSR_MSTATUS, MSTATUS_SIE); ++ ++ do_plic_test(); + + return 0; + } +@@ -47,8 +52,152 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) + { + const struct sbi_platform *plat = sbi_platform_ptr(scratch); + +- if (ext_irqfn != default_irqfn) +- csr_clear(CSR_MIE, MIP_MEIP); ++ //csr_clear(CSR_MIE, MIP_SEIP); ++ //csr_clear(CSR_MSTATUS, MSTATUS_SIE); + + sbi_platform_irqchip_exit(plat); + } ++ ++ ++/* ----------------- PLIC tests ---------------- */ ++ ++ ++#define MIE_MEIE (1UL << 11) // Machine External Interrupt Enable ++#define SIE_SEIE (1UL << 9) ++#define MIDELEG_SEIE (1UL << 9) // Delegate Machine External Interrupt to Supervisor ++#define PLIC_TIMER_PORT 4 ++// Base address of PLIC ++#define PLIC_BASE 0x0000000040800000UL ++#define PLIC_PRIORITY_OFFSET 0x0UL ++#define PLIC_PENDING_OFFSET 0x1000UL ++#define PLIC_ENABLE_OFFSET 0x2000UL ++#define PLIC_THRESHOLD_OFFSET 0x200000UL ++#define PLIC_CLAIM_OFFSET 0x200004UL ++ ++// Aux timer ++#define AUX_TIMER_BASE 0x40010000UL ++#define MTIMECMP_OFFSET 0x4000UL ++#define MTIME_OFFSET 0xBFF8UL ++ ++#define MSTATUS_MPP_MASK (3 << 11) ++#define MSTATUS_MPP_SUPERVISOR (1 << 11) ++ ++static void __attribute__((optimize("O0"))) switch_to_supervisor_mode(int (*target_address)(void)) ++{ ++ unsigned long mstatus; ++ ++ // Read the current mstatus ++ asm volatile("csrr %0, mstatus" : "=r"(mstatus)); ++ ++ // Set the MPP field to supervisor mode ++ mstatus = (mstatus & ~MSTATUS_MPP_MASK) | MSTATUS_MPP_SUPERVISOR; ++ ++ // Write back the modified mstatus ++ asm volatile("csrw mstatus, %0" : : "r"(mstatus)); ++ ++ // Set the mepc to the target address ++ asm volatile("csrw mepc, %0" : : "r"(target_address)); ++ ++ // Use mret to return to the specified address in supervisor mode ++ asm volatile("mret"); ++} ++ ++static int supervisor_mode_code(void) ++{ ++ sbi_printf("Hello from supervisor, waiting for interrupt\n"); ++ int count = 0; ++ while (1) { ++ if (count == 10000) { ++ sbi_printf("."); ++ count = 0; ++ } ++ count++; ++ } ++ return 0; ++} ++ ++ ++static void do_plic_test(void) ++{ ++ char *prefix = "\t"; ++ char *suffix = "\t"; ++ sbi_printf("--- TESTING PLIC ---\n"); ++ ++ volatile unsigned long *mtime = (unsigned long *)(AUX_TIMER_BASE + MTIME_OFFSET); ++ volatile unsigned long *mtimecmp = (unsigned long *)(AUX_TIMER_BASE + MTIMECMP_OFFSET); ++ ++ /* Disable auxiliar timer interrupt */ ++ *mtimecmp = 0xffffffffUL; ++ sbi_printf("Timer interrupt disabled\n"); ++ ++ sbi_printf("%sSIE%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_SIE)); ++ sbi_printf("%sSSTATUS%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_SSTATUS)); ++ sbi_printf("%sMIDELEG%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_MIDELEG)); ++ ++ /* Enable supervisor interrupt delegation */ ++ csr_set(CSR_SIE, SIE_SEIE); // Enable supervisor external interrupts ++ csr_set(CSR_SSTATUS, SSTATUS_SIE); // Enable global interrupts in supervisor mode ++ csr_set(CSR_MIDELEG, MIDELEG_SEIE); // Delegate machine interrupts to supervisor mode ++ sbi_printf("Enabled supervisor delegation:\n"); ++ ++ sbi_printf("%sSIE%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_SIE)); ++ sbi_printf("%sSSTATUS%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_SSTATUS)); ++ sbi_printf("%sMIDELEG%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_MIDELEG)); ++ ++ /* Configure PLIC aux timer input */ ++ volatile unsigned *plic_priority = (unsigned *)(PLIC_BASE + PLIC_PRIORITY_OFFSET + PLIC_TIMER_PORT * 4); ++ volatile unsigned *plic_enable = (unsigned *)(PLIC_BASE + PLIC_ENABLE_OFFSET); ++ volatile unsigned *plic_threshold = (unsigned *)(PLIC_BASE + PLIC_THRESHOLD_OFFSET); ++ ++ *plic_priority = PLIC_TIMER_PORT; ++ *plic_threshold = PLIC_TIMER_PORT - 1; ++ *plic_enable |= (1 << PLIC_TIMER_PORT); ++ ++ sbi_printf("Timer enabled in PLIC\n"); ++ ++ sbi_printf("%sMIE%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_MIE)); ++ sbi_printf("%sMSTATUS%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_MSTATUS)); ++ ++ /* Enable external timer interrupts */ ++ csr_set(CSR_MIE, MIE_MEIE); /* Needed? */ ++ csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ ++ ++ sbi_printf("External timer interrupts enabled in machine mode\n"); ++ ++ sbi_printf("%sMIE%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_MIE)); ++ sbi_printf("%sMSTATUS%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_MSTATUS)); ++ ++ /* Enable timer interrupt */ ++ *mtimecmp = *mtime + 10000; ++ ++ sbi_printf("Timer alarm programmed\n"); ++ sbi_printf("Switching to supervisor\n"); ++ ++ // Switch to supervisor mode and execute supervisor_mode_code ++ switch_to_supervisor_mode(&supervisor_mode_code); ++ ++ /* Never reached */ ++ ++ int count = 0; ++ // Main loop ++ while (1) { ++ if (count == 10000) { ++ sbi_printf("Still in machine mode\n"); ++ count = 0; ++ } ++ count++; ++ // Main application code ++ } ++} ++ ++ -- GitLab From 1faef5b452c2dd7efc2e0fad5945ab67818ff8b6 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 09:29:22 +0200 Subject: [PATCH 142/310] Don't enable machine interrupt --- opensbi-test-plic.patch | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch index 49c2950..c263014 100644 --- a/opensbi-test-plic.patch +++ b/opensbi-test-plic.patch @@ -1,5 +1,5 @@ diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c -index 0ae604a..fc0c286 100644 +index 0ae604a..11656e0 100644 --- a/lib/sbi/sbi_irqchip.c +++ b/lib/sbi/sbi_irqchip.c @@ -9,6 +9,9 @@ @@ -140,21 +140,21 @@ index 0ae604a..fc0c286 100644 + + sbi_printf("Timer enabled in PLIC\n"); + -+ sbi_printf("%sMIE%s: 0x%" PRILX "\n", -+ prefix, suffix, csr_read(CSR_MIE)); -+ sbi_printf("%sMSTATUS%s: 0x%" PRILX "\n", -+ prefix, suffix, csr_read(CSR_MSTATUS)); -+ -+ /* Enable external timer interrupts */ -+ csr_set(CSR_MIE, MIE_MEIE); /* Needed? */ -+ csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ -+ -+ sbi_printf("External timer interrupts enabled in machine mode\n"); -+ -+ sbi_printf("%sMIE%s: 0x%" PRILX "\n", -+ prefix, suffix, csr_read(CSR_MIE)); -+ sbi_printf("%sMSTATUS%s: 0x%" PRILX "\n", -+ prefix, suffix, csr_read(CSR_MSTATUS)); ++// sbi_printf("%sMIE%s: 0x%" PRILX "\n", ++// prefix, suffix, csr_read(CSR_MIE)); ++// sbi_printf("%sMSTATUS%s: 0x%" PRILX "\n", ++// prefix, suffix, csr_read(CSR_MSTATUS)); ++// ++// /* Enable external timer interrupts */ ++// csr_set(CSR_MIE, MIE_MEIE); /* Needed? */ ++// csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ ++// ++// sbi_printf("External timer interrupts enabled in machine mode\n"); ++// ++// sbi_printf("%sMIE%s: 0x%" PRILX "\n", ++// prefix, suffix, csr_read(CSR_MIE)); ++// sbi_printf("%sMSTATUS%s: 0x%" PRILX "\n", ++// prefix, suffix, csr_read(CSR_MSTATUS)); + + /* Enable timer interrupt */ + *mtimecmp = *mtime + 10000; -- GitLab From fd2b7667604cb92176608cb0a8658ed1ff9fd6b2 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 09:31:09 +0200 Subject: [PATCH 143/310] Improve progress print --- opensbi-test-plic.patch | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch index c263014..63cf7a5 100644 --- a/opensbi-test-plic.patch +++ b/opensbi-test-plic.patch @@ -1,5 +1,5 @@ diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c -index 0ae604a..11656e0 100644 +index 0ae604a..44601a4 100644 --- a/lib/sbi/sbi_irqchip.c +++ b/lib/sbi/sbi_irqchip.c @@ -9,6 +9,9 @@ @@ -25,7 +25,7 @@ index 0ae604a..11656e0 100644 return 0; } -@@ -47,8 +52,152 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) +@@ -47,8 +52,151 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) { const struct sbi_platform *plat = sbi_platform_ptr(scratch); @@ -84,13 +84,12 @@ index 0ae604a..11656e0 100644 +static int supervisor_mode_code(void) +{ + sbi_printf("Hello from supervisor, waiting for interrupt\n"); -+ int count = 0; ++ int i = 0; ++ char *s = "-\\|/"; + while (1) { -+ if (count == 10000) { -+ sbi_printf("."); -+ count = 0; -+ } -+ count++; ++ sbi_printf("\r%c", s[i++]); ++ if (i >= 4) ++ i = 0; + } + return 0; +} -- GitLab From baf45e6749940be446e750acf424f20a2eb1329c Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 09:44:53 +0200 Subject: [PATCH 144/310] Dump more registers in PLIC test --- opensbi-test-plic.patch | 56 ++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch index 63cf7a5..17b1b50 100644 --- a/opensbi-test-plic.patch +++ b/opensbi-test-plic.patch @@ -1,5 +1,5 @@ diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c -index 0ae604a..44601a4 100644 +index 0ae604a..a3df88c 100644 --- a/lib/sbi/sbi_irqchip.c +++ b/lib/sbi/sbi_irqchip.c @@ -9,6 +9,9 @@ @@ -61,6 +61,27 @@ index 0ae604a..44601a4 100644 +#define MSTATUS_MPP_MASK (3 << 11) +#define MSTATUS_MPP_SUPERVISOR (1 << 11) + ++static void dumpregs(void) ++{ ++ char *prefix = "\t"; ++ char *suffix = "\t"; ++ sbi_printf("Registers:\n"); ++ sbi_printf("%sMIE%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_MIE)); ++ sbi_printf("%sMIP%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_MIP)); ++ sbi_printf("%sMSTATUS%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_MSTATUS)); ++ sbi_printf("%sMIDELEG%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_MIDELEG)); ++ sbi_printf("%sSIE%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_SIE)); ++ sbi_printf("%sSIP%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_SIP)); ++ sbi_printf("%sSSTATUS%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_SSTATUS)); ++} ++ +static void __attribute__((optimize("O0"))) switch_to_supervisor_mode(int (*target_address)(void)) +{ + unsigned long mstatus; @@ -83,10 +104,12 @@ index 0ae604a..44601a4 100644 + +static int supervisor_mode_code(void) +{ -+ sbi_printf("Hello from supervisor, waiting for interrupt\n"); ++ sbi_printf("Hello from supervisor\n"); ++ sbi_printf("Waiting for interrupt...\n"); + int i = 0; + char *s = "-\\|/"; + while (1) { ++ for (volatile unsigned long j = 0; j < 100000; j++); + sbi_printf("\r%c", s[i++]); + if (i >= 4) + i = 0; @@ -97,8 +120,6 @@ index 0ae604a..44601a4 100644 + +static void do_plic_test(void) +{ -+ char *prefix = "\t"; -+ char *suffix = "\t"; + sbi_printf("--- TESTING PLIC ---\n"); + + volatile unsigned long *mtime = (unsigned long *)(AUX_TIMER_BASE + MTIME_OFFSET); @@ -108,12 +129,6 @@ index 0ae604a..44601a4 100644 + *mtimecmp = 0xffffffffUL; + sbi_printf("Timer interrupt disabled\n"); + -+ sbi_printf("%sSIE%s: 0x%" PRILX "\n", -+ prefix, suffix, csr_read(CSR_SIE)); -+ sbi_printf("%sSSTATUS%s: 0x%" PRILX "\n", -+ prefix, suffix, csr_read(CSR_SSTATUS)); -+ sbi_printf("%sMIDELEG%s: 0x%" PRILX "\n", -+ prefix, suffix, csr_read(CSR_MIDELEG)); + + /* Enable supervisor interrupt delegation */ + csr_set(CSR_SIE, SIE_SEIE); // Enable supervisor external interrupts @@ -121,12 +136,7 @@ index 0ae604a..44601a4 100644 + csr_set(CSR_MIDELEG, MIDELEG_SEIE); // Delegate machine interrupts to supervisor mode + sbi_printf("Enabled supervisor delegation:\n"); + -+ sbi_printf("%sSIE%s: 0x%" PRILX "\n", -+ prefix, suffix, csr_read(CSR_SIE)); -+ sbi_printf("%sSSTATUS%s: 0x%" PRILX "\n", -+ prefix, suffix, csr_read(CSR_SSTATUS)); -+ sbi_printf("%sMIDELEG%s: 0x%" PRILX "\n", -+ prefix, suffix, csr_read(CSR_MIDELEG)); ++ dumpregs(); + + /* Configure PLIC aux timer input */ + volatile unsigned *plic_priority = (unsigned *)(PLIC_BASE + PLIC_PRIORITY_OFFSET + PLIC_TIMER_PORT * 4); @@ -139,21 +149,9 @@ index 0ae604a..44601a4 100644 + + sbi_printf("Timer enabled in PLIC\n"); + -+// sbi_printf("%sMIE%s: 0x%" PRILX "\n", -+// prefix, suffix, csr_read(CSR_MIE)); -+// sbi_printf("%sMSTATUS%s: 0x%" PRILX "\n", -+// prefix, suffix, csr_read(CSR_MSTATUS)); -+// +// /* Enable external timer interrupts */ +// csr_set(CSR_MIE, MIE_MEIE); /* Needed? */ +// csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ -+// -+// sbi_printf("External timer interrupts enabled in machine mode\n"); -+// -+// sbi_printf("%sMIE%s: 0x%" PRILX "\n", -+// prefix, suffix, csr_read(CSR_MIE)); -+// sbi_printf("%sMSTATUS%s: 0x%" PRILX "\n", -+// prefix, suffix, csr_read(CSR_MSTATUS)); + + /* Enable timer interrupt */ + *mtimecmp = *mtime + 10000; @@ -161,6 +159,8 @@ index 0ae604a..44601a4 100644 + sbi_printf("Timer alarm programmed\n"); + sbi_printf("Switching to supervisor\n"); + ++ dumpregs(); ++ + // Switch to supervisor mode and execute supervisor_mode_code + switch_to_supervisor_mode(&supervisor_mode_code); + -- GitLab From 90df8a3ccb108760d1accb5acaee75b17bc08135 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 10:03:19 +0200 Subject: [PATCH 145/310] Add stvec trap --- opensbi-test-plic.patch | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch index 17b1b50..6a900dd 100644 --- a/opensbi-test-plic.patch +++ b/opensbi-test-plic.patch @@ -1,5 +1,5 @@ diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c -index 0ae604a..a3df88c 100644 +index 0ae604a..3708c3a 100644 --- a/lib/sbi/sbi_irqchip.c +++ b/lib/sbi/sbi_irqchip.c @@ -9,6 +9,9 @@ @@ -25,7 +25,7 @@ index 0ae604a..a3df88c 100644 return 0; } -@@ -47,8 +52,151 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) +@@ -47,8 +52,156 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) { const struct sbi_platform *plat = sbi_platform_ptr(scratch); @@ -117,6 +117,10 @@ index 0ae604a..a3df88c 100644 + return 0; +} + ++static void supervisor_trap_entry(void) ++{ ++ sbi_printf("\nSupervisor Trap Entry Reached!\n"); ++} + +static void do_plic_test(void) +{ @@ -134,6 +138,7 @@ index 0ae604a..a3df88c 100644 + csr_set(CSR_SIE, SIE_SEIE); // Enable supervisor external interrupts + csr_set(CSR_SSTATUS, SSTATUS_SIE); // Enable global interrupts in supervisor mode + csr_set(CSR_MIDELEG, MIDELEG_SEIE); // Delegate machine interrupts to supervisor mode ++ asm volatile("csrw stvec, %0" : : "r"(&supervisor_trap_entry)); + sbi_printf("Enabled supervisor delegation:\n"); + + dumpregs(); -- GitLab From 64cc2a982f80fced695e6c2b989c4c5364b2853b Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 10:09:06 +0200 Subject: [PATCH 146/310] Clear MIP before enabling timer --- opensbi-test-plic.patch | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch index 6a900dd..07dbc25 100644 --- a/opensbi-test-plic.patch +++ b/opensbi-test-plic.patch @@ -1,5 +1,5 @@ diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c -index 0ae604a..3708c3a 100644 +index 0ae604a..0314715 100644 --- a/lib/sbi/sbi_irqchip.c +++ b/lib/sbi/sbi_irqchip.c @@ -9,6 +9,9 @@ @@ -25,7 +25,7 @@ index 0ae604a..3708c3a 100644 return 0; } -@@ -47,8 +52,156 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) +@@ -47,8 +52,162 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) { const struct sbi_platform *plat = sbi_platform_ptr(scratch); @@ -158,6 +158,12 @@ index 0ae604a..3708c3a 100644 +// csr_set(CSR_MIE, MIE_MEIE); /* Needed? */ +// csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ + ++ /* ++ * Clear mip CSR before proceeding with init to avoid any spurious ++ * external interrupts in S-mode. ++ */ ++ csr_write(CSR_MIP, 0); ++ + /* Enable timer interrupt */ + *mtimecmp = *mtime + 10000; + -- GitLab From cc65500434ed98671d53a81069123d47086f11eb Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 10:12:41 +0200 Subject: [PATCH 147/310] Use csr_set to set stvec --- opensbi-test-plic.patch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch index 07dbc25..fdcf56a 100644 --- a/opensbi-test-plic.patch +++ b/opensbi-test-plic.patch @@ -1,5 +1,5 @@ diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c -index 0ae604a..0314715 100644 +index 0ae604a..000b89c 100644 --- a/lib/sbi/sbi_irqchip.c +++ b/lib/sbi/sbi_irqchip.c @@ -9,6 +9,9 @@ @@ -138,7 +138,7 @@ index 0ae604a..0314715 100644 + csr_set(CSR_SIE, SIE_SEIE); // Enable supervisor external interrupts + csr_set(CSR_SSTATUS, SSTATUS_SIE); // Enable global interrupts in supervisor mode + csr_set(CSR_MIDELEG, MIDELEG_SEIE); // Delegate machine interrupts to supervisor mode -+ asm volatile("csrw stvec, %0" : : "r"(&supervisor_trap_entry)); ++ csr_set(CSR_STVEC, &supervisor_trap_entry); + sbi_printf("Enabled supervisor delegation:\n"); + + dumpregs(); -- GitLab From 2f28f9268a32b3fa38398ff3275255b423cc1b20 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 10:13:45 +0200 Subject: [PATCH 148/310] Enable machine interrupts --- opensbi-test-plic.patch | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch index fdcf56a..3bd6ba3 100644 --- a/opensbi-test-plic.patch +++ b/opensbi-test-plic.patch @@ -154,9 +154,9 @@ index 0ae604a..000b89c 100644 + + sbi_printf("Timer enabled in PLIC\n"); + -+// /* Enable external timer interrupts */ -+// csr_set(CSR_MIE, MIE_MEIE); /* Needed? */ -+// csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ ++ /* Enable external timer interrupts */ ++ csr_set(CSR_MIE, MIE_MEIE); /* Needed? */ ++ csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ + + /* + * Clear mip CSR before proceeding with init to avoid any spurious -- GitLab From ea081176f74751aee73bc4ee8055b4608f443a27 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 10:20:11 +0200 Subject: [PATCH 149/310] Clear MIP before enabling interrupts --- opensbi-test-plic.patch | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch index 3bd6ba3..4697f00 100644 --- a/opensbi-test-plic.patch +++ b/opensbi-test-plic.patch @@ -1,5 +1,5 @@ diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c -index 0ae604a..000b89c 100644 +index 0ae604a..31d8082 100644 --- a/lib/sbi/sbi_irqchip.c +++ b/lib/sbi/sbi_irqchip.c @@ -9,6 +9,9 @@ @@ -154,16 +154,16 @@ index 0ae604a..000b89c 100644 + + sbi_printf("Timer enabled in PLIC\n"); + -+ /* Enable external timer interrupts */ -+ csr_set(CSR_MIE, MIE_MEIE); /* Needed? */ -+ csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ -+ + /* + * Clear mip CSR before proceeding with init to avoid any spurious + * external interrupts in S-mode. + */ + csr_write(CSR_MIP, 0); + ++ /* Enable external timer interrupts */ ++ csr_set(CSR_MIE, MIE_MEIE); /* Needed? */ ++ csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ ++ + /* Enable timer interrupt */ + *mtimecmp = *mtime + 10000; + -- GitLab From 706d858347b542ad2de161210edd0836a02701a3 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 11:29:12 +0200 Subject: [PATCH 150/310] Claim aux timer interrupt --- opensbi-test-plic.patch | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch index 4697f00..c9f8837 100644 --- a/opensbi-test-plic.patch +++ b/opensbi-test-plic.patch @@ -1,5 +1,5 @@ diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c -index 0ae604a..31d8082 100644 +index 0ae604a..177fb6b 100644 --- a/lib/sbi/sbi_irqchip.c +++ b/lib/sbi/sbi_irqchip.c @@ -9,6 +9,9 @@ @@ -25,7 +25,7 @@ index 0ae604a..31d8082 100644 return 0; } -@@ -47,8 +52,162 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) +@@ -47,8 +52,158 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) { const struct sbi_platform *plat = sbi_platform_ptr(scratch); @@ -147,22 +147,29 @@ index 0ae604a..31d8082 100644 + volatile unsigned *plic_priority = (unsigned *)(PLIC_BASE + PLIC_PRIORITY_OFFSET + PLIC_TIMER_PORT * 4); + volatile unsigned *plic_enable = (unsigned *)(PLIC_BASE + PLIC_ENABLE_OFFSET); + volatile unsigned *plic_threshold = (unsigned *)(PLIC_BASE + PLIC_THRESHOLD_OFFSET); ++ volatile unsigned *plic_claim = (unsigned *)(PLIC_BASE + PLIC_CLAIM_OFFSET); ++ volatile unsigned *plic_pending = (unsigned *)(PLIC_BASE + PLIC_PENDING_OFFSET); + ++ sbi_printf("Enabling timer in PLIC\n"); + *plic_priority = PLIC_TIMER_PORT; + *plic_threshold = PLIC_TIMER_PORT - 1; + *plic_enable |= (1 << PLIC_TIMER_PORT); + -+ sbi_printf("Timer enabled in PLIC\n"); ++ /* Clear interrupt */ ++ sbi_printf("Pending: %d\n", *plic_pending); ++ unsigned claim = *plic_claim; ++ sbi_printf("Claim: %d\n", claim); ++ *plic_claim = claim; ++ sbi_printf("Pending: %d\n", *plic_pending); + -+ /* -+ * Clear mip CSR before proceeding with init to avoid any spurious -+ * external interrupts in S-mode. -+ */ ++ sbi_printf("Clearing MIP\n"); + csr_write(CSR_MIP, 0); + + /* Enable external timer interrupts */ -+ csr_set(CSR_MIE, MIE_MEIE); /* Needed? */ -+ csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ ++ //sbi_printf("Enabling MEIE in MIE register\n"); ++ //csr_set(CSR_MIE, MIE_MEIE); /* Needed? */ ++ //sbi_printf("Enabling MIE in MSTATUS register\n"); ++ //csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ + + /* Enable timer interrupt */ + *mtimecmp = *mtime + 10000; @@ -172,21 +179,10 @@ index 0ae604a..31d8082 100644 + + dumpregs(); + -+ // Switch to supervisor mode and execute supervisor_mode_code + switch_to_supervisor_mode(&supervisor_mode_code); + + /* Never reached */ -+ -+ int count = 0; -+ // Main loop -+ while (1) { -+ if (count == 10000) { -+ sbi_printf("Still in machine mode\n"); -+ count = 0; -+ } -+ count++; -+ // Main application code -+ } ++ while (1); +} + + -- GitLab From 2c5f8a8ccfff3c9d198d2ae6e28e95d5557ecb0f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 11:42:12 +0200 Subject: [PATCH 151/310] Print registers in supervisor --- opensbi-test-plic.patch | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch index c9f8837..8f3df08 100644 --- a/opensbi-test-plic.patch +++ b/opensbi-test-plic.patch @@ -1,5 +1,5 @@ diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c -index 0ae604a..177fb6b 100644 +index 0ae604a..54449c7 100644 --- a/lib/sbi/sbi_irqchip.c +++ b/lib/sbi/sbi_irqchip.c @@ -9,6 +9,9 @@ @@ -25,7 +25,7 @@ index 0ae604a..177fb6b 100644 return 0; } -@@ -47,8 +52,158 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) +@@ -47,8 +52,161 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) { const struct sbi_platform *plat = sbi_platform_ptr(scratch); @@ -61,19 +61,21 @@ index 0ae604a..177fb6b 100644 +#define MSTATUS_MPP_MASK (3 << 11) +#define MSTATUS_MPP_SUPERVISOR (1 << 11) + -+static void dumpregs(void) ++static void dumpregs(int machine) +{ + char *prefix = "\t"; + char *suffix = "\t"; + sbi_printf("Registers:\n"); -+ sbi_printf("%sMIE%s: 0x%" PRILX "\n", -+ prefix, suffix, csr_read(CSR_MIE)); -+ sbi_printf("%sMIP%s: 0x%" PRILX "\n", -+ prefix, suffix, csr_read(CSR_MIP)); -+ sbi_printf("%sMSTATUS%s: 0x%" PRILX "\n", -+ prefix, suffix, csr_read(CSR_MSTATUS)); -+ sbi_printf("%sMIDELEG%s: 0x%" PRILX "\n", -+ prefix, suffix, csr_read(CSR_MIDELEG)); ++ if (machine) { ++ sbi_printf("%sMIE%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_MIE)); ++ sbi_printf("%sMIP%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_MIP)); ++ sbi_printf("%sMSTATUS%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_MSTATUS)); ++ sbi_printf("%sMIDELEG%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_MIDELEG)); ++ } + sbi_printf("%sSIE%s: 0x%" PRILX "\n", + prefix, suffix, csr_read(CSR_SIE)); + sbi_printf("%sSIP%s: 0x%" PRILX "\n", @@ -105,6 +107,7 @@ index 0ae604a..177fb6b 100644 +static int supervisor_mode_code(void) +{ + sbi_printf("Hello from supervisor\n"); ++ dumpregs(0); + sbi_printf("Waiting for interrupt...\n"); + int i = 0; + char *s = "-\\|/"; @@ -141,7 +144,7 @@ index 0ae604a..177fb6b 100644 + csr_set(CSR_STVEC, &supervisor_trap_entry); + sbi_printf("Enabled supervisor delegation:\n"); + -+ dumpregs(); ++ dumpregs(1); + + /* Configure PLIC aux timer input */ + volatile unsigned *plic_priority = (unsigned *)(PLIC_BASE + PLIC_PRIORITY_OFFSET + PLIC_TIMER_PORT * 4); @@ -177,7 +180,7 @@ index 0ae604a..177fb6b 100644 + sbi_printf("Timer alarm programmed\n"); + sbi_printf("Switching to supervisor\n"); + -+ dumpregs(); ++ dumpregs(1); + + switch_to_supervisor_mode(&supervisor_mode_code); + -- GitLab From 7091ca455c0333113f0ffbadc2de292cf986d7f7 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 12:28:17 +0200 Subject: [PATCH 152/310] Enable timer from supervisor --- JOURNAL.md | 2 +- opensbi-test-plic.patch | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index c73db79..110e04b 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -3463,7 +3463,7 @@ Let's set the stvec to zero, so it causes a machine exception. # 2024-08-21 -I tried with the new bitstream (ox_u55c_46619ef4.bit) setting the stvec to zero +I tried with the new bitstream (`ox_u55c_46619ef4.bit`) setting the stvec to zero and I can see the OpenSBI handler stopping, probably due to the jump to zero address: diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch index 8f3df08..f676341 100644 --- a/opensbi-test-plic.patch +++ b/opensbi-test-plic.patch @@ -1,5 +1,5 @@ diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c -index 0ae604a..54449c7 100644 +index 0ae604a..c446e60 100644 --- a/lib/sbi/sbi_irqchip.c +++ b/lib/sbi/sbi_irqchip.c @@ -9,6 +9,9 @@ @@ -25,7 +25,7 @@ index 0ae604a..54449c7 100644 return 0; } -@@ -47,8 +52,161 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) +@@ -47,8 +52,163 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) { const struct sbi_platform *plat = sbi_platform_ptr(scratch); @@ -61,6 +61,10 @@ index 0ae604a..54449c7 100644 +#define MSTATUS_MPP_MASK (3 << 11) +#define MSTATUS_MPP_SUPERVISOR (1 << 11) + ++static volatile unsigned long *mtime = (unsigned long *)(AUX_TIMER_BASE + MTIME_OFFSET); ++static volatile unsigned long *mtimecmp = (unsigned long *)(AUX_TIMER_BASE + MTIMECMP_OFFSET); ++ ++ +static void dumpregs(int machine) +{ + char *prefix = "\t"; @@ -108,6 +112,11 @@ index 0ae604a..54449c7 100644 +{ + sbi_printf("Hello from supervisor\n"); + dumpregs(0); ++ ++ /* Enable timer interrupt */ ++ *mtimecmp = *mtime + 10000; ++ ++ sbi_printf("Timer alarm programmed\n"); + sbi_printf("Waiting for interrupt...\n"); + int i = 0; + char *s = "-\\|/"; @@ -129,9 +138,6 @@ index 0ae604a..54449c7 100644 +{ + sbi_printf("--- TESTING PLIC ---\n"); + -+ volatile unsigned long *mtime = (unsigned long *)(AUX_TIMER_BASE + MTIME_OFFSET); -+ volatile unsigned long *mtimecmp = (unsigned long *)(AUX_TIMER_BASE + MTIMECMP_OFFSET); -+ + /* Disable auxiliar timer interrupt */ + *mtimecmp = 0xffffffffUL; + sbi_printf("Timer interrupt disabled\n"); @@ -174,10 +180,6 @@ index 0ae604a..54449c7 100644 + //sbi_printf("Enabling MIE in MSTATUS register\n"); + //csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ + -+ /* Enable timer interrupt */ -+ *mtimecmp = *mtime + 10000; -+ -+ sbi_printf("Timer alarm programmed\n"); + sbi_printf("Switching to supervisor\n"); + + dumpregs(1); -- GitLab From ae379f7b9a9a691242ce1660660f5fe66a372cf6 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 21 Aug 2024 12:29:52 +0200 Subject: [PATCH 153/310] Enable machine mode interrupts --- opensbi-test-plic.patch | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch index f676341..6222ef9 100644 --- a/opensbi-test-plic.patch +++ b/opensbi-test-plic.patch @@ -1,5 +1,5 @@ diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c -index 0ae604a..c446e60 100644 +index 0ae604a..7b1d95e 100644 --- a/lib/sbi/sbi_irqchip.c +++ b/lib/sbi/sbi_irqchip.c @@ -9,6 +9,9 @@ @@ -175,10 +175,10 @@ index 0ae604a..c446e60 100644 + csr_write(CSR_MIP, 0); + + /* Enable external timer interrupts */ -+ //sbi_printf("Enabling MEIE in MIE register\n"); -+ //csr_set(CSR_MIE, MIE_MEIE); /* Needed? */ -+ //sbi_printf("Enabling MIE in MSTATUS register\n"); -+ //csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ ++ sbi_printf("Enabling MEIE in MIE register\n"); ++ csr_set(CSR_MIE, MIE_MEIE); /* Needed? */ ++ sbi_printf("Enabling MIE in MSTATUS register\n"); ++ csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ + + sbi_printf("Switching to supervisor\n"); + -- GitLab From 9177f610a785f7c5552b52426e14d10974ebe12c Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 22 Aug 2024 17:12:42 +0200 Subject: [PATCH 154/310] Add FPGA baremetal tests --- JOURNAL.md | 20 +++++++++++++++++++- overlay.nix | 33 +++++++++++++++++++++++++++++++++ sa-fpga-crt.patch | 15 +++++++++++++++ sa-fpga-text-address.patch | 13 +++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 sa-fpga-crt.patch create mode 100644 sa-fpga-text-address.patch diff --git a/JOURNAL.md b/JOURNAL.md index 110e04b..73c49a7 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -3461,7 +3461,7 @@ No interrupts seem to arrive at the SIP register. Let's set the stvec to zero, so it causes a machine exception. -# 2024-08-21 +## 2024-08-21 I tried with the new bitstream (`ox_u55c_46619ef4.bit`) setting the stvec to zero and I can see the OpenSBI handler stopping, probably due to the jump to zero @@ -3589,3 +3589,21 @@ interrupt over to supervisor. Let's dump the machine registers too. Let's also disable the MEIE bit in MIE register. + +## 2024-08-22 + +I'm unable to delegate the external interrupt to supervisor. It is always +intercepted by OpenSBI in machine mode or nothing happens. + +I may still be missing some bit in some register that prevents the delegation +from working. So I tried to run the bare metal tests, specifically +`plic_supervisor` from here: + + https://gitlab.bsc.es/hwdesign/rtl/core-tile/sa-fpga/-/blob/6547e63f3aed312a4c36f093b5e6421c7f1704f6/fpga_core_bridge/simulator/tests/c_tests/plic_supervisor/plic_supervisor_test.c + +But I'm not able to make them run. I added them to Nix and created a bootable +image with objcopy, but nothing is displayed in the UART. I also tried to patch +the .text.init address to 0x100 without luck. + +I think the next experiment I will do is try to port the baremetal tests to +something I can load at `0x8000_0000` diff --git a/overlay.nix b/overlay.nix index 5813639..abac0e8 100644 --- a/overlay.nix +++ b/overlay.nix @@ -73,4 +73,37 @@ final: prev: url = "git@bscpm03.bsc.es:rarias/bitstreams.git"; rev = "ad901b0c21ffbdb310ff1dfb269f169f6ac6bde6"; }; + + # Baremetal tests for standalone FPGA + sa-fpga-tests = prev.pkgsStatic.stdenv.mkDerivation { + name = "sa-fpga-tests"; + src = builtins.fetchGit { + url = "git@gitlab-internal.bsc.es:hwdesign/rtl/core-tile/sa-fpga.git"; + rev = "6547e63f3aed312a4c36f093b5e6421c7f1704f6"; + ref = "ni/supervisor_tests"; + }; + dontConfigure = true; + patches = [ + #./sa-fpga-crt.patch + #./sa-fpga-text-address.patch + ]; + buildPhase = '' + cd fpga_core_bridge/simulator/tests/c_tests/ + make RISCV_PREFIX=riscv64-unknown-linux-musl- + + # Generate binary images to be loaded in memory + for f in *.riscv; do + # Don't copy 0x40000000 section + $OBJCOPY -R .tohost -O binary $f $f.bin + done + ''; + installPhase = '' + ls -lah + make install install_dir=$out + cp -a *.bin $out + ''; + dontFixup = true; + hardeningDisable = [ "all" ]; + }; + } diff --git a/sa-fpga-crt.patch b/sa-fpga-crt.patch new file mode 100644 index 0000000..d315030 --- /dev/null +++ b/sa-fpga-crt.patch @@ -0,0 +1,15 @@ +diff --git a/fpga_core_bridge/simulator/tests/c_tests/common/crt.S b/fpga_core_bridge/simulator/tests/c_tests/common/crt.S +index 3f5bb2c..bd738b1 100644 +--- a/fpga_core_bridge/simulator/tests/c_tests/common/crt.S ++++ b/fpga_core_bridge/simulator/tests/c_tests/common/crt.S +@@ -59,10 +59,6 @@ _start: + #else + bltz t0, 1f + #endif +-2: +- li a0, 1 +- sw a0, tohost, t0 +- j 2b + 1: + + #ifdef __riscv_flen diff --git a/sa-fpga-text-address.patch b/sa-fpga-text-address.patch new file mode 100644 index 0000000..1513561 --- /dev/null +++ b/sa-fpga-text-address.patch @@ -0,0 +1,13 @@ +diff --git a/fpga_core_bridge/simulator/tests/c_tests/common/test.ld b/fpga_core_bridge/simulator/tests/c_tests/common/test.ld +index 8321d86..f83a9ee 100644 +--- a/fpga_core_bridge/simulator/tests/c_tests/common/test.ld ++++ b/fpga_core_bridge/simulator/tests/c_tests/common/test.ld +@@ -26,7 +26,7 @@ SECTIONS + .tohost : { *(.tohost) } + + /* text: test code section */ +- . = 0x80000000; ++ . = 0x20020000; + .text.init : { *(.text.init) } + + . = ALIGN(0x1000); -- GitLab From 7b75cd256356082096ab3f68bfff525a1729d24a Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 23 Aug 2024 13:16:00 +0200 Subject: [PATCH 155/310] Add UART patch for FPGA in sa-fpga tests --- overlay.nix | 1 + sa-fpga-uart.patch | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 sa-fpga-uart.patch diff --git a/overlay.nix b/overlay.nix index abac0e8..a3266f0 100644 --- a/overlay.nix +++ b/overlay.nix @@ -86,6 +86,7 @@ final: prev: patches = [ #./sa-fpga-crt.patch #./sa-fpga-text-address.patch + ./sa-fpga-uart.patch ]; buildPhase = '' cd fpga_core_bridge/simulator/tests/c_tests/ diff --git a/sa-fpga-uart.patch b/sa-fpga-uart.patch new file mode 100644 index 0000000..eed755a --- /dev/null +++ b/sa-fpga-uart.patch @@ -0,0 +1,31 @@ +diff --git a/fpga_core_bridge/simulator/tests/c_tests/common/syscalls.c b/fpga_core_bridge/simulator/tests/c_tests/common/syscalls.c +index 278ea97..287e5fc 100644 +--- a/fpga_core_bridge/simulator/tests/c_tests/common/syscalls.c ++++ b/fpga_core_bridge/simulator/tests/c_tests/common/syscalls.c +@@ -592,8 +592,18 @@ int uart_is_transmit_empty() { + + // Function to write a character to the UART + void uart_write_char(char c) { +- while (!uart_is_transmit_empty()); ++ //while (!uart_is_transmit_empty()); ++ ++ /* Delay it a bit, as checking the transmit holding register doesn't seem to ++ * work in the FPGA */ ++ for (volatile long i = 0; i < 10000; i++) ++ ; ++ + *(volatile uint8_t *)(UART_BASE + UART_THR) = c; ++ ++ /* Make new line go back to the start of the line */ ++ if (c == '\n') ++ uart_write_char('\r'); + } + + // Function to write a string to the UART +@@ -602,4 +612,4 @@ void uart_write_string(const char* str) { + uart_write_char(*str++); + asm("fence"); + } +-} +\ No newline at end of file ++} -- GitLab From 44de310116402592b042fdefed08c1e915f53905 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 23 Aug 2024 16:06:21 +0200 Subject: [PATCH 156/310] Switch to ox_u55c_f6b4a076 bitstream --- lagarto-ox.nix | 2 +- overlay.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index d55b1f6..fda6afb 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -205,7 +205,7 @@ } "dtc -O dtb -o $out $dtsFile"; #bitstream = "${final.bitstreams}/lagarto-3-ox/gold.bit"; - bitstream = "${final.bitstreams}/lagarto-3-ox/ox_u55c_a234c132_two_uarts.bit"; + bitstream = "${final.bitstreams}/lagarto-3-ox/ox_u55c_f6b4a076_fix_delegation.bit"; bootrom = "${final.bitstreams}/lagarto-3-ox/large_bootrom.bin"; diff --git a/overlay.nix b/overlay.nix index a3266f0..64ad58f 100644 --- a/overlay.nix +++ b/overlay.nix @@ -71,7 +71,7 @@ final: prev: bitstreams = builtins.fetchGit { url = "git@bscpm03.bsc.es:rarias/bitstreams.git"; - rev = "ad901b0c21ffbdb310ff1dfb269f169f6ac6bde6"; + rev = "d02b4805002d43519a69d17d19488c065f3574f9"; }; # Baremetal tests for standalone FPGA -- GitLab From 8272212769a7595c725f4735fbc8474843234673 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 23 Aug 2024 16:06:21 +0200 Subject: [PATCH 157/310] Add custom bootrom --- bootrom/.gitignore | 1 + bootrom/Makefile | 23 ++++++++++++ bootrom/linker.ld | 19 ++++++++++ bootrom/rbootrom.S | 94 ++++++++++++++++++++++++++++++++++++++++++++++ overlay.nix | 14 +++++++ 5 files changed, 151 insertions(+) create mode 100644 bootrom/.gitignore create mode 100644 bootrom/Makefile create mode 100644 bootrom/linker.ld create mode 100644 bootrom/rbootrom.S diff --git a/bootrom/.gitignore b/bootrom/.gitignore new file mode 100644 index 0000000..d163863 --- /dev/null +++ b/bootrom/.gitignore @@ -0,0 +1 @@ +build/ \ No newline at end of file diff --git a/bootrom/Makefile b/bootrom/Makefile new file mode 100644 index 0000000..8ac94ef --- /dev/null +++ b/bootrom/Makefile @@ -0,0 +1,23 @@ +CC ?= riscv64-unknown-elf-gcc +OBJCOPY ?= riscv64-unknown-elf-objcopy +ROM_BASE ?= 0x00100 + +LDFLAGS=-Tlinker.ld -nostdlib -nostartfiles -nodefaultlibs -static -Wl,--no-gc-sections -Wl,--defsym=ROM_BASE=$(ROM_BASE) + +b=rbootrom + +all: $b.elf $b.bin + +$b.o: $b.S + $(CC) $(CFLAGS) -c -o $@ $< + +$b.elf: $b.o linker.ld + $(CC) $(LDFLAGS) -o $@ $< + +$b.bin: $b.elf + $(OBJCOPY) -O binary $b.elf $b.bin + +clean: + rm -f $b.elf $b.bin $b.o + +.PHONY: clean all diff --git a/bootrom/linker.ld b/bootrom/linker.ld new file mode 100644 index 0000000..9a955e7 --- /dev/null +++ b/bootrom/linker.ld @@ -0,0 +1,19 @@ +OUTPUT_ARCH( "riscv" ) +ENTRY(_start) + +SECTIONS +{ + ROM_BASE = DEFINED(ROM_BASE) ? ROM_BASE : 0x00100; /* Default to 0x00100 if ROM_BASE is not defined */ + + . = ROM_BASE; + .text.start : { *(.text.start) } + . = ROM_BASE + 0x80; + .text.hang : { *(.text.hang) } + /* + . = ROM_BASE + 0xC0; + .rodata.dtb : { *(.rodata.dtb) } + */ + + . = ALIGN(0x100); + .text : { *(.text) } +} diff --git a/bootrom/rbootrom.S b/bootrom/rbootrom.S new file mode 100644 index 0000000..c2c0d00 --- /dev/null +++ b/bootrom/rbootrom.S @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2024, Barcelona Supercomputing Center (BSC) + * SPDX-License-Identifier: MIT + * + * RBOOTROM v1.0 + * Modified by Rodrigo Arias Mallo + * + * This is a custom bootrom that prints some information to the UART when + * starting, as well as when it hangs. It assumes the UART is at UART_BASE and + * it will jump to DRAM_BASE to continue the boot. + */ + +#define DRAM_BASE 0x80000000 +#define UART_BASE 0x40001000 + +.macro PUTC, ch + li a0, \ch + jal putchar +.endm + +.section .text.start, "ax", @progbits +.globl _start +_start: + jal print_hello // Print initial message on HART 0 only + csrr a0, mhartid // Load HART ID into a0 + li s0, DRAM_BASE // Load next address into s0 + jr s0 // Jump to s0 + +.section .text.hang, "ax", @progbits +.globl _hang +_hang: + + PUTC '\n' + PUTC '\r' + PUTC 'R' + PUTC 'B' + PUTC 'O' + PUTC 'O' + PUTC 'T' + PUTC 'R' + PUTC 'O' + PUTC 'M' + PUTC ' ' + PUTC 'H' + PUTC 'A' + PUTC 'N' + PUTC 'G' + PUTC '\n' + PUTC '\r' + + /* Hang */ + csrr a0, mhartid +1: + wfi + j 1b + +.section .text, "ax", @progbits + +putchar: + // Wait a bit before writing to UART + li t0, 100000 // Delay a bit +1: + addi t0, t0, -1 // Decrement + bne t0, zero, 1b // Repeat if not zero + + // Write the character now + la t0, UART_BASE // Transmitter Holding Buffer + sb a0, 0(t0) // Write a0 byte + ret + +print_hello: + csrr t0, mhartid // Load HART ID into a0 + beq t0, zero, 1f // Print message on HART 0 only + ret +1: + // Identify bootroom + PUTC '\n' + PUTC '\r' + PUTC 'R' + PUTC 'B' + PUTC 'O' + PUTC 'O' + PUTC 'T' + PUTC 'R' + PUTC 'O' + PUTC 'M' + PUTC ' ' + PUTC 'v' + PUTC '1' + PUTC '.' + PUTC '0' + PUTC '\n' + PUTC '\r' + ret diff --git a/overlay.nix b/overlay.nix index 64ad58f..8bf3fca 100644 --- a/overlay.nix +++ b/overlay.nix @@ -107,4 +107,18 @@ final: prev: hardeningDisable = [ "all" ]; }; + rbootrom = prev.pkgsStatic.stdenv.mkDerivation { + name = "sa-fpga-tests"; + src = ./bootrom; + dontConfigure = true; + buildPhase = '' + make + ''; + installPhase = '' + mkdir $out + cp rbootrom.bin rbootrom.elf $out/ + ''; + dontFixup = true; + hardeningDisable = [ "all" ]; + }; } -- GitLab From 2371ac2fdab95d6908a589195a8a24db3189d96e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 23 Aug 2024 16:06:21 +0200 Subject: [PATCH 158/310] Switch to rbootrom --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index fda6afb..b4edb65 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -207,7 +207,7 @@ #bitstream = "${final.bitstreams}/lagarto-3-ox/gold.bit"; bitstream = "${final.bitstreams}/lagarto-3-ox/ox_u55c_f6b4a076_fix_delegation.bit"; - bootrom = "${final.bitstreams}/lagarto-3-ox/large_bootrom.bin"; + bootrom = "${final.rbootrom}/rbootrom.bin"; uboot = prev.ubootQemuRiscv64Smode.override { filesToInstall = [ "u-boot-nodtb.bin" ]; -- GitLab From 2dcb2ac20963b57fb8fa7bf0c2ac7d1d4abd167b Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 23 Aug 2024 16:45:03 +0200 Subject: [PATCH 159/310] Save return address before calling putchar --- bootrom/rbootrom.S | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/bootrom/rbootrom.S b/bootrom/rbootrom.S index c2c0d00..45c08fd 100644 --- a/bootrom/rbootrom.S +++ b/bootrom/rbootrom.S @@ -73,8 +73,8 @@ print_hello: beq t0, zero, 1f // Print message on HART 0 only ret 1: - // Identify bootroom - PUTC '\n' + mv s0, ra // Save return address + PUTC '\n' // Identify bootroom PUTC '\r' PUTC 'R' PUTC 'B' @@ -89,6 +89,41 @@ print_hello: PUTC '1' PUTC '.' PUTC '0' + PUTC ' ' + PUTC ':' + PUTC '^' + PUTC ')' + PUTC '\n' + PUTC '\r' + + // Print jumping address + PUTC 'J' + PUTC 'u' + PUTC 'm' + PUTC 'p' + PUTC 'i' + PUTC 'n' + PUTC 'g' + PUTC ' ' + PUTC 't' + PUTC 'o' + PUTC ' ' + PUTC '0' // TODO: Compute from DRAM_BASE + PUTC 'x' + PUTC '8' + PUTC '0' + PUTC '0' + PUTC '0' + PUTC '_' + PUTC '0' + PUTC '0' + PUTC '0' + PUTC '0' + PUTC '.' + PUTC '.' + PUTC '.' PUTC '\n' PUTC '\r' + + mv ra, s0 // Restore return address ret -- GitLab From 417a4d5c750001025c9a3b43e873de50386dd2e4 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 23 Aug 2024 16:46:40 +0200 Subject: [PATCH 160/310] Set the _hang trap in mtvec --- bootrom/rbootrom.S | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bootrom/rbootrom.S b/bootrom/rbootrom.S index 45c08fd..04454d8 100644 --- a/bootrom/rbootrom.S +++ b/bootrom/rbootrom.S @@ -22,6 +22,10 @@ .globl _start _start: jal print_hello // Print initial message on HART 0 only + + la t0, _hang + csrw mtvec, t0 // Set the machine trap vector + csrr a0, mhartid // Load HART ID into a0 li s0, DRAM_BASE // Load next address into s0 jr s0 // Jump to s0 -- GitLab From 806023778b8bf804cd84bda04a41fb44b641c2a4 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 23 Aug 2024 17:42:55 +0200 Subject: [PATCH 161/310] Initialize UART in bootrom Unless the UART is properly initialized, the console won't display any message until a next stage intializes it (OpenSBI) and then we will start to see messages after uploading the next bootroms. Follows the OpenSBI initialization for the UART setting the baud rate to 115200 and assuming a clock of 50 MHz. --- bootrom/rbootrom.S | 70 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 7 deletions(-) diff --git a/bootrom/rbootrom.S b/bootrom/rbootrom.S index 04454d8..9a7f02b 100644 --- a/bootrom/rbootrom.S +++ b/bootrom/rbootrom.S @@ -12,6 +12,28 @@ #define DRAM_BASE 0x80000000 #define UART_BASE 0x40001000 +#define UART_BAUDRATE 115200 +#define UART_CLOCK 50000000 + +#define UART_BDIV ((UART_CLOCK + 8 * UART_BAUDRATE) / (16 * UART_BAUDRATE)) +#define UART_SHIFT 2 + +#define UART_RBR_OFFSET (0<> 8) & 0xff // Set divisor high byte + sb t1, UART_DLM_OFFSET(t0) + + la t1, 0x03 // 8 bits, no parity, one stop bit + sb t1, UART_LCR_OFFSET(t0) + la t1, 0x01 // Enable FIFO + sb t1, UART_FCR_OFFSET(t0) + la t1, 0x00 // No modem control DTR RTS + sb t1, UART_MCR_OFFSET(t0) + + /* TODO: Clear line status */ + /* TODO: Read receive buffer */ + + la t1, 0x00 // Set scratchpad to 0 + sb t1, UART_SCR_OFFSET(t0) - // Write the character now - la t0, UART_BASE // Transmitter Holding Buffer - sb a0, 0(t0) // Write a0 byte ret print_hello: -- GitLab From 2fc69ebfc00c5d4b8facbbe54e6ba90bce496820 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 23 Aug 2024 21:22:22 +0200 Subject: [PATCH 162/310] Update bootrom gitignore --- bootrom/.gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bootrom/.gitignore b/bootrom/.gitignore index d163863..7cf238f 100644 --- a/bootrom/.gitignore +++ b/bootrom/.gitignore @@ -1 +1,3 @@ -build/ \ No newline at end of file +*.o +*.elf +*.bin -- GitLab From 26754fa6e46d3b49d1b0da68a950a8fe601dd3fa Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 26 Aug 2024 15:15:10 +0200 Subject: [PATCH 163/310] Update journal --- JOURNAL.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index 73c49a7..7078498 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -3607,3 +3607,20 @@ the .text.init address to 0x100 without luck. I think the next experiment I will do is try to port the baremetal tests to something I can load at `0x8000_0000` + +## 2024-08-26 + +I implemented a new bootrom as the original large bootrom is not initializing +the UART. This causes baremetal tests to fail until someone properly initializes +the UART (namely, OpenSBI). The new bootrom, named RBOOTROM, initializes the +UART and prints an initial message. It also sets a machine mode trap to catch +potential errors: + + $ picocom -qb 115200 $FPGACTL_UART + + RBOOTROM v1.0 :^) + Jumping to 0x8000_0000... + +While the bitstream with the potential fixes for the supervisor interrupts is +not available, I'll try to automate the testing by creating a pipeline that +automatically runs the tests in an FPGA. -- GitLab From 056f572ee9253b5ea680b6b38758e6c58dfb55a3 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 26 Aug 2024 15:20:16 +0200 Subject: [PATCH 164/310] Add CI pipeline --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..6260efd --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,6 @@ +build:lagarto-ox-rd: + stage: build + tags: + - nix + script: + - nix develop -L .#lagarto-ox-rd --command env -- GitLab From 8b491f519b005d39d0d5f523649c12330e4248b5 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 26 Aug 2024 17:14:53 +0200 Subject: [PATCH 165/310] Add trampoline scripts to CI --- .gitlab-ci.yml | 2 +- fpga/boot.sh | 8 ++++---- fpga/run-login.sh | 31 +++++++++++++++++++++++++++++++ fpga/run-node.sh | 30 ++++++++++++++++++++++++++++++ fpga/run-remotely.sh | 21 +++++++++++++++++++++ fpga/upload.sh | 12 +++++++----- fpga/verify.awk | 20 ++++++++++++++++++++ overlay.nix | 2 +- 8 files changed, 115 insertions(+), 11 deletions(-) create mode 100755 fpga/run-login.sh create mode 100755 fpga/run-node.sh create mode 100755 fpga/run-remotely.sh create mode 100644 fpga/verify.awk diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6260efd..33d9ada 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,4 +3,4 @@ build:lagarto-ox-rd: tags: - nix script: - - nix develop -L .#lagarto-ox-rd --command env + - nix develop -L .#lagarto-ox-rd --command fpga/run-remotely.sh fpgalogin1:ci diff --git a/fpga/boot.sh b/fpga/boot.sh index 41a6813..47f24e1 100755 --- a/fpga/boot.sh +++ b/fpga/boot.sh @@ -1,12 +1,12 @@ #!/usr/bin/bash -set -x +#set -x set -e source ./env.sh -#bitstream="-w system-acme_ea-4h2v.bit" +bitstream="-w bitstream.bit" -./fpgactl $bitstream -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img -R bootrom.bin +./fpgactl $bitstream -R bootrom.bin -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img -picocom -b 115200 /dev/ttyUSB2 +picocom -b 115200 $FPGACTL_UART diff --git a/fpga/run-login.sh b/fpga/run-login.sh new file mode 100755 index 0000000..afe8dca --- /dev/null +++ b/fpga/run-login.sh @@ -0,0 +1,31 @@ +#!/usr/bin/bash + +#set -x +set -e + +path="$1" + +# First determine if we already have jobs already +n=$(squeue --me -lh | wc -l) + +if [ "$n" -gt 1 ]; then + echo "Too many jobs queued already" >&2 + exit 1 +fi + +if [ "$n" == 0 ]; then + # No running jobs, so allocate a new job + echo salloc -N 1 --constraint=dmaqdma --no-shell +fi + +while [ "$n" != 1 ]; do + sleep 2 + n=$(squeue --me -lh) +done + +host=$(squeue -h -o %N) + +echo "Switching to $host" + +# Continue the execution there +ssh "$host" "$path/run-node.sh" "$path" diff --git a/fpga/run-node.sh b/fpga/run-node.sh new file mode 100755 index 0000000..635ca24 --- /dev/null +++ b/fpga/run-node.sh @@ -0,0 +1,30 @@ +#!/usr/bin/bash + +#set -x +set -e + +echo "Hello from $(hostname)" + +path="$1" + +cd "$path" + +# First kill any picocom instance +killall picocom || true + +# Setup the environment +. env.sh + +set -x + +# Then perform the boot +./fpgactl -w bitstream.bit -R bootrom.bin -b opensbi.bin -k kernel.bin -i initrd.bin # -r rootfs.img + +# Restart it again only loading the bootrom and OpenSBI (notice OpenSBI +# relocates itself so it will likely be gone by now) +sleep 1 && ./fpgactl -R bootrom.bin -b opensbi.bin & + +# Set dead switch +sleep 30 && killall picocom & + +picocom -q -x 10000 -b 115200 $FPGACTL_UART diff --git a/fpga/run-remotely.sh b/fpga/run-remotely.sh new file mode 100755 index 0000000..74cbe0f --- /dev/null +++ b/fpga/run-remotely.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +# Executes a pipeline in a remote machine taking the values from the environment +# Usage fpga/run-remotely.sh : + +set -e + +dst=fpgalogin1:nixos + +if [ "$1" != "" ]; then + dst="$1" +fi + +path=${dst#*:} + +# Copy all required elements to the destination machine +fpga/upload.sh "$dst" + +# Launch the pipeline from there +set +x +ssh fpgalogin1 "$path/run-login.sh" "$path" | awk -f fpga/verify.awk diff --git a/fpga/upload.sh b/fpga/upload.sh index 0c8807e..6e4df7c 100755 --- a/fpga/upload.sh +++ b/fpga/upload.sh @@ -3,16 +3,18 @@ set -e set -x -#dst=femu:nixos/ -dst=fpgalogin1:nixos/ +#dst=femu:nixos +dst=fpgalogin1:nixos if [ "$1" != "" ]; then dst="$1" fi -rsync -a fpga/fpgactl "$dst" -#rsync -a fpga/boot.sh "$dst" -rsync -a fpga/env.sh "$dst" +rsync -a fpga/run-login.sh "$dst/" +rsync -a fpga/run-node.sh "$dst/" +rsync -a fpga/fpgactl "$dst/" +rsync -a fpga/boot.sh "$dst/" +rsync -a fpga/env.sh "$dst/" rsync $(find "$OPENSBI" -name fw_payload.bin) "$dst/opensbi.bin" rsync "$KERNEL/Image" "$dst/kernel.bin" rsync "$INITRD/initrd" "$dst/initrd.bin" diff --git a/fpga/verify.awk b/fpga/verify.awk new file mode 100644 index 0000000..c9daa98 --- /dev/null +++ b/fpga/verify.awk @@ -0,0 +1,20 @@ +BEGIN { + bootrom_ok = 0 + opensbi_ok = 0 + test_ok = 0 +} +/RBOOTROM/ { bootrom_ok = 1 } +/^OpenSBI v/ { opensbi_ok = 1 } +/^TEST-RESULT-OK/ { test_ok = 1 } + { printf "line> "; print } +END { + printf "Test summary:\n" + printf " Bootrom: %s\n", bootrom_ok ? "OK" : "FAIL"; + printf " OpenSBI: %s\n", opensbi_ok ? "OK" : "FAIL"; + #printf " Result: %s\n", test_ok ? "OK" : "FAIL"; + + if (bootrom_ok && opensbi_ok) + exit 0; + else + exit 1; +} diff --git a/overlay.nix b/overlay.nix index 8bf3fca..af8e59e 100644 --- a/overlay.nix +++ b/overlay.nix @@ -108,7 +108,7 @@ final: prev: }; rbootrom = prev.pkgsStatic.stdenv.mkDerivation { - name = "sa-fpga-tests"; + name = "rbootrom"; src = ./bootrom; dontConfigure = true; buildPhase = '' -- GitLab From 2e2ba5b99642c475c8ce42c66e3aa7be3ba2eca9 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 27 Aug 2024 14:32:39 +0200 Subject: [PATCH 166/310] Use test result to pass the tests --- fpga/verify.awk | 4 ++-- opensbi-test-plic.patch | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fpga/verify.awk b/fpga/verify.awk index c9daa98..1db7959 100644 --- a/fpga/verify.awk +++ b/fpga/verify.awk @@ -11,9 +11,9 @@ END { printf "Test summary:\n" printf " Bootrom: %s\n", bootrom_ok ? "OK" : "FAIL"; printf " OpenSBI: %s\n", opensbi_ok ? "OK" : "FAIL"; - #printf " Result: %s\n", test_ok ? "OK" : "FAIL"; + printf " Result: %s\n", test_ok ? "OK" : "FAIL"; - if (bootrom_ok && opensbi_ok) + if (test_ok) exit 0; else exit 1; diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch index 6222ef9..427072d 100644 --- a/opensbi-test-plic.patch +++ b/opensbi-test-plic.patch @@ -131,7 +131,7 @@ index 0ae604a..7b1d95e 100644 + +static void supervisor_trap_entry(void) +{ -+ sbi_printf("\nSupervisor Trap Entry Reached!\n"); ++ sbi_printf("\nSupervisor Trap Entry Reached!\nTEST-RESULT-OK\n"); +} + +static void do_plic_test(void) -- GitLab From 65b91368d61009d1190c67c61598cb012a0bb436 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 28 Aug 2024 10:19:58 +0200 Subject: [PATCH 167/310] Update baremetal tests to sv_eirq branch --- overlay.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/overlay.nix b/overlay.nix index af8e59e..4c5fa66 100644 --- a/overlay.nix +++ b/overlay.nix @@ -79,8 +79,8 @@ final: prev: name = "sa-fpga-tests"; src = builtins.fetchGit { url = "git@gitlab-internal.bsc.es:hwdesign/rtl/core-tile/sa-fpga.git"; - rev = "6547e63f3aed312a4c36f093b5e6421c7f1704f6"; - ref = "ni/supervisor_tests"; + rev = "afe0372413a94fff279ca5d5002c3e999ac8defb"; + ref = "ft/sv_eirq"; }; dontConfigure = true; patches = [ -- GitLab From 5072f40a2f20567cbacf6baf633b09c73aa3c55f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 28 Aug 2024 16:23:17 +0200 Subject: [PATCH 168/310] Leave automatic allocation in fpgalogin1 --- fpga/run-login.sh | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/fpga/run-login.sh b/fpga/run-login.sh index afe8dca..cc95b40 100755 --- a/fpga/run-login.sh +++ b/fpga/run-login.sh @@ -1,9 +1,16 @@ #!/usr/bin/bash -#set -x +# There are several situations in which we may find the jobs: +# - There are no jobs queued or running +# - There is at least one job running +# - There is one job queued +# - There was a job running but ended and is now ending + +set -x set -e path="$1" +allocated= # First determine if we already have jobs already n=$(squeue --me -lh | wc -l) @@ -15,17 +22,33 @@ fi if [ "$n" == 0 ]; then # No running jobs, so allocate a new job - echo salloc -N 1 --constraint=dmaqdma --no-shell + salloc -N 1 --constraint=dmaqdma --no-shell -t 1-00 + allocated=1 + + # Wait until the job is running + while [ "$n" != 1 ]; do + sleep 2 + n=$(squeue --me -lh | grep RUNNING | wc -l) + done +else + # There is one job, ensure it is running + n=$(squeue --me -lh | grep RUNNING | wc -l) + + if [ "$n" != 1 ]; then + echo "The job is not running, stopping" >&2 + exit 1 + fi fi -while [ "$n" != 1 ]; do - sleep 2 - n=$(squeue --me -lh) -done +# If this point is reached there is one job running host=$(squeue -h -o %N) - echo "Switching to $host" # Continue the execution there ssh "$host" "$path/run-node.sh" "$path" + +# Cancel our job if it was successful +if [ "$allocated" ]; then + scancel --me +fi -- GitLab From ebcef525ba4693363696615b44c582b0c81ff452 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 29 Aug 2024 13:42:13 +0200 Subject: [PATCH 169/310] Update bitstream to try to fix delegation (again) --- lagarto-ox.nix | 2 +- overlay.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index b4edb65..ae96af0 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -205,7 +205,7 @@ } "dtc -O dtb -o $out $dtsFile"; #bitstream = "${final.bitstreams}/lagarto-3-ox/gold.bit"; - bitstream = "${final.bitstreams}/lagarto-3-ox/ox_u55c_f6b4a076_fix_delegation.bit"; + bitstream = "${final.bitstreams}/lagarto-3-ox/ox_u55c_450d0ff0_fix_delegation_v2.bit"; bootrom = "${final.rbootrom}/rbootrom.bin"; diff --git a/overlay.nix b/overlay.nix index 4c5fa66..bfbdbef 100644 --- a/overlay.nix +++ b/overlay.nix @@ -71,7 +71,7 @@ final: prev: bitstreams = builtins.fetchGit { url = "git@bscpm03.bsc.es:rarias/bitstreams.git"; - rev = "d02b4805002d43519a69d17d19488c065f3574f9"; + rev = "097dad3e888f630ad6ec7e7056ee1b3fcb03400b"; }; # Baremetal tests for standalone FPGA -- GitLab From 2204a8ea5678812bedba418d1b124244df9d74b8 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 29 Aug 2024 14:39:27 +0200 Subject: [PATCH 170/310] Change load address delta to 0x6000_0000 --- fpga/env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fpga/env.sh b/fpga/env.sh index 3768b63..7df5b3b 100644 --- a/fpga/env.sh +++ b/fpga/env.sh @@ -37,7 +37,7 @@ function setup_meep() # Setup mappings # Delta between where we load in the dma device and RAM - local delta_addr=-0x5ffe0000 + local delta_addr=-0x60000000 # See https://gitlab.bsc.es/hwdesign/fpga/integration-lab/fpga-tools/-/blob/6a63bcea6d1d59df3c7d62311aa4935efd54d3a3/boot_riscv/boot_sa.sh#L36-40 export FPGACTL_BOOTLOADER_ADDR=$((0x80000000+$delta_addr)) -- GitLab From b7d4a10a145d0de0d4129a7b8210d87786e9bcb6 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 29 Aug 2024 14:41:09 +0200 Subject: [PATCH 171/310] Don't load an external bootrom Unfortunalely the bootrom is included in the bitstream now. --- fpga/run-node.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fpga/run-node.sh b/fpga/run-node.sh index 635ca24..8a831cc 100755 --- a/fpga/run-node.sh +++ b/fpga/run-node.sh @@ -18,11 +18,11 @@ killall picocom || true set -x # Then perform the boot -./fpgactl -w bitstream.bit -R bootrom.bin -b opensbi.bin -k kernel.bin -i initrd.bin # -r rootfs.img +./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin # -r rootfs.img # Restart it again only loading the bootrom and OpenSBI (notice OpenSBI # relocates itself so it will likely be gone by now) -sleep 1 && ./fpgactl -R bootrom.bin -b opensbi.bin & +sleep 1 && ./fpgactl -b opensbi.bin & # Set dead switch sleep 30 && killall picocom & -- GitLab From 13b9cd86924ae109e8a274d1c7bd91dc4fd33491 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 29 Aug 2024 15:06:30 +0200 Subject: [PATCH 172/310] Don't enable machine interrupts in delegation test --- opensbi-test-plic.patch | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch index 427072d..ac25326 100644 --- a/opensbi-test-plic.patch +++ b/opensbi-test-plic.patch @@ -175,10 +175,10 @@ index 0ae604a..7b1d95e 100644 + csr_write(CSR_MIP, 0); + + /* Enable external timer interrupts */ -+ sbi_printf("Enabling MEIE in MIE register\n"); -+ csr_set(CSR_MIE, MIE_MEIE); /* Needed? */ -+ sbi_printf("Enabling MIE in MSTATUS register\n"); -+ csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ ++ //sbi_printf("Enabling MEIE in MIE register\n"); ++ //csr_set(CSR_MIE, MIE_MEIE); /* Needed? */ ++ //sbi_printf("Enabling MIE in MSTATUS register\n"); ++ //csr_set(CSR_MSTATUS, MSTATUS_MIE); /* Needed? */ + + sbi_printf("Switching to supervisor\n"); + -- GitLab From 58ddfd47206c0b26aa4f946ef05f049d206bc785 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 29 Aug 2024 15:45:59 +0200 Subject: [PATCH 173/310] Dump registers in baremetal plic supervisor test --- overlay.nix | 1 + sa-fpga-plic-registers.patch | 92 ++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 sa-fpga-plic-registers.patch diff --git a/overlay.nix b/overlay.nix index bfbdbef..49c8f83 100644 --- a/overlay.nix +++ b/overlay.nix @@ -87,6 +87,7 @@ final: prev: #./sa-fpga-crt.patch #./sa-fpga-text-address.patch ./sa-fpga-uart.patch + ./sa-fpga-plic-registers.patch ]; buildPhase = '' cd fpga_core_bridge/simulator/tests/c_tests/ diff --git a/sa-fpga-plic-registers.patch b/sa-fpga-plic-registers.patch new file mode 100644 index 0000000..2ae8488 --- /dev/null +++ b/sa-fpga-plic-registers.patch @@ -0,0 +1,92 @@ +diff --git a/fpga_core_bridge/simulator/tests/c_tests/plic_supervisor/plic_supervisor_test.c b/fpga_core_bridge/simulator/tests/c_tests/plic_supervisor/plic_supervisor_test.c +index 0cfa681..78d97cb 100644 +--- a/fpga_core_bridge/simulator/tests/c_tests/plic_supervisor/plic_supervisor_test.c ++++ b/fpga_core_bridge/simulator/tests/c_tests/plic_supervisor/plic_supervisor_test.c +@@ -68,6 +68,48 @@ uintptr_t handle_trap(uint64_t cause, uint64_t epc, uintptr_t regs[32]) + return epc; + } + ++static void dumpregs(int machine) ++{ ++ printf("Registers:"); ++ if (machine) { ++ uint64_t mie; ++ asm volatile("csrr %0, mie" : "=r"(mie)); ++ printf("\n MIE: "); ++ printhex(mie); ++ ++ uint64_t mip; ++ asm volatile("csrr %0, mip" : "=r"(mip)); ++ printf("\n MIP: "); ++ printhex(mip); ++ ++ uint64_t mstatus; ++ asm volatile("csrr %0, mstatus" : "=r"(mstatus)); ++ printf("\nMSTATUS: "); ++ printhex(mstatus); ++ ++ uint64_t mideleg; ++ asm volatile("csrr %0, mideleg" : "=r"(mideleg)); ++ printf("\nMIDELEG: "); ++ printhex(mideleg); ++ } ++ ++ uint64_t sie; ++ asm volatile("csrr %0, sie" : "=r"(sie)); ++ printf("\n SIE: "); ++ printhex(sie); ++ ++ uint64_t sip; ++ asm volatile("csrr %0, sip" : "=r"(sip)); ++ printf("\n SIP: "); ++ printhex(sip); ++ ++ uint64_t sstatus; ++ asm volatile("csrr %0, sstatus" : "=r"(sstatus)); ++ printf("\nSSTATUS: "); ++ printhex(sstatus); ++ printf("\n"); ++} ++ + // Define the bit positions for the external interrupt enable in mie and mideleg registers + #define SIE_SEIE (1 << 9) // Supervisor External Interrupt Enable + #define MIDELEG_MEIE (1 << 11) // Delegate Machine External Interrupt to Supervisor +@@ -156,16 +198,19 @@ void __attribute__((optimize("O0"))) switch_to_supervisor_mode(uint64_t* target_ + asm volatile("mret"); + } + +-uint64_t supervisor_mode_code() { +- int count = 0; +- while (1) { +- if (count == 10000) { +- uart_write_string("\nWaiting for interrupt in supervisor mode..."); +- count = 0; +- } +- count++; +- } +- return 0; ++uint64_t supervisor_mode_code() ++{ ++ uart_write_string("\nHello from supervisor mode..."); ++ dumpregs(0); ++ int count = 0; ++ while (1) { ++ if (count == 10000) { ++ uart_write_string("\nWaiting for interrupt in supervisor mode..."); ++ count = 0; ++ } ++ count++; ++ } ++ return 0; + } + + void main(void) { +@@ -181,6 +226,8 @@ void main(void) { + // Enable external timer interrupts + // enable_external_timer_interrupt(); + ++ dumpregs(1); ++ + // Switch to supervisor mode and execute supervisor_mode_code + switch_to_supervisor_mode(&supervisor_mode_code); + -- GitLab From 6be0f70c8d0358fdbf0ecff23dc726c4b38beef8 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 29 Aug 2024 17:14:04 +0200 Subject: [PATCH 174/310] Fix PLIC offsets for aux timer --- opensbi-test-plic.patch | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/opensbi-test-plic.patch b/opensbi-test-plic.patch index ac25326..d85a3c1 100644 --- a/opensbi-test-plic.patch +++ b/opensbi-test-plic.patch @@ -1,5 +1,5 @@ diff --git a/lib/sbi/sbi_irqchip.c b/lib/sbi/sbi_irqchip.c -index 0ae604a..7b1d95e 100644 +index 0ae604a..e34e90c 100644 --- a/lib/sbi/sbi_irqchip.c +++ b/lib/sbi/sbi_irqchip.c @@ -9,6 +9,9 @@ @@ -25,7 +25,7 @@ index 0ae604a..7b1d95e 100644 return 0; } -@@ -47,8 +52,163 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) +@@ -47,8 +52,170 @@ void sbi_irqchip_exit(struct sbi_scratch *scratch) { const struct sbi_platform *plat = sbi_platform_ptr(scratch); @@ -46,12 +46,12 @@ index 0ae604a..7b1d95e 100644 +#define MIDELEG_SEIE (1UL << 9) // Delegate Machine External Interrupt to Supervisor +#define PLIC_TIMER_PORT 4 +// Base address of PLIC -+#define PLIC_BASE 0x0000000040800000UL ++#define PLIC_BASE 0x40800000UL +#define PLIC_PRIORITY_OFFSET 0x0UL +#define PLIC_PENDING_OFFSET 0x1000UL -+#define PLIC_ENABLE_OFFSET 0x2000UL -+#define PLIC_THRESHOLD_OFFSET 0x200000UL -+#define PLIC_CLAIM_OFFSET 0x200004UL ++#define PLIC_ENABLE_OFFSET 0x2080UL ++#define PLIC_THRESHOLD_OFFSET 0x201000UL ++#define PLIC_CLAIM_OFFSET 0x201004UL + +// Aux timer +#define AUX_TIMER_BASE 0x40010000UL @@ -86,6 +86,8 @@ index 0ae604a..7b1d95e 100644 + prefix, suffix, csr_read(CSR_SIP)); + sbi_printf("%sSSTATUS%s: 0x%" PRILX "\n", + prefix, suffix, csr_read(CSR_SSTATUS)); ++ sbi_printf("%sSTVEC%s: 0x%" PRILX "\n", ++ prefix, suffix, csr_read(CSR_STVEC)); +} + +static void __attribute__((optimize("O0"))) switch_to_supervisor_mode(int (*target_address)(void)) @@ -129,9 +131,12 @@ index 0ae604a..7b1d95e 100644 + return 0; +} + -+static void supervisor_trap_entry(void) ++static void __attribute__((aligned(4))) __attribute__((interrupt ("supervisor"))) supervisor_trap_entry(void) +{ -+ sbi_printf("\nSupervisor Trap Entry Reached!\nTEST-RESULT-OK\n"); ++ sbi_printf("\nSupervisor Trap Entry Reached!\n"); ++ sbi_printf("\nTEST-RESULT-OK\n"); ++ while (1) { ++ } +} + +static void do_plic_test(void) @@ -144,10 +149,12 @@ index 0ae604a..7b1d95e 100644 + + + /* Enable supervisor interrupt delegation */ ++ + csr_set(CSR_SIE, SIE_SEIE); // Enable supervisor external interrupts + csr_set(CSR_SSTATUS, SSTATUS_SIE); // Enable global interrupts in supervisor mode + csr_set(CSR_MIDELEG, MIDELEG_SEIE); // Delegate machine interrupts to supervisor mode -+ csr_set(CSR_STVEC, &supervisor_trap_entry); ++ csr_write(CSR_STVEC, &supervisor_trap_entry); ++ + sbi_printf("Enabled supervisor delegation:\n"); + + dumpregs(1); -- GitLab From 9abb28e9c576a1cbad538d56b04c7f7840482565 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 29 Aug 2024 18:21:10 +0200 Subject: [PATCH 175/310] Update journal --- JOURNAL.md | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index 7078498..68ce345 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -3624,3 +3624,118 @@ potential errors: While the bitstream with the potential fixes for the supervisor interrupts is not available, I'll try to automate the testing by creating a pipeline that automatically runs the tests in an FPGA. + +## 2024-08-29 + +The `plic_supervisor` baremetal test seems to be working with bitstream +`ox_u55c_450d0ff0_fix_delegation_v2.bit`. Here is the output with registers +dumped: + + Set timer interval. + Registers: + MIE: 0000000000000200 + MIP: 0000000000000280 + MSTATUS: 8000000a00006002 + MIDELEG: 0000000000000200 + SIE: 0000000000000200 + SIP: 0000000000000200 + SSTATUS: 8000000200006002 + + Jumping to supervisor mode... + + Supervisor Trap Entry Reached + Cause: 8000000000000009 EPC: 000000008000111a + Hi from PLIC Interrupt Service Routine! + Claim interrupt: 0000000000000004 + Set timer interval. + + + Supervisor Trap Entry Reached + Cause: 8000000000000009 EPC: 0000000080001e88 + Hi from PLIC Interrupt Service Routine! + Claim interrupt: 0000000000000004 + Set timer interval. + +In our OpenSBI test however: + + OpenSBI v1.5 + ____ _____ ____ _____ + / __ \ / ____| _ \_ _| + | | | |_ __ ___ _ __ | (___ | |_) || | + | | | | '_ \ / _ \ '_ \ \___ \| _ < | | + | |__| | |_) | __/ | | |____) | |_) || |_ + \____/| .__/ \___|_| |_|_____/|____/_____| + | | + |_| + + --- TESTING PLIC --- + Timer interrupt disabled + Enabled supervisor delegation: + Registers: + MIE : 0x0000000000000200 + MIP : 0x0000000000000080 + MSTATUS : 0x8000000a00006082 + MIDELEG : 0x0000000000000222 + SIE : 0x0000000000000200 + SIP : 0x0000000000000000 + SSTATUS : 0x8000000200006002 + Enabling timer in PLIC + Pending: 16 + Claim: 4 + Pending: 0 + Clearing MIP + Switching to supervisor + Registers: + MIE : 0x0000000000000200 + MIP : 0x0000000000000080 + MSTATUS : 0x8000000a00006082 + MIDELEG : 0x0000000000000222 + SIE : 0x0000000000000200 + SIP : 0x0000000000000000 + SSTATUS : 0x8000000200006002 + Hello from supervisor + Registers: + SIE : 0x0000000000000200 + SIP : 0x0000000000000000 + SSTATUS : 0x8000000200006002 + Timer alarm programmed + Waiting for interrupt... + +Here are closer together: + + Baremetal: + MIE: 0000000000000200 + MIP: 0000000000000280 + MSTATUS: 8000000a00006002 + MIDELEG: 0000000000000200 + SIE: 0000000000000200 + SIP: 0000000000000200 + SSTATUS: 8000000200006002 + --------------------------------- + OpenSBI: + MIE: 0000000000000200 + MIP: 0000000000000080 * + MSTATUS: 8000000a00006082 * + MIDELEG: 0000000000000222 * + SIE: 0000000000000200 + SIP: 0000000000000000 * + SSTATUS: 8000000200006002 + +In the baremetal test, the interrupts are already pending before jumping to +supervisor. + +Let's try following all the steps exactly the same in OpenSBI. + +The supervisor trap address hold the proper function. + +The problem seems to be that the offsets I was using for the auxiliar timer were +not correct: + + -#define PLIC_ENABLE_OFFSET 0x2000UL + -#define PLIC_THRESHOLD_OFFSET 0x200000UL + -#define PLIC_CLAIM_OFFSET 0x200004UL + +#define PLIC_ENABLE_OFFSET 0x2080UL + +#define PLIC_THRESHOLD_OFFSET 0x201000UL + +#define PLIC_CLAIM_OFFSET 0x201004UL + +This seems to be required now as we have two contexts. The test is working now. -- GitLab From 93014c68b8c3fb86efe7dff168c746ae5ed2d0ee Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 11:35:32 +0200 Subject: [PATCH 176/310] Disable PLIC test for now Let's focus on the rest of the boot process. --- JOURNAL.md | 10 ++++++++++ lagarto-ox.nix | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/JOURNAL.md b/JOURNAL.md index 68ce345..c62bf7e 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -3739,3 +3739,13 @@ not correct: +#define PLIC_CLAIM_OFFSET 0x201004UL This seems to be required now as we have two contexts. The test is working now. + +## 2024-08-30 + +We finally have delegation to supervisor working, so now I can go back and focus +on booting NixOS. We will probably require JTAG or some other means to debug all +problems with the boot. + +Now that we have a CI pipeline, let's try making the whole boot process +automatic, so I don't have to type anything. This basically requires setting the +environment in U-Boot. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index ae96af0..d5c6400 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -334,7 +334,7 @@ ./opensbi-timer-debug.patch #./opensbi-enable-meip.patch #./opensbi-enable-seip.patch - ./opensbi-test-plic.patch + #./opensbi-test-plic.patch # Working delegation test, disabled for now ./opensbi-dump-mregs.patch #./opensbi-dont-delegate.patch #./ox-alveo-platform-plic.patch -- GitLab From 8a123d93bf4ec3b6009e550122286dbfba328970 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 11:54:45 +0200 Subject: [PATCH 177/310] Place environment in U-Boot --- lagarto-ox.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index d5c6400..e367f8d 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -221,6 +221,15 @@ ./uboot-debug-ext-interrupts.patch ./uboot-exception-extras.patch ]; + # Copy our environment to board/emulation/qemu-riscv/environ.env + preConfigure = '' + cp ${final.uboot-env} board/emulation/qemu-riscv/environ.env + ''; + postConfigure = '' + echo --------------------------- generated config: + cat .config + echo --------------------------- + ''; # # CONFIG_SERIAL_PRESENT=n # CONFIG_SYS_NS16550=n @@ -248,6 +257,7 @@ CONFIG_CMD_MEMTEST=y CONFIG_CMD_EXCEPTION=y CONFIG_CMD_TIMER=y + CONFIG_ENV_SOURCE_FILE="environ" '' # # Enable debug logs # + @@ -259,6 +269,7 @@ # '' ; extraMakeFlags = [ + "V=1" #"KCPPFLAGS=-DLOG_DEBUG" #"EXT_DTB=${final.ox-dtb}" ]; -- GitLab From 742ce37cecd4c739c9cba8ab9619a491744dbf32 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 13:46:00 +0200 Subject: [PATCH 178/310] Automatically load U-Boot environment on boot --- lagarto-ox.nix | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index e367f8d..39b87be 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -230,6 +230,11 @@ cat .config echo --------------------------- ''; + postBuild = '' + echo --------------------------- generated env starts + cat include/generated/env.in + echo --------------------------- generated env ends + ''; # # CONFIG_SERIAL_PRESENT=n # CONFIG_SYS_NS16550=n @@ -280,17 +285,14 @@ initrd = "${config.system.build.initialRamdisk}/initrd"; rev = if self ? rev then self.rev else throw ("Refusing to build from a dirty Git tree!"); - in prev.runCommand "uboot.txt" {} '' - cat > $out < + #fdt mknode / pmem@0x140000000 + #fdt set /pmem@0x140000000 compatible "pmem-region" + #fdt set /pmem@0x140000000 reg <0x1 0x40000000 0x0 0xc0000000> # Reduce memory to 3 GiB [0x80000000, 0x140000000) - fdt set /memory@80000000 reg <0x0 0x80000000 0x0 0xc0000000> + #fdt set /memory@80000000 reg <0x0 0x80000000 0x0 0xc0000000> # Set kernel bootcmd options. # rdinit=/preinit boot custom preinit script @@ -307,14 +309,14 @@ # tp_printk write ftrace events to console # trace_buf_size=1M set ftrace buffer to 1M # - setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 systemd.log_level=debug systemd.log_target=console rdinit=/preinit init=${init}" - + in prev.runCommand "uboot.txt" {} '' + cat > $out <> $out - echo "setenv ramdisk_size $(stat --format %s $(readlink -f ${initrd}))" >> $out - echo 'booti ''${kernel_addr_r} ''${ramdisk_addr_r}:''${ramdisk_size} ''${fdtcontroladdr}' >> $out ''; opensbi = prev.opensbi.overrideAttrs (old: rec { @@ -342,7 +344,7 @@ "FW_PAYLOAD_FDT_ADDR=0xc0000000" ]; patches = [ - ./opensbi-timer-debug.patch + ./opensbi-timer-debug.patch # Print calls to machine trap #./opensbi-enable-meip.patch #./opensbi-enable-seip.patch #./opensbi-test-plic.patch # Working delegation test, disabled for now -- GitLab From 1ede19807e395b4f6a8ce0851540dd9b038c2b43 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 13:46:33 +0200 Subject: [PATCH 179/310] Increase boot timeout to 5 minutes --- fpga/run-node.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fpga/run-node.sh b/fpga/run-node.sh index 8a831cc..6f1de88 100755 --- a/fpga/run-node.sh +++ b/fpga/run-node.sh @@ -25,6 +25,6 @@ set -x sleep 1 && ./fpgactl -b opensbi.bin & # Set dead switch -sleep 30 && killall picocom & +sleep 300 && killall picocom & -picocom -q -x 10000 -b 115200 $FPGACTL_UART +picocom -q -x 300000 -b 115200 $FPGACTL_UART -- GitLab From 51ceb85d691fba256244970e35edc7b50b14acb5 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 14:06:15 +0200 Subject: [PATCH 180/310] Print FDT before booting in U-Boot --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 39b87be..ae7ffc4 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -315,7 +315,7 @@ xtrace=yes bootargs=root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 rdinit=/preinit init=${init} ramdisk_size=$(stat --format %s $(readlink -f ${initrd})) - bootcmd=booti \''${kernel_addr_r} \''${ramdisk_addr_r}:\''${ramdisk_size} \''${fdtcontroladdr} + bootcmd=fdt print; booti \''${kernel_addr_r} \''${ramdisk_addr_r}:\''${ramdisk_size} \''${fdtcontroladdr} EOF ''; -- GitLab From d72b5ff5e9932cf02426f82842286530c93ef28f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 14:14:27 +0200 Subject: [PATCH 181/310] Disable the CLINT for now --- ox-plic.dts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ox-plic.dts b/ox-plic.dts index a5fdff6..e0191f4 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -224,17 +224,17 @@ * property is described in * Documentation/devicetree/bindings/riscv/cpus.yaml */ - clint: clint@40002000 { - /* MTIME and MTIMECMP address and size pairs */ - reg = <0x0 0x40002000 0x0 0x8>, <0x0 0x40002008 0x0 0x8>; - interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; - /*<&CPU0 0x3>, <&CPU0 0x7>,*/ - /*<&onic_pool 0x3>, <&onic_pool 0x7>,*/ - /*<&SERIAL 0x3>, <&SERIAL 0x7>*/ - - /* Allows using the "generic" platform in OpenSBI. */ - compatible = "riscv,aclint-mtimer"; - }; +// clint: clint@40002000 { +// /* MTIME and MTIMECMP address and size pairs */ +// reg = <0x0 0x40002000 0x0 0x8>, <0x0 0x40002008 0x0 0x8>; +// interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; +// /*<&CPU0 0x3>, <&CPU0 0x7>,*/ +// /*<&onic_pool 0x3>, <&onic_pool 0x7>,*/ +// /*<&SERIAL 0x3>, <&SERIAL 0x7>*/ +// +// /* Allows using the "generic" platform in OpenSBI. */ +// compatible = "riscv,aclint-mtimer"; +// }; /* Guesswork: There must be a timer at 0x40170000 as it is * initialized in OpenSBI. It seems to drive the console. */ -- GitLab From dce8b590611db812bda4c0d431118536afa54838 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 14:29:23 +0200 Subject: [PATCH 182/310] Print interrupt type from mcause --- opensbi-timer-debug.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensbi-timer-debug.patch b/opensbi-timer-debug.patch index 0f4b635..cc47238 100644 --- a/opensbi-timer-debug.patch +++ b/opensbi-timer-debug.patch @@ -49,7 +49,7 @@ index b4f3a17..cde2073 100644 sbi_trap_set_context(scratch, tcntx); if (mcause & MCAUSE_IRQ_MASK) { -+ sbi_printf("i"); ++ sbi_printf("i(%lu)", mcause & ~MCAUSE_IRQ_MASK); if (sbi_hart_has_extension(sbi_scratch_thishart_ptr(), SBI_HART_EXT_SMAIA)) rc = sbi_trap_aia_irq(); -- GitLab From a7c6a6f1e635134439aa06447fb24b6e3081306d Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 15:10:46 +0200 Subject: [PATCH 183/310] Add CLINT to device tree at 0x4010_0000 --- ox-plic.dts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ox-plic.dts b/ox-plic.dts index e0191f4..2bcce4c 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -224,6 +224,12 @@ * property is described in * Documentation/devicetree/bindings/riscv/cpus.yaml */ + clint: clint@40100000 { + reg = <0x0 0x40100000 0x0 0x00010000>; + reg-names = "control"; + interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; + compatible = "riscv,clint0"; + }; // clint: clint@40002000 { // /* MTIME and MTIMECMP address and size pairs */ // reg = <0x0 0x40002000 0x0 0x8>, <0x0 0x40002008 0x0 0x8>; -- GitLab From 101aa1b35290246d38ff081fd1be702e3d7b7a4e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 15:20:11 +0200 Subject: [PATCH 184/310] Don't print machine mode interrupts Get are getting a lot of machine mode interrupts, which should be delegated to supervisor directly. Disable them for now as otherwise they will flood the log. --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index ae7ffc4..3986687 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -344,7 +344,7 @@ "FW_PAYLOAD_FDT_ADDR=0xc0000000" ]; patches = [ - ./opensbi-timer-debug.patch # Print calls to machine trap + #./opensbi-timer-debug.patch # Print calls to machine trap #./opensbi-enable-meip.patch #./opensbi-enable-seip.patch #./opensbi-test-plic.patch # Working delegation test, disabled for now -- GitLab From 46d897a57492603fccd0b3281a24a25b339c8949 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 17:31:52 +0200 Subject: [PATCH 185/310] Use a custom stage2 script with strace --- lagarto-ox.nix | 24 ++++++++ stage-2-init.sh | 152 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100755 stage-2-init.sh diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 3986687..30025a5 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -11,6 +11,30 @@ # gcc.tune = "generic"; #}; + system.build.bootStage2 = let + useHostResolvConf = config.networking.resolvconf.enable && config.networking.useHostResolvConf; + bootStage2 = pkgs.substituteAll { + src = ./stage-2-init.sh; + shellDebug = "${pkgs.bashInteractive}/bin/bash"; + shell = "${pkgs.bash}/bin/bash"; + inherit (config.boot) readOnlyNixStore systemdExecutable extraSystemdUnitPaths; + inherit (config.system.nixos) distroName; + isExecutable = true; + inherit useHostResolvConf; + inherit (config.system.build) earlyMountScript; + path = lib.makeBinPath ([ + pkgs.coreutils + pkgs.util-linux + pkgs.strace + ] ++ lib.optional useHostResolvConf pkgs.openresolv); + postBootCommands = pkgs.writeText "local-cmds" + '' + ${config.boot.postBootCommands} + ${config.powerManagement.powerUpCommands} + ''; + }; + in lib.mkForce bootStage2; + boot.kernelPackages = pkgs.linuxPackages_latest; boot = { kernelPatches = [ diff --git a/stage-2-init.sh b/stage-2-init.sh new file mode 100755 index 0000000..5a44539 --- /dev/null +++ b/stage-2-init.sh @@ -0,0 +1,152 @@ +#! @shell@ + +set -x + +echo "HELLOOOOOOOOOOOO FROM STAGE 2" + +systemConfig=@systemConfig@ + +export HOME=/root PATH="@path@" + + +if [ "${IN_NIXOS_SYSTEMD_STAGE1:-}" != true ]; then + # Process the kernel command line. + for o in $(>>\e[0m" + echo + + + # Normally, stage 1 mounts the root filesystem read/writable. + # However, in some environments, stage 2 is executed directly, and the + # root is read-only. So make it writable here. + if [ -z "$container" ]; then + mount -n -o remount,rw none / + fi +fi + + +# Likewise, stage 1 mounts /proc, /dev and /sys, so if we don't have a +# stage 1, we need to do that here. +if [ ! -e /proc/1 ]; then + specialMount() { + local device="$1" + local mountPoint="$2" + local options="$3" + local fsType="$4" + + # We must not overwrite this mount because it's bind-mounted + # from stage 1's /run + if [ "${IN_NIXOS_SYSTEMD_STAGE1:-}" = true ] && [ "${mountPoint}" = /run ]; then + return + fi + + install -m 0755 -d "$mountPoint" + mount -n -t "$fsType" -o "$options" "$device" "$mountPoint" + } + source @earlyMountScript@ +fi + + +if [ "${IN_NIXOS_SYSTEMD_STAGE1:-}" = true ] || [ ! -c /dev/kmsg ] ; then + echo "booting system configuration ${systemConfig}" +else + echo "booting system configuration $systemConfig" > /dev/kmsg +fi + + +# Make /nix/store a read-only bind mount to enforce immutability of +# the Nix store. Note that we can't use "chown root:nixbld" here +# because users/groups might not exist yet. +# Silence chown/chmod to fail gracefully on a readonly filesystem +# like squashfs. +chown -f 0:30000 /nix/store +chmod -f 1775 /nix/store +if [ -n "@readOnlyNixStore@" ]; then + if ! [[ "$(findmnt --noheadings --output OPTIONS /nix/store)" =~ ro(,|$) ]]; then + if [ -z "$container" ]; then + mount --bind /nix/store /nix/store + else + mount --rbind /nix/store /nix/store + fi + mount -o remount,ro,bind /nix/store + fi +fi + + +if [ "${IN_NIXOS_SYSTEMD_STAGE1:-}" != true ]; then + # Use /etc/resolv.conf supplied by systemd-nspawn, if applicable. + if [ -n "@useHostResolvConf@" ] && [ -e /etc/resolv.conf ]; then + resolvconf -m 1000 -a host &1 {logErrFd}>&2 + if test -w /dev/kmsg; then + exec > >(tee -i /proc/self/fd/"$logOutFd" | while read -r line; do + if test -n "$line"; then + echo "<7>stage-2-init: $line" > /dev/kmsg + fi + done) 2>&1 + else + mkdir -p /run/log + exec > >(tee -i /run/log/stage-2-init.log) 2>&1 + fi +fi + + +# Required by the activation script +install -m 0755 -d /etc +if [ ! -h "/etc/nixos" ]; then + install -m 0755 -d /etc/nixos +fi +install -m 01777 -d /tmp + + +# Run the script that performs all configuration activation that does +# not have to be done at boot time. +echo "running activation script..." +strace -f $systemConfig/activate + +echo "ACTIVATE WORKED!!!" + +# Record the boot configuration. +ln -sfn "$systemConfig" /run/booted-system + + +# Run any user-specified commands. +@shell@ @postBootCommands@ + + +# Ensure systemd doesn't try to populate /etc, by forcing its first-boot +# heuristic off. It doesn't matter what's in /etc/machine-id for this purpose, +# and systemd will immediately fill in the file when it starts, so just +# creating it is enough. This `: >>` pattern avoids forking and avoids changing +# the mtime if the file already exists. +: >> /etc/machine-id + + +# No need to restore the stdout/stderr streams we never redirected and +# especially no need to start systemd +if [ "${IN_NIXOS_SYSTEMD_STAGE1:-}" != true ]; then + # Reset the logging file descriptors. + exec 1>&$logOutFd 2>&$logErrFd + exec {logOutFd}>&- {logErrFd}>&- + + + # Start systemd in a clean environment. + echo "starting systemd..." + exec @systemdExecutable@ "$@" +fi -- GitLab From 5a97c527f44a0a663f71fa61a022402e856ee94d Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 18:17:41 +0200 Subject: [PATCH 186/310] Don't redirect stage2 output --- stage-2-init.sh | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/stage-2-init.sh b/stage-2-init.sh index 5a44539..c496adf 100755 --- a/stage-2-init.sh +++ b/stage-2-init.sh @@ -91,19 +91,19 @@ if [ "${IN_NIXOS_SYSTEMD_STAGE1:-}" != true ]; then fi - # Log the script output to /dev/kmsg or /run/log/stage-2-init.log. - # Only at this point are all the necessary prerequisites ready for these commands. - exec {logOutFd}>&1 {logErrFd}>&2 - if test -w /dev/kmsg; then - exec > >(tee -i /proc/self/fd/"$logOutFd" | while read -r line; do - if test -n "$line"; then - echo "<7>stage-2-init: $line" > /dev/kmsg - fi - done) 2>&1 - else - mkdir -p /run/log - exec > >(tee -i /run/log/stage-2-init.log) 2>&1 - fi +# # Log the script output to /dev/kmsg or /run/log/stage-2-init.log. +# # Only at this point are all the necessary prerequisites ready for these commands. +# exec {logOutFd}>&1 {logErrFd}>&2 +# if test -w /dev/kmsg; then +# exec > >(tee -i /proc/self/fd/"$logOutFd" | while read -r line; do +# if test -n "$line"; then +# echo "<7>stage-2-init: $line" > /dev/kmsg +# fi +# done) 2>&1 +# else +# mkdir -p /run/log +# exec > >(tee -i /run/log/stage-2-init.log) 2>&1 +# fi fi @@ -142,8 +142,8 @@ ln -sfn "$systemConfig" /run/booted-system # especially no need to start systemd if [ "${IN_NIXOS_SYSTEMD_STAGE1:-}" != true ]; then # Reset the logging file descriptors. - exec 1>&$logOutFd 2>&$logErrFd - exec {logOutFd}>&- {logErrFd}>&- + #exec 1>&$logOutFd 2>&$logErrFd + #exec {logOutFd}>&- {logErrFd}>&- # Start systemd in a clean environment. -- GitLab From cd1c340a34c5828122e225ce1eb0e822a02ffc0b Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 18:21:33 +0200 Subject: [PATCH 187/310] Disable preinit script --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 30025a5..8ac3b2c 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -337,7 +337,7 @@ cat > $out < Date: Fri, 30 Aug 2024 18:37:15 +0200 Subject: [PATCH 188/310] Prepare CI pipeline to boot NixOS Increase the timeout to 30 minutes and load the rootfs --- fpga/run-node.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fpga/run-node.sh b/fpga/run-node.sh index 6f1de88..01ab16e 100755 --- a/fpga/run-node.sh +++ b/fpga/run-node.sh @@ -18,13 +18,16 @@ killall picocom || true set -x # Then perform the boot -./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin # -r rootfs.img +./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img # Restart it again only loading the bootrom and OpenSBI (notice OpenSBI # relocates itself so it will likely be gone by now) sleep 1 && ./fpgactl -b opensbi.bin & +timeout=$((30 * 60)) # Always stop after 30 min +timeout_silent=$((10 * 60)) # Stop if 10 min without output + # Set dead switch -sleep 300 && killall picocom & +sleep $timeout && killall picocom & -picocom -q -x 300000 -b 115200 $FPGACTL_UART +picocom -q -x $(($timeout_silent*1000)) -b 115200 $FPGACTL_UART -- GitLab From 2bf9e94ca641588a76580d02629090cbe19fb4c8 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 18:53:58 +0200 Subject: [PATCH 189/310] Build the full system in CI --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 33d9ada..b5e4351 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,4 +3,4 @@ build:lagarto-ox-rd: tags: - nix script: - - nix develop -L .#lagarto-ox-rd --command fpga/run-remotely.sh fpgalogin1:ci + - nix develop -L .#lagarto-ox --command fpga/run-remotely.sh fpgalogin1:ci -- GitLab From 10fb6d0a0afde79338c3d35fadacd2f2477d70f2 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 19:06:54 +0200 Subject: [PATCH 190/310] Disable awk verification It seems to be causing problems with the CI output log. --- fpga/run-remotely.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fpga/run-remotely.sh b/fpga/run-remotely.sh index 74cbe0f..550db92 100755 --- a/fpga/run-remotely.sh +++ b/fpga/run-remotely.sh @@ -18,4 +18,5 @@ fpga/upload.sh "$dst" # Launch the pipeline from there set +x -ssh fpgalogin1 "$path/run-login.sh" "$path" | awk -f fpga/verify.awk +#ssh fpgalogin1 "$path/run-login.sh" "$path" | awk -f fpga/verify.awk +ssh fpgalogin1 "$path/run-login.sh" "$path" -- GitLab From 7573145dccaacc30c8cdc32d3cb9bd8f6f83cfdc Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 19:56:55 +0200 Subject: [PATCH 191/310] Disable nscd call in activation script --- lagarto-ox.nix | 31 +++- update-users-groups.pl | 379 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 409 insertions(+), 1 deletion(-) create mode 100644 update-users-groups.pl diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 8ac3b2c..bfcade3 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -1,4 +1,4 @@ -{ config, lib, pkgs, modulesPath, self, ... }: +{ config, lib, utils, pkgs, modulesPath, self, ... }: { imports = [ @@ -11,6 +11,35 @@ # gcc.tune = "generic"; #}; + system.activationScripts.users = lib.mkForce ( + let + cfg = config.users; + spec = pkgs.writeText "users-groups.json" (builtins.toJSON { + inherit (cfg) mutableUsers; + users = lib.mapAttrsToList (_: u: + { + inherit (u) + name uid group description home homeMode createHome isSystemUser + password hashedPasswordFile hashedPassword + autoSubUidGidRange subUidRanges subGidRanges + initialPassword initialHashedPassword expires; + shell = utils.toShellPath u.shell; + }) cfg.users; + groups = lib.attrValues cfg.groups; + }); + in + if !config.systemd.sysusers.enable then { + supportsDryActivation = true; + text = '' + install -m 0700 -d /root + install -m 0755 -d /home + + ${pkgs.perl.withPackages (p: [ p.FileSlurp p.JSON ])}/bin/perl \ + -w ${./update-users-groups.pl} ${spec} + ''; + } else "" # keep around for backwards compatibility + ); + system.build.bootStage2 = let useHostResolvConf = config.networking.resolvconf.enable && config.networking.useHostResolvConf; bootStage2 = pkgs.substituteAll { diff --git a/update-users-groups.pl b/update-users-groups.pl new file mode 100644 index 0000000..3f859ca --- /dev/null +++ b/update-users-groups.pl @@ -0,0 +1,379 @@ +use strict; +use warnings; +use File::Path qw(make_path); +use File::Slurp; +use Getopt::Long; +use JSON; +use Time::Piece; + +# Keep track of deleted uids and gids. +my $uidMapFile = "/var/lib/nixos/uid-map"; +my $uidMap = -e $uidMapFile ? decode_json(read_file($uidMapFile)) : {}; + +my $gidMapFile = "/var/lib/nixos/gid-map"; +my $gidMap = -e $gidMapFile ? decode_json(read_file($gidMapFile)) : {}; + +my $is_dry = ($ENV{'NIXOS_ACTION'} // "") eq "dry-activate"; +GetOptions("dry-activate" => \$is_dry); +make_path("/var/lib/nixos", { mode => 0755 }) unless $is_dry; + +sub updateFile { + my ($path, $contents, $perms) = @_; + return if $is_dry; + write_file($path, { atomic => 1, binmode => ':utf8', perms => $perms // 0644 }, $contents) or die; +} + +# Converts an ISO date to number of days since 1970-01-01 +sub dateToDays { + my ($date) = @_; + my $time = Time::Piece->strptime($date, "%Y-%m-%d"); + return $time->epoch / 60 / 60 / 24; +} + +sub nscdInvalidate { + system("echo", "nscd", "--invalidate", $_[0]) unless $is_dry; +} + +sub hashPassword { + my ($password) = @_; + my $salt = ""; + my @chars = ('.', '/', 0..9, 'A'..'Z', 'a'..'z'); + $salt .= $chars[rand 64] for (1..8); + return crypt($password, '$6$' . $salt . '$'); +} + +sub dry_print { + if ($is_dry) { + print STDERR ("$_[1] $_[2]\n") + } else { + print STDERR ("$_[0] $_[2]\n") + } +} + + +# Functions for allocating free GIDs/UIDs. FIXME: respect ID ranges in +# /etc/login.defs. +sub allocId { + my ($used, $prevUsed, $idMin, $idMax, $up, $getid) = @_; + my $id = $up ? $idMin : $idMax; + while ($id >= $idMin && $id <= $idMax) { + if (!$used->{$id} && !$prevUsed->{$id} && !defined &$getid($id)) { + $used->{$id} = 1; + return $id; + } + $used->{$id} = 1; + if ($up) { $id++; } else { $id--; } + } + die "$0: out of free UIDs or GIDs\n"; +} + +my (%gidsUsed, %uidsUsed, %gidsPrevUsed, %uidsPrevUsed); + +sub allocGid { + my ($name) = @_; + my $prevGid = $gidMap->{$name}; + if (defined $prevGid && !defined $gidsUsed{$prevGid}) { + dry_print("reviving", "would revive", "group '$name' with GID $prevGid"); + $gidsUsed{$prevGid} = 1; + return $prevGid; + } + return allocId(\%gidsUsed, \%gidsPrevUsed, 400, 999, 0, sub { my ($gid) = @_; getgrgid($gid) }); +} + +sub allocUid { + my ($name, $isSystemUser) = @_; + my ($min, $max, $up) = $isSystemUser ? (400, 999, 0) : (1000, 29999, 1); + my $prevUid = $uidMap->{$name}; + if (defined $prevUid && $prevUid >= $min && $prevUid <= $max && !defined $uidsUsed{$prevUid}) { + dry_print("reviving", "would revive", "user '$name' with UID $prevUid"); + $uidsUsed{$prevUid} = 1; + return $prevUid; + } + return allocId(\%uidsUsed, \%uidsPrevUsed, $min, $max, $up, sub { my ($uid) = @_; getpwuid($uid) }); +} + +# Read the declared users/groups +my $spec = decode_json(read_file($ARGV[0])); + +# Don't allocate UIDs/GIDs that are manually assigned. +foreach my $g (@{$spec->{groups}}) { + $gidsUsed{$g->{gid}} = 1 if defined $g->{gid}; +} + +foreach my $u (@{$spec->{users}}) { + $uidsUsed{$u->{uid}} = 1 if defined $u->{uid}; +} + +# Likewise for previously used but deleted UIDs/GIDs. +$uidsPrevUsed{$_} = 1 foreach values %{$uidMap}; +$gidsPrevUsed{$_} = 1 foreach values %{$gidMap}; + + +# Read the current /etc/group. +sub parseGroup { + chomp; + my @f = split(':', $_, -4); + my $gid = $f[2] eq "" ? undef : int($f[2]); + $gidsUsed{$gid} = 1 if defined $gid; + return ($f[0], { name => $f[0], password => $f[1], gid => $gid, members => $f[3] }); +} + +my %groupsCur = -f "/etc/group" ? map { parseGroup } read_file("/etc/group", { binmode => ":utf8" }) : (); + +# Read the current /etc/passwd. +sub parseUser { + chomp; + my @f = split(':', $_, -7); + my $uid = $f[2] eq "" ? undef : int($f[2]); + $uidsUsed{$uid} = 1 if defined $uid; + return ($f[0], { name => $f[0], fakePassword => $f[1], uid => $uid, + gid => $f[3], description => $f[4], home => $f[5], shell => $f[6] }); +} +my %usersCur = -f "/etc/passwd" ? map { parseUser } read_file("/etc/passwd", { binmode => ":utf8" }) : (); + +# Read the groups that were created declaratively (i.e. not by groups) +# in the past. These must be removed if they are no longer in the +# current spec. +my $declGroupsFile = "/var/lib/nixos/declarative-groups"; +my %declGroups; +$declGroups{$_} = 1 foreach split / /, -e $declGroupsFile ? read_file($declGroupsFile, { binmode => ":utf8" }) : ""; + +# Idem for the users. +my $declUsersFile = "/var/lib/nixos/declarative-users"; +my %declUsers; +$declUsers{$_} = 1 foreach split / /, -e $declUsersFile ? read_file($declUsersFile, { binmode => ":utf8" }) : ""; + + +# Generate a new /etc/group containing the declared groups. +my %groupsOut; +foreach my $g (@{$spec->{groups}}) { + my $name = $g->{name}; + my $existing = $groupsCur{$name}; + + my %members = map { ($_, 1) } @{$g->{members}}; + + if (defined $existing) { + $g->{gid} = $existing->{gid} if !defined $g->{gid}; + if ($g->{gid} != $existing->{gid}) { + dry_print("warning: not applying", "warning: would not apply", "GID change of group ‘$name’ ($existing->{gid} -> $g->{gid}) in /etc/group"); + $g->{gid} = $existing->{gid}; + } + $g->{password} = $existing->{password}; # do we want this? + if ($spec->{mutableUsers}) { + # Merge in non-declarative group members. + foreach my $uname (split /,/, $existing->{members} // "") { + $members{$uname} = 1 if !defined $declUsers{$uname}; + } + } + } else { + $g->{gid} = allocGid($name) if !defined $g->{gid}; + $g->{password} = "x"; + } + + $g->{members} = join ",", sort(keys(%members)); + $groupsOut{$name} = $g; + + $gidMap->{$name} = $g->{gid}; +} + +# Update the persistent list of declarative groups. +updateFile($declGroupsFile, join(" ", sort(keys %groupsOut))); + +# Merge in the existing /etc/group. +foreach my $name (keys %groupsCur) { + my $g = $groupsCur{$name}; + next if defined $groupsOut{$name}; + if (!$spec->{mutableUsers} || defined $declGroups{$name}) { + dry_print("removing group", "would remove group", "‘$name’"); + } else { + $groupsOut{$name} = $g; + } +} + + +# Rewrite /etc/group. FIXME: acquire lock. +my @lines = map { join(":", $_->{name}, $_->{password}, $_->{gid}, $_->{members}) . "\n" } + (sort { $a->{gid} <=> $b->{gid} } values(%groupsOut)); +updateFile($gidMapFile, to_json($gidMap, {canonical => 1})); +updateFile("/etc/group", \@lines); +nscdInvalidate("group"); + +# Generate a new /etc/passwd containing the declared users. +my %usersOut; +foreach my $u (@{$spec->{users}}) { + my $name = $u->{name}; + + # Resolve the gid of the user. + if ($u->{group} =~ /^[0-9]$/) { + $u->{gid} = $u->{group}; + } elsif (defined $groupsOut{$u->{group}}) { + $u->{gid} = $groupsOut{$u->{group}}->{gid} // die; + } else { + warn "warning: user ‘$name’ has unknown group ‘$u->{group}’\n"; + $u->{gid} = 65534; + } + + my $existing = $usersCur{$name}; + if (defined $existing) { + $u->{uid} = $existing->{uid} if !defined $u->{uid}; + if ($u->{uid} != $existing->{uid}) { + dry_print("warning: not applying", "warning: would not apply", "UID change of user ‘$name’ ($existing->{uid} -> $u->{uid}) in /etc/passwd"); + $u->{uid} = $existing->{uid}; + } + } else { + $u->{uid} = allocUid($name, $u->{isSystemUser}) if !defined $u->{uid}; + + if (!defined $u->{hashedPassword}) { + if (defined $u->{initialPassword}) { + $u->{hashedPassword} = hashPassword($u->{initialPassword}); + } elsif (defined $u->{initialHashedPassword}) { + $u->{hashedPassword} = $u->{initialHashedPassword}; + } + } + } + + # Ensure home directory incl. ownership and permissions. + if ($u->{createHome} and !$is_dry) { + make_path($u->{home}, { mode => oct($u->{homeMode}) }) if ! -e $u->{home}; + chown $u->{uid}, $u->{gid}, $u->{home}; + chmod oct($u->{homeMode}), $u->{home}; + } + + if (defined $u->{hashedPasswordFile}) { + if (-e $u->{hashedPasswordFile}) { + $u->{hashedPassword} = read_file($u->{hashedPasswordFile}); + chomp $u->{hashedPassword}; + } else { + warn "warning: password file ‘$u->{hashedPasswordFile}’ does not exist\n"; + } + } elsif (defined $u->{password}) { + $u->{hashedPassword} = hashPassword($u->{password}); + } + + if (!defined $u->{shell}) { + if (defined $existing) { + $u->{shell} = $existing->{shell}; + } else { + warn "warning: no declarative or previous shell for ‘$name’, setting shell to nologin\n"; + $u->{shell} = "/run/current-system/sw/bin/nologin"; + } + } + + $u->{fakePassword} = $existing->{fakePassword} // "x"; + $usersOut{$name} = $u; + + $uidMap->{$name} = $u->{uid}; +} + +# Update the persistent list of declarative users. +updateFile($declUsersFile, join(" ", sort(keys %usersOut))); + +# Merge in the existing /etc/passwd. +foreach my $name (keys %usersCur) { + my $u = $usersCur{$name}; + next if defined $usersOut{$name}; + if (!$spec->{mutableUsers} || defined $declUsers{$name}) { + dry_print("removing user", "would remove user", "‘$name’"); + } else { + $usersOut{$name} = $u; + } +} + +# Rewrite /etc/passwd. FIXME: acquire lock. +@lines = map { join(":", $_->{name}, $_->{fakePassword}, $_->{uid}, $_->{gid}, $_->{description}, $_->{home}, $_->{shell}) . "\n" } + (sort { $a->{uid} <=> $b->{uid} } (values %usersOut)); +updateFile($uidMapFile, to_json($uidMap, {canonical => 1})); +updateFile("/etc/passwd", \@lines); +nscdInvalidate("passwd"); + + +# Rewrite /etc/shadow to add new accounts or remove dead ones. +my @shadowNew; +my %shadowSeen; + +foreach my $line (-f "/etc/shadow" ? read_file("/etc/shadow", { binmode => ":utf8" }) : ()) { + chomp $line; + # struct name copied from `man 3 shadow` + my ($sp_namp, $sp_pwdp, $sp_lstch, $sp_min, $sp_max, $sp_warn, $sp_inact, $sp_expire, $sp_flag) = split(':', $line, -9); + my $u = $usersOut{$sp_namp};; + next if !defined $u; + $sp_pwdp = "!" if !$spec->{mutableUsers}; + $sp_pwdp = $u->{hashedPassword} if defined $u->{hashedPassword} && !$spec->{mutableUsers}; # FIXME + $sp_expire = dateToDays($u->{expires}) if defined $u->{expires}; + chomp $sp_pwdp; + push @shadowNew, join(":", $sp_namp, $sp_pwdp, $sp_lstch, $sp_min, $sp_max, $sp_warn, $sp_inact, $sp_expire, $sp_flag) . "\n"; + $shadowSeen{$sp_namp} = 1; +} + +foreach my $u (values %usersOut) { + next if defined $shadowSeen{$u->{name}}; + my $hashedPassword = "!"; + $hashedPassword = $u->{hashedPassword} if defined $u->{hashedPassword}; + my $expires = ""; + $expires = dateToDays($u->{expires}) if defined $u->{expires}; + # FIXME: set correct value for sp_lstchg. + push @shadowNew, join(":", $u->{name}, $hashedPassword, "1::::", $expires, "") . "\n"; +} + +updateFile("/etc/shadow", \@shadowNew, 0640); +{ + my $uid = getpwnam "root"; + my $gid = getgrnam "shadow"; + my $path = "/etc/shadow"; + (chown($uid, $gid, $path) || die "Failed to change ownership of $path: $!") unless $is_dry; +} + +# Rewrite /etc/subuid & /etc/subgid to include default container mappings + +my $subUidMapFile = "/var/lib/nixos/auto-subuid-map"; +my $subUidMap = -e $subUidMapFile ? decode_json(read_file($subUidMapFile)) : {}; + +my (%subUidsUsed, %subUidsPrevUsed); + +$subUidsPrevUsed{$_} = 1 foreach values %{$subUidMap}; + +sub allocSubUid { + my ($name, @rest) = @_; + + # TODO: No upper bounds? + my ($min, $max, $up) = (100000, 100000 * 100, 1); + my $prevId = $subUidMap->{$name}; + if (defined $prevId && !defined $subUidsUsed{$prevId}) { + $subUidsUsed{$prevId} = 1; + return $prevId; + } + + my $id = allocId(\%subUidsUsed, \%subUidsPrevUsed, $min, $max, $up, sub { my ($uid) = @_; getpwuid($uid) }); + my $offset = $id - 100000; + my $count = $offset * 65536; + my $subordinate = 100000 + $count; + return $subordinate; +} + +my @subGids; +my @subUids; +foreach my $u (values %usersOut) { + my $name = $u->{name}; + + foreach my $range (@{$u->{subUidRanges}}) { + my $value = join(":", ($name, $range->{startUid}, $range->{count})); + push @subUids, $value; + } + + foreach my $range (@{$u->{subGidRanges}}) { + my $value = join(":", ($name, $range->{startGid}, $range->{count})); + push @subGids, $value; + } + + if($u->{autoSubUidGidRange}) { + my $subordinate = allocSubUid($name); + $subUidMap->{$name} = $subordinate; + my $value = join(":", ($name, $subordinate, 65536)); + push @subUids, $value; + push @subGids, $value; + } +} + +updateFile("/etc/subuid", join("\n", @subUids) . "\n"); +updateFile("/etc/subgid", join("\n", @subGids) . "\n"); +updateFile($subUidMapFile, encode_json($subUidMap) . "\n"); -- GitLab From 7a77a3ba806a71ed1e9946f4ab5ac2e02ea81ea0 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 20:41:12 +0200 Subject: [PATCH 192/310] Enable haveged service --- lagarto-ox.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index bfcade3..12b18a9 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -11,6 +11,7 @@ # gcc.tune = "generic"; #}; + services.haveged.enable = true; system.activationScripts.users = lib.mkForce ( let cfg = config.users; -- GitLab From 4d70cb33de7a835c3579afc0db818ed82f4bde3f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 21:21:30 +0200 Subject: [PATCH 193/310] Use jitterentropy-rngd instead of haveged --- lagarto-ox.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 12b18a9..644b3eb 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -11,7 +11,8 @@ # gcc.tune = "generic"; #}; - services.haveged.enable = true; + #services.haveged.enable = true; + services.jitterentropy-rngd.enable = true; system.activationScripts.users = lib.mkForce ( let cfg = config.users; -- GitLab From e02d5a3999d0a74d1a2f4aae18401396a52027ec Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 21:59:45 +0200 Subject: [PATCH 194/310] Autologin as root in case the user is not working --- lagarto-ox.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 644b3eb..142698b 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -225,6 +225,8 @@ wantedBy = lib.mkForce [ ]; }; + services.getty.autologinUser = lib.mkForce "root"; + sdImage = { # The image will be loaded as-is in memory, so no compression compressImage = false; -- GitLab From 1c95d96519f4715384d0f0dfcdf8eacb477de3e6 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 23:07:15 +0200 Subject: [PATCH 195/310] Disable nscd daemon too --- lagarto-ox.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 142698b..391532a 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -13,6 +13,8 @@ #services.haveged.enable = true; services.jitterentropy-rngd.enable = true; + + # Prevent executing the nscd program as it seems to hang the CPU system.activationScripts.users = lib.mkForce ( let cfg = config.users; @@ -42,6 +44,10 @@ } else "" # keep around for backwards compatibility ); + # Also disable the nscd daemon + services.nscd.enable = false; + system.nssModules = lib.mkForce []; # Required + system.build.bootStage2 = let useHostResolvConf = config.networking.resolvconf.enable && config.networking.useHostResolvConf; bootStage2 = pkgs.substituteAll { -- GitLab From a463d88b76912633e132d92c9fcaf0f326925650 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 30 Aug 2024 23:59:42 +0200 Subject: [PATCH 196/310] Disable strace for the activation script --- stage-2-init.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stage-2-init.sh b/stage-2-init.sh index c496adf..0ef8cc6 100755 --- a/stage-2-init.sh +++ b/stage-2-init.sh @@ -118,7 +118,8 @@ install -m 01777 -d /tmp # Run the script that performs all configuration activation that does # not have to be done at boot time. echo "running activation script..." -strace -f $systemConfig/activate +#strace -f $systemConfig/activate +$systemConfig/activate echo "ACTIVATE WORKED!!!" -- GitLab From 6802d867db936dd378c0382af2aa527d7b8b5271 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Sat, 31 Aug 2024 00:00:04 +0200 Subject: [PATCH 197/310] Remove debug from the bootcmd line --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 391532a..d5cec88 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -376,7 +376,7 @@ cat > $out < Date: Mon, 2 Sep 2024 10:21:01 +0200 Subject: [PATCH 198/310] Disable DHCP client --- lagarto-ox.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index d5cec88..7a8b9ff 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -217,6 +217,7 @@ # No network services.openssh.enable = false; + networking.useDHCP = false; # Run getty on /dev/console and restartt until it works systemd.services."serial-getty@console" = { -- GitLab From 1aa899c8f0123f3c562ae06564cafa31fdd72598 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 3 Sep 2024 14:58:44 +0200 Subject: [PATCH 199/310] Print md5 checksum in fpgactl --- fpga/fpgactl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fpga/fpgactl b/fpga/fpgactl index 6a2540e..b54bbd6 100755 --- a/fpga/fpgactl +++ b/fpga/fpgactl @@ -133,6 +133,7 @@ function load_file_in_memory() # {{{ address=$(($address_hex)) total_size=$(stat --format "%s" "$file") + md5sum=$(md5sum "$file" | cut -d' ' -f1) # Previous tests... #strace -f dma-to-device -d /dev/${qdmadev}-MM-1 -a "$address" -s $((8*1024*1024)) -f "$file" @@ -145,7 +146,7 @@ function load_file_in_memory() # {{{ #dd if="$file" bs=8M seek="${address}" oflag=seek_bytes of=/dev/${qdmadev}-MM-1 status=none - printf "loaded '%s' at 0x%x with size %d\n" "$file" "$address" "$total_size" >&2 + printf "loaded '%s' at 0x%x with size %d and md5 %s\n" "$file" "$address" "$total_size" "$md5sum" >&2 } # }}} function do_boot_only() # {{{ { -- GitLab From 3ee0933d7b318c8272ca91bf070f54bfc121a0ed Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 3 Sep 2024 13:06:33 +0200 Subject: [PATCH 200/310] Add plictool to dump PLIC state --- lagarto-ox.nix | 1 + overlay.nix | 16 ++++++ tools/plictool.c | 131 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 tools/plictool.c diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 7a8b9ff..c168b65 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -166,6 +166,7 @@ extraUtilsCommands = '' cp -a ${pkgs.csrtool}/bin/csrtool $out/bin cp -a ${pkgs.memtool}/bin/memtool $out/bin + cp -a ${pkgs.plictool}/bin/plictool $out/bin ''; # Write a counter to the DMA region, so we can check the kernel is not diff --git a/overlay.nix b/overlay.nix index 49c8f83..bc60e91 100644 --- a/overlay.nix +++ b/overlay.nix @@ -69,6 +69,22 @@ final: prev: ''; }; + plictool = prev.pkgsStatic.stdenv.mkDerivation { + name = "plictool"; + src = ./tools/plictool.c; + unpackPhase = '' + cp ${./tools/plictool.c} plictool.c + ''; + dontConfigure = true; + buildPhase = '' + $CC -static plictool.c -o plictool + ''; + installPhase = '' + mkdir -p $out/bin + cp plictool $out/bin/ + ''; + }; + bitstreams = builtins.fetchGit { url = "git@bscpm03.bsc.es:rarias/bitstreams.git"; rev = "097dad3e888f630ad6ec7e7056ee1b3fcb03400b"; diff --git a/tools/plictool.c b/tools/plictool.c new file mode 100644 index 0000000..08f267e --- /dev/null +++ b/tools/plictool.c @@ -0,0 +1,131 @@ +/* Copyright (c) 2024 Barcelona Supercomputing Center (BSC) + * SPDX-License-Identifier: MIT + * Author: Rodrigo Arias Mallo */ + +/* Small utility to dump the state of the PLIC. */ + +/* Changelog: + * v0.0.1 (2024-09-03): Initial version. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +const char *plic_address_str = "0x40800000"; +long ncontexts = 15872L; +long maxsources = 1024L; +long nsources = 1024L; + +static void +usage(void) +{ + printf("plictool v0.0.1 - Rodrigo Arias Mallo \n"); + + printf( +"Usage: plictool [-a addr] [-s nsources] [-c ncontexts]\n"); + exit(1); +} + +void dump_sources(void *base) +{ + for (long s = 0; s < nsources; s++) { + int printed_source = 0; + + uint32_t *pending_reg = base + 0x1000L + (s / 32L) * 4L; + long shift = s % 32L; + uint32_t pending = ((*pending_reg) >> shift) & 1; + uint32_t *priority_reg = base + 0x0000L + (s * 4L); + uint32_t priority = *priority_reg; + + const char *fmt = + "src=%ld pend=%u prio=%u"; + + if (pending || priority) { + printf(fmt, s, pending, priority); + printed_source = 1; + } + + int first_context = 1; + for (long c = 0; c < ncontexts; c++) { + uint32_t *enable_reg = base + 0x2000L + 0x80L * c + (s / 32L) * 4L; + uint32_t enabled = ((*enable_reg) >> shift) & 1; + uint32_t *threshold_reg = base + 0x200000L + (c * 0x1000L); + uint32_t threshold = *threshold_reg; + + if (!enabled) + continue; + + if (!printed_source) { + printf(fmt, s, pending, priority); + printed_source = 1; + } + + printf(" ctx=%ld thre=%u", c, threshold); + } + + if (printed_source) + printf("\n"); + } +} + +int main(int argc, char *argv[]) +{ + int opt; + + while ((opt = getopt(argc, argv, "a:hs:c:")) != -1) { + switch (opt) { + case 'a': + plic_address_str = optarg; + break; + case 's': + nsources = atol(optarg); + break; + case 'c': + ncontexts = atol(optarg); + break; + case 'h': + default: /* '?' */ + usage(); + } + } + + off_t plic_address = (off_t) strtoll(plic_address_str, NULL, 16); + + printf("plic=0x%08x nsources=%ld ncontexts=%ld\n", + plic_address, nsources, ncontexts); + + int fd = open("/dev/mem", O_RDWR | O_SYNC); + + if (fd == -1) { + perror("cannot open /dev/mem"); + exit(1); + } + + size_t map_size = 0x4000000UL; + void *map_base = mmap(0, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, + fd, plic_address); + + if (map_base == MAP_FAILED) { + perror("mmap failed"); + if (errno == EPERM) { + fprintf(stderr, "Have you disabled 'CONFIG_STRICT_DEVMEM' and " + "'CONFIG_IO_STRICT_DEVMEM' in the kernel config?\n" + "Hint: zgrep STRICT_DEVMEM /proc/config.gz\n"); + } + exit(1); + } + + dump_sources(map_base); + + munmap(map_base, map_size); + + close(fd); + + return 0; +} -- GitLab From fa6227928e5017cf69e9e02668e7226455bacc3c Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 3 Sep 2024 16:09:53 +0200 Subject: [PATCH 201/310] Enable aux timer and test uart in DT --- ox-plic.dts | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/ox-plic.dts b/ox-plic.dts index 2bcce4c..e550464 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -128,17 +128,18 @@ current-speed = <115200>; status = "okay"; }; -// /* The serial for interrupt tests */ -// uart_testing: serial@40003000 { -// compatible = "ns16550"; -// reg = <0x0 0x40003000 0x0 0x1000>; -//// interrupts = <1>; /* Output interrupt 1 */ -//// interrupt-parent = <&PLIC>; -// reg-shift = <2>; -// clock-frequency = <50000000>; -// current-speed = <115200>; -// status = "okay"; -// }; + /* The serial for interrupt tests */ + uart_testing: serial@40003000 { + compatible = "ns16550"; + reg = <0x0 0x40003000 0x0 0x1000>; + reg-shift = <2>; + /* Output interrupt 1 (the first one) */ + interrupts = <1>; + interrupt-parent = <&PLIC>; + clock-frequency = <50000000>; + current-speed = <115200>; + status = "okay"; + }; // ethernet0 { // xlnx,rxmem = <0x000005f2>; @@ -211,7 +212,7 @@ */ interrupts-extended = <&HLIC0 11>, <&HLIC0 9>; reg = < 0x0 0x40800000 0x0 0x00400000>; - riscv,ndev = <3>; + riscv,ndev = <4>; //riscv,max-priority = <0x7>; phandle = <0x3>; }; @@ -230,6 +231,13 @@ interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; compatible = "riscv,clint0"; }; + aux_timer: clint@40010000 { + reg = <0x0 0x40010000 0x0 0x00010000>; + reg-names = "control"; + interrupts = <4>; /* PLIC input source 4 */ + interrupt-parent = <&PLIC>; + compatible = "riscv,clint0"; + }; // clint: clint@40002000 { // /* MTIME and MTIMECMP address and size pairs */ // reg = <0x0 0x40002000 0x0 0x8>, <0x0 0x40002008 0x0 0x8>; -- GitLab From 1620684b8ebbacff4d6f548d0d16c2d14ab5b7d2 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 3 Sep 2024 16:14:16 +0200 Subject: [PATCH 202/310] Disable aux timer as OpenSBI now fails --- ox-plic.dts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ox-plic.dts b/ox-plic.dts index e550464..f40168c 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -231,13 +231,13 @@ interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; compatible = "riscv,clint0"; }; - aux_timer: clint@40010000 { - reg = <0x0 0x40010000 0x0 0x00010000>; - reg-names = "control"; - interrupts = <4>; /* PLIC input source 4 */ - interrupt-parent = <&PLIC>; - compatible = "riscv,clint0"; - }; +// aux_timer: clint@40010000 { +// reg = <0x0 0x40010000 0x0 0x00010000>; +// reg-names = "control"; +// interrupts = <4>; /* PLIC input source 4 */ +// interrupt-parent = <&PLIC>; +// compatible = "riscv,clint0"; +// }; // clint: clint@40002000 { // /* MTIME and MTIMECMP address and size pairs */ // reg = <0x0 0x40002000 0x0 0x8>, <0x0 0x40002008 0x0 0x8>; -- GitLab From 32a7b2f3b78cc67fa8ead3340868654aa680abe3 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 3 Sep 2024 16:55:34 +0200 Subject: [PATCH 203/310] Enable ethernet node in DT --- ox-plic.dts | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/ox-plic.dts b/ox-plic.dts index f40168c..6d2309e 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -141,16 +141,18 @@ status = "okay"; }; -// ethernet0 { -// xlnx,rxmem = <0x000005f2>; -// carv,mtu = <0x000005dc>; -// carv,no-mac; -// device_type = "network"; -// local-mac-address = [02 05 00 01 00 05]; -// axistream-connected = <0x000000fe>; -// compatible = "xlnx,xxv-ethernet-1.0-carv"; -// memory-region = <ð_pool>; -// }; + ethernet0 { + xlnx,rxmem = <0x000005f2>; + carv,mtu = <0x000005dc>; + carv,no-mac; + device_type = "network"; + local-mac-address = [02 05 00 01 00 05]; + axistream-connected = <0x000000fe>; + //compatible = "xlnx,xxv-ethernet-1.0-carv"; + compatible = "xlnx,axi-ethernet-1.00.a"; + memory-region = <ð_pool>; + }; + // dma@40400000 { // xlnx,include-dre; // phandle = <0x000000fe>; -- GitLab From 158e232520739c3b2662e058fa6258dd1d19a7ef Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 4 Sep 2024 10:48:48 +0200 Subject: [PATCH 204/310] Update journal --- JOURNAL.md | 148 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index c62bf7e..4ca71fe 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -3749,3 +3749,151 @@ problems with the boot. Now that we have a CI pipeline, let's try making the whole boot process automatic, so I don't have to type anything. This basically requires setting the environment in U-Boot. + + <<< NixOS Stage 1 >>> + + + An error occurred in stage 1 of the boot process, which must mount the + root filesystem on `/mnt-root' and then start stage 2. Press one + of the following keys: + + i) to launch an interactive shell + f) to start an interactive shell having pid 1 (needed if you want to + start stage 2's init manually) + [ 22.365260] stage-1-init: [Thu Jan 1 00:00:22 UTC 1970] An error occurred in stage 1 of the boot process, which must mount the + r) to reboot immediately + *) to ignore the error and continue + [ 22.526780] stage-1-init: [Thu Jan 1 00:00:22 UTC 1970] root filesystem on `/mnt-root' and then start stage 2. Press one + [ 22.611640] stage-1-init: [Thu Jan 1 00:00:22 UTC 1970] of the following keys: + [ 22.697460] stage-1-init: [Thu Jan 1 00:00:22 UTC 1970] i) to launch an interactive shell + [ 22.788100] stage-1-init: [Thu Jan 1 00:00:22 UTC 1970] f) to start an interactive shell having pid 1 (needed if you want to + [ 22.874060] stage-1-init: [Thu Jan 1 00:00:22 UTC 1970] start stage 2's init manually) + [ 22.957940] stage-1-init: [Thu Jan 1 00:00:22 UTC 1970] r) to reboot immediately + [ 23.042520] stage-1-init: [Thu Jan 1 00:00:22 UTC 1970] *) to ignore the error and continue + iStarting interactive shell... + [ 32.314080] stage-1-init: [Thu Jan 1 00:00:32 UTC 1970] Starting interactive shell... + ~ # cat /proc/sys/kernel/random + random/ randomize_va_space + ~ # cat /proc/sys/kernel/random/entropy_avail + 0 + +Let's see what happens with strace. + +## 2024-09-02 + +Interestingly, I managed to reach the login console and run some commands after +fully booting NixOS. I just needed to disable nscd and enable a daemon to fill +the entropy pull, which is depleted on boot. + +I'm not looking at the nscd binary, which works ok when running `nscd -V` but +hangs the CPU when running `nscd --invalidate group`. I tried executing it under +GDB and then ^C, but it doesn't respond. + +As always, we don't have JTAG support ready, so this is going to be an absolute +pain to debug. I could bisect the code and try to guess at which point it must +be failing. But each attempt will take around 30 minutes, so it is extremely +expensive. + +It would be nice to reproduce this in the initrd shell, which loads much faster. + + # not strictly required, but you'll likely want the log anyway + (gdb) set logging on + + # ask gdb to not stop every screen-full + (gdb) set height 0 + + (gdb) while 1 + > x/i $pc + > stepi + > end + +Interestingly, I can step up to the end of the program, but it seems to be +failing: + + (gdb) r + Starting program: /nix/store/nh1f85icnvlqs3dc8lv2ya0ylmljsfax-system-path/bin/nscd --invalidate group + [Thread debugging using libthread_db enabled] + Using host libthread_db library "/nix/store/47a03qi49pwlk3hxpfwx2vq671mlqn57-glibc-riscv64-unknown-linux-gnu-2.39-52/lib/libthread_db.so.1". + + Breakpoint 1, 0x0000002aaaaaf738 in main () + (gdb) while 1 + >x/i $pc + >nexti + >end + => 0x2aaaaaf738 : jal 0x2aaaaaf160 + 0x0000002aaaaaf73c in main () + => 0x2aaaaaf73c : auipc a0,0x19 + 0x0000002aaaaaf740 in main () + => 0x2aaaaaf740 : ld a0,1580(a0) + 0x0000002aaaaaf744 in main () + => 0x2aaaaaf744 : jal 0x2aaaaaf690 + 0x0000002aaaaaf748 in main () + => 0x2aaaaaf748 : li a5,0 + 0x0000002aaaaaf74c in main () + => 0x2aaaaaf74c : addi a4,sp,8 + 0x0000002aaaaaf750 in main () + => 0x2aaaaaf750 : li a3,0 + 0x0000002aaaaaf754 in main () + => 0x2aaaaaf754 : mv a2,s1 + 0x0000002aaaaaf758 in main () + => 0x2aaaaaf758 : mv a1,s0 + 0x0000002aaaaaf75c in main () + => 0x2aaaaaf75c : auipc a0,0x19 + 0x0000002aaaaaf760 in main () + => 0x2aaaaaf760 : addi a0,a0,-1852 + 0x0000002aaaaaf764 in main () + => 0x2aaaaaf764 : jal 0x2aaaaaef30 + [Inferior 1 (process 1962) exited with code 01] + No registers. + +Using passwd instead hangs the CPU before reaching the main: + + [root@nixos-riscv:~]# gdb --args $(which nscd) --invalidate passwd + GNU gdb (GDB) 14.2 + Copyright (C) 2023 Free Software Foundation, Inc. + License GPLv3+: GNU GPL version 3 or later + This is free software: you are free to change and redistribute it. + There is NO WARRANTY, to the extent permitted by law. + Type "show copying" and "show warranty" for details. + This GDB was configured as "riscv64-unknown-linux-gnu". + Type "show configuration" for configuration details. + For bug reporting instructions, please see: + . + Find the GDB manual and other documentation resources online at: + . + + For help, type "help". + Type "apropos word" to search for commands related to "word"... + Reading symbols from /run/current-system/sw/bin/nscd... + (No debugging symbols found in /run/current-system/sw/bin/nscd) + (gdb) b main + Breakpoint 1 at 0x5738 + (gdb) b argp_parse + Breakpoint 2 at 0x4f38 + (gdb) c + The program is not being run. + (gdb) r + Starting program: /nix/store/nh1f85icnvlqs3dc8lv2ya0ylmljsfax-system-path/bin/nscd --invalidate passwd + +It's probably not deterministic. + +I have also disabled kaslr by adding nokaslr to the bootargs. + +Anyway, doesn't make much sense to debug this on userland and pay 30 minutes for +each test. I'll wait until we have proper JTAG support or a similar way to dump +the registers on a crash. + +# 2024-09-03 + +Wrote a small tool `plictool` to dump the state of the PLIC: + + ~ # plictool -c 2 + plic=0x40800000 nsources=1024 ncontexts=2 + src=1 pend=0 prio=1 ctx=1 thre=1 + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + src=33 pend=0 prio=0 ctx=1 thre=1 + +Interestingly, the auxiliar UART interrupts don't seem to be working very well. +Also, is there another source 33 enabled? -- GitLab From 272fd211b2bae14eed6735af8a5aa37f1f6454e0 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 4 Sep 2024 11:46:18 +0200 Subject: [PATCH 205/310] Adjust RTC frequency to 32786 Hz --- JOURNAL.md | 17 +++++++++++++++++ ox-plic.dts | 6 ++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 4ca71fe..267342b 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -3897,3 +3897,20 @@ Wrote a small tool `plictool` to dump the state of the PLIC: Interestingly, the auxiliar UART interrupts don't seem to be working very well. Also, is there another source 33 enabled? + +I have also noticed that the clock is running with the wrong frequency. +Everything is about twice as slow. For example, `sleep 1` takes 2 seconds. +So I assume the kernel thinks the clock goes faster than it actually does. + +The timer is at 50 KHz: + + timebase-frequency = <50000>; /* 50 kHz */ + +So, the frequency of this "clock" is done by using the "general" clock which +goes at 50 MHz, and then using a simple counter that counts up to 1525, so the +frequency is: + + 32786.88524590164 Hz + +Assuming that it actually works well. Let's try that and see if we can have a +more realistic reading for the CLINT interrupt frequency. diff --git a/ox-plic.dts b/ox-plic.dts index 6d2309e..09b9442 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -16,8 +16,10 @@ cpus { #address-cells = <1>; #size-cells = <0>; - /* Timer */ - timebase-frequency = <50000>; /* 50 kHz */ + + // The RTC timer is clocked at the CPU frequency / 1525, so + // around 32786.88 Hz + timebase-frequency = <32786>; CPU0: cpu@0 { clock-frequency = <50000000>; /* 50 MHz */ device_type = "cpu"; -- GitLab From b075d6fa2af06c4367df31d3d92b22e1194c4aaa Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 4 Sep 2024 12:43:46 +0200 Subject: [PATCH 206/310] Add problems with the PLIC to the journal --- JOURNAL.md | 121 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index 267342b..76dbaea 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -3914,3 +3914,124 @@ frequency is: Assuming that it actually works well. Let's try that and see if we can have a more realistic reading for the CLINT interrupt frequency. + +Something is going on with the plictool: + + ~ # plictool + plic=0x40800000 nsources=1024 ncontexts=15872 + src=1 pend=1 prio=1 ctx=1 thre=0 <--- zero + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + src=33 pend=0 prio=0 ctx=1 thre=2 + ~ # plictool + plic=0x40800000 nsources=1024 ncontexts=15872 + src=1 pend=1 prio=1 ctx=1 thre=2 <--- now changed to 2??? + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + src=33 pend=0 prio=0 ctx=1 thre=2 + +Either the register is changing its value on its own, or the tool is causing it. + +Let's revert the threshold to 0, but writing to the threshold address, should be +at 0x40a01000. + + ~ # devmem 0x40a01000 + 0x00000002 + ~ # devmem 0x40a01000 + 0x00000002 + ~ # devmem 0x40a01000 + 0x00000002 + ~ # devmem 0x40a01000 8 0 + ~ # devmem 0x40a01000 + 0x00000000 + ~ # devmem 0x40a01000 + 0x00000000 + ~ # devmem 0x40a01000 + 0x00000000 + ~ # plictool -c 2 -s 5 <-- limiting the sources and contexts + plic=0x40800000 nsources=5 ncontexts=2 + src=1 pend=1 prio=1 ctx=1 thre=0 <-- good + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + ~ # plictool -c 2 -s 5 + plic=0x40800000 nsources=5 ncontexts=2 + src=1 pend=1 prio=1 ctx=1 thre=0 + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + ~ # plictool -c 2 -s 5 + plic=0x40800000 nsources=5 ncontexts=2 + src=1 pend=1 prio=1 ctx=1 thre=0 <--- stable + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + ~ # cat /proc/interrupts + CPU0 + 1: 1 SiFive PLIC 1 Edge ttyS1 + 5: 195413 RISC-V INTC 5 Edge riscv-timer + ~ # cat /proc/int^Crupts + + ~ # echo a > /dev/ttyS1 + ~ # cat /proc/interrupts + CPU0 + 1: 1 SiFive PLIC 1 Edge ttyS1 + 5: 197102 RISC-V INTC 5 Edge riscv-timer + ~ # plictool -c 2 -s 5 + plic=0x40800000 nsources=5 ncontexts=2 + src=1 pend=1 <--- wtf prio=1 ctx=1 thre=0 + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + ~ # devmem 0x40a01004 + 0x00000000 <--- the claim register continues to give 0 instead of 1 + ~ # devmem 0x40a01004 + 0x00000000 + ~ # devmem 0x40a01004 + 0x00000000 + ~ # devmem 0x40a01004 + 0x00000000 + ~ # devmem 0x40a01004 + 0x00000000 + ~ # devmem 0x40a01004 8 5 <-- try writing 5 + ~ # devmem 0x40a01004 + 0x00000000 <--- no change + ~ # devmem 0x40a01004 + 0x00000000 + ~ # devmem 0x40a01004 + 0x00000000 + ~ # devmem 0x40a01004 + 0x00000000 + ~ # devmem 0x40a01004 8 1 <-- try with 1 + ~ # devmem 0x40a01004 + 0x00000000 + ~ # devmem 0x40a01004 + 0x00000000 + ~ # devmem 0x40a01004 + 0x00000000 + ~ # plictool -c 2 -s 5 + plic=0x40800000 nsources=5 ncontexts=2 + src=1 pend=0 <--- now it is gone prio=1 ctx=1 thre=0 + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + ~ # cat /proc/interrupts + CPU0 + 1: 3 <-- and we have >1 SiFive PLIC 1 Edge ttyS1 + 5: 213775 RISC-V INTC 5 Edge riscv-timer + ~ # echo a > /dev/ttyS1 + ~ # cat /proc/interrupts + CPU0 + 1: 3 <--- stuck again SiFive PLIC 1 Edge ttyS1 + 5: 214551 RISC-V INTC 5 Edge riscv-timer + ~ # plictool -c 2 -s 5 + plic=0x40800000 nsources=5 ncontexts=2 + src=1 pend=1 prio=1 ctx=1 thre=1 <--- again set to 1 + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + +Clearly something is not working well in the PLIC. + -- GitLab From 7e48cfe72e1834b673358c570c85cf2107c1ea69 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 4 Sep 2024 12:44:53 +0200 Subject: [PATCH 207/310] Print contexts in another line with active info Make it clear to determine if a source is configured to emit interruptions to a given context by printing the active=1 attribute when the priority is larger than the threshold of the context. --- tools/plictool.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/plictool.c b/tools/plictool.c index 08f267e..c517e81 100644 --- a/tools/plictool.c +++ b/tools/plictool.c @@ -6,6 +6,7 @@ /* Changelog: * v0.0.1 (2024-09-03): Initial version. + * v0.0.2 (2024-09-04): Print contexts in another line and on/off information. */ #include @@ -25,7 +26,7 @@ long nsources = 1024L; static void usage(void) { - printf("plictool v0.0.1 - Rodrigo Arias Mallo \n"); + printf("plictool v0.0.2 - Rodrigo Arias Mallo \n"); printf( "Usage: plictool [-a addr] [-s nsources] [-c ncontexts]\n"); @@ -44,7 +45,7 @@ void dump_sources(void *base) uint32_t priority = *priority_reg; const char *fmt = - "src=%ld pend=%u prio=%u"; + "src=%ld pend=%u prio=%u\n"; if (pending || priority) { printf(fmt, s, pending, priority); @@ -66,11 +67,10 @@ void dump_sources(void *base) printed_source = 1; } - printf(" ctx=%ld thre=%u", c, threshold); - } + int active = priority > threshold ? 1 : 0; - if (printed_source) - printf("\n"); + printf(" ctx=%ld thre=%u active=%d\n", c, threshold, active); + } } } -- GitLab From a6a1d75b7a958523e4621e9f71f6e2ec5e5081ee Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 4 Sep 2024 16:15:15 +0200 Subject: [PATCH 208/310] Enable debug1 for now --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index c168b65..02fedcd 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -378,7 +378,7 @@ cat > $out < Date: Wed, 4 Sep 2024 16:17:21 +0200 Subject: [PATCH 209/310] Improve plictool output and write manual --- tools/plictool.c | 74 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 9 deletions(-) diff --git a/tools/plictool.c b/tools/plictool.c index c517e81..d2705c6 100644 --- a/tools/plictool.c +++ b/tools/plictool.c @@ -6,9 +6,12 @@ /* Changelog: * v0.0.1 (2024-09-03): Initial version. - * v0.0.2 (2024-09-04): Print contexts in another line and on/off information. + * v0.0.2 (2024-09-04): Print contexts in another line and masked information. + * v0.0.3 (2024-09-04): Make output format more clear and add manual. */ +#define VERSION "v0.0.3" + #include #include #include @@ -26,10 +29,49 @@ long nsources = 1024L; static void usage(void) { - printf("plictool v0.0.2 - Rodrigo Arias Mallo \n"); - printf( -"Usage: plictool [-a addr] [-s nsources] [-c ncontexts]\n"); +"NAME\n" +" plictool - dump PLIC information\n" +"\n" +"SYNOPSIS\n" +" plictool [-a addr] [-s nsources] [-c ncontexts]\n" +"\n" +"DESCRIPTION\n" +" Dumps information about the RISC-V PLIC interrupt controller.\n" +" Optional arguments:\n" +"\n" +" -a addr The starting address of the PLIC.\n" +" The default is 0x40800000.\n" +"\n" +" -s nsources Limit the number of sources to print.\n" +" The default is 1024.\n" +"\n" +" -c ncontexts Limit the number of contexts to print.\n" +" The default is 15872.\n" +"\n" +"OUTPUT FORMAT\n" +" Information is printed for each source that has some bit set\n" +" in the pending or priority registers, or it has some context\n" +" enabled. The the following attributes are printed:\n" +"\n" +" src source number\n" +" pend pending bit\n" +" prio priority of the source\n" +"\n" +" Additionally, for each enabled context of the source the\n" +" following attributes are printed for that context:\n" +"\n" +" ctx context number\n" +" thre threshold value\n" +" masked the source is masked (prio <= thre)\n" +"\n" +"AUTHOR\n" +" Rodrigo Arias Mallo \n" +"\n" +"SEE ALSO\n" +" See https://github.com/riscv/riscv-plic-spec/\n" +"\n" +); exit(1); } @@ -45,7 +87,7 @@ void dump_sources(void *base) uint32_t priority = *priority_reg; const char *fmt = - "src=%ld pend=%u prio=%u\n"; + "src=%ld pend=%u prio=%u"; if (pending || priority) { printf(fmt, s, pending, priority); @@ -53,6 +95,8 @@ void dump_sources(void *base) } int first_context = 1; + int enabled_contexts = 0; + int unmasked = 0; for (long c = 0; c < ncontexts; c++) { uint32_t *enable_reg = base + 0x2000L + 0x80L * c + (s / 32L) * 4L; uint32_t enabled = ((*enable_reg) >> shift) & 1; @@ -67,10 +111,21 @@ void dump_sources(void *base) printed_source = 1; } - int active = priority > threshold ? 1 : 0; - printf(" ctx=%ld thre=%u active=%d\n", c, threshold, active); + int is_masked = priority <= threshold; + + if (!is_masked) + unmasked++; + + const char *masked = is_masked ? "masked" : "unmasked"; + + printf("\n ctx=%ld thre=%u %s", + c, threshold, masked); + enabled_contexts++; } + + if (printed_source) + printf("\n"); } } @@ -95,11 +150,12 @@ int main(int argc, char *argv[]) } } - off_t plic_address = (off_t) strtoll(plic_address_str, NULL, 16); + unsigned long long plic_address = strtoull(plic_address_str, NULL, 16); - printf("plic=0x%08x nsources=%ld ncontexts=%ld\n", + printf("plictool "VERSION" addr=0x%08llx nsrc=%ld nctx=%ld\n", plic_address, nsources, ncontexts); + //int fd = open("test.bin", O_RDWR | O_SYNC); int fd = open("/dev/mem", O_RDWR | O_SYNC); if (fd == -1) { -- GitLab From 6f12ec6372ed6ac6682b0a59d53d465538cf3cf2 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 5 Sep 2024 07:41:32 +0200 Subject: [PATCH 210/310] Add more PLIC results to the journal --- JOURNAL.md | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index 76dbaea..7cd6c1e 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -4035,3 +4035,75 @@ at 0x40a01000. Clearly something is not working well in the PLIC. +Let's try to dump the pending and priority registers of the PLIC with devmem, +just to check that the plictool is not doing something wrong. + + devmem 0x40801000 # Dump pending bits of sources 0-31 + devmem 0x40801004 # Dump pending bits of sources 32-63 + devmem 0x40a01000 # Dump priority threshold of context 1 + devmem 0x40a00000 # Dump priority threshold of context 0 + +Interesting output: + + ~ # plictool -c2 + plictool v0.0.3 addr=0x40800000 nsrc=1024 nctx=2 + src=1 pend=0 prio=1 + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + ~ # plictool -c2 + plictool v0.0.3 addr=0x40800000 nsrc=1024 nctx=2 + src=1 pend=0 prio=1 + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + ~ # plictool -c2 + plictool v0.0.3 addr=0x40800000 nsrc=1024 nctx=2 + src=1 pend=0 prio=1 + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + ~ # plictool -c2 + plictool v0.0.3 addr=0x40800000 nsrc=1024 nctx=2 + src=1 pend=0 prio=1 + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + ~ # dd if=/dev/ttyS1 bs=1 count=1 of=/dev/null & + ~ # plictool -c2 + plictool v0.0.3 addr=0x40800000 nsrc=1024 nctx=2 + src=1 pend=1 prio=1 + ctx=1 thre=0 unmasked + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + src=33 pend=0 prio=0 + ctx=1 thre=1 masked + ~ # plictool -c2 + plictool v0.0.3 addr=0x40800000 nsrc=1024 nctx=2 + src=1 pend=1 prio=1 + ctx=1 thre=1 masked + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + src=33 pend=0 prio=0 + ctx=1 thre=1 masked + ~ # plictool -c2 + plictool v0.0.3 addr=0x40800000 nsrc=1024 nctx=2 + src=1 pend=1 prio=1 + ctx=1 thre=1 masked + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + src=33 pend=0 prio=0 + ctx=1 thre=1 masked + ~ # plictool -c2 + plictool v0.0.3 addr=0x40800000 nsrc=1024 nctx=2 + src=1 pend=1 prio=1 + ctx=1 thre=1 masked + src=2 pend=0 prio=1 + src=3 pend=0 prio=1 + src=4 pend=1 prio=1 + src=33 pend=0 prio=0 + ctx=1 thre=1 masked + -- GitLab From 6239191835366eac2a80a4426bb4312d2b67e7b3 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 5 Sep 2024 07:41:54 +0200 Subject: [PATCH 211/310] Add custom Ethernet module --- ethernet-driver-build.patch | 21 ++++++++++++ ethernet-driver-kbuild.patch | 7 ++++ ethernet-driver-poll.patch | 23 +++++++++++++ lagarto-ox.nix | 65 +++++++++++++++++++++++------------- 4 files changed, 92 insertions(+), 24 deletions(-) create mode 100644 ethernet-driver-build.patch create mode 100644 ethernet-driver-kbuild.patch create mode 100644 ethernet-driver-poll.patch diff --git a/ethernet-driver-build.patch b/ethernet-driver-build.patch new file mode 100644 index 0000000..a0736a2 --- /dev/null +++ b/ethernet-driver-build.patch @@ -0,0 +1,21 @@ +diff --git a/drivers/xxvnet_carv.c b/drivers/xxvnet_carv.c +index d3f60f9..9fd4c21 100644 +--- a/drivers/xxvnet_carv.c ++++ b/drivers/xxvnet_carv.c +@@ -36,6 +36,7 @@ + #include + #include + #include ++#include + + #include "xxvnet_carv.h" + +@@ -1511,7 +1512,7 @@ axienet_dma_probe(struct platform_device *pdev, struct net_device *ndev) + spin_lock_init(&q->tx_lock); + spin_lock_init(&q->rx_lock); + +- netif_napi_add(ndev, &lp->napi, axienet_rx_poll, AXIENET_NAPI_WEIGHT); ++ netif_napi_add(ndev, &lp->napi, axienet_rx_poll); + + return 0; + } diff --git a/ethernet-driver-kbuild.patch b/ethernet-driver-kbuild.patch new file mode 100644 index 0000000..32fa1af --- /dev/null +++ b/ethernet-driver-kbuild.patch @@ -0,0 +1,7 @@ +diff --git a/drivers/Kbuild b/drivers/Kbuild +index 28d6c0f..7f02860 100644 +--- a/drivers/Kbuild ++++ b/drivers/Kbuild +@@ -1,2 +1 @@ + obj-m := xxvnet_carv.o +-obj-m := xilinx_dma.o diff --git a/ethernet-driver-poll.patch b/ethernet-driver-poll.patch new file mode 100644 index 0000000..db2b8a9 --- /dev/null +++ b/ethernet-driver-poll.patch @@ -0,0 +1,23 @@ +diff --git a/drivers/xxvnet_carv.c b/drivers/xxvnet_carv.c +index eb664bb..d3f60f9 100644 +--- a/drivers/xxvnet_carv.c ++++ b/drivers/xxvnet_carv.c +@@ -1435,12 +1435,12 @@ static void axienet_poll_controller(struct net_device *ndev) + { + struct axienet_local *lp = netdev_priv(ndev); + +- disable_irq(lp->tx_irq); +- disable_irq(lp->rx_irq); +- axienet_rx_irq(lp->tx_irq, ndev); +- axienet_tx_irq(lp->rx_irq, ndev); +- enable_irq(lp->tx_irq); +- enable_irq(lp->rx_irq); ++ disable_irq(lp->dq->tx_irq); ++ disable_irq(lp->dq->rx_irq); ++ axienet_rx_irq(lp->dq->tx_irq, ndev); ++ axienet_tx_irq(lp->dq->rx_irq, ndev); ++ enable_irq(lp->dq->tx_irq); ++ enable_irq(lp->dq->rx_irq); + } + #endif + diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 02fedcd..8ac9b50 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -74,6 +74,14 @@ boot.kernelPackages = pkgs.linuxPackages_latest; boot = { + extraModulePackages = [ + # Add the custom Ethernet module + pkgs.xilinx-axienet-carv + ]; + kernelModules = [ + # Load the Ethernet module by default + "xilinx_axienet_carv" + ]; kernelPatches = [ { name = assert false; "sbi-early-console"; @@ -418,29 +426,38 @@ #./ox-alveo-platform-plic.patch ]; }); -# opensbi = prev.opensbi.overrideAttrs (old: { -# #NIX_DEBUG=5; -# # Compile dts to dtb form -# #nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ prev.buildPackages.dtc ]; -# #preBuild = '' -# # dtc -O dtb -o ox.dtb ${./ox.dts} -# #''; -# makeFlags = old.makeFlags ++ [ -# "PLATFORM=fpga/sargantana_alveo" -# #"PLATFORM_RISCV_ISA=rv64imafd" # No compressed instructions -# "PLATFORM_RISCV_ISA=rv64g" # No compressed instructions -# #"PLATFORM_RISCV_ABI=lp64d" -# "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin" -# #"FW_FDT_PATH=ox.dtb" -# ]; -# patches = [ ./alveo-uart-sbi-hack.patch ]; -# }); -# opensbi = prev.opensbi.overrideAttrs (old: { -# makeFlags = old.makeFlags ++ [ -# "PLATFORM=fpga/openpiton" -# "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin" -# ]; -# patches = [ ./opensbi-lagarto-hun.patch ]; -# }); + + # Custom kernel driver for the Ethernet + xilinx-axienet-carv = let + kernel = config.boot.kernelPackages.kernel; + in prev.stdenv.mkDerivation rec { + pname = "xilinx-axienet-carv"; + version = src.shortRev; + src = builtins.fetchGit { + url = "git@gitlab-internal.bsc.es:meep/meep-os/lagarto-openpiton-sdk.git"; + rev = "d2ae2e788bf1cc60676599184a9ec1128cc81d81"; + ref = "master"; + }; + patches = [ + ./ethernet-driver-poll.patch + ./ethernet-driver-build.patch + ./ethernet-driver-kbuild.patch + ]; + preConfigure = '' + export sourceRoot=$PWD/drivers + cd drivers + #rm Makefile + ''; + + nativeBuildInputs = kernel.moduleBuildDependencies; + makeFlags = kernel.makeFlags ++ [ + "-C" + "${kernel.dev}/lib/modules/${kernel.modDirVersion}/build" + "M=$(PWD)" + ]; + buildFlags = [ "modules" ]; + installFlags = [ "INSTALL_MOD_PATH=${placeholder "out"}" ]; + installTargets = [ "modules_install" ]; + }; }) ]; } -- GitLab From 7817c4be2dab78b13da210a25b2239c0be04cf6f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 5 Sep 2024 09:44:39 +0200 Subject: [PATCH 212/310] Enable DMA node in device tree --- ox-plic.dts | 78 ++++++++++++++++++++++++++--------------------------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/ox-plic.dts b/ox-plic.dts index 09b9442..f3a0a68 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -95,12 +95,12 @@ compatible = "shared-dma-pool"; }; }; -// eth0_clk: eth0_clk { -// compatible = "fixed-clock"; -// #clock-cells = <0x00000000>; -// clock-frequency = <0x09502f90>; -// phandle = <0x00000002>; -// }; + dma_clk: dma_clk { + compatible = "fixed-clock"; + #clock-cells = <0x00000000>; + clock-frequency = <0x09502f90>; + phandle = <0x00000002>; + }; pmem@100000000 { /* volatile; This property indicates that this region is * actually backed by non-persistent memory. This lets the OS @@ -149,43 +149,41 @@ carv,no-mac; device_type = "network"; local-mac-address = [02 05 00 01 00 05]; - axistream-connected = <0x000000fe>; - //compatible = "xlnx,xxv-ethernet-1.0-carv"; - compatible = "xlnx,axi-ethernet-1.00.a"; + axistream-connected = <&axi_dma>; + compatible = "xlnx,xxv-ethernet-1.0-carv"; memory-region = <ð_pool>; }; -// dma@40400000 { -// xlnx,include-dre; -// phandle = <0x000000fe>; -// #dma-cells = <0x00000001>; -// compatible = "xlnx,axi-dma-1.00.a"; -// clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk", "m_axi_s2mm_aclk", "m_axi_sg_aclk"; -// clocks = <ð0_clk>, <ð0_clk>, <ð0_clk>, <ð0_clk>; -// reg = <0x00000000 0x40400000 0x00000000 0x00400000>; -// interrupt-names = "mm2s_introut", "s2mm_introut"; -// interrupt-parent = <&PLIC>; -// interrupts = <2 3>; -// xlnx,addrwidth = <0x00000028>; -// xlnx,include-sg; -// xlnx,sg-length-width = <0x00000017>; -// dma-channel@40400000 { -// compatible = "xlnx,axi-dma-mm2s-channel"; -// dma-channels = <0x00000000>; -// interrupts = <0x00000002>; -// xlnx,datawidth = <0x00000040>; -// xlnx,device-id = <0x00000000>; -// xlnx,include-dre; -// }; -// dma-channel@40400030 { -// compatible = "xlnx,axi-dma-s2mm-channel"; -// dma-channels = <0x00000001>; -// interrupts = <0x00000003>; -// xlnx,datawidth = <0x00000040>; -// xlnx,device-id = <0x00000000>; -// xlnx,include-dre; -// }; -// }; + axi_dma: dma@40400000 { + xlnx,include-dre; + #dma-cells = <0x00000001>; + compatible = "xlnx,axi-dma-1.00.a"; + clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk", "m_axi_s2mm_aclk", "m_axi_sg_aclk"; + clocks = <&dma_clk>, <&dma_clk>, <&dma_clk>, <&dma_clk>; + reg = <0x00000000 0x40400000 0x00000000 0x00400000>; + interrupt-names = "mm2s_introut", "s2mm_introut"; + interrupt-parent = <&PLIC>; + interrupts = <2 3>; + xlnx,addrwidth = <0x28>; + xlnx,include-sg; + xlnx,sg-length-width = <0x17>; + dma-channel@40400000 { + compatible = "xlnx,axi-dma-mm2s-channel"; + dma-channels = <0>; + interrupts = <2>; + xlnx,datawidth = <0x40>; + xlnx,device-id = <0x0>; + xlnx,include-dre; + }; + dma-channel@40400030 { + compatible = "xlnx,axi-dma-s2mm-channel"; + dma-channels = <1>; + interrupts = <3>; + xlnx,datawidth = <0x40>; + xlnx,device-id = <0x0>; + xlnx,include-dre; + }; + }; /* Platform-Level Interrupt Controller: Delivers interrupts to * HARTs. */ -- GitLab From 1787d2cede62b4c1dcdd94ec0b5ea44e0704de56 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 5 Sep 2024 09:48:36 +0200 Subject: [PATCH 213/310] Load the modules in initrd --- lagarto-ox.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 8ac9b50..c64c408 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -78,10 +78,7 @@ # Add the custom Ethernet module pkgs.xilinx-axienet-carv ]; - kernelModules = [ - # Load the Ethernet module by default - "xilinx_axienet_carv" - ]; + kernelModules = config.boot.initrd.kernelModules; kernelPatches = [ { name = assert false; "sbi-early-console"; @@ -147,7 +144,10 @@ initrd = { # Avoid zstd as we don't have the tools in "cucu" machine compressor = "gzip"; - kernelModules = [ ]; + kernelModules = [ + # Load the Ethernet module by default + "xilinx_axienet_carv" + ]; # Custom init script extraFiles = { -- GitLab From bc94e6355f6a68cd916239f317c7d4b3c2b0f14b Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 5 Sep 2024 10:01:30 +0200 Subject: [PATCH 214/310] Fix Ethernet module file name --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index c64c408..3fa772e 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -146,7 +146,7 @@ compressor = "gzip"; kernelModules = [ # Load the Ethernet module by default - "xilinx_axienet_carv" + "xxvnet_carv" ]; # Custom init script -- GitLab From 67d57cdd034cb5cd23fbafe6d15e91acf4fddb11 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 5 Sep 2024 10:06:41 +0200 Subject: [PATCH 215/310] Add Xilinx DMA module --- lagarto-ox.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 3fa772e..2e76474 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -145,6 +145,8 @@ # Avoid zstd as we don't have the tools in "cucu" machine compressor = "gzip"; kernelModules = [ + # DMA for Ethernet + "xilinx_dma" # Load the Ethernet module by default "xxvnet_carv" ]; -- GitLab From ef0c4d3d94371e8b91ba1e684000f9283e4a10a6 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 5 Sep 2024 10:58:17 +0200 Subject: [PATCH 216/310] Set MAC address to zeros --- ox-plic.dts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ox-plic.dts b/ox-plic.dts index f3a0a68..b233d59 100644 --- a/ox-plic.dts +++ b/ox-plic.dts @@ -148,7 +148,10 @@ carv,mtu = <0x000005dc>; carv,no-mac; device_type = "network"; - local-mac-address = [02 05 00 01 00 05]; + // 02:$node:00:01:00:$fpga -> 02:05:00:01:00:02 + // 10.5.1.$N/16 -> 10.5.1.184/16 + // N = 150 + ($node - 1) * 8 + $fpga + local-mac-address = [00 00 00 00 00 00]; axistream-connected = <&axi_dma>; compatible = "xlnx,xxv-ethernet-1.0-carv"; memory-region = <ð_pool>; -- GitLab From d2745c53b18b026af37d3940f39f9e4d77e7aac0 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 5 Sep 2024 11:04:51 +0200 Subject: [PATCH 217/310] Prepare preinit for Ethernet --- lagarto-ox.nix | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 2e76474..9dbc792 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -164,9 +164,26 @@ "/preinit".source = pkgs.writeScript "preinit" '' #!${config.system.build.extraUtils}/bin/ash - set -x export PATH=${config.system.build.extraUtils}/bin - csrtool all-in-order + # csrtool all-in-order + + if read -t 3 -p 'press enter for shell'; then + ash + fi + + set -x + + ip addr + cat /proc/interrupts + + modprobe xxvnet_carv + + plictool -c2 + plictool -c2 + plictool -c2 + + ip addr + exec /init ''; }; -- GitLab From bbf85438b311a3f006ab590b7305c1bb47b2ac30 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 5 Sep 2024 14:53:00 +0200 Subject: [PATCH 218/310] Update journal --- JOURNAL.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index 7cd6c1e..fc95c45 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -4107,3 +4107,20 @@ Interesting output: src=33 pend=0 prio=0 ctx=1 thre=1 masked +## 2024-09-05 + + devmem 0x4080000C 32 127 # Write 127 to source 3 priority + devmem 0x40a01000 32 0 # Write context 1 threshold to 0 + + # devmem 0x40a01004 # Read claim register context 1 + # devmem 0x40a01004 # Read claim register context 1 + # 0x201004 + + devmem 0x40800008 # Print source 3 priority + devmem 0x40a01000 # Print context 1 threshold + + ifconfig eth0 hw ether 02:05:00:01:00:02 + +- Report specific PLIC issues +- Enable aux timer +- Broken RTC frequency. -- GitLab From ee9ecac8ed887e41ef2f4335652d23910712ae4b Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 5 Sep 2024 14:52:47 +0200 Subject: [PATCH 219/310] Add PLIC tests to preinit --- lagarto-ox.nix | 92 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 82 insertions(+), 10 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 9dbc792..b34eb4c 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -173,16 +173,88 @@ set -x - ip addr - cat /proc/interrupts - - modprobe xxvnet_carv - - plictool -c2 - plictool -c2 - plictool -c2 - - ip addr + ( + echo "--- Testing threshold register init value" + # Ensure that reading a few times the threshold value + # always gives the same initial value 0 + t1=$(devmem 0x40a00000) # Read context 1 threshold value + t2=$(devmem 0x40a00000) # Read context 1 threshold value + t3=$(devmem 0x40a00000) # Read context 1 threshold value + found="$t1 $t2 $t3" + expected="0x00000000 0x00000000 0x00000000" + if [ "$found" = "$expected" ]; then + echo "--- Threshold init value: OK" + else + echo "found =$found" + echo "expected=$expected" + echo "--- Threshold init value: FAIL" + fi + ) + + ( + echo "--- Testing threshold register stability" + # Write the priority register of an interrupt and ensure + # the threshold register didn't change + devmem 0x40a00000 32 0 # Write context 1 threshold value 0 + t1=$(devmem 0x40a00000) # Read context 1 threshold value + devmem 0x40800010 32 5 # Write source 4 priority value 5 + t2=$(devmem 0x40a00000) # Read context 1 threshold value + found="$t1 $t2" + expected="0x00000000 0x00000000" + if [ "$found" = "$expected" ]; then + echo "--- Threshold stability: OK" + else + echo "found =$found" + echo "expected=$expected" + echo "--- Threshold stability: FAIL" + fi + ) + + ( + echo "--- Testing claim register" + + # Use aux timer on source 4 for this one + pending=$(devmem 0x40801000) # Dump pending bits of sources 0-31 + + # Ensure the aux timer is pending + if [ "$pending" = "0x00000018" ]; then + + # Make sure the priority is higher than the threshold + devmem 0x40800010 32 0x10 # Write source 4 priority value 16 + devmem 0x40802080 32 0x10 # Enable source 4 in context 1 + # Writing the threshold has to be last, otherwise it will change + devmem 0x40a00000 32 0 # Write context 1 threshold value 0 + c1=$(devmem 0x40a01004) # Claim context 1 + c2=$(devmem 0x40a01004) # Claim context 1 + c3=$(devmem 0x40a01004) # Claim context 1 + found="$c1 $c2 $c3" + expected="0x00000004 0x00000004 0x00000004" + if [ "$found" = "$expected" ]; then + echo "--- Testing claim register: OK" + else + echo "found =$found" + echo "expected=$expected" + echo "--- Testing claim register: FAIL" + fi + else + echo "unknown pending bits: $pending" + echo "--- Testing claim register: SKIP" + fi + ) + + echo "all done, waiting forever..." + while [ 1 ]; do sleep 9999; done + + # ip addr + # cat /proc/interrupts + + # modprobe xxvnet_carv + + # plictool -c2 + # plictool -c2 + # plictool -c2 + + # ip addr exec /init ''; -- GitLab From 4f70ac11acb882c73d450c6ad3b1c33f9e166153 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 5 Sep 2024 14:57:54 +0200 Subject: [PATCH 220/310] Run /preinit before init --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index b34eb4c..9f2a164 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -477,7 +477,7 @@ cat > $out < Date: Thu, 5 Sep 2024 15:06:34 +0200 Subject: [PATCH 221/310] Run PLIC test from init directly --- lagarto-ox.nix | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 9f2a164..4ca6934 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -162,14 +162,9 @@ ash ''; - "/preinit".source = pkgs.writeScript "preinit" '' + "/testplic".source = pkgs.writeScript "testplic" '' #!${config.system.build.extraUtils}/bin/ash export PATH=${config.system.build.extraUtils}/bin - # csrtool all-in-order - - if read -t 3 -p 'press enter for shell'; then - ash - fi set -x @@ -241,9 +236,15 @@ echo "--- Testing claim register: SKIP" fi ) + set +x + echo "all done, dropping to a shell..." + ash + ''; - echo "all done, waiting forever..." - while [ 1 ]; do sleep 9999; done + "/preinit".source = pkgs.writeScript "preinit" '' + #!${config.system.build.extraUtils}/bin/ash + export PATH=${config.system.build.extraUtils}/bin + # csrtool all-in-order # ip addr # cat /proc/interrupts @@ -273,8 +274,15 @@ # while [ 1 ]; do xxd -s $((0x1bfff0000 - 0x60000000)) \ # -l 4 /dev/qdma34000-MM-1; sleep 0.2; done preDeviceCommands = '' - echo "Creating a heartbeat counter at 0x1bfff0000" - sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' & + if read -t 3 -p 'Press enter for shell... '; then + ash + fi + + echo "Running tests..." + sh /testplic + + # echo "Creating a heartbeat counter at 0x1bfff0000" + # sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' & '' + # Disable proactive compaction. May be better to disable CONFIG_COMPACTION. @@ -477,7 +485,7 @@ cat > $out < Date: Thu, 5 Sep 2024 15:12:44 +0200 Subject: [PATCH 222/310] Don't load any modules for now --- lagarto-ox.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 4ca6934..bf569b0 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -146,9 +146,9 @@ compressor = "gzip"; kernelModules = [ # DMA for Ethernet - "xilinx_dma" + #"xilinx_dma" # Load the Ethernet module by default - "xxvnet_carv" + #"xxvnet_carv" ]; # Custom init script -- GitLab From 5fdaab9f8f5608a5029556f72dde5cb3473d4312 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 5 Sep 2024 15:24:02 +0200 Subject: [PATCH 223/310] Fix plictest --- lagarto-ox.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index bf569b0..4505759 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -191,8 +191,8 @@ # Write the priority register of an interrupt and ensure # the threshold register didn't change devmem 0x40a00000 32 0 # Write context 1 threshold value 0 - t1=$(devmem 0x40a00000) # Read context 1 threshold value devmem 0x40800010 32 5 # Write source 4 priority value 5 + t1=$(devmem 0x40a00000) # Read context 1 threshold value t2=$(devmem 0x40a00000) # Read context 1 threshold value found="$t1 $t2" expected="0x00000000 0x00000000" @@ -212,7 +212,7 @@ pending=$(devmem 0x40801000) # Dump pending bits of sources 0-31 # Ensure the aux timer is pending - if [ "$pending" = "0x00000018" ]; then + if [ "$pending" = "0x00000010" ]; then # Make sure the priority is higher than the threshold devmem 0x40800010 32 0x10 # Write source 4 priority value 16 -- GitLab From 2f48ad5f404c5deb129dfaca26856627f2515971 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 5 Sep 2024 16:59:06 +0200 Subject: [PATCH 224/310] Define DT constants in a C header file Allows writing baremetal programs that can read the correct offsets of devices or clock frequencies. --- dts/.gitignore | 2 ++ dts/Makefile | 17 +++++++++++++++++ ox-plic.dts => dts/lagarto_ox.dts | 18 ++++++++++-------- dts/lagarto_ox.h | 9 +++++++++ lagarto-ox.nix | 22 ++++++++++++++++------ 5 files changed, 54 insertions(+), 14 deletions(-) create mode 100644 dts/.gitignore create mode 100644 dts/Makefile rename ox-plic.dts => dts/lagarto_ox.dts (95%) create mode 100644 dts/lagarto_ox.h diff --git a/dts/.gitignore b/dts/.gitignore new file mode 100644 index 0000000..2008a4a --- /dev/null +++ b/dts/.gitignore @@ -0,0 +1,2 @@ +*.pp.dts +*.dtb diff --git a/dts/Makefile b/dts/Makefile new file mode 100644 index 0000000..7f7d3be --- /dev/null +++ b/dts/Makefile @@ -0,0 +1,17 @@ +CC?=gcc +DTC?=dtc + +CPPFLAGS=-E -C -P -nostdinc -undef -x assembler-with-cpp + +all: lagarto_ox.dtb + +clean: + rm -f *.pp.dts *.dtb + +%.pp.dts: %.dts *.h + $(CC) $(CPPFLAGS) $< -o $@ + +%.dtb: %.pp.dts + dtc -O dtb -o $@ $^ + +.PRECIOUS: %.pp.dts diff --git a/ox-plic.dts b/dts/lagarto_ox.dts similarity index 95% rename from ox-plic.dts rename to dts/lagarto_ox.dts index b233d59..5a4776d 100644 --- a/ox-plic.dts +++ b/dts/lagarto_ox.dts @@ -1,3 +1,5 @@ +#include "lagarto_ox.h" + /dts-v1/; / { #address-cells = <2>; @@ -19,9 +21,9 @@ // The RTC timer is clocked at the CPU frequency / 1525, so // around 32786.88 Hz - timebase-frequency = <32786>; + timebase-frequency = ; CPU0: cpu@0 { - clock-frequency = <50000000>; /* 50 MHz */ + clock-frequency = ; /* 50 MHz */ device_type = "cpu"; reg = <0>; status = "okay"; @@ -120,14 +122,14 @@ * consoles */ /* The serial for the kernel console */ - uart_console: serial@40001000 { + uart_console: serial@UART0_ADDR_HEX { compatible = "ns16550"; - reg = <0x0 0x40001000 0x0 0x1000>; + reg = <0x0 UART0_ADDR 0x0 0x1000>; reg-shift = <2>; /* No interrupts for this UART, use console=hvc0 */ /* This clock is the SERIAL_CLK */ - clock-frequency = <50000000>; - current-speed = <115200>; + clock-frequency = ; + current-speed = ; status = "okay"; }; /* The serial for interrupt tests */ @@ -138,8 +140,8 @@ /* Output interrupt 1 (the first one) */ interrupts = <1>; interrupt-parent = <&PLIC>; - clock-frequency = <50000000>; - current-speed = <115200>; + clock-frequency = ; + current-speed = ; status = "okay"; }; diff --git a/dts/lagarto_ox.h b/dts/lagarto_ox.h new file mode 100644 index 0000000..4a93006 --- /dev/null +++ b/dts/lagarto_ox.h @@ -0,0 +1,9 @@ +#define CPU_CLOCK_FREQUENCY 50000000 // 50 MHz +#define RTC_CLOCK_FREQUENCY 32786 + +#define UART_SPEED 115200 +#define UART0_ADDR 0x40001000 +#define UART0_ADDR_HEX 40001000 + +#define PLIC_ADDR 0x40800000 +#define UART_ADDR 0x40001000 diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 4505759..01f54f9 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -370,11 +370,21 @@ # }; #})); - ox-dtb = prev.runCommand "ox.dtb" { - dtsFile = ./ox-plic.dts; - #dtsFile = ./xavi.dts; + ox-dtb = prev.stdenv.mkDerivation rec { + name = "ox.dtb"; + src = ./dts; + dontConfigure = true; nativeBuildInputs = [ prev.buildPackages.dtc ]; - } "dtc -O dtb -o $out $dtsFile"; + buildPhase = '' + make lagarto_ox.dtb + ''; + installPhase = '' + mkdir $out + cp lagarto_ox.* $out + ''; + dontFixup = true; + hardeningDisable = [ "all" ]; + }; #bitstream = "${final.bitstreams}/lagarto-3-ox/gold.bit"; bitstream = "${final.bitstreams}/lagarto-3-ox/ox_u55c_450d0ff0_fix_delegation_v2.bit"; @@ -448,7 +458,7 @@ extraMakeFlags = [ "V=1" #"KCPPFLAGS=-DLOG_DEBUG" - #"EXT_DTB=${final.ox-dtb}" + #"EXT_DTB=${final.ox-dtb}/lagarto_ox.dtb" ]; }; @@ -509,7 +519,7 @@ #"PLATFORM_RISCV_ISA=rv64g" # No compressed instructions #"PLATFORM_RISCV_ABI=lp64d" "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin" - "FW_FDT_PATH=${final.ox-dtb}" + "FW_FDT_PATH=${final.ox-dtb}/lagarto_ox.dtb" # Ensure it doesn't overlap from the 0x80200000 where the kernel will be # placed. -- GitLab From 75ea21f9fea68bf798cbc4ec0c822065767fbc01 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 5 Sep 2024 17:12:35 +0200 Subject: [PATCH 225/310] Move clock comments to lagarto_ox.h --- dts/lagarto_ox.dts | 4 +--- dts/lagarto_ox.h | 6 +++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dts/lagarto_ox.dts b/dts/lagarto_ox.dts index 5a4776d..48a2442 100644 --- a/dts/lagarto_ox.dts +++ b/dts/lagarto_ox.dts @@ -19,11 +19,9 @@ #address-cells = <1>; #size-cells = <0>; - // The RTC timer is clocked at the CPU frequency / 1525, so - // around 32786.88 Hz timebase-frequency = ; CPU0: cpu@0 { - clock-frequency = ; /* 50 MHz */ + clock-frequency = ; device_type = "cpu"; reg = <0>; status = "okay"; diff --git a/dts/lagarto_ox.h b/dts/lagarto_ox.h index 4a93006..3491ebc 100644 --- a/dts/lagarto_ox.h +++ b/dts/lagarto_ox.h @@ -1,4 +1,8 @@ -#define CPU_CLOCK_FREQUENCY 50000000 // 50 MHz +/* CPU is at 50 MHz */ +#define CPU_CLOCK_FREQUENCY 50000000 + +/* The RTC timer is clocked at the CPU frequency / 1525, so + * around 32786.88 Hz */ #define RTC_CLOCK_FREQUENCY 32786 #define UART_SPEED 115200 -- GitLab From 93edc72700559efcfe291e58af9f4af6a3ea7330 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 5 Sep 2024 17:24:43 +0200 Subject: [PATCH 226/310] Don't load the rootfs for now --- .gitlab-ci.yml | 2 +- fpga/run-node.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b5e4351..33d9ada 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,4 +3,4 @@ build:lagarto-ox-rd: tags: - nix script: - - nix develop -L .#lagarto-ox --command fpga/run-remotely.sh fpgalogin1:ci + - nix develop -L .#lagarto-ox-rd --command fpga/run-remotely.sh fpgalogin1:ci diff --git a/fpga/run-node.sh b/fpga/run-node.sh index 01ab16e..178eedb 100755 --- a/fpga/run-node.sh +++ b/fpga/run-node.sh @@ -18,7 +18,8 @@ killall picocom || true set -x # Then perform the boot -./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img +#./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img +./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin # Restart it again only loading the bootrom and OpenSBI (notice OpenSBI # relocates itself so it will likely be gone by now) -- GitLab From 08a304a711e1bf039c2deb9e979e9c45c5ab5f1f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 6 Sep 2024 07:48:22 +0200 Subject: [PATCH 227/310] Remove old DT for Lagarto Ox --- ox.dts | 58 ---------------------------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 ox.dts diff --git a/ox.dts b/ox.dts deleted file mode 100644 index f9e2e94..0000000 --- a/ox.dts +++ /dev/null @@ -1,58 +0,0 @@ -/dts-v1/; - -/ { - #address-cells = <1>; - #size-cells = <1>; - compatible = "riscv,rv64i"; - model = "Barcelona Supercomputing Center - OX"; - chosen { - bootargs = "earlycon=sbi console=ttyS0,115200n8"; - }; - cpus { - #address-cells = <1>; - #size-cells = <0>; - timebase-frequency = <50000>; // 32.768 kHz - CPU0: cpu@0 { - //clock-frequency = <25000000>; // 25 MHz - clock-frequency = <50000000>; // 50 MHz - device_type = "cpu"; - reg = <0>; - status = "okay"; - compatible = "riscv"; - riscv,isa = "rv64imafd"; - mmu-type = "riscv,sv39"; - tlb-split; - // HLIC - hart local interrupt controller - CPU0_intc: interrupt-controller { - #address-cells = <1>; - #interrupt-cells = <1>; - interrupt-controller; - compatible = "riscv,cpu-intc"; - }; - }; - }; - - uart0: serial@40001000 { - compatible = "ns16750"; - //reg = <0x40001000 0x1000>; - reg = <0x00000000 0x40001000 0x00000000 0x00001000>; - interrupts = <0>; - port-number = <0>; - reg-shift = <2>; - reg-io-width = <4>; - clock-frequency = <25000000>; - current-speed = <115200>; - status = "okay"; - }; - - memory@80000000 { - device_type = "memory"; - reg = <0x80000000 0x40000000>; - }; - soc { - #address-cells = <1>; - #size-cells = <1>; - compatible = "simple-bus"; - ranges; - }; -}; -- GitLab From e2c770208eeef4b4d9416475207b3fcbe143812e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 6 Sep 2024 08:11:24 +0200 Subject: [PATCH 228/310] Move other tools to tools/ --- configuration.nix | 2 +- lagarto-ox.nix | 7 ++-- overlay.nix | 66 +++--------------------------------- tools/.gitignore | 4 +++ tools/Makefile | 20 +++++++++++ csrtool.c => tools/csrtool.c | 0 memtool.c => tools/memtool.c | 0 unalign.c => tools/unalign.c | 0 8 files changed, 31 insertions(+), 68 deletions(-) create mode 100644 tools/.gitignore create mode 100644 tools/Makefile rename csrtool.c => tools/csrtool.c (100%) rename memtool.c => tools/memtool.c (100%) rename unalign.c => tools/unalign.c (100%) diff --git a/configuration.nix b/configuration.nix index 669fe60..0110598 100644 --- a/configuration.nix +++ b/configuration.nix @@ -36,5 +36,5 @@ }; #environment.systemPackages = with pkgs; [ vim gdb neofetch gcc bintools ]; - #environment.systemPackages = with pkgs; [ rvb unalignedCheck ]; + #environment.systemPackages = with pkgs; [ rvb riscv-tools ]; } diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 01f54f9..3e5e404 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -261,12 +261,9 @@ ''; }; - # Add the csrtool to the initrd so we can change the - # in-order/out-of-order, and memtool to stress the memory. + # Add riscv-tools to initrd extraUtilsCommands = '' - cp -a ${pkgs.csrtool}/bin/csrtool $out/bin - cp -a ${pkgs.memtool}/bin/memtool $out/bin - cp -a ${pkgs.plictool}/bin/plictool $out/bin + cp -a ${pkgs.riscv-tools}/bin/* $out/bin ''; # Write a counter to the DMA region, so we can check the kernel is not diff --git a/overlay.nix b/overlay.nix index bc60e91..9cf6788 100644 --- a/overlay.nix +++ b/overlay.nix @@ -21,68 +21,10 @@ final: prev: ]; }); - unalignedCheck = prev.stdenv.mkDerivation { - name = "unaligned-check"; - src = ./unalign.c; - unpackPhase = '' - cp ${./unalign.c} unalign.c - ''; - dontConfigure = true; - buildPhase = '' - $CC unalign.c -o unalign_check - ''; - installPhase = '' - mkdir -p $out/bin - cp unalign_check $out/bin/ - ''; - }; - - csrtool = prev.pkgsStatic.stdenv.mkDerivation { - name = "csrtool"; - src = ./csrtool.c; - unpackPhase = '' - cp ${./csrtool.c} csrtool.c - ''; - dontConfigure = true; - buildPhase = '' - $CC -static csrtool.c -o csrtool - ''; - installPhase = '' - mkdir -p $out/bin - cp csrtool $out/bin/ - ''; - }; - - memtool = prev.pkgsStatic.stdenv.mkDerivation { - name = "memtool"; - src = ./memtool.c; - unpackPhase = '' - cp ${./memtool.c} memtool.c - ''; - dontConfigure = true; - buildPhase = '' - $CC -static memtool.c -o memtool - ''; - installPhase = '' - mkdir -p $out/bin - cp memtool $out/bin/ - ''; - }; - - plictool = prev.pkgsStatic.stdenv.mkDerivation { - name = "plictool"; - src = ./tools/plictool.c; - unpackPhase = '' - cp ${./tools/plictool.c} plictool.c - ''; - dontConfigure = true; - buildPhase = '' - $CC -static plictool.c -o plictool - ''; - installPhase = '' - mkdir -p $out/bin - cp plictool $out/bin/ - ''; + riscv-tools = prev.pkgsStatic.stdenv.mkDerivation { + name = "riscv-tools"; + src = ./tools; + makeFlags = [ "PREFIX=${placeholder "out"}" ]; }; bitstreams = builtins.fetchGit { diff --git a/tools/.gitignore b/tools/.gitignore new file mode 100644 index 0000000..c79a82f --- /dev/null +++ b/tools/.gitignore @@ -0,0 +1,4 @@ +plictool +csrtool +memtool +*.bin diff --git a/tools/Makefile b/tools/Makefile new file mode 100644 index 0000000..c57de84 --- /dev/null +++ b/tools/Makefile @@ -0,0 +1,20 @@ +CFLAGS=-static +PREFIX?=/usr/local +bin=plictool csrtool memtool unalign + +all: $(bin) + +clean: + rm -f $(bin) + +install: + mkdir -p $(PREFIX)/bin + cp -a $(bin) $(PREFIX)/bin + +plictool: plictool.c + +csrtool: csrtool.c + +memtool: memtool.c + +unalign: unalign.c diff --git a/csrtool.c b/tools/csrtool.c similarity index 100% rename from csrtool.c rename to tools/csrtool.c diff --git a/memtool.c b/tools/memtool.c similarity index 100% rename from memtool.c rename to tools/memtool.c diff --git a/unalign.c b/tools/unalign.c similarity index 100% rename from unalign.c rename to tools/unalign.c -- GitLab From 3dad4fb3d18ff0906c4eacc18a1449284a09fa99 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 6 Sep 2024 08:22:14 +0200 Subject: [PATCH 229/310] Move patches to its own directory --- lagarto-hun.nix | 2 +- lagarto-ox.nix | 28 +++++++++---------- overlay.nix | 8 +++--- .../alveo-uart-sbi-hack.patch | 0 .../busybox-debug.patch | 0 .../ethernet-driver-build.patch | 0 .../ethernet-driver-kbuild.patch | 0 .../ethernet-driver-poll.patch | 0 .../opensbi-dont-delegate.patch | 0 .../opensbi-dump-mregs.patch | 0 .../opensbi-enable-meip.patch | 0 .../opensbi-enable-seip.patch | 0 .../opensbi-lagarto-hun.patch | 0 .../opensbi-test-plic.patch | 0 .../opensbi-timer-debug.patch | 0 .../ox-alveo-platform-plic.patch | 0 .../ox-alveo-platform.patch | 0 .../sa-fpga-crt.patch | 0 .../sa-fpga-plic-registers.patch | 0 .../sa-fpga-text-address.patch | 0 .../sa-fpga-uart.patch | 0 .../u-boot-debug.patch | 0 .../uboot-debug-ext-interrupts.patch | 0 .../uboot-exception-extras.patch | 0 24 files changed, 19 insertions(+), 19 deletions(-) rename alveo-uart-sbi-hack.patch => patches/alveo-uart-sbi-hack.patch (100%) rename busybox-debug.patch => patches/busybox-debug.patch (100%) rename ethernet-driver-build.patch => patches/ethernet-driver-build.patch (100%) rename ethernet-driver-kbuild.patch => patches/ethernet-driver-kbuild.patch (100%) rename ethernet-driver-poll.patch => patches/ethernet-driver-poll.patch (100%) rename opensbi-dont-delegate.patch => patches/opensbi-dont-delegate.patch (100%) rename opensbi-dump-mregs.patch => patches/opensbi-dump-mregs.patch (100%) rename opensbi-enable-meip.patch => patches/opensbi-enable-meip.patch (100%) rename opensbi-enable-seip.patch => patches/opensbi-enable-seip.patch (100%) rename opensbi-lagarto-hun.patch => patches/opensbi-lagarto-hun.patch (100%) rename opensbi-test-plic.patch => patches/opensbi-test-plic.patch (100%) rename opensbi-timer-debug.patch => patches/opensbi-timer-debug.patch (100%) rename ox-alveo-platform-plic.patch => patches/ox-alveo-platform-plic.patch (100%) rename ox-alveo-platform.patch => patches/ox-alveo-platform.patch (100%) rename sa-fpga-crt.patch => patches/sa-fpga-crt.patch (100%) rename sa-fpga-plic-registers.patch => patches/sa-fpga-plic-registers.patch (100%) rename sa-fpga-text-address.patch => patches/sa-fpga-text-address.patch (100%) rename sa-fpga-uart.patch => patches/sa-fpga-uart.patch (100%) rename u-boot-debug.patch => patches/u-boot-debug.patch (100%) rename uboot-debug-ext-interrupts.patch => patches/uboot-debug-ext-interrupts.patch (100%) rename uboot-exception-extras.patch => patches/uboot-exception-extras.patch (100%) diff --git a/lagarto-hun.nix b/lagarto-hun.nix index 51359ba..a1a98a8 100644 --- a/lagarto-hun.nix +++ b/lagarto-hun.nix @@ -126,7 +126,7 @@ "PLATFORM=fpga/openpiton" "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin" ]; - patches = [ ./opensbi-lagarto-hun.patch ]; + patches = [ ./patches/opensbi-lagarto-hun.patch ]; }); }) ]; } diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 3e5e404..6f71896 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -357,7 +357,7 @@ nixpkgs.overlays = [ (final: prev: { #busybox = prev.busybox.overrideAttrs (old: { # # Print some debug lines on switch_root to see where it hangs. - # patches = (old.patches or []) ++ [ ./busybox-debug.patch ]; + # patches = (old.patches or []) ++ [ ./patches/busybox-debug.patch ]; #}); linuxPackages_latest = prev.linuxPackages_latest; @@ -396,9 +396,9 @@ # rev = "f80a22a480f0e4157647bacf90e663be457c72c4"; #}; patches = [ - #./u-boot-debug.patch - ./uboot-debug-ext-interrupts.patch - ./uboot-exception-extras.patch + #./patches/u-boot-debug.patch + ./patches/uboot-debug-ext-interrupts.patch + ./patches/uboot-exception-extras.patch ]; # Copy our environment to board/emulation/qemu-riscv/environ.env preConfigure = '' @@ -523,13 +523,13 @@ "FW_PAYLOAD_FDT_ADDR=0xc0000000" ]; patches = [ - #./opensbi-timer-debug.patch # Print calls to machine trap - #./opensbi-enable-meip.patch - #./opensbi-enable-seip.patch - #./opensbi-test-plic.patch # Working delegation test, disabled for now - ./opensbi-dump-mregs.patch - #./opensbi-dont-delegate.patch - #./ox-alveo-platform-plic.patch + #./patches/opensbi-timer-debug.patch # Print calls to machine trap + #./patches/opensbi-enable-meip.patch + #./patches/opensbi-enable-seip.patch + #./patches/opensbi-test-plic.patch # Working delegation test, disabled for now + ./patches/opensbi-dump-mregs.patch + #./patches/opensbi-dont-delegate.patch + #./patches/ox-alveo-platform-plic.patch ]; }); @@ -545,9 +545,9 @@ ref = "master"; }; patches = [ - ./ethernet-driver-poll.patch - ./ethernet-driver-build.patch - ./ethernet-driver-kbuild.patch + ./patches/ethernet-driver-poll.patch + ./patches/ethernet-driver-build.patch + ./patches/ethernet-driver-kbuild.patch ]; preConfigure = '' export sourceRoot=$PWD/drivers diff --git a/overlay.nix b/overlay.nix index 9cf6788..c88a431 100644 --- a/overlay.nix +++ b/overlay.nix @@ -42,10 +42,10 @@ final: prev: }; dontConfigure = true; patches = [ - #./sa-fpga-crt.patch - #./sa-fpga-text-address.patch - ./sa-fpga-uart.patch - ./sa-fpga-plic-registers.patch + #./patches/sa-fpga-crt.patch + #./patches/sa-fpga-text-address.patch + ./patches/sa-fpga-uart.patch + ./patches/sa-fpga-plic-registers.patch ]; buildPhase = '' cd fpga_core_bridge/simulator/tests/c_tests/ diff --git a/alveo-uart-sbi-hack.patch b/patches/alveo-uart-sbi-hack.patch similarity index 100% rename from alveo-uart-sbi-hack.patch rename to patches/alveo-uart-sbi-hack.patch diff --git a/busybox-debug.patch b/patches/busybox-debug.patch similarity index 100% rename from busybox-debug.patch rename to patches/busybox-debug.patch diff --git a/ethernet-driver-build.patch b/patches/ethernet-driver-build.patch similarity index 100% rename from ethernet-driver-build.patch rename to patches/ethernet-driver-build.patch diff --git a/ethernet-driver-kbuild.patch b/patches/ethernet-driver-kbuild.patch similarity index 100% rename from ethernet-driver-kbuild.patch rename to patches/ethernet-driver-kbuild.patch diff --git a/ethernet-driver-poll.patch b/patches/ethernet-driver-poll.patch similarity index 100% rename from ethernet-driver-poll.patch rename to patches/ethernet-driver-poll.patch diff --git a/opensbi-dont-delegate.patch b/patches/opensbi-dont-delegate.patch similarity index 100% rename from opensbi-dont-delegate.patch rename to patches/opensbi-dont-delegate.patch diff --git a/opensbi-dump-mregs.patch b/patches/opensbi-dump-mregs.patch similarity index 100% rename from opensbi-dump-mregs.patch rename to patches/opensbi-dump-mregs.patch diff --git a/opensbi-enable-meip.patch b/patches/opensbi-enable-meip.patch similarity index 100% rename from opensbi-enable-meip.patch rename to patches/opensbi-enable-meip.patch diff --git a/opensbi-enable-seip.patch b/patches/opensbi-enable-seip.patch similarity index 100% rename from opensbi-enable-seip.patch rename to patches/opensbi-enable-seip.patch diff --git a/opensbi-lagarto-hun.patch b/patches/opensbi-lagarto-hun.patch similarity index 100% rename from opensbi-lagarto-hun.patch rename to patches/opensbi-lagarto-hun.patch diff --git a/opensbi-test-plic.patch b/patches/opensbi-test-plic.patch similarity index 100% rename from opensbi-test-plic.patch rename to patches/opensbi-test-plic.patch diff --git a/opensbi-timer-debug.patch b/patches/opensbi-timer-debug.patch similarity index 100% rename from opensbi-timer-debug.patch rename to patches/opensbi-timer-debug.patch diff --git a/ox-alveo-platform-plic.patch b/patches/ox-alveo-platform-plic.patch similarity index 100% rename from ox-alveo-platform-plic.patch rename to patches/ox-alveo-platform-plic.patch diff --git a/ox-alveo-platform.patch b/patches/ox-alveo-platform.patch similarity index 100% rename from ox-alveo-platform.patch rename to patches/ox-alveo-platform.patch diff --git a/sa-fpga-crt.patch b/patches/sa-fpga-crt.patch similarity index 100% rename from sa-fpga-crt.patch rename to patches/sa-fpga-crt.patch diff --git a/sa-fpga-plic-registers.patch b/patches/sa-fpga-plic-registers.patch similarity index 100% rename from sa-fpga-plic-registers.patch rename to patches/sa-fpga-plic-registers.patch diff --git a/sa-fpga-text-address.patch b/patches/sa-fpga-text-address.patch similarity index 100% rename from sa-fpga-text-address.patch rename to patches/sa-fpga-text-address.patch diff --git a/sa-fpga-uart.patch b/patches/sa-fpga-uart.patch similarity index 100% rename from sa-fpga-uart.patch rename to patches/sa-fpga-uart.patch diff --git a/u-boot-debug.patch b/patches/u-boot-debug.patch similarity index 100% rename from u-boot-debug.patch rename to patches/u-boot-debug.patch diff --git a/uboot-debug-ext-interrupts.patch b/patches/uboot-debug-ext-interrupts.patch similarity index 100% rename from uboot-debug-ext-interrupts.patch rename to patches/uboot-debug-ext-interrupts.patch diff --git a/uboot-exception-extras.patch b/patches/uboot-exception-extras.patch similarity index 100% rename from uboot-exception-extras.patch rename to patches/uboot-exception-extras.patch -- GitLab From c5b9700655d1e348966d48b780085862f34be28c Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 6 Sep 2024 08:27:07 +0200 Subject: [PATCH 230/310] Move other patched scripts to patches/ --- lagarto-ox.nix | 4 ++-- stage-2-init.sh => patches/stage-2-init.sh | 0 update-users-groups.pl => patches/update-users-groups.pl | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename stage-2-init.sh => patches/stage-2-init.sh (100%) rename update-users-groups.pl => patches/update-users-groups.pl (100%) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 6f71896..a1c4033 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -39,7 +39,7 @@ install -m 0755 -d /home ${pkgs.perl.withPackages (p: [ p.FileSlurp p.JSON ])}/bin/perl \ - -w ${./update-users-groups.pl} ${spec} + -w ${./patches/update-users-groups.pl} ${spec} ''; } else "" # keep around for backwards compatibility ); @@ -51,7 +51,7 @@ system.build.bootStage2 = let useHostResolvConf = config.networking.resolvconf.enable && config.networking.useHostResolvConf; bootStage2 = pkgs.substituteAll { - src = ./stage-2-init.sh; + src = ./patches/stage-2-init.sh; shellDebug = "${pkgs.bashInteractive}/bin/bash"; shell = "${pkgs.bash}/bin/bash"; inherit (config.boot) readOnlyNixStore systemdExecutable extraSystemdUnitPaths; diff --git a/stage-2-init.sh b/patches/stage-2-init.sh similarity index 100% rename from stage-2-init.sh rename to patches/stage-2-init.sh diff --git a/update-users-groups.pl b/patches/update-users-groups.pl similarity index 100% rename from update-users-groups.pl rename to patches/update-users-groups.pl -- GitLab From 81e866e68d506ae03f9a84c3f193b699a8d33c9f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 6 Sep 2024 08:28:51 +0200 Subject: [PATCH 231/310] Remove unused files --- stage2.log | 668 ----------------------------------------------------- xavi.dts | 159 ------------- 2 files changed, 827 deletions(-) delete mode 100644 stage2.log delete mode 100644 xavi.dts diff --git a/stage2.log b/stage2.log deleted file mode 100644 index 1ad0652..0000000 --- a/stage2.log +++ /dev/null @@ -1,668 +0,0 @@ -OpenSBI v1.4 - ____ _____ ____ _____ - / __ \ / ____| _ \_ _| - | | | |_ __ ___ _ __ | (___ | |_) || | - | | | | '_ \ / _ \ '_ \ \___ \| _ < | | - | |__| | |_) | __/ | | |____) | |_) || |_ - \____/| .__/ \___|_| |_|_____/|____/_____| - | | - |_| - -Platform Name : ox (Rodrigo NixOS version) -Platform Features : medeleg -Platform HART Count : 1 -Platform IPI Device : --- -Platform Timer Device : axi_timer @ 50000000Hz -Platform Console Device : uart8250 -Platform HSM Device : --- -Platform PMU Device : --- -Platform Reboot Device : --- -Platform Shutdown Device : --- -Platform Suspend Device : --- -Platform CPPC Device : --- -Firmware Base : 0x80000000 -Firmware Size : 178 KB -Firmware RW Offset : 0x20000 -Firmware RW Size : 50 KB -Firmware Heap Offset : 0x24000 -Firmware Heap Size : 34 KB (total), 2 KB (reserved), 8 KB (used), 23 KB (free) -Firmware Scratch Size : 4096 B (total), 280 B (used), 3816 B (free) -Runtime SBI Version : 2.0 - -Domain0 Name : root -Domain0 Boot HART : 0 -Domain0 HARTs : 0* -Domain0 Region00 : 0x0000000040000000-0x0000000040000fff M: (I,R,W) S/U: (R,W) -Domain0 Region01 : 0x0000000080020000-0x000000008002ffff M: (R,W) S/U: () -Domain0 Region02 : 0x0000000080000000-0x000000008001ffff M: (R,X) S/U: () -Domain0 Region03 : 0x0000000000000000-0xffffffffffffffff M: () S/U: (R,W,X) -Domain0 Next Address : 0x0000000080200000 -Domain0 Next Arg1 : 0x0000000080013000 -Domain0 Next Mode : S-mode -Domain0 SysReset : yes -Domain0 SysSuspend : yes - -Boot HART ID : 0 -Boot HART Domain : root -Boot HART Priv Version : v1.10 -Boot HART Base ISA : rv64imafdc -Boot HART ISA Extensions : zicntr,zihpm -Boot HART PMP Count : 0 -Boot HART PMP Granularity : 0 bits -Boot HART PMP Address Bits: 0 -Boot HART MHPM Info : 29 (0xfffffff8) -Boot HART MIDELEG : 0x0000000000000222 -Boot HART MEDELEG : 0x000000000000b109 - - -Core: 12 devices, 8 uclasses, devicetree: board -Loading Environment from nowhere... OK -In: serial,usbkbd -Out: serial,vidconsole -Err: serial,vidconsole -No working controllers found -Net: No ethernet found. -Working FDT set to 80013000 -Hit any key to stop autoboot: 0 - -Device 0: unknown device - -Device 1: unknown device -scanning bus for devices... - -Device 0: unknown device -starting USB... -No working controllers found -No ethernet found. -No ethernet found. -=> setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi boot.trace console=hvc0 ${debugargs} init=/nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init" -=> setenv ramdisk_size 12563951 -=> booti ${kernel_addr_r} ${ramdisk_addr_r}:${ramdisk_size} ${fdtcontroladdr} -Moving Image from 0x84000000 to 0x80200000, end=83044650 -## Flattened Device Tree blob at 80013000 - Booting using the fdt blob at 0x80013000 -Working FDT set to 80013000 - Using Device Tree in place at 0000000080013000, end 000000008001696f -Working FDT set to 80013000 - -Starting kernel ... - -[ 0.000000] Linux version 6.9.7 (nixbld@localhost) (riscv64-unknown-linux-gnu-gcc (GCC) 13.3.0, GNU ld (GNU Binutils) 2.41) #1-NixOS Thu Jun 27 11:52:32 UTC 2024 -[ 0.000000] Machine model: Barcelona Supercomputing Center - Lagarto Ox (NixOS) -[ 0.000000] SBI specification v2.0 detected -[ 0.000000] SBI implementation ID=0x1 Version=0x10004 -[ 0.000000] SBI TIME extension detected -[ 0.000000] SBI IPI extension detected -[ 0.000000] SBI RFENCE extension detected -[ 0.000000] SBI DBCN extension detected -[ 0.000000] earlycon: sbi0 at I/O port 0x0 (options '') -[ 0.000000] printk: legacy bootconsole [sbi0] enabled -[ 0.000000] Reserved memory: created DMA memory pool at 0x0000000060000000, size 256 MiB -[ 0.000000] OF: reserved mem: initialized node dma_pool@60000000, compatible id shared-dma-pool -[ 0.000000] OF: reserved mem: 0x0000000060000000..0x000000006fffffff (262144 KiB) map non-reusable dma_pool@60000000 -[ 0.000000] Reserved memory: created DMA memory pool at 0x0000000070000000, size 256 MiB -[ 0.000000] OF: reserved mem: initialized node dma_pool@70000000, compatible id shared-dma-pool -[ 0.000000] OF: reserved mem: 0x0000000070000000..0x000000007fffffff (262144 KiB) map non-reusable dma_pool@70000000 -[ 0.000000] cma: Reserved 16 MiB at 0x00000000fee00000 on node -1 -[ 0.000000] Zone ranges: -[ 0.000000] DMA32 [mem 0x0000000080000000-0x00000000ffefffff] -[ 0.000000] Normal empty -[ 0.000000] Movable zone start for each node -[ 0.000000] Early memory node ranges -[ 0.000000] node 0: [mem 0x0000000080000000-0x00000000ffefffff] -[ 0.000000] Initmem setup node 0 [mem 0x0000000080000000-0x00000000ffefffff] -[ 0.000000] On node 0, zone DMA32: 256 pages in unavailable ranges -[ 0.000000] Falling back to deprecated "riscv,isa" -[ 0.000000] riscv: base ISA extensions adfim -[ 0.000000] riscv: ELF capabilities adfim -[ 0.000000] pcpu-alloc: s0 r0 d131072 u131072 alloc=1*131072 -[ 0.000000] pcpu-alloc: [0] 0 -[ 0.000000] Kernel command line: root=/dev/ram0 loglevel=7 debug rw earlycon=sbi boot.trace console=hvc0 init=/nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init -[ 0.000000] Dentry cache hash table entries: 262144 (order: 9, 2097152 bytes, linear) -[ 0.000000] Inode-cache hash table entries: 131072 (order: 8, 1048576 bytes, linear) -[ 0.000000] Built 1 zonelists, mobility grouping on. Total pages: 515844 -[ 0.000000] mem auto-init: stack:all(zero), heap alloc:off, heap free:off -[ 0.000000] Virtual kernel memory layout: -[ 0.000000] fixmap : 0xffffffc6fea00000 - 0xffffffc6ff000000 (6144 kB) -[ 0.000000] pci io : 0xffffffc6ff000000 - 0xffffffc700000000 ( 16 MB) -[ 0.000000] vmemmap : 0xffffffc700000000 - 0xffffffc800000000 (4096 MB) -[ 0.000000] vmalloc : 0xffffffc800000000 - 0xffffffd800000000 ( 64 GB) -[ 0.000000] modules : 0xffffffff02e45000 - 0xffffffff80000000 (2001 MB) -[ 0.000000] lowmem : 0xffffffd800000000 - 0xffffffd87ff00000 (2047 MB) -[ 0.000000] kernel : 0xffffffff80000000 - 0xffffffffffffffff (2047 MB) -[ 0.000000] Memory: 1675400K/2096128K available (17075K kernel code, 9047K rwdata, 10240K rodata, 8737K init, 917K bss, 404344K reserved, 16384K cma-reserved) -[ 0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=1, Nodes=1 -[ 0.000000] ftrace: allocating 46961 entries in 184 pages -[ 0.000000] ftrace: allocated 184 pages with 4 groups -[ 0.000000] trace event string verifier disabled -[ 0.000000] RCU Tasks Rude: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1. -[ 0.000000] RCU Tasks Trace: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1. -[ 0.000000] NR_IRQS: 64, nr_irqs: 64, preallocated irqs: 0 -[ 0.000000] riscv-intc: 64 local interrupts mapped -[ 0.000000] clocksource: riscv_clocksource: mask: 0xffffffffffffffff max_cycles: 0x179dd7f66, max_idle_ns: 56421785867800 ns -[ 0.000020] sched_clock: 64 bits at 50kHz, resolution 20000ns, wraps every 70368744170000ns -[ 0.015140] kfence: initialized - using 2097152 bytes for 255 objects at 0x(____ptrval____)-0x(____ptrval____) -[ 0.037080] Console: colour dummy device 80x25 -[ 0.043380] Calibrating delay loop (skipped), value calculated using timer frequency.. 0.10 BogoMIPS (lpj=200) -[ 0.054600] pid_max: default: 32768 minimum: 301 -[ 0.206980] LSM: initializing lsm=capability,landlock,yama -[ 0.537640] landlock: Up and running. -[ 0.542180] Yama: becoming mindful. -[ 0.584380] Mount-cache hash table entries: 4096 (order: 3, 32768 bytes, linear) -[ 0.593160] Mountpoint-cache hash table entries: 4096 (order: 3, 32768 bytes, linear) -[ 0.885440] riscv: ELF compat mode unsupported -[ 0.885820] ASID allocator disabled (0 bits) -[ 0.992540] devtmpfs: initialized -[ 1.122280] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns -[ 1.133360] futex hash table entries: 256 (order: 1, 12288 bytes, linear) -[ 1.233120] pinctrl core: initialized pinctrl subsystem -[ 1.366300] NET: Registered PF_NETLINK/PF_ROUTE protocol family -[ 1.426680] DMA: preallocated 256 KiB GFP_KERNEL pool for atomic allocations -[ 1.441220] DMA: preallocated 256 KiB GFP_KERNEL|GFP_DMA32 pool for atomic allocations -[ 1.452360] audit: initializing netlink subsys (disabled) -[ 1.468360] audit: type=2000 audit(1.200:1): state=initialized audit_enabled=0 res=1 -[ 1.526400] thermal_sys: Registered thermal governor 'step_wise' -[ 1.527680] cpuidle: using governor ladder -[ 2.024460] cpu0: Ratio of byte access time to unaligned word access is 0.00, unaligned accesses are slow -[ 2.301780] HugeTLB: registered 1.00 GiB page size, pre-allocated 0 pages -[ 2.309580] HugeTLB: 16380 KiB vmemmap can be freed for a 1.00 GiB page -[ 2.317240] HugeTLB: registered 2.00 MiB page size, pre-allocated 0 pages -[ 2.324960] HugeTLB: 28 KiB vmemmap can be freed for a 2.00 MiB page -[ 2.807240] iommu: Default domain type: Translated -[ 2.813020] iommu: DMA domain TLB invalidation policy: strict mode -[ 2.938240] SCSI subsystem initialized -[ 2.966020] libata version 3.00 loaded. -[ 3.006820] usbcore: registered new interface driver usbfs -[ 3.028200] usbcore: registered new interface driver hub -[ 3.042520] usbcore: registered new device driver usb -[ 3.162860] Advanced Linux Sound Architecture Driver Initialized. -[ 3.331140] vgaarb: loaded -[ 3.374160] clocksource: Switched to clocksource riscv_clocksource -[ 18.084500] VFS: Disk quotas dquot_6.6.0 -[ 18.099980] VFS: Dquot-cache hash table entries: 512 (order 0, 4096 bytes) -[ 18.129800] netfs: FS-Cache loaded -[ 18.887880] NET: Registered PF_INET protocol family -[ 18.905140] IP idents hash table entries: 32768 (order: 6, 262144 bytes, linear) -[ 20.307580] tcp_listen_portaddr_hash hash table entries: 1024 (order: 3, 32768 bytes, linear) -[ 20.317920] Table-perturb hash table entries: 65536 (order: 6, 262144 bytes, linear) -[ 20.327840] TCP established hash table entries: 16384 (order: 5, 131072 bytes, linear) -[ 20.339640] TCP bind hash table entries: 16384 (order: 8, 1048576 bytes, linear) -[ 20.371220] TCP: Hash tables configured (established 16384 bind 16384) -[ 20.399500] MPTCP token hash table entries: 2048 (order: 4, 98304 bytes, linear) -[ 20.421380] UDP hash table entries: 1024 (order: 4, 98304 bytes, linear) -[ 20.432460] UDP-Lite hash table entries: 1024 (order: 4, 98304 bytes, linear) -[ 20.459500] NET: Registered PF_UNIX/PF_LOCAL protocol family -[ 20.506800] RPC: Registered named UNIX socket transport module. -[ 20.514160] RPC: Registered udp transport module. -[ 20.520000] RPC: Registered tcp transport module. -[ 20.525720] RPC: Registered tcp-with-tls transport module. -[ 20.532220] RPC: Registered tcp NFSv4.1 backchannel transport module. -[ 20.541120] NET: Registered PF_XDP protocol family -[ 20.547220] PCI: CLS 0 bytes, default 64 -[ 20.570800] Trying to unpack rootfs image as initramfs... -[ 20.733220] Initialise system trusted keyrings -[ 20.772720] workingset: timestamp_bits=46 max_order=19 bucket_order=0 -[ 22.193520] NFS: Registering the id_resolver key type -[ 22.220780] Key type id_resolver registered -[ 22.226940] Key type id_legacy registered -[ 22.259900] nfs4filelayout_init: NFSv4 File Layout Driver Registering... -[ 22.267740] nfs4flexfilelayout_init: NFSv4 Flexfile Layout Driver Registering... -[ 22.310900] 9p: Installing v9fs 9p2000 file system support -[ 30.612000] NET: Registered PF_ALG protocol family -[ 30.625120] Key type asymmetric registered -[ 30.635440] Asymmetric key parser 'x509' registered -[ 30.676100] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 244) -[ 30.712080] io scheduler mq-deadline registered -[ 30.717760] io scheduler kyber registered -[ 30.797740] riscv-plic 40800000.plic: mapped 3 interrupts with 1 handlers for 2 contexts. -[ 40.296340] printk: legacy console [hvc0] enabled -[ 40.296340] printk: legacy console [hvc0] enabled -[ 40.307600] printk: legacy bootconsole [sbi0] disabled -[ 40.307600] printk: legacy bootconsole [sbi0] disabled -[ 40.341220] Serial: 8250/16550 driver, 4 ports, IRQ sharing disabled -[ 40.697320] of_serial 40001000.serial: error -ENXIO: IRQ index 0 not found -[ 40.844360] 40001000.serial: ttyS0 at MMIO 0x40001000 (irq = 0, base_baud = 3125000) is a 16550 -[ 41.039860] 40003000.serial: ttyS1 at MMIO 0x40003000 (irq = 1, base_baud = 3125000) is a 16550 -[ 41.147500] SuperH (H)SCI(F) driver initialized -[ 42.464640] loop: module loaded -[ 42.621480] nd_pmem namespace0.0: unable to guarantee persistence of writes -[ 42.797320] pmem0: p1 p2 -[ 43.117840] usbcore: registered new interface driver uas -[ 43.151240] usbcore: registered new interface driver usb-storage -[ 43.187600] usbcore: registered new interface driver usbserial_generic -[ 43.198760] usbserial: USB Serial support registered for generic -[ 43.240820] mousedev: PS/2 mouse device common for all mice -[ 43.421760] sdhci: Secure Digital Host Controller Interface driver -[ 43.429680] sdhci: Copyright(c) Pierre Ossman -[ 43.461820] Synopsys Designware Multimedia Card Interface Driver -[ 43.497400] sdhci-pltfm: SDHCI platform and OF driver helper -[ 43.539140] hid: raw HID events driver (C) Jiri Kosina -[ 43.569620] usbcore: registered new interface driver usbhid -[ 43.576400] usbhid: USB HID core driver -[ 43.621260] riscv-pmu-sbi: SBI PMU extension is available -[ 43.633420] riscv-pmu-sbi: 16 firmware and 31 hardware counters -[ 43.640860] riscv-pmu-sbi: Perf sampling/filtering is not supported as sscof extension is not available -[ 43.792480] drop_monitor: Initializing network drop monitor service -[ 43.915420] NET: Registered PF_INET6 protocol family -[ 49.587120] Initramfs unpacking failed: invalid magic at start of compressed archive -[ 57.799900] Freeing initrd memory: 300428K -[ 57.863740] Segment Routing with IPv6 -[ 57.873220] In-situ OAM (IOAM) with IPv6 -[ 57.882320] sit: IPv6, IPv4 and MPLS over IPv4 tunneling driver -[ 57.964280] NET: Registered PF_PACKET protocol family -[ 57.993060] 9pnet: Installing 9P2000 support -[ 58.003340] Key type dns_resolver registered -[ 58.013600] start plist test -[ 58.094820] end plist test -[ 59.433600] registered taskstats version 1 -[ 59.447940] Loading compiled-in X.509 certificates -[ 62.396240] debug_vm_pgtable: [debug_vm_pgtable ]: Validating architecture page table helpers -[ 62.449040] Key type .fscrypt registered -[ 62.454140] Key type fscrypt-provisioning registered -[ 62.540060] hid_bpf: error while preloading HID BPF dispatcher: -22 -[ 62.552660] clk: Disabling unused clocks -[ 62.564820] PM: genpd: Disabling unused power domains -[ 62.571280] ALSA device list: -[ 62.575900] No soundcards found. -[ 63.179280] Freeing unused kernel image (initmem) memory: 8736K -[ 63.281280] Checked W+X mappings: passed, no W+X pages found -[ 63.289780] Run /init as init process -[ 63.295560] with arguments: -[ 63.299640] /init -[ 63.303080] with environment: -[ 63.307320] HOME=/ -[ 63.310820] TERM=linux - -<<< NixOS Stage 1 >>> - -+ IFS='=' -+ echo console hvc0 -+ set -- console hvc0 -+ params=hvc0 -+ IFS=, -+ echo hvc0 -+ set -- hvc0 -+ console=hvc0 -+ IFS='=' -+ echo init /nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init -+ set -- init /nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init -+ stage2Init=/nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init -+ echo /nix/store/988axh0bq3wqp90gms4b4a0hkfwvkd3i-extra-utils/bin/modprobe[ 71.111740] stage-1-init: [Thu Jan 1 00:01:10 UTC 1970] + IFS='=' - -+ basename dm_mod -+ info 'loading module dm_mod...' -+ '[[' -n 1 ]] -+ echo 'loading module dm_mod...' -loading module dm_mod... -+ modprobe dm_mod -[ 71.713020] stage-1-init: [Thu Jan 1 00:01:11 UTC 1970] + echo console hvc0 -[ 72.282000] stage-1-init: [Thu Jan 1 00:01:11 UTC 1970] + set -- console hvc0 -[ 72.828720] stage-1-init: [Thu Jan 1 00:01:12 UTC 1970] + params=hvc0 -[ 73.359520] stage-1-init: [Thu Jan 1 00:01:12 UTC 1970] + IFS=, -[ 73.889960] stage-1-init: [Thu Jan 1 00:01:13 UTC 1970] + echo hvc0 -[ 74.448740] stage-1-init: [Thu Jan 1 00:01:13 UTC 1970] + set -- hvc0 -[ 74.988080] stage-1-init: [Thu Jan 1 00:01:14 UTC 1970] + console=hvc0 -[ 75.523700] stage-1-init: [Thu Jan 1 00:01:14 UTC 1970] + IFS='=' -[ 76.152720] stage-1-init: [Thu Jan 1 00:01:15 UTC 1970] + echo init /nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init -[ 76.800900] stage-1-init: [Thu Jan 1 00:01:16 UTC 1970] + set -- init /nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init -[ 77.441300] stage-1-init: [Thu Jan 1 00:01:16 UTC 1970] + stage2Init=/nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init -[ 78.060720] stage-1-init: [Thu Jan 1 00:01:17 UTC 1970] + echo /nix/store/988axh0bq3wqp90gms4b4a0hkfwvkd3i-extra-utils/bin/modprobe -[ 78.613460] stage-1-init: [Thu Jan 1 00:01:17 UTC 1970] + basename dm_mod -[ 79.520040] stage-1-init: [Thu Jan 1 00:01:18 UTC 1970] + info 'loading module dm_mod...' -[ 79.958040] stage-1-init: [Thu Jan 1 00:01:19 UTC 1970] + '[[' -n 1 ]] -[ 80.571720] stage-1-init: [Thu Jan 1 00:01:19 UTC 1970] + echo 'loading module dm_mod...' -[ 80.607660] device-mapper: ioctl: 4.48.0-ioctl (2023-03-01) initialised: dm-devel@lists.linux.dev -+ echo 'Creating a heartbeat counter at 0x1bfff0000' -Creating a heartbeat counter at 0x1bfff0000 -+ echo 0 -+ info 'running udev...' -+ '[[' -n 1 ]] -+ echo 'running udev...' -running udev... -+ ln -sfn /proc/self/fd /dev/fd -+ sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' -[ 81.473120] stage-1-init: [Thu Jan 1 00:01:20 UTC 1970] loading module dm_mod... -+ ln -sfn /proc/self/fd/0 /dev/stdin -[ 82.329900] stage-1-init: [Thu Jan 1 00:01:21 UTC 1970] + modprobe dm_mod -+ ln -sfn /proc/self/fd/1 /dev/stdout -[ 83.264540] stage-1-init: [Thu Jan 1 00:01:22 UTC 1970] + echo 'Creating a heartbeat counter at 0x1bfff0000' -+ ln -sfn /proc/self/fd/2 /dev/stderr -+ mkdir -p /etc/systemd -[ 84.156700] stage-1-init: [Thu Jan 1 00:01:23 UTC 1970] Creating a heartbeat counter at 0x1bfff0000 -+ ln -sfn /nix/store/rwgnr0zbwm045ijjkhq9mmyqymffan17-link-units /etc/systemd/network -[ 84.989640] stage-1-init: [Thu Jan 1 00:01:24 UTC 1970] + echo 0 -+ mkdir -p /etc/udev -[ 85.867560] stage-1-init: [Thu Jan 1 00:01:25 UTC 1970] + info 'running udev...' -+ ln -sfn /nix/store/2lwmlx07crc2zys49kkwrfm3zhkj6zfr-udev-rules /etc/udev/rules.d -[ 86.717780] stage-1-init: [Thu Jan 1 00:01:26 UTC 1970] + '[[' -n 1 ]] -+ mkdir -p /dev/.mdadm -[ 87.589900] stage-1-init: [Thu Jan 1 00:01:26 UTC 1970] + echo 'running udev...' -+ systemd-udevd --daemon -[ 88.423760] stage-1-init: [Thu Jan 1 00:01:27 UTC 1970] running udev... -[ 89.309480] stage-1-init: [Thu Jan 1 00:01:28 UTC 1970] + ln -sfn /proc/self/fd /dev/fd -[ 90.287460] stage-1-init: [Thu Jan 1 00:01:29 UTC 1970] + sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' -Starting systemd-udevd version 255.6 -[ 91.248620] stage-1-init: [Thu Jan 1 00:01:30 UTC 1970] + ln -sfn /proc/self/fd/0 /dev/stdin -+ udevadm trigger '--action=add' -[ 92.431600] stage-1-init: [Thu Jan 1 00:01:31 UTC 1970] + ln -sfn /proc/self/fd/1 /dev/stdout -[ 93.623640] stage-1-init: [Thu Jan 1 00:01:32 UTC 1970] + ln -sfn /proc/self/fd/2 /dev/stderr -[ 94.768040] stage-1-init: [Thu Jan 1 00:01:34 UTC 1970] + mkdir -p /etc/systemd -[ 95.720180] stage-1-init: [Thu Jan 1 00:01:34 UTC 1970] + ln -sfn /nix/store/rwgnr0zbwm045ijjkhq9mmyqymffan17-link-units /etc/systemd/network -[ 96.587820] stage-1-init: [Thu Jan 1 00:01:35 UTC 1970] + mkdir -p /etc/udev -[ 97.519760] stage-1-init: [Thu Jan 1 00:01:36 UTC 1970] + ln -sfn /nix/store/2lwmlx07crc2zys49kkwrfm3zhkj6zfr-udev-rules /etc/udev/rules.d -[ 98.364160] stage-1-init: [Thu Jan 1 00:01:37 UTC 1970] + mkdir -p /dev/.mdadm -[ 99.194820] stage-1-init: [Thu Jan 1 00:01:38 UTC 1970] + systemd-udevd --daemon -[ 100.050720] stage-1-init: [Thu Jan 1 00:01:39 UTC 1970] Starting systemd-udevd version 255.6 -[ 100.907560] stage-1-init: [Thu Jan 1 00:01:40 UTC 1970] + udevadm trigger '--action=add' -+ udevadm settle -[ 204.792900] stage-1-init: [Thu Jan 1 00:03:24 UTC 1970] + udevadm settle -+ kbd_mode -u -C /dev/console -kbd_mode: KDSKBMODE: Inappropriate ioctl for device -+ printf '\033%%G' -+ loadkmap -[ 305.593520] stage-1-init: [Thu Jan 1 00:05:04 UTC 1970] + kbd_mode -u -C /dev/console -[ 306.507560] stage-1-init: [Thu Jan 1 00:05:05 UTC 1970] kbd_mode: KDSKBMODE: Inappropriate ioctl for device -+ info 'starting device mapper and LVM...' -+ '[[' -n 1 ]] -+ echo 'starting device mapper and LVM...' -starting device mapper and LVM... -+ lvm vgchange -ay -[ 307.384260] stage-1-init: [Thu Jan 1 00:05:06 UTC 1970] + printf '\033%%G' -[ 308.487820] stage-1-init: [Thu Jan 1 00:05:07 UTC 1970] + loadkmap -[ 309.589740] random: lvm: uninitialized urandom read (4 bytes read) -[ 309.676340] stage-1-init: [Thu Jan 1 00:05:08 UTC 1970] + info 'starting device mapper and LVM...' -[ 310.899120] stage-1-init: [Thu Jan 1 00:05:10 UTC 1970] + '[[' -n 1 ]] -[ 311.760340] stage-1-init: [Thu Jan 1 00:05:11 UTC 1970] + echo 'starting device mapper and LVM...' -[ 312.620080] stage-1-init: [Thu Jan 1 00:05:11 UTC 1970] starting device mapper and LVM... -[ 313.468280] stage-1-init: [Thu Jan 1 00:05:12 UTC 1970] + lvm vgchange -ay -+ test -n -+ test -e /sys/power/resume -a -e /sys/power/disk -+ '[' -n ] -+ mkdir -p /mnt-root -[ 317.268920] stage-1-init: [Thu Jan 1 00:05:16 UTC 1970] + test -n -+ exec -+ read -u 3 mountPoint -+ read -u 3 device -+ read -u 3 fsType -+ read -u 3 options -+ pseudoDevice= -+ test -z -+ waitDevice /dev/disk/by-label/NIXOS_SD -+ local 'device=/dev/disk/by-label/NIXOS_SD' -+ local IFS -+ '[' ext4 '=' bcachefs ] -+ test '!' -e /dev/disk/by-label/NIXOS_SD -+ udevadm settle -[ 318.217580] stage-1-init: [Thu Jan 1 00:05:17 UTC 1970] + test -e /sys/power/resume -a -e /sys/power/disk -[ 319.069060] stage-1-init: [Thu Jan 1 00:05:18 UTC 1970] + '[' -n ] -[ 319.916780] stage-1-init: [Thu Jan 1 00:05:19 UTC 1970] + mkdir -p /mnt-root -[ 320.769780] stage-1-init: [Thu Jan 1 00:05:20 UTC 1970] + exec -+ '[' -n ] -+ '[' / '=' / ] -+ '[' /dev/disk/by-label/NIXOS_SD '=' tmpfs ] -+ escapeFstab / -+ local 'original=/' -+ local 'escaped=/' -+ echo / -[ 321.672440] stage-1-init: [Thu Jan 1 00:05:20 UTC 1970] + read -u 3 mountPoint -+ escapeFstab x-initrd.mount -+ local 'original=x-initrd.mount' -+ local 'escaped=x-initrd.mount' -+ echo x-initrd.mount -+ mountFS /dev/disk/by-label/NIXOS_SD / x-initrd.mount ext4 -+ local 'device=/dev/disk/by-label/NIXOS_SD' -+ local 'mountPoint=/' -+ local 'options=x-initrd.mount' -+ local 'fsType=ext4' -+ '[' ext4 '=' auto ] -+ IFS=, -+ '[' x- '!=' x- ] -+ local 'optionsFiltered=' -[ 322.609040] stage-1-init: [Thu Jan 1 00:05:21 UTC 1970] + read -u 3 device -+ echo -+ sed -E 's#\<(lowerdir|upperdir|workdir)=#\1=/mnt-root#g' -[ 323.548540] stage-1-init: [Thu Jan 1 00:05:22 UTC 1970] + read -u 3 fsType -+ local 'optionsPrefixed=' -+ echo '/dev/disk/by-label/NIXOS_SD /mnt-root/ ext4 ' -+ checkFS /dev/disk/by-label/NIXOS_SD ext4 -+ local 'device=/dev/disk/by-label/NIXOS_SD' -+ local 'fsType=ext4' -+ '[' '!' -b /dev/disk/by-label/NIXOS_SD ] -+ '[' ext4 '=' iso9660 -o ext4 '=' udf ] -+ '[' ext4 '=' btrfs -o ext4 '=' zfs -o ext4 '=' bcachefs ] -+ '[' ext4 '=' apfs ] -+ '[' ext4 '=' nilfs2 ] -+ '[' ext4 '=' squashfs ] -+ '[' ext4 '=' erofs ] -+ '[' ext4 '=' auto ] -+ mount -+ grep -q '^/dev/disk/by-label/NIXOS_SD on ' -[ 324.688960] stage-1-init: [Thu Jan 1 00:05:23 UTC 1970] + read -u 3 options -+ test -z 1 -a '(' ext4 '=' ext3 -o ext4 '=' ext4 -o ext4 '=' reiserfs -o ext4 '=' xfs -o ext4 '=' jfs -o ext4 '=' f2fs ')' -+ echo 'checking /dev/disk/by-label/NIXOS_SD...' -checking /dev/disk/by-label/NIXOS_SD... -+ fsck -V -a /dev/disk/by-label/NIXOS_SD -[ 325.609200] stage-1-init: [Thu Jan 1 00:05:24 UTC 1970] + pseudoDevice= -fsck (busybox 1.36.1) -[fsck.ext4 (1) -- /mnt-root/] fsck.ext4 -a /dev/disk/by-label/NIXOS_SD -[ 326.476140] stage-1-init: [Thu Jan 1 00:05:25 UTC 1970] + test -z -NIXOS_SD: recovering journal -NIXOS_SD: clean, 52833/122160 files, 393133/491520 blocks -[ 327.380080] stage-1-init: [Thu Jan 1 00:05:26 UTC 1970] + waitDevice /dev/disk/by-label/NIXOS_SD -+ fsckResult=0 -+ test 2 '=' 0 -+ test 4 '=' 0 -+ test 0 -ge 8 -+ return 0 -+ '[' ext4 '=' overlay ] -+ info 'mounting /dev/disk/by-label/NIXOS_SD on /...' -+ '[[' -n 1 ]] -+ echo 'mounting /dev/disk/by-label/NIXOS_SD on /...' -mounting /dev/disk/by-label/NIXOS_SD on /... -+ makeMountPoint /dev/disk/by-label/NIXOS_SD / -+ local 'device=/dev/disk/by-label/NIXOS_SD' -+ local 'mountPoint=/' -+ local 'options=' -+ local 'IFS=,' -+ '[' -d /dev/disk/by-label/NIXOS_SD ] -+ mkdir -m 0755 -p /mnt-root/ -[ 328.351960] stage-1-init: [Thu Jan 1 00:05:27 UTC 1970] + local 'device=/dev/disk/by-label/NIXOS_SD' -+ local 'n=0' -+ true -+ mount /mnt-root/ -[ 329.204940] stage-1-init: [Thu Jan 1 00:05:28 UTC 1970] + local IFS -[ 329.524640] EXT4-fs (pmem0p2): mounted filesystem 44444444-4444-4444-8888-888888888888 r/w with ordered data mode. Quota mode: none. -+ break -+ mount /mnt-root/ -o remount, -[ 330.096240] stage-1-init: [Thu Jan 1 00:05:29 UTC 1970] + '[' ext4 '=' bcachefs ] -[ 330.423880] EXT4-fs (pmem0p2): re-mounted 44444444-4444-4444-8888-888888888888 r/w. Quota mode: none. -+ '[' / '==' / ] -+ '[' -f /mnt-root/etc/NIXOS_LUSTRATE ] -+ true -+ read -u 3 mountPoint -+ exec -+ '[' -e /mnt-root/iso ] -+ udevadm info --export '--export-prefix=ROOT_' '--device-id-of-file=/mnt-root' -[ 331.044220] stage-1-init: [Thu Jan 1 00:05:30 UTC 1970] + test '!' -e /dev/disk/by-label/NIXOS_SD -[ 331.904160] stage-1-init: [Thu Jan 1 00:05:31 UTC 1970] + udevadm settle -[ 332.755700] stage-1-init: [Thu Jan 1 00:05:32 UTC 1970] + '[' -n ] -[ 333.603600] stage-1-init: [Thu Jan 1 00:05:32 UTC 1970] + '[' / '=' / ] -+ eval 'ROOT_MAJOR=259' 'ROOT_MINOR=2' -+ ROOT_MAJOR=259 ROOT_MINOR=2 -+ '[' 259 -a 2 -a 259 '!=' 0 ] -+ mkdir -p /run/udev/rules.d -[ 334.527860] stage-1-init: [Thu Jan 1 00:05:33 UTC 1970] + '[' /dev/disk/by-label/NIXOS_SD '=' tmpfs ] -+ echo 'ACTION=="add|change", SUBSYSTEM=="block", ENV{MAJOR}=="259", ENV{MINOR}=="2", SYMLINK+="root"' -+ udevadm control --exit -[ 335.392500] stage-1-init: [Thu Jan 1 00:05:34 UTC 1970] + escapeFstab / -[ 336.237560] stage-1-init: [Thu Jan 1 00:05:35 UTC 1970] + local 'original=/' -[ 337.080820] stage-1-init: [Thu Jan 1 00:05:36 UTC 1970] + local 'escaped=/' -+ + evalexec - 'exec 8>&- 9>&-' -+ exec -[ 338.100000] stage-1-init: [Thu Jan 1 00:05:37 UTC 1970] + echo / -+ pgrep -v -f ^@ -[ 338.990840] stage-1-init: [Thu Jan 1 00:05:38 UTC 1970] + escapeFstab x-initrd.mount -[ 339.835920] stage-1-init: [Thu Jan 1 00:05:39 UTC 1970] + local 'original=x-initrd.mount' -+ readlink /proc/1/exe -[ 340.702820] stage-1-init: [Thu Jan 1 00:05:39 UTC 1970] + local 'escaped=x-initrd.mount' -+ '[' 1 -eq 1 ] -+ continue -+ readlink /proc/2/exe -[ 341.543740] stage-1-init: [Thu Jan 1 00:05:40 UTC 1970] + echo x-initrd.mount -+ continue -+ readlink /proc/3/exe -[ 342.457600] stage-1-init: [Thu Jan 1 00:05:41 UTC 1970] + mountFS /dev/disk/by-label/NIXOS_SD / x-initrd.mount ext4 -+ continue -+ readlink /proc/4/exe -[ 343.332000] stage-1-init: [Thu Jan 1 00:05:42 UTC 1970] + local 'device=/dev/disk/by-label/NIXOS_SD' -+ continue -+ readlink /proc/5/exe -+ [ 344.178760] stage-1-init: [Thu Jan 1 00:05:43 UTC 1970] + local 'mountPoint=/' -continue -+ readlink /proc/6/exe -+ continue -+ readlink /proc/7/exe -[ 345.050720] stage-1-init: [Thu Jan 1 00:05:44 UTC 1970] + local 'options=x-initrd.mount' -+ continue -+ readlink /proc/8/exe -[ 345.927260] stage-1-init: [Thu Jan 1 00:05:45 UTC 1970] + local 'fsType=ext4' -+ continue -+ readlink /proc/9/exe -[ 346.777360] stage-1-init: [Thu Jan 1 00:05:46 UTC 1970] + '[' ext4 '=' auto ] -+ continue -+ readlink /proc/10/exe -[ 347.588680] stage-1-init: [Thu Jan 1 00:05:46 UTC 1970] + IFS=, -+ continue -+ readlink /proc/11/exe -[ 348.448200] stage-1-init: [Thu Jan 1 00:05:47 UTC 1970] + '[' x- '!=' x- ] -+ continue -+ readlink /proc/12/exe -[ 349.316640] stage-1-init: [Thu Jan 1 00:05:48 UTC 1970] + local 'optionsFiltered=' -+ continue -+ readlink /proc/13/exe -[ 350.152600] stage-1-init: [Thu Jan 1 00:05:49 UTC 1970] + echo -+ continue -+ readlink /proc/14/exe -[ 351.035880] stage-1-init: [Thu Jan 1 00:05:50 UTC 1970] + sed -E 's#\<(lowerdir|upperdir|workdir)=#\1=/mnt-root#g' -+ continue -+ readlink /proc/15/exe -[ 351.897160] stage-1-init: [Thu Jan 1 00:05:51 UTC 1970] + local 'optionsPrefixed=' -+ continue -+ readlink /proc/16/exe -[ 352.813200] stage-1-init: [Thu Jan 1 00:05:52 UTC 1970] + echo '/dev/disk/by-label/NIXOS_SD /mnt-root/ ext4 ' -+ continue -+ readlink /proc/17/exe -+ continue -+ readlink /proc/18/exe[ 353.712780] stage-1-init: [Thu Jan 1 00:05:52 UTC 1970] + checkFS /dev/disk/by-label/NIXOS_SD ext4 - -+ continue -+ readlink /proc/19/exe -[ 354.601220] stage-1-init: [Thu Jan 1 00:05:53 UTC 1970] + local 'device=/dev/disk/by-label/NIXOS_SD' -+ continue -+ readlink /proc/20/exe -[ 355.468760] stage-1-init: [Thu Jan 1 00:05:54 UTC 1970] + local 'fsType=ext4' -+ continue -+ readlink /proc/21/exe -[ 356.367400] stage-1-init: [Thu Jan 1 00:05:55 UTC 1970] + '[' '!' -b /dev/disk/by-label/NIXOS_SD ] -+ continue -+ readlink /proc/22/exe -[ 357.265240] stage-1-init: [Thu Jan 1 00:05:56 UTC 1970] + '[' ext4 '=' iso9660 -o ext4 '=' udf ] -+ continue -+ readlink /proc/23/exe -[ 358.205080] stage-1-init: [Thu Jan 1 00:05:57 UTC 1970] + '[' ext4 '=' btrfs -o ext4 '=' zfs -o ext4 '=' bcachefs ] -+ continue -+ readlink /proc/24/exe -[ 359.079760] stage-1-init: [Thu Jan 1 00:05:58 UTC 1970] + '[' ext4 '=' apfs ] -+ continue -+ readlink /proc/25/exe -[ 359.919800] stage-1-init: [Thu Jan 1 00:05:59 UTC 1970] + '[' ext4 '=' nilfs2 ] -+ continue -+ readlink /proc/26/exe -+ continue -[ 360.787940] stage-1-init: [Thu Jan 1 00:06:00 UTC 1970] + '[' ext4 '=' squashfs ] -+ readlink /proc/27/exe -+ continue -+ readlink /proc/28/exe -[ 361.648260] stage-1-init: [Thu Jan 1 00:06:00 UTC 1970] + '[' ext4 '=' erofs ] -+ continue -+ readlink /proc/29/exe -[ 362.507320] stage-1-init: [Thu Jan 1 00:06:01 UTC 1970] + '[' ext4 '=' auto ] -+ continue -+ readlink /proc/30/exe -[ 363.316580] stage-1-init: [Thu Jan 1 00:06:02 UTC 1970] + mount -+ continue -+ readlink /proc/31/exe -[ 364.199960] stage-1-init: [Thu Jan 1 00:06:03 UTC 1970] + grep -q '^/dev/disk/by-label/NIXOS_SD on ' -+ continue -+ readlink /proc/32/exe -[ 365.232880] stage-1-init: [Thu Jan 1 00:06:04 UTC 1970] + test -z 1 -a '(' ext4 '=' ext3 -o ext4 '=' ext4 -o ext4 '=' reiserfs -o ext4 '=' xfs -o ext4 '=' jfs -o ext4 '=' f2fs ')' -+ continue -+ readlink /proc/33/exe -[ 366.162000] stage-1-init: [Thu Jan 1 00:06:05 UTC 1970] + echo 'checking /dev/disk/by-label/NIXOS_SD...' -+ continue -+ readlink /proc/34/exe -[ 367.042760] stage-1-init: [Thu Jan 1 00:06:06 UTC 1970] checking /dev/disk/by-label/NIXOS_SD... -+ continue -+ readlink /proc/35/exe -+ continue -+ readlink[ 367.939480] stage-1-init: [Thu Jan 1 00:06:07 UTC 1970] + fsck -V -a /dev/disk/by-label/NIXOS_SD - /proc/36/exe -+ continue -+ readlink /proc/37/exe -[ 368.785380] stage-1-init: [Thu Jan 1 00:06:08 UTC 1970] fsck (busybox 1.36.1) -+ continue -+ readlink /proc/38/exe -[ 369.728280] stage-1-init: [Thu Jan 1 00:06:09 UTC 1970] [fsck.ext4 (1) -- /mnt-root/] fsck.ext4 -a /dev/disk/by-label/NIXOS_SD -+ continue -+ readlink /proc/39/exe -[ 370.620960] stage-1-init: [Thu Jan 1 00:06:09 UTC 1970] NIXOS_SD: recovering journal -+ continue -+ readlink /proc/40/exe -[ 371.553240] stage-1-init: [Thu Jan 1 00:06:10 UTC 1970] NIXOS_SD: clean, 52833/122160 files, 393133/491520 blocks -+ continue -+ readlink /proc/47/exe -[ 372.419760] stage-1-init: [Thu Jan 1 00:06:11 UTC 1970] + fsckResult=0 -+ continue -+ readlink /proc/48/exe -[ 373.252460] stage-1-init: [Thu Jan 1 00:06:12 UTC 1970] + test 2 '=' 0 -+ continue -+ readlink /proc/74/exe -[ 374.083140] stage-1-init: [Thu Jan 1 00:06:13 UTC 1970] + test 4 '=' 0 -+ '[' 74 -eq 1 ] -+ kill -9 74 -+ readlink /proc/75/exe -[ 374.961120] stage-1-init: [Thu Jan 1 00:06:14 UTC 1970] + test 0 -ge 8 -+ '[' 75 -eq 1 ] -+ kill -9 75 -+ readlink /proc/102/exe -+ '[' 102 -eq 1 ] -+ kill -9 102 -+ readlink /proc/137/exe -+ continue -+ readlink /proc/674/exe -+ continue -+ readlink /proc/675/exe -+ continue -+ test -n -+ echo /sbin/modprobe -+ '[' '!' -e /mnt-root//nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init ] -+ mkdir -m 0755 -p /mnt-root/proc /mnt-root/sys /mnt-root/dev /mnt-root/run -+ mount --move /proc /mnt-root/proc -+ mount --move /sys /mnt-root/sys -+ mount --move /dev /mnt-root/dev -+ mount --move /run /mnt-root/run -+ type -P switch_root -+ exec env -i /nix/store/988axh0bq3wqp90gms4b4a0hkfwvkd3i-extra-utils/bin/switch_root /mnt-root /nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git/init - -<<< NixOS Stage 2 >>> - -[ 384.203680] EXT4-fs (pmem0p2): re-mounted 44444444-4444-4444-8888-888888888888 r/w. Quota mode: none. -[ 384.287600] booting system configuration /nix/store/xwqaqpc66ijvay9wxnm5nqmi30f2lp1i-nixos-system-nixos-riscv-24.11pre-git -running activation script... -[ 388.163860] stage-2-init: running activation script... -[ 391.643500] random: perl: uninitialized urandom read (4 bytes read) -[ 391.884800] random: perl: uninitialized urandom read (4 bytes read) -[ 425.302000] random: perl: uninitialized urandom read (4 bytes read) - - diff --git a/xavi.dts b/xavi.dts deleted file mode 100644 index b7a7936..0000000 --- a/xavi.dts +++ /dev/null @@ -1,159 +0,0 @@ -/dts-v1/; -/ { - #address-cells = <0x00000002>; - #size-cells = <0x00000002>; - compatible = "riscv,rv64i"; - model = "Barcelona Supercomputing Center - Lagarto Ox"; - chosen { - bootargs = "earlyprintk ignore_loglevel earlycon=sbi console=hvc0 root=/dev/pmem0p1 ro init=/bin/bash"; - }; - cpus { - #address-cells = <0x00000001>; - #size-cells = <0x00000000>; - timebase-frequency = <0x0000c350>; - cpu@0 { - clock-frequency = <0x02faf080>; - device_type = "cpu"; - reg = <0x00000000>; - status = "okay"; - compatible = "riscv"; - riscv,isa = "rv64imafd"; - mmu-type = "riscv,sv39"; - tlb-split; - phandle = <0x00000004>; - interrupt-controller { - #interrupt-cells = <0x00000001>; - interrupt-controller; - compatible = "riscv,cpu-intc"; - phandle = <0x00000005>; - }; - }; - }; - memory@80000000 { - device_type = "memory"; - reg = <0x00000000 0x80000000 0x00000000 0x77e00000>; - }; - reserved-memory { - #address-cells = <0x00000002>; - #size-cells = <0x00000002>; - ranges; - dma_pool@60000000 { - reg = <0x00000000 0x60000000 0x00000000 0x10000000>; - compatible = "shared-dma-pool"; - phandle = <0x00000001>; - }; - dma_pool@70000000 { - reg = <0x00000000 0x70000000 0x00000000 0x10000000>; - compatible = "shared-dma-pool"; - phandle = <0x00000006>; - }; - }; - eth0_clk { - compatible = "fixed-clock"; - #clock-cells = <0x00000000>; - clock-frequency = <0x09502f90>; - phandle = <0x00000002>; - }; - pmem@1bff00000 { - volatile; - compatible = "pmem-region"; - reg = <0x00000001 0xbff00000 0x00000000 0xc0100000>; - }; - pmem2@1b7f00000 { - #address-cells = <0x00000002>; - #size-cells = <0x00000002>; - volatile; - compatible = "pmem-region"; - reg = <0x00000001 0xb7f00000 0x00000000 0x08000000>; - }; - pmem3@f7e00000 { - volatile; - compatible = "pmem-region"; - reg = <0x00000000 0xf7e00000 0x00000000 0xc0100000>; - }; - soc { - #address-cells = <0x00000002>; - #size-cells = <0x00000002>; - compatible = "BSC,Lagarto-bare-soc", "simple-bus"; - ranges; - serial@40001000 { - compatible = "ns16750"; - reg = <0x00000000 0x40001000 0x00000000 0x00000100>; - interrupts = <0x00000000>; - port-number = <0x00000000>; - reg-shift = <0x00000002>; - clock-frequency = <0x017d7840>; - current-speed = <0x0001c200>; - status = "okay"; - phandle = <0x00000007>; - }; - ethernet0 { - xlnx,rxmem = <0x000005f2>; - carv,mtu = <0x000005dc>; - carv,no-mac; - device_type = "network"; - local-mac-address = [02 05 00 01 00 05]; - axistream-connected = <0x000000fe>; - compatible = "xlnx,xxv-ethernet-1.0-carv"; - memory-region = <0x00000001>; - }; - dma@40020000 { - xlnx,include-dre; - phandle = <0x000000fe>; - #dma-cells = <0x00000001>; - compatible = "xlnx,axi-dma-1.00.a"; - clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk", "m_axi_s2mm_aclk", "m_axi_sg_aclk"; - clocks = <0x00000002 0x00000002 0x00000002 0x00000002>; - reg = <0x00000000 0x40200000 0x00000000 0x00400000>; - interrupt-names = "mm2s_introut", "s2mm_introut"; - interrupt-parent = <0x00000003>; - interrupts = <0x00000002 0x00000003>; - xlnx,addrwidth = <0x00000028>; - xlnx,include-sg; - xlnx,sg-length-width = <0x00000017>; - dma-channel@40020000 { - compatible = "xlnx,axi-dma-mm2s-channel"; - dma-channels = <0x00000001>; - interrupts = <0x00000002>; - xlnx,datawidth = <0x00000040>; - xlnx,device-id = <0x00000000>; - xlnx,include-dre; - }; - dma-channel@40020030 { - compatible = "xlnx,axi-dma-s2mm-channel"; - dma-channels = <0x00000001>; - interrupts = <0x00000003>; - xlnx,datawidth = <0x00000040>; - xlnx,device-id = <0x00000000>; - xlnx,include-dre; - }; - }; - clint@40002000 { - reg-names = "control"; - interrupts-extended = <0x00000004 0x00000003 0x00000004 0x00000007 0x00000005 0x00000003 0x00000005 0x00000007 0x00000006 0x00000003 0x00000006 0x00000007 0x00000007 0x00000003 0x00000007 0x00000007>; - compatible = "riscv,clint0"; - #interrupt-cells = <0x00000001>; - reg = <0x00000000 0x40002000 0x00000000 0x000c0000>; - }; - plic@fff1100000 { - #address-cells = <0x00000000>; - compatible = "riscv,plic0"; - #interrupt-cells = <0x00000001>; - reg = <0x000000ff 0xf1100000 0x00000000 0x04000000>; - riscv,ndev = <0x00000003>; - riscv,max-priority = <0x00000007>; - interrupt-controller; - phandle = <0x00000003>; - }; - }; - __symbols__ { - CPU0 = "/cpus/cpu@0"; - CPU0_intc = "/cpus/cpu@0/interrupt-controller"; - eth_pool = "/reserved-memory/dma_pool@60000000"; - meep_pool = "/reserved-memory/dma_pool@70000000"; - eth0_clk = "/eth0_clk"; - uart0 = "/soc/serial@40001000"; - dma_eth = "/soc/dma@40020000"; - PLIC0 = "/soc/plic@fff1100000"; - }; -}; -- GitLab From 42cd8a46de2371e56c2384a2595588a3bb68c72c Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 6 Sep 2024 08:42:02 +0200 Subject: [PATCH 232/310] Ignore CR in picocom Hopefully this fixes the problem in the live log output of the GitLab CI where the lines are not displayed. --- fpga/run-node.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fpga/run-node.sh b/fpga/run-node.sh index 178eedb..30f1d2e 100755 --- a/fpga/run-node.sh +++ b/fpga/run-node.sh @@ -31,4 +31,8 @@ timeout_silent=$((10 * 60)) # Stop if 10 min without output # Set dead switch sleep $timeout && killall picocom & -picocom -q -x $(($timeout_silent*1000)) -b 115200 $FPGACTL_UART +# Note: --imap igncr is broken so we replace it with LF. +# See https://github.com/npat-efault/picocom/pull/114 +# It looks like picocom is abandoned, we may want to switch to minicom or +# stty+cat +picocom --imap crlf -q -x $(($timeout_silent*1000)) -b 115200 $FPGACTL_UART -- GitLab From a397ede43dc7b64717b05bdb8f641874cc030fd5 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 12 Sep 2024 10:30:56 +0200 Subject: [PATCH 233/310] Switch the default devshell to lagarto-ox-rd --- flake.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index ed621bf..6df0a36 100644 --- a/flake.nix +++ b/flake.nix @@ -58,7 +58,7 @@ # A development shell with QEMU ready to boot the RISC-V system in an x86 # machine. - devShells.x86_64-linux.default = + devShells.x86_64-linux.qemu-lagarto-hun = let nixosconf = self.nixosConfigurations.qemu-nc; syspkgs = nixosconf.pkgs; @@ -70,6 +70,7 @@ NIXOS_SYSTEM_TOPLEVEL = toplevel; OPENSBI = syspkgs.opensbi-uboot; }; + devShells.x86_64-linux.lagarto-hun = let nixosconf = self.nixosConfigurations.lagarto-hun; @@ -127,5 +128,8 @@ TOPLEVEL = ""; ROOTFS = ""; }); + + devShells.x86_64-linux.default = + self.outputs.devShells.x86_64-linux.lagarto-ox-rd; }; } -- GitLab From 9daa796b10a68fa16e979b4132691ee38cda5ac3 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 18 Sep 2024 11:41:20 +0200 Subject: [PATCH 234/310] Try to save a gcroot will build dependencies --- flake.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/flake.nix b/flake.nix index 6df0a36..d4236b6 100644 --- a/flake.nix +++ b/flake.nix @@ -100,6 +100,16 @@ nixosconf = self.nixosConfigurations.lagarto-ox; syspkgs = nixosconf.pkgs; build = nixosconf.config.system.build; + + buildRoots = savePkgs: + let + pkgs = syspkgs; + buildInner = { package, attr }: pkgs.linkFarmFromDrvs "${package.name}.${attr}" package.${attr}; + in pkgs.linkFarmFromDrvs "top-level" (pkgs.lib.mapCartesianProduct buildInner { + package = savePkgs; + attr = [ "buildInputs" "nativeBuildInputs" ]; + }); + in syspkgs.mkShell { pname = "lagarto-ox-shell"; TOPLEVEL = build.toplevel; @@ -110,6 +120,7 @@ UBOOT_ENV = syspkgs.uboot-env; BITSTREAM = syspkgs.bitstream; BOOTROM = syspkgs.bootrom; + GCROOT = buildRoots [ build.toplevel build.kernel ]; shellHook = '' echo "Here are the current system pieces:" echo " TOPLEVEL = $TOPLEVEL" @@ -120,6 +131,7 @@ echo " UBOOT_ENV = $UBOOT_ENV" echo " BITSTREAM = $BITSTREAM" echo " BOOTROM = $BOOTROM" + echo " GCROOT = $GCROOT" ''; }; -- GitLab From 877428c2fe102f5771c76cb13a97989be5174d57 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 18 Sep 2024 14:07:49 +0200 Subject: [PATCH 235/310] Only save kernel and opensbi build inputs --- flake.nix | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/flake.nix b/flake.nix index d4236b6..af50ba0 100644 --- a/flake.nix +++ b/flake.nix @@ -8,6 +8,16 @@ let system = "x86_64-linux"; nixosSystem = import (nixpkgs + "/nixos/lib/eval-config.nix"); + buildRoots = { pkgs, save }: + let + buildInner = { package, attr }: + if (attr == "") + then pkgs.linkFarmFromDrvs "${package.name}" [ package ] + else pkgs.linkFarmFromDrvs "${package.name}.${attr}" package.${attr}; + in pkgs.linkFarmFromDrvs "top-level" (pkgs.lib.mapCartesianProduct buildInner { + package = save; + attr = [ "" "buildInputs" "nativeBuildInputs" ]; + }); in { #overlay = import ./overlay.nix; nixosConfigurations = { @@ -100,17 +110,7 @@ nixosconf = self.nixosConfigurations.lagarto-ox; syspkgs = nixosconf.pkgs; build = nixosconf.config.system.build; - - buildRoots = savePkgs: - let - pkgs = syspkgs; - buildInner = { package, attr }: pkgs.linkFarmFromDrvs "${package.name}.${attr}" package.${attr}; - in pkgs.linkFarmFromDrvs "top-level" (pkgs.lib.mapCartesianProduct buildInner { - package = savePkgs; - attr = [ "buildInputs" "nativeBuildInputs" ]; - }); - - in syspkgs.mkShell { + in syspkgs.mkShell rec { pname = "lagarto-ox-shell"; TOPLEVEL = build.toplevel; OPENSBI = syspkgs.opensbi; @@ -120,7 +120,7 @@ UBOOT_ENV = syspkgs.uboot-env; BITSTREAM = syspkgs.bitstream; BOOTROM = syspkgs.bootrom; - GCROOT = buildRoots [ build.toplevel build.kernel ]; + GCROOT = buildRoots { pkgs = syspkgs; save = [ KERNEL OPENSBI ]; }; shellHook = '' echo "Here are the current system pieces:" echo " TOPLEVEL = $TOPLEVEL" @@ -136,9 +136,13 @@ }; devShells.x86_64-linux.lagarto-ox-rd = - self.outputs.devShells.x86_64-linux.lagarto-ox.overrideAttrs (old:{ + let + nixosconf = self.nixosConfigurations.lagarto-ox; + syspkgs = nixosconf.pkgs; + in self.outputs.devShells.x86_64-linux.lagarto-ox.overrideAttrs (old:{ TOPLEVEL = ""; ROOTFS = ""; + GCROOT = buildRoots { pkgs = syspkgs; save = [ old.OPENSBI ]; }; }); devShells.x86_64-linux.default = -- GitLab From 4562173d41854eaad63489b0f23b15f1d81d709b Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 18 Sep 2024 14:30:18 +0200 Subject: [PATCH 236/310] Remove commit from U-Boot for now It causes the rebuild of U-Boot and OpenSBI, as we are now bundling the environment inside the U-Boot image. --- lagarto-ox.nix | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index a1c4033..8e1dfd6 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -462,9 +462,6 @@ uboot-env = let init = "${config.system.build.toplevel}/init"; initrd = "${config.system.build.initialRamdisk}/initrd"; - rev = if self ? rev then self.rev - else throw ("Refusing to build from a dirty Git tree!"); - # Create pmem of 3 GiB [0x140000000, 0x200000000) #fdt mknode / pmem@0x140000000 #fdt set /pmem@0x140000000 compatible "pmem-region" @@ -490,7 +487,6 @@ # in prev.runCommand "uboot.txt" {} '' cat > $out < Date: Wed, 18 Sep 2024 14:35:00 +0200 Subject: [PATCH 237/310] Save stdenv in gcroot --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index af50ba0..77ba7a2 100644 --- a/flake.nix +++ b/flake.nix @@ -120,7 +120,7 @@ UBOOT_ENV = syspkgs.uboot-env; BITSTREAM = syspkgs.bitstream; BOOTROM = syspkgs.bootrom; - GCROOT = buildRoots { pkgs = syspkgs; save = [ KERNEL OPENSBI ]; }; + GCROOT = buildRoots { pkgs = syspkgs; save = [ syspkgs.stdenv KERNEL OPENSBI ]; }; shellHook = '' echo "Here are the current system pieces:" echo " TOPLEVEL = $TOPLEVEL" @@ -142,7 +142,7 @@ in self.outputs.devShells.x86_64-linux.lagarto-ox.overrideAttrs (old:{ TOPLEVEL = ""; ROOTFS = ""; - GCROOT = buildRoots { pkgs = syspkgs; save = [ old.OPENSBI ]; }; + GCROOT = buildRoots { pkgs = syspkgs; save = [ syspkgs.stdenv old.OPENSBI ]; }; }); devShells.x86_64-linux.default = -- GitLab From 11ed3dc731280e716d4174289589cf78734b6bde Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 18 Sep 2024 14:48:17 +0200 Subject: [PATCH 238/310] Save GC roots using all drv attributes --- flake.nix | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/flake.nix b/flake.nix index 77ba7a2..9e50405 100644 --- a/flake.nix +++ b/flake.nix @@ -8,16 +8,8 @@ let system = "x86_64-linux"; nixosSystem = import (nixpkgs + "/nixos/lib/eval-config.nix"); - buildRoots = { pkgs, save }: - let - buildInner = { package, attr }: - if (attr == "") - then pkgs.linkFarmFromDrvs "${package.name}" [ package ] - else pkgs.linkFarmFromDrvs "${package.name}.${attr}" package.${attr}; - in pkgs.linkFarmFromDrvs "top-level" (pkgs.lib.mapCartesianProduct buildInner { - package = save; - attr = [ "" "buildInputs" "nativeBuildInputs" ]; - }); + mkRoots = pkgs: list: pkgs.writeText "gcroots.json" + (builtins.toJSON (map (x: { drv = x; attrs = x.drvAttrs; }) list)); in { #overlay = import ./overlay.nix; nixosConfigurations = { @@ -120,7 +112,7 @@ UBOOT_ENV = syspkgs.uboot-env; BITSTREAM = syspkgs.bitstream; BOOTROM = syspkgs.bootrom; - GCROOT = buildRoots { pkgs = syspkgs; save = [ syspkgs.stdenv KERNEL OPENSBI ]; }; + GCROOT = mkRoots syspkgs [ syspkgs.stdenv KERNEL OPENSBI ]; shellHook = '' echo "Here are the current system pieces:" echo " TOPLEVEL = $TOPLEVEL" @@ -142,7 +134,7 @@ in self.outputs.devShells.x86_64-linux.lagarto-ox.overrideAttrs (old:{ TOPLEVEL = ""; ROOTFS = ""; - GCROOT = buildRoots { pkgs = syspkgs; save = [ syspkgs.stdenv old.OPENSBI ]; }; + GCROOT = mkRoots syspkgs [ syspkgs.stdenv old.OPENSBI ]; }); devShells.x86_64-linux.default = -- GitLab From 88a4e239a144f715fcd1da660b0c622ce0f7c5ce Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 18 Sep 2024 15:12:14 +0200 Subject: [PATCH 239/310] Save commit in shell variable --- flake.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flake.nix b/flake.nix index 9e50405..3219d44 100644 --- a/flake.nix +++ b/flake.nix @@ -80,6 +80,7 @@ build = nixosconf.config.system.build; in syspkgs.mkShell { pname = "lagarto-hun-shell"; + COMMIT = if self ? rev then self.rev else "dirty"; TOPLEVEL = build.toplevel; OPENSBI = syspkgs.opensbi; KERNEL = build.kernel; @@ -88,6 +89,7 @@ UBOOT_ENV = syspkgs.uboot-env; shellHook = '' echo "Here are the current system pieces:" + echo " COMMIT = $COMMIT" echo " TOPLEVEL = $TOPLEVEL" echo " KERNEL = $KERNEL" echo " OPENSBI = $OPENSBI" @@ -104,6 +106,7 @@ build = nixosconf.config.system.build; in syspkgs.mkShell rec { pname = "lagarto-ox-shell"; + COMMIT = if self ? rev then self.rev else "dirty"; TOPLEVEL = build.toplevel; OPENSBI = syspkgs.opensbi; KERNEL = build.kernel; @@ -115,6 +118,7 @@ GCROOT = mkRoots syspkgs [ syspkgs.stdenv KERNEL OPENSBI ]; shellHook = '' echo "Here are the current system pieces:" + echo " COMMIT = $COMMIT" echo " TOPLEVEL = $TOPLEVEL" echo " KERNEL = $KERNEL" echo " OPENSBI = $OPENSBI" -- GitLab From b28e317e56d6bcb56178ecd67a5dd4878dd6d077 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 18 Sep 2024 15:13:46 +0200 Subject: [PATCH 240/310] Build the whole system in CI --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 33d9ada..b5e4351 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,4 +3,4 @@ build:lagarto-ox-rd: tags: - nix script: - - nix develop -L .#lagarto-ox-rd --command fpga/run-remotely.sh fpgalogin1:ci + - nix develop -L .#lagarto-ox --command fpga/run-remotely.sh fpgalogin1:ci -- GitLab From 8846d95281666453eced3d7bace248b0402f8a0e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 18 Sep 2024 15:14:54 +0200 Subject: [PATCH 241/310] Load rootfs with fpgactl --- fpga/run-node.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fpga/run-node.sh b/fpga/run-node.sh index 30f1d2e..670f5a9 100755 --- a/fpga/run-node.sh +++ b/fpga/run-node.sh @@ -18,8 +18,8 @@ killall picocom || true set -x # Then perform the boot -#./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img -./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin +./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img +#./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin # Restart it again only loading the bootrom and OpenSBI (notice OpenSBI # relocates itself so it will likely be gone by now) -- GitLab From b6f15f5b6c6c3dca3acf8ebf7e29bbacd66daba1 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 18 Sep 2024 15:16:49 +0200 Subject: [PATCH 242/310] Don't stop the boot after PLIC tests --- lagarto-ox.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 8e1dfd6..746d485 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -237,8 +237,8 @@ fi ) set +x - echo "all done, dropping to a shell..." - ash + #echo "all done, dropping to a shell..." + #ash ''; "/preinit".source = pkgs.writeScript "preinit" '' -- GitLab From f8cce17fc2a161c76167ab2e390e1649d4f07bb9 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 18 Sep 2024 15:48:02 +0200 Subject: [PATCH 243/310] Add riscv-tools to gcroot --- flake.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 3219d44..4a8a533 100644 --- a/flake.nix +++ b/flake.nix @@ -115,7 +115,9 @@ UBOOT_ENV = syspkgs.uboot-env; BITSTREAM = syspkgs.bitstream; BOOTROM = syspkgs.bootrom; - GCROOT = mkRoots syspkgs [ syspkgs.stdenv KERNEL OPENSBI ]; + GCROOT = mkRoots syspkgs [ + syspkgs.stdenv KERNEL OPENSBI syspkgs.riscv-tools + ]; shellHook = '' echo "Here are the current system pieces:" echo " COMMIT = $COMMIT" -- GitLab From 507f70cf55a26e8e7b5b0969e74020fac693d393 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 18 Sep 2024 16:34:17 +0200 Subject: [PATCH 244/310] Print a message at login --- configuration.nix | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/configuration.nix b/configuration.nix index 0110598..9d5c20b 100644 --- a/configuration.nix +++ b/configuration.nix @@ -37,4 +37,23 @@ #environment.systemPackages = with pkgs; [ vim gdb neofetch gcc bintools ]; #environment.systemPackages = with pkgs; [ rvb riscv-tools ]; + + services.getty.helpLine = '' + + + __________________ + < Welcome to NixOS > + ------------------ + \ ^__^ + \ (oo)\_______ + (__)\ )\/\ + ||----w | + || || + + + If you can read this message then then you have + successfully booted NixOS into the login shell. + + + ''; } -- GitLab From a8859380a6ca7dee1973d49bbe6f07abd754c352 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 18 Sep 2024 18:13:39 +0200 Subject: [PATCH 245/310] Use minimal profile Disables a bunch of options, including boot.enableContainers which removes some unneeded modules from loading. --- configuration.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration.nix b/configuration.nix index 9d5c20b..372c3dd 100644 --- a/configuration.nix +++ b/configuration.nix @@ -3,7 +3,7 @@ { imports = [ #"${modulesPath}/profiles/base.nix" - #"${modulesPath}/profiles/minimal.nix" + "${modulesPath}/profiles/minimal.nix" ]; nixpkgs = { -- GitLab From 5e2430c48b5a89ea20880df30ed4c504f0e272af Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 19 Sep 2024 10:06:48 +0200 Subject: [PATCH 246/310] Disable all firmware from the image --- lagarto-ox.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 746d485..853b52f 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -11,6 +11,9 @@ # gcc.tune = "generic"; #}; + # We don't need any firmware + hardware.firmware = lib.mkForce []; + #services.haveged.enable = true; services.jitterentropy-rngd.enable = true; -- GitLab From 2fab6b8ec04ee0a171b74e867ef955ae4a58b457 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 19 Sep 2024 11:01:48 +0200 Subject: [PATCH 247/310] Update bitstream to ox_u55c_87a14c32 This bitstream attempts to fix the PLIC issues with the threshold register, so we should see the PLIC tests success now. See: https://gitlab.bsc.es/hwdesign/fpga/integration-lab/fpga-shell/-/issues/147 --- lagarto-ox.nix | 2 +- overlay.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 853b52f..94710ee 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -387,7 +387,7 @@ }; #bitstream = "${final.bitstreams}/lagarto-3-ox/gold.bit"; - bitstream = "${final.bitstreams}/lagarto-3-ox/ox_u55c_450d0ff0_fix_delegation_v2.bit"; + bitstream = "${final.bitstreams}/lagarto-3-ox/ox_u55c_87a14c32_fix_threshold.bit"; bootrom = "${final.rbootrom}/rbootrom.bin"; diff --git a/overlay.nix b/overlay.nix index c88a431..9c9ffe7 100644 --- a/overlay.nix +++ b/overlay.nix @@ -29,7 +29,7 @@ final: prev: bitstreams = builtins.fetchGit { url = "git@bscpm03.bsc.es:rarias/bitstreams.git"; - rev = "097dad3e888f630ad6ec7e7056ee1b3fcb03400b"; + rev = "2f899627a226890c6f9820aa44e34c2ecea03faf"; }; # Baremetal tests for standalone FPGA -- GitLab From fcb6313187f1f0a36be9f78697e7628bca1dfd89 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 19 Sep 2024 11:25:39 +0200 Subject: [PATCH 248/310] Add rvb and riscv-tools to rootfs --- configuration.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration.nix b/configuration.nix index 372c3dd..f73c328 100644 --- a/configuration.nix +++ b/configuration.nix @@ -36,7 +36,7 @@ }; #environment.systemPackages = with pkgs; [ vim gdb neofetch gcc bintools ]; - #environment.systemPackages = with pkgs; [ rvb riscv-tools ]; + environment.systemPackages = with pkgs; [ rvb riscv-tools ]; services.getty.helpLine = '' -- GitLab From 6d4db58e7d4b4187957d057c9a44a21b46dc7a30 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 19 Sep 2024 12:42:52 +0200 Subject: [PATCH 249/310] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'bscpkgs': 'git+https://git.sr.ht/~rodarima/bscpkgs?ref=riscv-benchmarks&rev=f5515a80bfed28cd4e2e737858ba4124a7e53333' (2024-05-30) → 'git+https://git.sr.ht/~rodarima/bscpkgs?ref=riscv-benchmarks&rev=9b197d12f1edf52efcb6e3dad421f2a8fd5af2a7' (2024-09-19) --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index cf746de..de983a2 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1717064246, - "narHash": "sha256-zgwnjGVBXTvItXIYtaMIdf2acHiNQOJ6Fba4OFZRYT4=", + "lastModified": 1726742476, + "narHash": "sha256-M0ubiqx3al1TX7Eyq6eSIBax22XXCqGlds/9naIlBFw=", "ref": "riscv-benchmarks", - "rev": "f5515a80bfed28cd4e2e737858ba4124a7e53333", - "revCount": 939, + "rev": "9b197d12f1edf52efcb6e3dad421f2a8fd5af2a7", + "revCount": 941, "type": "git", "url": "https://git.sr.ht/~rodarima/bscpkgs" }, -- GitLab From 46f699f1c256e1fa57dabc2b940bc75100216ebf Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 19 Sep 2024 14:18:30 +0200 Subject: [PATCH 250/310] Add perf command to image --- configuration.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/configuration.nix b/configuration.nix index f73c328..220477e 100644 --- a/configuration.nix +++ b/configuration.nix @@ -36,7 +36,10 @@ }; #environment.systemPackages = with pkgs; [ vim gdb neofetch gcc bintools ]; - environment.systemPackages = with pkgs; [ rvb riscv-tools ]; + environment.systemPackages = with pkgs; [ + rvb riscv-tools + config.boot.kernelPackages.perf + ]; services.getty.helpLine = '' -- GitLab From 8047a6a4ebfa19d2a7fab0be51a75761b4b5d4ac Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 25 Sep 2024 10:18:14 +0200 Subject: [PATCH 251/310] Add llvm-epi clang compiler --- overlay.nix | 4 + pkgs/llvm-epi/clang.nix | 124 ++++++++++++++++++++++++++++ pkgs/llvm-epi/default.nix | 46 +++++++++++ pkgs/llvm-epi/include-cstdint.patch | 10 +++ 4 files changed, 184 insertions(+) create mode 100644 pkgs/llvm-epi/clang.nix create mode 100644 pkgs/llvm-epi/default.nix create mode 100644 pkgs/llvm-epi/include-cstdint.patch diff --git a/overlay.nix b/overlay.nix index 9c9ffe7..b7e9cc6 100644 --- a/overlay.nix +++ b/overlay.nix @@ -3,6 +3,10 @@ final: prev: # Changes to packages from nixpkgs { + clangEpi = final.callPackage ./pkgs/llvm-epi/default.nix { openmp = null; }; + clangEpiUnwrapped = final.callPackage ./pkgs/llvm-epi/clang.nix { }; + stdenvClangEpi = final.stdenv.override { cc = final.buildPackages.clangEpi; allowedRequisites = null; }; + blis = ((prev.blis.override { blas64 = true; withArchitecture = "generic"; diff --git a/pkgs/llvm-epi/clang.nix b/pkgs/llvm-epi/clang.nix new file mode 100644 index 0000000..e3e85f1 --- /dev/null +++ b/pkgs/llvm-epi/clang.nix @@ -0,0 +1,124 @@ +{ + stdenv +, llvmPackages_latest +, lib +, fetchFromGitHub +, cmake +, bash +, python3 +, perl +, which +, elfutils +, libffi +, zlib +, pkg-config +, enableDebug ? false +, gitUrl ? "https://repo.hca.bsc.es/gitlab/rferrer/llvm-epi.git" +, gitBranch ? "EPI-0.7" +, gitCommit ? "479518dc58dfceb23fc90667a5d6253e429f0fc2" +}: + +let + llvmPackages = llvmPackages_latest; + llvmStdenv = llvmPackages.stdenv; + # needed to set the rpath of libstdc++ for clang-tblgen + gcc = stdenv.cc; + + git = rec { + version = src.shortRev; + src = builtins.fetchGit { + url = gitUrl; + ref = gitBranch; + rev = gitCommit; + }; + }; + + source = git; + +in llvmStdenv.mkDerivation rec { + pname = "clang-epi"; + inherit (source) src version; + + enableParallelBuilding = true; + isClang = true; + + patches = if (gitBranch == "EPI-0.7") then [ + ./include-cstdint.patch + ] else [ + ]; + + # See https://reviews.llvm.org/D135402 + env.LDFLAGS = "-Wl,--undefined-version"; + + passthru = { + CC = "clang"; + CXX = "clang++"; + }; + + nativeBuildInputs = [ + zlib + gcc.cc.lib # Required for libstdc++.so.6 + ]; + + buildInputs = [ + which + bash + python3 + perl + cmake + llvmPackages.lld + elfutils + libffi + pkg-config + zlib + ]; + + # Error with -D_FORTIFY_SOURCE=2, see https://bugs.gentoo.org/636604: + # /build/source/compiler-rt/lib/tsan/dd/dd_interceptors.cpp:225:20: + # error: redefinition of 'realpath' + # Requires disabling the "fortify" set of flags, however, for performance we + # disable all: + hardeningDisable = [ "all" ]; + + cmakeBuildType = if enableDebug then "Debug" else "Release"; + + dontStrip = enableDebug; + + dontUseCmakeBuildDir = true; + + # Fix shebangs, /usr/bin/env doesn't exist + prePatch = '' + patchShebangs clang/utils/EPI/generate-epi-builtins-def.py + ''; + + # Fix the host triple, as it has changed in a newer config.guess: + # https://git.savannah.gnu.org/gitweb/?p=config.git;a=commitdiff;h=ca9bfb8cc75a2be1819d89c664a867785c96c9ba + preConfigure = '' + mkdir -p build + cd build + cmakeDir="../llvm" + cmakeFlagsArray=( + "-DLLVM_HOST_TRIPLE=${llvmStdenv.targetPlatform.config}" + "-DLLVM_DEFAULT_TARGET_TRIPLE=riscv64-unknown-linux-gnu" + "-DLLVM_TARGETS_TO_BUILD=RISCV" + "-DLLVM_BUILD_LLVM_DYLIB=ON" + "-DLLVM_LINK_LLVM_DYLIB=ON" + # Required to run clang-ast-dump and clang-tblgen during build + "-DCMAKE_BUILD_RPATH=$PWD/lib:${zlib}/lib:${gcc.cc.lib}/lib" + "-DLLVM_ENABLE_LLD=ON" + "-DCMAKE_CXX_FLAGS_DEBUG=-g -ggnu-pubnames" + "-DCMAKE_EXE_LINKER_FLAGS_DEBUG=-Wl,--gdb-index" + "-DLLVM_LIT_ARGS=-sv --xunit-xml-output=xunit.xml" + "-DLLVM_ENABLE_PROJECTS=clang;lld" + "-DLLVM_ENABLE_ASSERTIONS=ON" + "-DLLVM_INSTALL_TOOLCHAIN_ONLY=ON" + "-DCMAKE_INSTALL_BINDIR=bin" + "-DLLVM_ENABLE_ZLIB=FORCE_ON" + "-DLLVM_ENABLE_LIBXML2=OFF" + # Set the rpath to include external libraries (zlib) both on build and + # install + "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON" + "-DCMAKE_INSTALL_RPATH=${zlib}/lib:${gcc.cc.lib}/lib" + ) + ''; +} diff --git a/pkgs/llvm-epi/default.nix b/pkgs/llvm-epi/default.nix new file mode 100644 index 0000000..9fe7c8e --- /dev/null +++ b/pkgs/llvm-epi/default.nix @@ -0,0 +1,46 @@ +{ + stdenv +, lib +, gcc +, clangEpiUnwrapped +, openmp ? null +, wrapCCWith +, llvmPackages_latest +, ompss2rt ? null +}: + +let + # We need to replace the lld linker from bintools with our linker just built, + # otherwise we run into incompatibility issues when mixing compiler and linker + # versions. + bintools-unwrapped = llvmPackages_latest.tools.bintools-unwrapped.override { + lld = clangEpiUnwrapped; + }; + bintools = llvmPackages_latest.tools.bintools.override { + bintools = bintools-unwrapped; + }; + targetConfig = stdenv.targetPlatform.config; + inherit gcc; + cc = clangEpiUnwrapped; +in wrapCCWith { + inherit cc bintools; + # extraPackages adds packages to depsTargetTargetPropagated + extraPackages = lib.optional (openmp != null) openmp; + extraBuildCommands = '' + echo "-target ${targetConfig}" >> $out/nix-support/cc-cflags + echo "-B${gcc.cc}/lib/gcc/${targetConfig}/${gcc.version}" >> $out/nix-support/cc-cflags + echo "-L${gcc.cc}/lib/gcc/${targetConfig}/${gcc.version}" >> $out/nix-support/cc-ldflags + echo "-L${gcc.cc.lib}/lib" >> $out/nix-support/cc-ldflags + + for dir in ${gcc.cc}/include/c++/*; do + echo "-isystem $dir" >> $out/nix-support/libcxx-cxxflags + done + for dir in ${gcc.cc}/include/c++/*/${targetConfig}; do + echo "-isystem $dir" >> $out/nix-support/libcxx-cxxflags + done + + echo "--gcc-toolchain=${gcc}" >> $out/nix-support/cc-cflags + + wrap clang++ $wrapper $ccPath/clang++ + ''; +} diff --git a/pkgs/llvm-epi/include-cstdint.patch b/pkgs/llvm-epi/include-cstdint.patch new file mode 100644 index 0000000..aed75d7 --- /dev/null +++ b/pkgs/llvm-epi/include-cstdint.patch @@ -0,0 +1,10 @@ +--- a/llvm/include/llvm/Support/Signals.h 2024-09-25 08:34:21.257642944 +0200 ++++ b/llvm/include/llvm/Support/Signals.h 2024-09-25 08:35:12.593556793 +0200 +@@ -15,6 +15,7 @@ + #define LLVM_SUPPORT_SIGNALS_H + + #include ++#include + + namespace llvm { + class StringRef; -- GitLab From f8eb8b8d5270502b22bf5e1fa0aa1ed1cf0b3d2c Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 25 Sep 2024 10:20:55 +0200 Subject: [PATCH 252/310] Add rvb package --- overlay.nix | 3 ++ pkgs/rvb/Makefile | 79 +++++++++++++++++++++++++++++++++++++++++ pkgs/rvb/Makefile.in | 83 ++++++++++++++++++++++++++++++++++++++++++++ pkgs/rvb/default.nix | 35 +++++++++++++++++++ 4 files changed, 200 insertions(+) create mode 100644 pkgs/rvb/Makefile create mode 100644 pkgs/rvb/Makefile.in create mode 100644 pkgs/rvb/default.nix diff --git a/overlay.nix b/overlay.nix index b7e9cc6..1c8c788 100644 --- a/overlay.nix +++ b/overlay.nix @@ -7,6 +7,9 @@ final: prev: clangEpiUnwrapped = final.callPackage ./pkgs/llvm-epi/clang.nix { }; stdenvClangEpi = final.stdenv.override { cc = final.buildPackages.clangEpi; allowedRequisites = null; }; + rvb = final.callPackage ./pkgs/rvb/default.nix { }; + rvb-clang = final.callPackage ./pkgs/rvb/default.nix { stdenv = final.stdenvClangEpi; }; + blis = ((prev.blis.override { blas64 = true; withArchitecture = "generic"; diff --git a/pkgs/rvb/Makefile b/pkgs/rvb/Makefile new file mode 100644 index 0000000..39a3524 --- /dev/null +++ b/pkgs/rvb/Makefile @@ -0,0 +1,79 @@ +include Makefile.in + +HPC_BENCHMARKS_DIRS=\ + axpy \ + jacobi-2d \ + somier \ +# Require submodules: +# fft \ +# fftp \ +# Require vehave: +# lulesh \ +# Missing compare_array_double: +# gemm \ +# Broken, not found: +# spmv + +DESKTOP_BENCHMARKS_DIRS=\ + blackscholes \ + canneal \ + particlefilter \ + streamcluster \ + swaptions + #pathfinder + +MICRO_BENCHMARKS_DIRS= + #BuffCopyUnit \ + #BuffCopyStrided \ + #BuffCopyIndexed \ + #FpuMicroKernel \ + #InstrNopBalance \ + #MemArithBalance \ + #LatencyVrgather + +.PHONY: default all clean $(HPC_BENCHMARKS_DIRS) $(MICRO_BENCHMARKS_DIRS) + +all: base + +default: + @cd common; make; cd .. + @$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir);) + @$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir);) + @$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir);) + +all-types: + @cd common; make all; cd .. + @$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) all ;) + @$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) all ;) + @$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) all ;) + +base: + @cd common; make all; cd .. + @$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) base ;) + @$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) base ;) + @$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) base ;) + +install: + @$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) install ;) + @$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) install ;) + @$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) install ;) + +fftp: + ${MAKE} -C third_party fftw + ${MAKE} -C hpc_benchmarks/fftp all + ${MAKE} -C hpc_benchmarks/fftp/test all + +spmv-ellpack: + rm -rf hpc_benchmarks/spmv-ellpack/spmv/build + mkdir -p hpc_benchmarks/spmv-ellpack/spmv/build + cd hpc_benchmarks/spmv-ellpack/spmv/build;\ + ../configure riscv;\ + INDEX64=1 EPI_EXT=07 PATH=${EPI_LLVM_HOME}/bin:${PATH} make + +clean: + @cd common; make clean; cd .. + @$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) clean ;) + @$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) clean ;) + @$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) clean ;) + @rm -rf hpc_benchmarks/spmv-ellpack/spmv/build + diff --git a/pkgs/rvb/Makefile.in b/pkgs/rvb/Makefile.in new file mode 100644 index 0000000..8c8b4dc --- /dev/null +++ b/pkgs/rvb/Makefile.in @@ -0,0 +1,83 @@ +#Compile all benchmarks with individual settings defined in their Makefiles + +_default-target: default + +# RVB_ROOT defined as argument + +# TODO: RVB_COMMON_DIR should substitute COMMON_DIR +COMMON_DIR=$(RVB_ROOT)/common +RVB_COMMON_DIR=$(RVB_ROOT)/common + +#CC=clang +#CXX=clang++ + +# Needs EPI clang +#MEPI=-mepi +MEPI= + +#VREPORT_FLAGS=-Rpass=loop-vectorize -Rpass-missed=loop-vectorize -Rpass-analysis=loop-vectorize +#Available CFLAGS conditional compilation: +# -DUSE_MALLOC_HP, redefines the use of malloc and free + +#Makefile: General compiler flags: CFLAGS, CFLAGS_, CFLAGS_, ... +#CFLAGS=-B ${LLVM_BIN} -Wall -Wextra -march=rv64g -O2 -I${RVB_COMMON_DIR} +CFLAGS=-B ${LLVM_BIN} -Wall -Wextra -O2 -I${RVB_COMMON_DIR} +LDFLAGS= + +#Only scalar instructions +CFLAGS_SCALAR=${CFLAGS} -DRVB_USE_SCALAR +LDFLAGS_SCALAR= + +#NOVEC=-fno-vectorize +NOVEC= + +#Vector instructions using intrinsics +CFLAGS_VECTORIAL=${CFLAGS} ${NOVEC} $(MEPI) +LDFLAGS_VECTORIAL= + +# TODO: RVV should substitute VECTORIAL (?) +#Vector instructions using intrinsics +CFLAGS_RVV=${CFLAGS} -DRVB_USE_RVV ${NOVEC} $(MEPI) +LDFLAGS_RVV=${LDFLAGS} + +#Vector instructions only when code is annotate +CFLAGS_EXPLICIT_AUTOVECTORIZATION=${CFLAGS} -fopenmp-simd ${NOVEC} $(MEPI) ${VREPORT_FLAGS} +LDFLAGS_EXPLICIT_AUTOVECTORIZATION= + +# TODO: OMP substitutes EXPLICIT_AUTOVECTORIZATION +#Vector instructions only when code is annotate +CFLAGS_OMP=${CFLAGS} -DRVB_USE_OMP -fopenmp-simd $(MEPI) ${VREPORT_FLAGS} +LDFLAGS_OMP=${LDFLAGS} + +#Vector instructions when compiler decides +CFLAGS_AUTOVECTORIZATION=${CFLAGS} -fopenmp-simd $(MEPI) ${VREPORT_FLAGS} +LDFLAGS_AUTOVECTORIZATION= + +# TODO: AUTOVECT shoud substitue AUTOVECTORIZATION (?) +#Vector instructions when compiler decides +CFLAGS_AUTOVECT=${CFLAGS} -DRVB_USE_AUTOVECT -fopenmp-simd $(MEPI) ${VREPORT_FLAGS} +LDFLAGS_AUTOVECT=${LDFLAGS} + +# CBLAS library compile and link flags +CFLAGS_CBLAS=${CFLAGS} -DRVB_USE_CBLAS -I${CBLAS_INC} -I${LLVM_INC} +LDFLAGS_CBLAS=${LDFLAGS} -L${CBLAS_LIB} -lblis -Wl,-rpath,${CBLAS_LIB} -fopenmp + +# BARE-METAL compile and link flags +# You may consider to disable OpenPiton Stream: -DDISABLE_OPSTREAM +CFLAGS_BMETAL=${CFLAGS} -DRVB_USE_BMETAL -B ${RVB_BMETAL_DIR} \ + --target=riscv64-unknown-elf \ + -I../ \ + -I/apps/riscv/llvm/EPI-0.7/cross/development/riscv64-unknown-linux-gnu/sysroot/usr/include \ + -I/usr/include/riscv64-linux-gnu \ + -I/usr/include \ + -I${RVB_BMETAL_DIR}/env \ + -I${RVB_BMETAL_DIR} \ + -DPREALLOCATE=1 -mcmodel=medany \ + -static -std=gnu99 \ + -ffast-math \ + -fno-common \ + -fno-builtin-printf \ + ${RVB_BMETAL_DIR}/syscalls.c \ + ${RVB_BMETAL_DIR}/crt.S \ + -static -nostdlib -T \ + ${RVB_BMETAL_DIR}/test.ld diff --git a/pkgs/rvb/default.nix b/pkgs/rvb/default.nix new file mode 100644 index 0000000..bf93628 --- /dev/null +++ b/pkgs/rvb/default.nix @@ -0,0 +1,35 @@ +{ + stdenv +, blis +, gitBranch ? "nix-fixes" +, gitURL ? "git@gitlab-internal.bsc.es:rarias/risc-v-benchmarks.git" +, gitCommit ? "da202d6f818421b72e06c39b5417ad2f8f6ca23c" +}: + +stdenv.mkDerivation rec { + pname = "rvb"; + version = "${src.shortRev}"; + + src = builtins.fetchGit { + url = gitURL; + ref = gitBranch; + rev = gitCommit; + submodules = true; + }; + + buildInputs = [ blis ]; + configurePhase = '' + export RVB_ROOT=$(readlink -f .) + export CBLAS_HOME=${blis} + export CBLAS_INC=${blis}/include/blis + export CBLAS_LIB=${blis}/lib + rm Makefile.in + ln -s ${./Makefile.in} Makefile.in + rm Makefile + ln -s ${./Makefile} Makefile + ''; + enableParallelBuilding = false; + hardeningDisable = [ "all" ]; + installFlags = [ "DESTDIR=$(out)" ]; + dontStrip = true; +} -- GitLab From b26c992176e92252ad6344e4209e6eb60c52a0f1 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 25 Sep 2024 10:22:17 +0200 Subject: [PATCH 253/310] Add gcroots to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e2e5d7b..3ee4c55 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *result* nixos-riscv.qcow2 misc/ +gcroots/ -- GitLab From 6ee4ad237ebdd964cced26327d95bca211400c89 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 25 Sep 2024 10:24:04 +0200 Subject: [PATCH 254/310] Switch to bscpkgs master branch --- flake.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 4a8a533..f4ac664 100644 --- a/flake.nix +++ b/flake.nix @@ -1,8 +1,6 @@ { inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; - inputs.bscpkgs.url = "git+https://git.sr.ht/~rodarima/bscpkgs?ref=riscv-benchmarks"; - # When changing bscpkgs, take directly from disk to avoid having to push: - #inputs.bscpkgs.url = "/home/Computational/rarias/bscpkgs"; + inputs.bscpkgs.url = "git+https://git.sr.ht/~rodarima/bscpkgs"; outputs = {self, nixpkgs, ...}@inputs: let -- GitLab From c4af9e7c73b33b10f2a614e29a46243ec4b9a273 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 25 Sep 2024 10:23:17 +0200 Subject: [PATCH 255/310] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'bscpkgs': 'git+https://git.sr.ht/~rodarima/bscpkgs?ref=riscv-benchmarks&rev=9b197d12f1edf52efcb6e3dad421f2a8fd5af2a7' (2024-09-19) → 'git+https://git.sr.ht/~rodarima/bscpkgs?ref=refs/heads/master&rev=de89197a4a7b162db7df9d41c9d07759d87c5709' (2024-04-24) --- flake.lock | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index de983a2..410fe19 100644 --- a/flake.lock +++ b/flake.lock @@ -5,16 +5,15 @@ "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1726742476, - "narHash": "sha256-M0ubiqx3al1TX7Eyq6eSIBax22XXCqGlds/9naIlBFw=", - "ref": "riscv-benchmarks", - "rev": "9b197d12f1edf52efcb6e3dad421f2a8fd5af2a7", - "revCount": 941, + "lastModified": 1713974364, + "narHash": "sha256-ilZTVWSaNP1ibhQIIRXE+q9Lj2XOH+F9W3Co4QyY1eU=", + "ref": "refs/heads/master", + "rev": "de89197a4a7b162db7df9d41c9d07759d87c5709", + "revCount": 937, "type": "git", "url": "https://git.sr.ht/~rodarima/bscpkgs" }, "original": { - "ref": "riscv-benchmarks", "type": "git", "url": "https://git.sr.ht/~rodarima/bscpkgs" } -- GitLab From 8780098a29f654121e5abf2d5aa41d3fa6ed0fc7 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 25 Sep 2024 13:08:40 +0200 Subject: [PATCH 256/310] Add stream benchmark --- overlay.nix | 1 + pkgs/stream/default.nix | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 pkgs/stream/default.nix diff --git a/overlay.nix b/overlay.nix index 1c8c788..2b32356 100644 --- a/overlay.nix +++ b/overlay.nix @@ -9,6 +9,7 @@ final: prev: rvb = final.callPackage ./pkgs/rvb/default.nix { }; rvb-clang = final.callPackage ./pkgs/rvb/default.nix { stdenv = final.stdenvClangEpi; }; + stream = final.callPackage ./pkgs/stream/default.nix { }; blis = ((prev.blis.override { blas64 = true; diff --git a/pkgs/stream/default.nix b/pkgs/stream/default.nix new file mode 100644 index 0000000..84671c6 --- /dev/null +++ b/pkgs/stream/default.nix @@ -0,0 +1,33 @@ +{ + stdenv +, fetchFromGitHub +}: + +stdenv.mkDerivation rec { + pname = "stream"; + version = "4dbce1d0"; + + src = fetchFromGitHub { + owner = "jeffhammond"; + repo = "STREAM"; + rev = "4dbce1d0fdb7410e8f21b48f3381bc0a1341967f"; + sha256 = "sha256-sBwdPeaMyI/wH1Nq0yQtb/kvi5913e0azXaulOJIG3A="; + }; + + # CFLAGS='-O2 -fopenmp -DSTREAM_ARRAY_SIZE=40000000' + buildPhase = '' + set -x + make stream_c.exe CC=$CC FC=$FC CFLAGS=-O2 + set +x + ''; + + #nativeBuildInputs = [ gfortran ]; + dontStrip = true; + dontConfigure = true; + enableParallelBuilding = false; + hardeningDisable = [ "all" ]; + installPhase = '' + mkdir -p $out/bin + cp -a stream_c.exe $out/bin/stream + ''; +} -- GitLab From 9d1ad6f1b3c71db6e94b292f1c6f24a86cad3821 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 25 Sep 2024 13:08:54 +0200 Subject: [PATCH 257/310] Add stream benchmark to rootfs --- configuration.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration.nix b/configuration.nix index 220477e..54276f1 100644 --- a/configuration.nix +++ b/configuration.nix @@ -37,7 +37,7 @@ #environment.systemPackages = with pkgs; [ vim gdb neofetch gcc bintools ]; environment.systemPackages = with pkgs; [ - rvb riscv-tools + rvb riscv-tools stream config.boot.kernelPackages.perf ]; -- GitLab From 7a875a4665145fc171e6d13ad6f6d7e255866e02 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 26 Sep 2024 11:05:24 +0200 Subject: [PATCH 258/310] Use seedrng to increase entropy --- JOURNAL.md | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++ lagarto-ox.nix | 14 ++++++++-- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index fc95c45..b899bd5 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -4124,3 +4124,77 @@ Interesting output: - Report specific PLIC issues - Enable aux timer - Broken RTC frequency. + +Thu, 26 Sep 2024 10:26:26 +0200 + +## 2024-09-26 + +The seedrng tool can add some entropy in the kernel from a saved seed in the FS. +The source is available here: + +https://git.zx2c4.com/seedrng/tree/seedrng.c + +But it seems to be included with busybox, so it should be already included in +our initrd: + + ~ # which seedrng + /nix/store/8a4i33qxnpwn0q7hs1vx3q5h4y2cym7n-extra-utils/bin/seedrng + ~ # cat /proc/sys/kernel/random/entropy_avail + 0 + ~ # seedrng + seedrng: can't create directory '/var/lib/seedrng': No such file or directory + ~ # mkdir -p /var/lib/seedrng + ~ # seedrng + Saving 256 bits of non-creditable seed for next boot + ~ # cat /proc/sys/kernel/random/entropy_avail + 0 + ~ # ls /var/lib/seedrng + seed.no-credit + ~ # hexdump /var/lib/seedrng/seed.no-credit + 0000000 caba 9c5c c19a 5b1a 97fc 0a2e a805 9608 + 0000010 d860 b3b9 4ade b781 ce23 2fce 19a5 d1b1 + 0000020 + +Okay, so now I can magically convert it into creditable seed: + + ~ # mv /var/lib/seedrng/seed.no-credit /var/lib/seedrng/seed.credit + ~ # cat /proc/sys/kernel/random/entropy_avail + 0 + ~ # seedrng + Seeding 256 bits and crediting + [ 135.171201] random: crng init done + Saving 256 bits of creditable seed for next boot + ~ # cat /proc/sys/kernel/random/entropy_avail + 256 + +Nice. + +So, I can remove the jitter daemon and use this instead, assuming it won't go +down. It doesn't seem to go down: + + ~ # cat /proc/sys/kernel/random/entropy_avail + 256 + ~ # hexdump -n $((1024*1024)) /dev/random > /dev/null + ~ # cat /proc/sys/kernel/random/entropy_avail + 256 + +Another option may be to use a fake HW RNG by pointing to the address of some +timer register: + + https://github.com/torvalds/linux/blob/master/drivers/char/hw_random/timeriomem-rng.c + https://www.kernel.org/doc/Documentation/devicetree/bindings/rng/timeriomem_rng.txt + +Let's try the seedrng approach first. + + ~ # seedrng -h + seedrng: invalid option -- 'h' + BusyBox v1.36.1 () multi-call binary. + + Usage: seedrng [-d DIR] [-n] + + Seed the kernel RNG from seed files + + -d DIR Use seed files in DIR (default: /var/lib/seedrng) + -n Do not credit randomness, even if creditable + +I can work with this. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 94710ee..3a760eb 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -274,12 +274,20 @@ # while [ 1 ]; do xxd -s $((0x1bfff0000 - 0x60000000)) \ # -l 4 /dev/qdma34000-MM-1; sleep 0.2; done preDeviceCommands = '' + # Seed RNG + seedrng -d /tmp || true + mv /tmp/seed.no-credit /tmp/seed.credit || true + seedrng -d /tmp || true + echo "Available entropy: $(cat /proc/sys/kernel/random/entropy_avail)" + + # Last chance to enter a shell if read -t 3 -p 'Press enter for shell... '; then - ash + allowShell=1 + fail fi - echo "Running tests..." - sh /testplic + # echo "Running tests..." + # sh /testplic # echo "Creating a heartbeat counter at 0x1bfff0000" # sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' & -- GitLab From 2e2b957528ff7306ba2437ff9aa24b375fa9b7b4 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 26 Sep 2024 11:21:49 +0200 Subject: [PATCH 259/310] Disable jitterentropy daemon --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 3a760eb..371e146 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -15,7 +15,7 @@ hardware.firmware = lib.mkForce []; #services.haveged.enable = true; - services.jitterentropy-rngd.enable = true; + #services.jitterentropy-rngd.enable = true; # Prevent executing the nscd program as it seems to hang the CPU system.activationScripts.users = lib.mkForce ( -- GitLab From d694f3c8dabf000ecaab89161e35aa3c6bac9190 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 26 Sep 2024 11:31:47 +0200 Subject: [PATCH 260/310] Fix RTC frequency for now --- dts/lagarto_ox.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dts/lagarto_ox.h b/dts/lagarto_ox.h index 3491ebc..1d5df29 100644 --- a/dts/lagarto_ox.h +++ b/dts/lagarto_ox.h @@ -3,7 +3,12 @@ /* The RTC timer is clocked at the CPU frequency / 1525, so * around 32786.88 Hz */ -#define RTC_CLOCK_FREQUENCY 32786 +//#define RTC_CLOCK_FREQUENCY 32786 + +/* FIXME: The real RTC frequency is around half that, as the divider was wrongly + * configured. So for now lets use the real frequency: + * 50e6 / (1525*2) = 16393.44262295082 -> 16393 Hz */ +#define RTC_CLOCK_FREQUENCY 16393 #define UART_SPEED 115200 #define UART0_ADDR 0x40001000 -- GitLab From 3478cf293160d64a2e762ca5e2da8350324fdc85 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 26 Sep 2024 12:56:11 +0200 Subject: [PATCH 261/310] Update journal --- JOURNAL.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index b899bd5..9f876a4 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -4198,3 +4198,54 @@ Let's try the seedrng approach first. -n Do not credit randomness, even if creditable I can work with this. + +It seems to be working. + +After correcting the RTC, now I get a more or less good boot time: + + [root@nixos-riscv:~]# systemd-analyze + Startup finished in 4min 49.469s (kernel) + 10min 48.260s (userspace) = 15min 37.729s + multi-user.target reached after 10min 48.024s in userspace. + +Still, udevd is killing the CPU: + + [root@nixos-riscv:~]# vmstat 1 + procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu----- + r b swpd free buff cache si so bi bo in cs us sy id wa st + 2 0 0 585072 16580 59200 0 0 0 12 164 142 49 51 0 0 0 + 4 0 0 585072 16580 59200 0 0 0 0 159 190 46 54 0 0 0 + 1 0 0 585072 16580 59200 0 0 0 0 157 66 49 51 0 0 0 + 5 0 0 585072 16580 59200 0 0 0 0 156 60 75 25 0 0 0 + 2 0 0 585072 16580 59200 0 0 0 0 165 200 30 70 0 0 0 + 2 0 0 585072 16580 59216 0 0 0 4 157 200 31 69 0 0 0 + 2 0 0 585072 16580 59216 0 0 0 0 157 194 39 61 0 0 0 + 2 0 0 585072 16580 59216 0 0 0 0 158 201 53 47 0 0 0 + 2 0 0 585072 16580 59216 0 0 0 0 156 173 62 38 0 0 0 + 3 0 0 585072 16580 59216 0 0 0 0 162 192 64 36 0 0 0 + 4 0 0 585072 16580 59232 0 0 0 0 163 205 35 65 0 0 0 + 1 0 0 585072 16580 59232 0 0 0 0 165 182 36 64 0 0 0 + 1 0 0 585072 16580 59232 0 0 0 0 155 44 76 24 0 0 0 + 3 0 0 585072 16580 59232 0 0 0 0 157 149 46 54 0 0 0 + 2 0 0 585072 16580 59232 0 0 0 0 158 197 34 66 0 0 0 + 2 0 0 585072 16580 59232 0 0 0 0 158 198 30 70 0 0 0 + 2 0 0 585072 16580 59232 0 0 0 0 156 196 48 52 0 0 0 + 2 0 0 585072 16580 59232 0 0 0 0 158 179 58 42 0 0 0 + 3 0 0 585072 16580 59232 0 0 0 0 158 176 67 33 0 0 0 + 2 0 0 585072 16580 59488 0 0 256 0 159 195 36 64 0 0 0 + 1 0 0 585072 16580 59620 0 0 128 0 156 134 55 45 0 0 0 + +Not sure which device is malfunctioning. + + [root@nixos-riscv:~]# udevadm monitor + monitor will print the received events for: + UDEV - the event which udev sends out after rule processing + KERNEL - the kernel uevent + + UDEV [1738.570531] add /devices/virtual/tty/ptyaa (tty) + UDEV [1741.983817] add /devices/virtual/tty/ptyab (tty) + UDEV [1745.434070] add /devices/virtual/tty/ptyac (tty) + UDEV [1748.846075] add /devices/virtual/tty/ptyad (tty) + UDEV [1752.238682] add /devices/virtual/tty/ptyae (tty) + UDEV [1755.665632] add /devices/virtual/tty/ptyaf (tty) + UDEV [1759.047259] add /devices/virtual/tty/ptyb0 (tty) + -- GitLab From 6994472a92cd0aaa2e8b0acf96ba0dd8218b0089 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 26 Sep 2024 12:56:28 +0200 Subject: [PATCH 262/310] Disable more daemons we dont need --- configuration.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configuration.nix b/configuration.nix index 54276f1..f8f5644 100644 --- a/configuration.nix +++ b/configuration.nix @@ -35,6 +35,11 @@ }; }; + systemd.oomd.enable = false; + networking.firewall.enable = false; + networking.dhcpcd.enable = false; + services.timesyncd.enable = false; + #environment.systemPackages = with pkgs; [ vim gdb neofetch gcc bintools ]; environment.systemPackages = with pkgs; [ rvb riscv-tools stream -- GitLab From d33775e6260b1a303d3bd31de26b9f5cad389c96 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 26 Sep 2024 12:56:50 +0200 Subject: [PATCH 263/310] Disable ethernet module for now --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 371e146..81b181e 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -79,7 +79,7 @@ boot = { extraModulePackages = [ # Add the custom Ethernet module - pkgs.xilinx-axienet-carv + #pkgs.xilinx-axienet-carv ]; kernelModules = config.boot.initrd.kernelModules; kernelPatches = [ -- GitLab From 61cf697854d497ed74ceb3aca8e4608f57da9f16 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 26 Sep 2024 12:57:04 +0200 Subject: [PATCH 264/310] Rename CI job to lagarto-ox --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b5e4351..3cdbd5d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -build:lagarto-ox-rd: +build:lagarto-ox: stage: build tags: - nix -- GitLab From e8f80e7509f8e9805da917f0271efb7cce5ad15f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 27 Sep 2024 09:14:14 +0200 Subject: [PATCH 265/310] Disable udev --- JOURNAL.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ lagarto-ox.nix | 4 ++-- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 9f876a4..27d7870 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -4249,3 +4249,48 @@ Not sure which device is malfunctioning. UDEV [1755.665632] add /devices/virtual/tty/ptyaf (tty) UDEV [1759.047259] add /devices/virtual/tty/ptyb0 (tty) +## 2024-09-27 + +Here is the FPGA startup services by duration: + + [root@nixos-riscv:~]# systemd-analyze blame + 3min 59.980s systemd-udev-trigger.service + 2min 6.780s suid-sgid-wrappers.service + 2min 2.677s mount-pstore.service + 1min 46.770s user@0.service + 1min 10.554s systemd-journald.service + 1min 5.538s resolvconf.service + 48.660s systemd-logind.service + 44.747s systemd-sysctl.service + 43.971s sys-fs-fuse-connections.mount + 43.358s dev-mqueue.mount + 43.197s dev-hugepages.mount + 43.043s sys-kernel-debug.mount + 42.959s kmod-static-nodes.service + 42.568s network-setup.service + 42.546s modprobe@configfs.service + 42.285s systemd-tmpfiles-setup-dev-early.service + 40.980s modprobe@efi_pstore.service + 40.510s sys-kernel-config.mount + 39.891s modprobe@fuse.service + 36.103s systemd-random-seed.service + 30.686s systemd-udevd.service + 30.059s systemd-modules-load.service + 29.682s systemd-journal-catalog-update.service + 29.091s systemd-remount-fs.service + 26.090s systemd-tmpfiles-setup.service + 23.570s systemd-journal-flush.service + 21.042s systemd-tmpfiles-setup-dev.service + 19.736s audit.service + 17.794s systemd-update-utmp.service + 15.329s dbus.service + 10.177s modprobe@drm.service + 9.866s systemd-update-done.service + 8.302s user-runtime-dir@0.service + 6.615s systemd-user-sessions.service + 5.384s network-local-commands.service + 4.933s getty@tty1.service + +It doesn't make much sense to have udev on the FPGA, as we are not going to +hotplug anything. Maybe I can just disable the service, and save 4 minutes (at +least). diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 81b181e..7b2d8eb 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -14,8 +14,8 @@ # We don't need any firmware hardware.firmware = lib.mkForce []; - #services.haveged.enable = true; - #services.jitterentropy-rngd.enable = true; + # No need for udev + services.udev.enable = false; # Prevent executing the nscd program as it seems to hang the CPU system.activationScripts.users = lib.mkForce ( -- GitLab From e971ce4ae46d14c5fef6620ca6000b222f6c4b8e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 27 Sep 2024 09:48:47 +0200 Subject: [PATCH 266/310] Delay console after network.target Let's see if we can get rid of the messages after boot. --- lagarto-ox.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 7b2d8eb..36644d8 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -338,6 +338,7 @@ # Run getty on /dev/console and restartt until it works systemd.services."serial-getty@console" = { enable = true; + after = [ "network.target" ]; wantedBy = [ "getty.target" ]; # to start at boot serviceConfig.Restart = "always"; }; -- GitLab From d4df444a1fc6c9eb243a5ab016e1768e87cb5335 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 27 Sep 2024 11:53:52 +0200 Subject: [PATCH 267/310] Show name of systemd units instead of description --- JOURNAL.md | 3 +++ lagarto-ox.nix | 13 +++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 27d7870..5070077 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -4294,3 +4294,6 @@ Here is the FPGA startup services by duration: It doesn't make much sense to have udev on the FPGA, as we are not going to hotplug anything. Maybe I can just disable the service, and save 4 minutes (at least). + +Disabling udev by just setting `services.udev.enable = false` doesn't work. It +still gets activated by the kernel socket. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 36644d8..c850fbd 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -14,8 +14,17 @@ # We don't need any firmware hardware.firmware = lib.mkForce []; - # No need for udev - services.udev.enable = false; + # Doesn't work, it gets activated via the kernel socket. + # # No need for udev + # services.udev.enable = false; + # systemd.suppressedSystemUnits = [ + # "systemd-udev-trigger.service" + # ]; + + # Output the unit name so we can remove it + systemd.extraConfig = '' + StatusUnitFormat=name + ''; # Prevent executing the nscd program as it seems to hang the CPU system.activationScripts.users = lib.mkForce ( -- GitLab From 58c6d56e96553f202d44ac0add9664f8ccd34acc Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 27 Sep 2024 14:11:33 +0200 Subject: [PATCH 268/310] Allow running an interactive shell in stage 2 --- patches/stage-2-init.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/patches/stage-2-init.sh b/patches/stage-2-init.sh index 0ef8cc6..78d416f 100755 --- a/patches/stage-2-init.sh +++ b/patches/stage-2-init.sh @@ -2,7 +2,7 @@ set -x -echo "HELLOOOOOOOOOOOO FROM STAGE 2" +echo "--- starting stage 2 ---" systemConfig=@systemConfig@ @@ -17,6 +17,9 @@ if [ "${IN_NIXOS_SYSTEMD_STAGE1:-}" != true ]; then # Show each command. set -x ;; + debug2) + bash -i + ;; esac done @@ -121,7 +124,7 @@ echo "running activation script..." #strace -f $systemConfig/activate $systemConfig/activate -echo "ACTIVATE WORKED!!!" +echo "activation script OK" # Record the boot configuration. ln -sfn "$systemConfig" /run/booted-system -- GitLab From edb597ad6a8150f50e6436973d56a7315b5c49fd Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 27 Sep 2024 14:12:21 +0200 Subject: [PATCH 269/310] Enable stage 2 shell --- lagarto-ox.nix | 3 ++- patches/stage-2-init.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index c850fbd..e1df79c 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -498,6 +498,7 @@ # systemd.log_target=console # NixOS interesting options: # debug1 enable debug shell in stage 1 + # debug2 enable debug shell in stage 2 (custom) # boot.trace enable set -x in stage 1 # boot.tracedebug enable set -x in stage 2 # Ftrace interesting options: @@ -509,7 +510,7 @@ in prev.runCommand "uboot.txt" {} '' cat > $out < Date: Fri, 27 Sep 2024 15:45:14 +0200 Subject: [PATCH 270/310] Use setsid to get job control in debug shell --- lagarto-ox.nix | 1 + patches/stage-2-init.sh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index e1df79c..aa5ea7d 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -65,6 +65,7 @@ bootStage2 = pkgs.substituteAll { src = ./patches/stage-2-init.sh; shellDebug = "${pkgs.bashInteractive}/bin/bash"; + bashInteractive = "${pkgs.bashInteractive}"; shell = "${pkgs.bash}/bin/bash"; inherit (config.boot) readOnlyNixStore systemdExecutable extraSystemdUnitPaths; inherit (config.system.nixos) distroName; diff --git a/patches/stage-2-init.sh b/patches/stage-2-init.sh index d22470b..1c48d21 100755 --- a/patches/stage-2-init.sh +++ b/patches/stage-2-init.sh @@ -18,7 +18,8 @@ if [ "${IN_NIXOS_SYSTEMD_STAGE1:-}" != true ]; then set -x ;; debug2) - @shellDebug@ -i + export PATH="@bashInteractive@/bin:@systemConfig@/sw/bin:$PATH" + setsid bash -c "exec bash -l /dev/hvc0 2>/dev/hvc0" ;; esac done -- GitLab From 290679d3e3a2ea885a5b5cf147dab217e61dd209 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 27 Sep 2024 16:07:49 +0200 Subject: [PATCH 271/310] Reduce timeout to 3 minutes --- fpga/fpgactl | 1 - fpga/run-node.sh | 7 +------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/fpga/fpgactl b/fpga/fpgactl index b54bbd6..ef40daf 100755 --- a/fpga/fpgactl +++ b/fpga/fpgactl @@ -97,7 +97,6 @@ function do_cpu_release() # {{{ dma-ctl "${qdmadev}" reg write bar 2 0x0 0x3 elif [ "$model" == "ox" ]; then dma-ctl "${qdmadev}" reg write bar 2 0x0 0x1 - sleep 1 fi } # }}} function copy_by_dma() # {{{ diff --git a/fpga/run-node.sh b/fpga/run-node.sh index 670f5a9..98da578 100755 --- a/fpga/run-node.sh +++ b/fpga/run-node.sh @@ -19,14 +19,9 @@ set -x # Then perform the boot ./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img -#./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin - -# Restart it again only loading the bootrom and OpenSBI (notice OpenSBI -# relocates itself so it will likely be gone by now) -sleep 1 && ./fpgactl -b opensbi.bin & timeout=$((30 * 60)) # Always stop after 30 min -timeout_silent=$((10 * 60)) # Stop if 10 min without output +timeout_silent=$((3 * 60)) # Stop if 3 min without output # Set dead switch sleep $timeout && killall picocom & -- GitLab From cae948e923928a7a66dbed13c1d9f9de68b2e762 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 30 Sep 2024 09:32:42 +0200 Subject: [PATCH 272/310] Switch to ttyS0 as console device --- JOURNAL.md | 14 ++++++++++++++ lagarto-ox.nix | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/JOURNAL.md b/JOURNAL.md index 5070077..c2ebb27 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -4297,3 +4297,17 @@ least). Disabling udev by just setting `services.udev.enable = false` doesn't work. It still gets activated by the kernel socket. + +## 2024-09-30 + +Managed to launch an interactive shell from stage2, before we run systemd. It +boots in less than 5 minutes. + +Now I can run some benchmarks there. + +Let's see if I can enable flow control on the serial console. That would be +helpful. + +For that I need to first switch to the 8250/16550 driver. Let's try switch to +the ttyS0 without enabling flow control yet. We probably need to change the +stage1 and stage2 scripts to follow the proper console device. diff --git a/lagarto-ox.nix b/lagarto-ox.nix index aa5ea7d..096f268 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -511,7 +511,7 @@ in prev.runCommand "uboot.txt" {} '' cat > $out < Date: Mon, 30 Sep 2024 11:55:57 +0200 Subject: [PATCH 273/310] Add claim support in plictool --- JOURNAL.md | 31 +++++++++++++++++++++++++++++++ tools/plictool.c | 46 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index c2ebb27..2ca183d 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -4311,3 +4311,34 @@ helpful. For that I need to first switch to the 8250/16550 driver. Let's try switch to the ttyS0 without enabling flow control yet. We probably need to change the stage1 and stage2 scripts to follow the proper console device. + +It works, but very slowly: + + [ 0.000000] Kernel command line: root=/dev/ram0 loglevel=7 rw earlycon=sbi console=ttyS0,115200n8 debug2 ... + ... + [ 42.069358] Serial: 8250/16550 driver, 4 ports, IRQ sharing disabled + [ 42.473860] of_serial 40001000.serial: error -ENXIO: IRQ index 0 not found + [ 42.559263] printk: legacy console [ttyS0] disabled + [ 42.614225] 40001000.serial: ttyS0 at MMIO 0x40001000 (irq = 0, base_baud = 3125000) is a 16550 + [ 42.645519] printk: legacy console [ttyS0] enabled + [ 42.645519] printk: legacy console [ttyS0] enabled + [ 42.656865] printk: legacy bootconsole [sbi0] disabled + [ 42.656865] printk: legacy bootconsole [sbi0] disabled + [ 42.896358] 40003000.serial: ttyS1 at MMIO 0x40003000 (irq = 1, base_baud = 3125000) is a 16550 + [ 42.999450] SuperH (H)SCI(F) driver initialized + ... + ~ # stty -a + speed 115200 baud;stty: standard input + line = 0; + intr = ^C; quit = ^\; erase = ^?; kill = ^U; eof = ^D; eol = ; + eol2 = ; swtch = ; start = ^Q; stop = ^S; susp = ^Z; rprnt = ^R; + werase = ^W; lnext = ^V; flush = ^O; min = 1; time = 0; + -parenb -parodd -cmspar cs8 hupcl -cstopb cread clocal -crtscts + -ignbrk -brkint -ignpar -parmrk -inpck -istrip -inlcr -igncr icrnl ixon -ixoff + -iuclc -ixany -imaxbel -iutf8 + opost -olcuc -ocrnl onlcr -onocr -onlret -ofill -ofdel nl0 cr0 tab0 bs0 vt0 ff0 + isig icanon iexten echo echoe echok -echonl -noflsh -xcase -tostop -echoprt + echoctl echoke -flusho -extproc + +Let's add the ability to claim a context in the plictool, as I suspect the claim +may be broken. diff --git a/tools/plictool.c b/tools/plictool.c index d2705c6..da598e6 100644 --- a/tools/plictool.c +++ b/tools/plictool.c @@ -8,9 +8,10 @@ * v0.0.1 (2024-09-03): Initial version. * v0.0.2 (2024-09-04): Print contexts in another line and masked information. * v0.0.3 (2024-09-04): Make output format more clear and add manual. + * v0.0.4 (2024-09-30): Implement support for claiming an interrupt. */ -#define VERSION "v0.0.3" +#define VERSION "v0.0.4" #include #include @@ -21,10 +22,12 @@ #include #include +enum op { DUMP, CLAIM } operation = DUMP; const char *plic_address_str = "0x40800000"; long ncontexts = 15872L; long maxsources = 1024L; long nsources = 1024L; +long claim_ctx = 0; static void usage(void) @@ -34,7 +37,7 @@ usage(void) " plictool - dump PLIC information\n" "\n" "SYNOPSIS\n" -" plictool [-a addr] [-s nsources] [-c ncontexts]\n" +" plictool [-a addr] [-s nsources] [-c ncontexts] [-C ctx]\n" "\n" "DESCRIPTION\n" " Dumps information about the RISC-V PLIC interrupt controller.\n" @@ -49,6 +52,8 @@ usage(void) " -c ncontexts Limit the number of contexts to print.\n" " The default is 15872.\n" "\n" +" -C ctx Claim the interrupt on the given context.\n" +"\n" "OUTPUT FORMAT\n" " Information is printed for each source that has some bit set\n" " in the pending or priority registers, or it has some context\n" @@ -75,7 +80,31 @@ usage(void) exit(1); } -void dump_sources(void *base) +static uint32_t +read_reg(void *base, size_t offset) +{ + volatile uint32_t *p = base + offset; + return *p; +} + +static void +write_reg(void *base, size_t offset, uint32_t value) +{ + volatile uint32_t *p = base + offset; + *p = value; +} + +static void +claim_interrupt(void *base, long ctx) +{ + size_t offset = 0x200004L + ctx * 0x1000; + uint32_t value = read_reg(base, offset); + printf("ctx=%ld claim=%u\n", ctx, value); + write_reg(base, offset, value); +} + +static void +dump_sources(void *base) { for (long s = 0; s < nsources; s++) { int printed_source = 0; @@ -133,7 +162,7 @@ int main(int argc, char *argv[]) { int opt; - while ((opt = getopt(argc, argv, "a:hs:c:")) != -1) { + while ((opt = getopt(argc, argv, "a:hs:c:C:")) != -1) { switch (opt) { case 'a': plic_address_str = optarg; @@ -144,6 +173,10 @@ int main(int argc, char *argv[]) case 'c': ncontexts = atol(optarg); break; + case 'C': + operation = CLAIM; + claim_ctx = atol(optarg); + break; case 'h': default: /* '?' */ usage(); @@ -177,7 +210,10 @@ int main(int argc, char *argv[]) exit(1); } - dump_sources(map_base); + if (operation == CLAIM) + claim_interrupt(map_base, claim_ctx); + else if (operation == DUMP) + dump_sources(map_base); munmap(map_base, map_size); -- GitLab From 9dda6459f5458d3c07fa5d855d1dd8980cb8cb0e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 30 Sep 2024 12:03:09 +0200 Subject: [PATCH 274/310] Reduce the number of contexts to 2 by default --- tools/plictool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/plictool.c b/tools/plictool.c index da598e6..6041d65 100644 --- a/tools/plictool.c +++ b/tools/plictool.c @@ -24,7 +24,7 @@ enum op { DUMP, CLAIM } operation = DUMP; const char *plic_address_str = "0x40800000"; -long ncontexts = 15872L; +long ncontexts = 2L; long maxsources = 1024L; long nsources = 1024L; long claim_ctx = 0; -- GitLab From af666c44ef70ea5154d86229e1aa4d32edb0f681 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 2 Oct 2024 07:33:57 +0200 Subject: [PATCH 275/310] Add plic claim baremetal test --- overlay.nix | 5 +- patches/sa-fpga-add-plic-claim-test.patch | 92 +++++++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 patches/sa-fpga-add-plic-claim-test.patch diff --git a/overlay.nix b/overlay.nix index 2b32356..8527ad1 100644 --- a/overlay.nix +++ b/overlay.nix @@ -45,8 +45,8 @@ final: prev: name = "sa-fpga-tests"; src = builtins.fetchGit { url = "git@gitlab-internal.bsc.es:hwdesign/rtl/core-tile/sa-fpga.git"; - rev = "afe0372413a94fff279ca5d5002c3e999ac8defb"; - ref = "ft/sv_eirq"; + rev = "720be4f1f5dd0ef963135992578be2ab55fb5537"; + ref = "main"; }; dontConfigure = true; patches = [ @@ -54,6 +54,7 @@ final: prev: #./patches/sa-fpga-text-address.patch ./patches/sa-fpga-uart.patch ./patches/sa-fpga-plic-registers.patch + ./patches/sa-fpga-add-plic-claim-test.patch ]; buildPhase = '' cd fpga_core_bridge/simulator/tests/c_tests/ diff --git a/patches/sa-fpga-add-plic-claim-test.patch b/patches/sa-fpga-add-plic-claim-test.patch new file mode 100644 index 0000000..b9b6cb4 --- /dev/null +++ b/patches/sa-fpga-add-plic-claim-test.patch @@ -0,0 +1,92 @@ +commit 1a2c5f12e7676930123cfe7853f1805cf3680c25 +Author: Rodrigo Arias Mallo +Date: Tue Oct 1 12:59:38 2024 +0200 + + Add claim test + +diff --git a/fpga_core_bridge/simulator/tests/c_tests/Makefile b/fpga_core_bridge/simulator/tests/c_tests/Makefile +index f744131..91d61b6 100644 +--- a/fpga_core_bridge/simulator/tests/c_tests/Makefile ++++ b/fpga_core_bridge/simulator/tests/c_tests/Makefile +@@ -23,6 +23,7 @@ bmarks = \ + plic \ + plic_supervisor \ + plic_threshold \ ++ plic_claim \ + clint_supervisor \ + uart + +diff --git a/fpga_core_bridge/simulator/tests/c_tests/plic_claim/plic_claim.c b/fpga_core_bridge/simulator/tests/c_tests/plic_claim/plic_claim.c +new file mode 100644 +index 0000000..352adb9 +--- /dev/null ++++ b/fpga_core_bridge/simulator/tests/c_tests/plic_claim/plic_claim.c +@@ -0,0 +1,68 @@ ++#include "util.h" ++ ++#define PLIC_BASE 0x40800000 ++ ++int main(void) ++{ ++ uart_init(); ++ ++ /* This test requires the auxiliar timer to cause a pending ++ * interrupt in the source 4. All the other pending bits must be ++ * zero. */ ++ ++ uint32_t src = 4; ++ uint32_t mask = 1 << src; ++ ++ /* 0x001000: Interrupt Source #0 to #31 Pending Bits */ ++ volatile uint32_t *pending = PLIC_BASE + 0x001000; ++ ++ /* Manually enable the pending register on both timer and serial */ ++ *pending = (1<<4) | (1<<1); ++ ++ uint32_t p; ++ /* Wait for a interrupt on the serial line */ ++ while ((p = *pending) != (1<<4) | (1<<1)) { ++ printf("waiting, pending="); ++ printhex(p); ++ printf("\n"); ++ } ++ ++ /* 0x002080: Interrupt Source #0 to #31 Enable Bits on context 1 */ ++ volatile uint32_t *enable = PLIC_BASE + 0x002080; ++ *enable = (1<<4) | (1<<1); /* Enable source 4 (aux timer) and 1 ++ (serial) */ ++ ++ /* 0x000004: Interrupt source 1 priority */ ++ /* 0x000008: Interrupt source 2 priority */ ++ /* 0x00000c: Interrupt source 3 priority */ ++ /* 0x000010: Interrupt source 4 priority */ ++ for (uint32_t i = 1; i <= 4; i++) { ++ volatile uint32_t *priority = PLIC_BASE + i * 4; ++ *priority = 1; /* Make priority larger than threshold */ ++ } ++ ++ /* 0x201000: Priority threshold for context 1 */ ++ volatile uint32_t *threshold = PLIC_BASE + 0x201000; ++ *threshold = 0; /* Make threshold small */ ++ ++ /* Now the context 1 must be receiving interrupts from the aux ++ * timer. Let's try to claim the interrupt. */ ++ ++ /* Read claim */ ++ ++ /* 0x201004: Interrupt Claim Process for context 1 */ ++ volatile uint32_t *claim = PLIC_BASE + 0x201004; ++ while (1) { ++ uint32_t c = *claim; ++ if (c == src) ++ break; ++ printf("ERROR: unexpected claim found, expecting 4: "); ++ printhex(c); ++ printf("\n"); ++ } ++ ++ while (1) ++ printf("SUCCESS: Claim test succeeded\n"); ++ ++ return 0; ++} -- GitLab From 407edf31ac1e3f24e7db88c84c95f7ab01c50abd Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 2 Oct 2024 07:35:19 +0200 Subject: [PATCH 276/310] Allow claiming a custom value in plictool --- tools/plictool.c | 105 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 91 insertions(+), 14 deletions(-) diff --git a/tools/plictool.c b/tools/plictool.c index 6041d65..3b51124 100644 --- a/tools/plictool.c +++ b/tools/plictool.c @@ -22,12 +22,23 @@ #include #include -enum op { DUMP, CLAIM } operation = DUMP; +enum op { LIST, CLAIM } operation = LIST; const char *plic_address_str = "0x40800000"; long ncontexts = 2L; long maxsources = 1024L; long nsources = 1024L; long claim_ctx = 0; +uint32_t claim_value = 0; +uint32_t *claim_pvalue = NULL; + +struct source_ctx { + unsigned enabled; + unsigned threshold; + unsigned masked; +}; + +struct source { +}; static void usage(void) @@ -37,12 +48,17 @@ usage(void) " plictool - dump PLIC information\n" "\n" "SYNOPSIS\n" -" plictool [-a addr] [-s nsources] [-c ncontexts] [-C ctx]\n" +" plictool [-a addr] [-L] [-s nsources] [-c ncontexts]\n" +" plictool [-a addr] [-C] [-c context] [-w value]\n" "\n" "DESCRIPTION\n" " Dumps information about the RISC-V PLIC interrupt controller.\n" " Optional arguments:\n" "\n" +" -L Show PLIC information (default operation).\n" +"\n" +" -C context Claim an interrupt on the given context.\n" +"\n" " -a addr The starting address of the PLIC.\n" " The default is 0x40800000.\n" "\n" @@ -54,6 +70,8 @@ usage(void) "\n" " -C ctx Claim the interrupt on the given context.\n" "\n" +" -w value Use this value to claim an interrupt.\n" +"\n" "OUTPUT FORMAT\n" " Information is printed for each source that has some bit set\n" " in the pending or priority registers, or it has some context\n" @@ -95,18 +113,67 @@ write_reg(void *base, size_t offset, uint32_t value) } static void -claim_interrupt(void *base, long ctx) +claim_interrupt(void *base, uint32_t ctx, uint32_t *value) { size_t offset = 0x200004L + ctx * 0x1000; - uint32_t value = read_reg(base, offset); - printf("ctx=%ld claim=%u\n", ctx, value); - write_reg(base, offset, value); + uint32_t r = read_reg(base, offset); + uint32_t w = value ? *value : r; + write_reg(base, offset, w); + printf("claim ctx=%u r=%u w=%u\n", ctx, r, w); } static void -dump_sources(void *base) +source_init(struct source *src, long ncontexts) { + memset(src, 0, sizeof(struct source)); + src->ctx = calloc(ncontexts, sizeof(struct source_ctx)); + if (src->ctx == NULL) { + perror("calloc failed"); + exit(1); + } +} + +static void +source_reset(struct source *src, long ncontexts) +{ + memset(src, 0, sizeof(struct source)); + memset(src->ctx, 0, ncontexts * sizeof(struct source_ctx)); +} + + +static void +source_free(struct source *src, long ncontexts) +{ + free(src->ctx); +} + +static void +source_read(struct source *src, void *base, long s, long ncontexts) +{ + uint32_t pending_reg = read_reg(base, 0x1000L + (s / 32L) * 4L) + long shift = s % 32L; + src->pending = (pending_reg >> shift) & 1; + src->priority = read_reg(base, 0x0000L + (s * 4L)); + + for (long c = 0; c < ncontexts; c++) { + struct source_ctx *ctx = &src->ctx[c]; + + size_t off_en = 0x2000L + 0x80L * c + (s / 32L) * 4L; + uint32_t enabled_reg = reg_read(base, off_en); + ctx->enabled = (enable_reg >> shift) & 1; + ctx->threshold = read_reg(base, 0x200000L + (c * 0x1000L)); + ctx->masked = priority <= threshold; + } +} + +static void +list_sources(void *base) +{ + struct source src; + + source_init(&src, ncontexts); for (long s = 0; s < nsources; s++) { + source_read(&src, base, s, ncontexts); int printed_source = 0; uint32_t *pending_reg = base + 0x1000L + (s / 32L) * 4L; @@ -160,13 +227,20 @@ dump_sources(void *base) int main(int argc, char *argv[]) { + const char *memfile = "/dev/mem"; int opt; - while ((opt = getopt(argc, argv, "a:hs:c:C:")) != -1) { + while ((opt = getopt(argc, argv, "f:a:hLs:c:C:w:")) != -1) { switch (opt) { + case 'f': + memfile = optarg; + break; case 'a': plic_address_str = optarg; break; + case 'L': /* default operation */ + operation = LIST; + break; case 's': nsources = atol(optarg); break; @@ -177,6 +251,10 @@ int main(int argc, char *argv[]) operation = CLAIM; claim_ctx = atol(optarg); break; + case 'w': + claim_value = atol(optarg); + claim_pvalue = &claim_value; + break; case 'h': default: /* '?' */ usage(); @@ -188,11 +266,10 @@ int main(int argc, char *argv[]) printf("plictool "VERSION" addr=0x%08llx nsrc=%ld nctx=%ld\n", plic_address, nsources, ncontexts); - //int fd = open("test.bin", O_RDWR | O_SYNC); - int fd = open("/dev/mem", O_RDWR | O_SYNC); + int fd = open(memfile, O_RDWR | O_SYNC); if (fd == -1) { - perror("cannot open /dev/mem"); + fprintf(stderr, "cannot open %s: %s", memfile, strerror(errno)); exit(1); } @@ -211,9 +288,9 @@ int main(int argc, char *argv[]) } if (operation == CLAIM) - claim_interrupt(map_base, claim_ctx); - else if (operation == DUMP) - dump_sources(map_base); + claim_interrupt(map_base, claim_ctx, claim_pvalue); + else if (operation == LIST) + list_sources(map_base); munmap(map_base, map_size); -- GitLab From 1cf4e576d67fa4a92ad72da1d06ab19a684ee74c Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 2 Oct 2024 14:50:33 +0200 Subject: [PATCH 277/310] Add more operations to plictool --- tools/Makefile | 2 +- tools/plictool.c | 412 ++++++++++++++++++++++++++++------------------- 2 files changed, 251 insertions(+), 163 deletions(-) diff --git a/tools/Makefile b/tools/Makefile index c57de84..fd445cb 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -1,4 +1,4 @@ -CFLAGS=-static +CFLAGS=-Wall -static PREFIX?=/usr/local bin=plictool csrtool memtool unalign diff --git a/tools/plictool.c b/tools/plictool.c index 3b51124..e1caa1b 100644 --- a/tools/plictool.c +++ b/tools/plictool.c @@ -2,102 +2,58 @@ * SPDX-License-Identifier: MIT * Author: Rodrigo Arias Mallo */ -/* Small utility to dump the state of the PLIC. */ +/* Small utility to manage the PLIC. */ /* Changelog: * v0.0.1 (2024-09-03): Initial version. * v0.0.2 (2024-09-04): Print contexts in another line and masked information. * v0.0.3 (2024-09-04): Make output format more clear and add manual. * v0.0.4 (2024-09-30): Implement support for claiming an interrupt. + * v0.0.5 (2024-10-02): Support other read/write operations. */ -#define VERSION "v0.0.4" +#define VERSION "v0.0.5" #include #include #include +#include #include #include #include #include #include -enum op { LIST, CLAIM } operation = LIST; +int operation; const char *plic_address_str = "0x40800000"; long ncontexts = 2L; long maxsources = 1024L; long nsources = 1024L; -long claim_ctx = 0; -uint32_t claim_value = 0; -uint32_t *claim_pvalue = NULL; + +long context = -1; +long source = -1; +long value = -1; +bool value_set = NULL; + +struct ctx { + uint32_t threshold; +}; struct source_ctx { - unsigned enabled; - unsigned threshold; - unsigned masked; + bool enabled; + bool masked; + uint32_t threshold; + const char *state; }; struct source { + bool pending; + bool show; + uint32_t priority; + struct source_ctx *ctx; + long ncontexts; }; -static void -usage(void) -{ - printf( -"NAME\n" -" plictool - dump PLIC information\n" -"\n" -"SYNOPSIS\n" -" plictool [-a addr] [-L] [-s nsources] [-c ncontexts]\n" -" plictool [-a addr] [-C] [-c context] [-w value]\n" -"\n" -"DESCRIPTION\n" -" Dumps information about the RISC-V PLIC interrupt controller.\n" -" Optional arguments:\n" -"\n" -" -L Show PLIC information (default operation).\n" -"\n" -" -C context Claim an interrupt on the given context.\n" -"\n" -" -a addr The starting address of the PLIC.\n" -" The default is 0x40800000.\n" -"\n" -" -s nsources Limit the number of sources to print.\n" -" The default is 1024.\n" -"\n" -" -c ncontexts Limit the number of contexts to print.\n" -" The default is 15872.\n" -"\n" -" -C ctx Claim the interrupt on the given context.\n" -"\n" -" -w value Use this value to claim an interrupt.\n" -"\n" -"OUTPUT FORMAT\n" -" Information is printed for each source that has some bit set\n" -" in the pending or priority registers, or it has some context\n" -" enabled. The the following attributes are printed:\n" -"\n" -" src source number\n" -" pend pending bit\n" -" prio priority of the source\n" -"\n" -" Additionally, for each enabled context of the source the\n" -" following attributes are printed for that context:\n" -"\n" -" ctx context number\n" -" thre threshold value\n" -" masked the source is masked (prio <= thre)\n" -"\n" -"AUTHOR\n" -" Rodrigo Arias Mallo \n" -"\n" -"SEE ALSO\n" -" See https://github.com/riscv/riscv-plic-spec/\n" -"\n" -); - exit(1); -} - static uint32_t read_reg(void *base, size_t offset) { @@ -112,14 +68,89 @@ write_reg(void *base, size_t offset, uint32_t value) *p = value; } +uint32_t +claim_get(void *base, uint32_t ctx) +{ + return read_reg(base, 0x200004L + ctx * 0x1000); +} + +static void +claim_set(void *base, uint32_t ctx, uint32_t value) +{ + write_reg(base, 0x200004L + ctx * 0x1000, value); +} + +uint32_t +thre_get(void *base, uint32_t ctx) +{ + return read_reg(base, 0x200000L + (ctx * 0x1000L)); +} + +static void +thre_set(void *base, uint32_t ctx, uint32_t value) +{ + write_reg(base, 0x200000L + (ctx * 0x1000L), value); +} + +uint32_t +prio_get(void *base, uint32_t s) +{ + return read_reg(base, s * 4L); +} + +static void +prio_set(void *base, uint32_t s, uint32_t value) +{ + write_reg(base, s * 4L, value); +} + +uint32_t +pending_get(void *base, uint32_t s) +{ + uint32_t offset = 0x1000L + (s / 32L) * 4L; + uint32_t pending = read_reg(base, offset); + long shift = s % 32L; + + return (pending >> shift) & 1; +} + +static void +pending_set(void *base, uint32_t s, uint32_t value) +{ + uint32_t offset = 0x1000L + (s / 32L) * 4L; + uint32_t pending = read_reg(base, offset); + long shift = s % 32L; + + if (value) + pending |= (1L << shift); + else + pending &= ~(1L << shift); + + write_reg(base, offset, pending); +} + +uint32_t +enable_get(void *base, uint32_t c, uint32_t s) +{ + size_t off_en = 0x2000L + 0x80L * c + (s / 32L) * 4L; + uint32_t enabled_reg = read_reg(base, off_en); + long shift = s % 32L; + return (enabled_reg >> shift) & 1; +} + static void -claim_interrupt(void *base, uint32_t ctx, uint32_t *value) +enable_set(void *base, uint32_t c, uint32_t s, uint32_t value) { - size_t offset = 0x200004L + ctx * 0x1000; - uint32_t r = read_reg(base, offset); - uint32_t w = value ? *value : r; - write_reg(base, offset, w); - printf("claim ctx=%u r=%u w=%u\n", ctx, r, w); + size_t off_en = 0x2000L + 0x80L * c + (s / 32L) * 4L; + uint32_t enabled_reg = read_reg(base, off_en); + long shift = s % 32L; + + if (value) + enabled_reg |= (1L << shift); + else + enabled_reg &= ~(1L << shift); + + write_reg(base, off_en, enabled_reg); } static void @@ -127,6 +158,7 @@ source_init(struct source *src, long ncontexts) { memset(src, 0, sizeof(struct source)); src->ctx = calloc(ncontexts, sizeof(struct source_ctx)); + src->ncontexts = ncontexts; if (src->ctx == NULL) { perror("calloc failed"); exit(1); @@ -134,95 +166,98 @@ source_init(struct source *src, long ncontexts) } static void -source_reset(struct source *src, long ncontexts) +source_reset(struct source *src) { - memset(src, 0, sizeof(struct source)); - memset(src->ctx, 0, ncontexts * sizeof(struct source_ctx)); + src->pending = false; + src->show = false; + src->priority = 0; + + memset(src->ctx, 0, src->ncontexts * sizeof(struct source_ctx)); } static void -source_free(struct source *src, long ncontexts) +source_free(struct source *src) { free(src->ctx); } static void -source_read(struct source *src, void *base, long s, long ncontexts) +source_read(struct source *src, void *base, long s) { - uint32_t pending_reg = read_reg(base, 0x1000L + (s / 32L) * 4L) + uint32_t pending_reg = read_reg(base, 0x1000L + (s / 32L) * 4L); long shift = s % 32L; src->pending = (pending_reg >> shift) & 1; src->priority = read_reg(base, 0x0000L + (s * 4L)); - for (long c = 0; c < ncontexts; c++) { + bool ctx_show = 0; + for (long c = 0; c < src->ncontexts; c++) { struct source_ctx *ctx = &src->ctx[c]; size_t off_en = 0x2000L + 0x80L * c + (s / 32L) * 4L; - uint32_t enabled_reg = reg_read(base, off_en); - ctx->enabled = (enable_reg >> shift) & 1; + uint32_t enabled_reg = read_reg(base, off_en); + ctx->enabled = (enabled_reg >> shift) & 1; ctx->threshold = read_reg(base, 0x200000L + (c * 0x1000L)); - ctx->masked = priority <= threshold; + ctx->masked = src->priority <= ctx->threshold; + ctx_show = ctx_show || ctx->enabled; + + if (!ctx->enabled) + ctx->state = "-"; + else if (ctx->masked) + ctx->state = "masked"; + else + ctx->state = "firing"; } + + /* Show the source if it has some bit to non-zero */ + src->show = src->pending || src->priority || ctx_show; } static void list_sources(void *base) { - struct source src; - - source_init(&src, ncontexts); - for (long s = 0; s < nsources; s++) { - source_read(&src, base, s, ncontexts); - int printed_source = 0; - - uint32_t *pending_reg = base + 0x1000L + (s / 32L) * 4L; - long shift = s % 32L; - uint32_t pending = ((*pending_reg) >> shift) & 1; - uint32_t *priority_reg = base + 0x0000L + (s * 4L); - uint32_t priority = *priority_reg; - - const char *fmt = - "src=%ld pend=%u prio=%u"; - - if (pending || priority) { - printf(fmt, s, pending, priority); - printed_source = 1; - } - - int first_context = 1; - int enabled_contexts = 0; - int unmasked = 0; - for (long c = 0; c < ncontexts; c++) { - uint32_t *enable_reg = base + 0x2000L + 0x80L * c + (s / 32L) * 4L; - uint32_t enabled = ((*enable_reg) >> shift) & 1; - uint32_t *threshold_reg = base + 0x200000L + (c * 0x1000L); - uint32_t threshold = *threshold_reg; - - if (!enabled) - continue; + printf("Source\tPend\tPrio"); + for (long i = 0; i < ncontexts; i++) { + uint32_t threshold = read_reg(base, 0x200000L + (i * 0x1000L)); + printf("\tC%ld(%u)", i, threshold); + } + printf("\n"); - if (!printed_source) { - printf(fmt, s, pending, priority); - printed_source = 1; - } + struct source s; + source_init(&s, ncontexts); + for (long i = 0; i < nsources; i++) { + source_reset(&s); + source_read(&s, base, i); + if (!s.show) + continue; - int is_masked = priority <= threshold; + printf("%ld\t%s\t%u", i, s.pending ? "yes" : "-", s.priority); - if (!is_masked) - unmasked++; + for (long j = 0; j < ncontexts; j++) + printf("\t%s", s.ctx[j].state); - const char *masked = is_masked ? "masked" : "unmasked"; + printf("\n"); + } - printf("\n ctx=%ld thre=%u %s", - c, threshold, masked); - enabled_contexts++; - } + source_free(&s); +} - if (printed_source) - printf("\n"); - } +static void usage(void) +{ + printf("plictool "VERSION" -- Rodrigo Arias Mallo \n"); + + fprintf(stderr, +"Usage:\n" +" plictool [-a addr] [-L] [-n nsrc] [-x nctx] # List (default)\n" +" plictool [-a addr] -C ctx [-w value] # Claim\n" +" plictool [-a addr] -T ctx [-w value] # Threshold\n" +" plictool [-a addr] -I src [-w value] # Priority\n" +" plictool [-a addr] -P src [-w value] # Pending\n" +" plictool [-a addr] -E src -c ctx [-w value] # Enabled\n" +" plictool -v # Version\n" +); + exit(1); } int main(int argc, char *argv[]) @@ -230,41 +265,70 @@ int main(int argc, char *argv[]) const char *memfile = "/dev/mem"; int opt; - while ((opt = getopt(argc, argv, "f:a:hLs:c:C:w:")) != -1) { + while ((opt = getopt(argc, argv, "f:a:LC:T:P:I:E:n:x:c:w:vh")) != -1) { switch (opt) { - case 'f': - memfile = optarg; - break; - case 'a': - plic_address_str = optarg; - break; - case 'L': /* default operation */ - operation = LIST; - break; - case 's': - nsources = atol(optarg); - break; - case 'c': - ncontexts = atol(optarg); - break; - case 'C': - operation = CLAIM; - claim_ctx = atol(optarg); - break; - case 'w': - claim_value = atol(optarg); - claim_pvalue = &claim_value; - break; - case 'h': - default: /* '?' */ - usage(); + /* Common flags */ + case 'f': + memfile = optarg; + break; + case 'a': + plic_address_str = optarg; + break; + case 'n': + nsources = atol(optarg); + break; + case 'x': + ncontexts = atol(optarg); + break; + case 'C': /* claim */ + case 'T': /* threshold */ + operation = opt; + context = atol(optarg); + break; + case 'P': /* pending */ + case 'I': /* priority */ + case 'E': /* enable */ + operation = opt; + source = atol(optarg); + break; + case 'L': /* list */ + operation = opt; + break; + case 'c': + context = atol(optarg); + break; + case 'w': + value = atol(optarg); + value_set = true; + break; + case 'v': + printf("plictool "VERSION"\n"); + exit(0); + case 'h': + default: /* '?' */ + usage(); + break; + } + } + + if (operation == 'P' || operation == 'I' || operation == 'E') { + if (source < 0) { + fprintf(stderr, "missing source\n"); + exit(1); + } + } + + if (operation == 'C' || operation == 'T' || operation == 'E') { + if (context < 0) { + fprintf(stderr, "missing context\n"); + exit(1); } } unsigned long long plic_address = strtoull(plic_address_str, NULL, 16); - printf("plictool "VERSION" addr=0x%08llx nsrc=%ld nctx=%ld\n", - plic_address, nsources, ncontexts); + //printf("plictool "VERSION" addr=0x%08llx nsrc=%ld nctx=%ld\n", + // plic_address, nsources, ncontexts); int fd = open(memfile, O_RDWR | O_SYNC); @@ -287,10 +351,34 @@ int main(int argc, char *argv[]) exit(1); } - if (operation == CLAIM) - claim_interrupt(map_base, claim_ctx, claim_pvalue); - else if (operation == LIST) + if (operation == 'C') { /* claim */ + if (value_set) + claim_set(map_base, context, value); + else + printf("%u\n", claim_get(map_base, context)); + } else if (operation == 'T') { /* threshold */ + if (value_set) + thre_set(map_base, context, value); + else + printf("%u\n", thre_get(map_base, context)); + } else if (operation == 'I') { /* priority */ + if (value_set) + prio_set(map_base, source, value); + else + printf("%u\n", prio_get(map_base, source)); + } else if (operation == 'P') { /* pending */ + if (value_set) + pending_set(map_base, source, value); + else + printf("%u\n", pending_get(map_base, source)); + } else if (operation == 'E') { /* enable */ + if (value_set) + enable_set(map_base, context, source, value); + else + printf("%u\n", enable_get(map_base, context, source)); + } else /* list */ { list_sources(map_base); + } munmap(map_base, map_size); -- GitLab From f8886b28dded7490a9a2cb56143e98f745584dfa Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 2 Oct 2024 14:52:10 +0200 Subject: [PATCH 278/310] Go back to using console=hvc0 --- lagarto-ox.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 096f268..bf8bca6 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -494,6 +494,7 @@ # Set kernel bootcmd options. # rdinit=/preinit boot custom preinit script + # console=ttyS0,115200n8 use serial driver (slow) # Systemd options # systemd.log_level=debug # systemd.log_target=console @@ -511,7 +512,7 @@ in prev.runCommand "uboot.txt" {} '' cat > $out < Date: Fri, 4 Oct 2024 09:01:41 +0200 Subject: [PATCH 279/310] Update JOURNAL --- JOURNAL.md | 798 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 798 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index 2ca183d..e7e8668 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -4342,3 +4342,801 @@ It works, but very slowly: Let's add the ability to claim a context in the plictool, as I suspect the claim may be broken. + +## 2024-10-02 + +Interesting output with ftrace: + + + <<< NixOS Stage 1 >>> + + + An error occurred in stage 1 of the boot process, which must mount the + root filesystem on `/mnt-root' and then start stage 2. Press one + of the following keys: + + i) to launch an interactive shell + f) to start an interactive shell having pid 1 (needed if you want to + start stage 2's init manually) + r) to reboot immediately + *) to ignore the error and continue + iStarting interactive shell... + ~ # mount -t tracefs nodev /sys/kernel/tracing/ + ~ # cd /sys/kernel/tracing/ + /sys/kernel/tracing # cat current_tracer + function + /sys/kernel/tracing # cat enabled_functions + plic_irq_unmask (1) + plic_irq_mask (1) + plic_dying_cpu (1) + plic_starting_cpu (1) + plic_irq_domain_translate (1) + plic_irq_domain_alloc (1) + plic_parse_context_parent (1) + plic_irq_set_type (1) + plic_toggle.constprop.0 (1) + plic_irq_eoi (1) + plic_probe (1) + plic_irq_suspend (1) + plic_handle_irq (1) + plic_irq_enable (1) + plic_irq_disable (1) + plic_irq_resume (1) + /sys/kernel/tracing # cat tracing_on + 1 + /sys/kernel/tracing # cat trac + trace trace_options tracing_max_latency + trace_clock trace_pipe tracing_on + trace_marker trace_stat/ tracing_thresh + trace_marker_raw tracing_cpumask + /sys/kernel/tracing # cat trace + # tracer: function + # + # entries-in-buffer/entries-written: 1586/1586 #P:1 + ... + /sys/kernel/tracing # cat options/function-trace + 1 + /sys/kernel/tracing # plictool + Source Pend Prio C0(7) C1(0) + 1 - 1 - - + 2 - 1 - - + 3 - 1 - - + 4 yes 1 - - + /sys/kernel/tracing # cat tracing_on + 1 + /sys/kernel/tracing # echo 0>tracing_on + + /sys/kernel/tracing # cat tracing_on + 1 + /sys/kernel/tracing # echo 0 > tracing_on + /sys/kernel/tracing # cat tracing_on + 0 + /sys/kernel/tracing # echo plic_handle_irq > set_ftrace_filter + /sys/kernel/tracing # cat enabled_functions + plic_handle_irq (1) + /sys/kernel/tracing # echo plic_handle_irq,plic_irq_eoi,plic_irq_enable,plic_irq + _disable > set_ftrace_filter ^C + + /sys/kernel/tracing # echo plic_irq_eoi >> set_ftrace_filter + /sys/kernel/tracing # cat enabled_functions + plic_irq_eoi (1) + plic_handle_irq (1) + /sys/kernel/tracing # echo plic_irq_enable >> set_ftrace_filter + /sys/kernel/tracing # plictool + Source Pend Prio C0(7) C1(0) + 1 - 1 - - + 2 - 1 - - + 3 - 1 - - + 4 yes 1 - - + /sys/kernel/tracing # ^Ct enabled_functions + + /sys/kernel/tracing # echo hi > /dev/ttyS1 + ^Z^C + /sys/kernel/tracing # plictool + Source Pend Prio C0(7) C1(0) + 1 yes 1 - - + 2 - 1 - - + 3 - 1 - - + 4 yes 1 - - + /sys/kernel/tracing # echo plic_irq_enable >> set_f^Cace_filter + + /sys/kernel/tracing # cat tracing_on + 0 + /sys/kernel/tracing # echo 1 > tracing_on + /sys/kernel/tracing # cat tracing_on + 1 + /sys/kernel/tracing # cat trace_pipe & + /sys/kernel/tracing # fg + cat trace_pipe + ^Z[1]+ Stopped cat trace_pipe + /sys/kernel/tracing # bg + [1] cat trace_pipe + /sys/kernel/tracing # echo hi > /dev/ttyS1 + ash-91 [000] d..2. 669.966205: plic_irq_enable <-irq_enable + ash-91 [000] d..2. 669.966815: plic_irq_enable <-irq_enable + + *** local echo: yes *** + + *** local echo: no *** + ^C + /sys/kernel/tracing # echo 0 > tracing_on + /sys/kernel/tracing # cat tracing_on + 0 + /sys/kernel/tracing # echo 'plic_*' >> set_ftrace_filter + /sys/kernel/tracing # cat enabled_functions + plic_irq_unmask (1) + plic_irq_mask (1) + plic_dying_cpu (1) + plic_starting_cpu (1) + plic_irq_domain_translate (1) + plic_irq_domain_alloc (1) + plic_parse_context_parent (1) + plic_irq_set_type (1) + plic_toggle.constprop.0 (1) + plic_irq_eoi (1) + plic_probe (1) + plic_irq_suspend (1) + plic_handle_irq (1) + plic_irq_enable (1) + plic_irq_disable (1) + plic_irq_resume (1) + /sys/kernel/tracing # echo 1 > tracing_on + /sys/kernel/tracing # cat trace_pipe & + /sys/kernel/tracing # cat: can't open 'trace_pipe': Device or resource busy + + [2]+ Done(1) cat trace_pipe + /sys/kernel/tracing # ps + PID USER TIME COMMAND + 1 0 0:33 {init} /nix/store/y477q6jlbg9b53knnclgib65fbzpazkj-extra-u + 2 0 0:00 [kthreadd] + 3 0 0:00 [pool_workqueue_] + 4 0 0:00 [kworker/R-slub_] + 5 0 0:00 [kworker/R-netns] + 6 0 0:00 [kworker/0:0-eve] + 7 0 0:00 [kworker/0:0H] + 8 0 0:03 [kworker/u4:0-ev] + 9 0 0:00 [kworker/R-mm_pe] + 10 0 0:41 [kworker/u4:1-ev] + 11 0 0:00 [rcu_tasks_rude_] + 12 0 0:00 [rcu_tasks_trace] + 13 0 0:00 [ksoftirqd/0] + 14 0 0:00 [kworker/0:1-eve] + 15 0 0:01 [kdevtmpfs] + 16 0 0:00 [kworker/R-inet_] + 17 0 0:00 [kauditd] + 18 0 0:00 [khungtaskd] + 20 0 0:00 [oom_reaper] + 21 0 0:00 [kworker/R-write] + 22 0 0:01 [kcompactd0] + 23 0 0:00 [ksmd] + 24 0 0:00 [kworker/R-kinte] + 25 0 0:00 [kworker/R-kbloc] + 26 0 0:00 [kworker/R-blkcg] + 27 0 0:00 [kworker/R-ata_s] + 28 0 0:00 [kworker/R-devfr] + 29 0 0:00 [watchdogd] + 30 0 0:01 [kworker/u4:3-ev] + 31 0 0:00 [kworker/R-rpcio] + 32 0 0:00 [kworker/R-xprti] + 33 0 0:00 [kswapd0] + 34 0 0:00 [kworker/R-nfsio] + 35 0 0:00 [kworker/R-kthro] + 36 0 0:09 [khvcd] + 37 0 0:00 [kworker/R-uas] + 38 0 0:00 [kworker/R-mld] + 39 0 0:00 [kworker/R-ipv6_] + 46 0 0:00 [kworker/R-kstrp] + 47 0 0:00 [kworker/u5:0] + 73 0 0:00 tee -i /proc/self/fd/8 + 74 0 0:00 {init} /nix/store/y477q6jlbg9b53knnclgib65fbzpazkj-extra-u + 91 0 0:46 /nix/store/y477q6jlbg9b53knnclgib65fbzpazkj-extra-utils/bi + 113 0 0:00 cat trace_pipe + 117 0 0:00 ps + /sys/kernel/tracing # fg + cat trace_pipe + ^C + /sys/kernel/tracing # cat trace_pipe & + /sys/kernel/tracing # echo 1 > tracing_on + /sys/kernel/tracing # echo hi > /dev/ttyS1 + ash-91 [000] d..2. 748.552248: plic_irq_enable <-irq_enable + ash-91 [000] d..2. 748.552370: plic_toggle.constprop.0 <-plic_irq_enable + ash-91 [000] d..2. 748.552797: plic_irq_disable <-irq_disable + ash-91 [000] d..2. 748.552858: plic_toggle.constprop.0 <-plic_irq_disable + ash-91 [000] d..2. 748.553102: plic_irq_enable <-irq_enable + ash-91 [000] d..2. 748.553163: plic_toggle.constprop.0 <-plic_irq_enable + ^C + /sys/kernel/tracing # ash-91 [000] d..2. 762.013481: plic_irq_disable <-__irq_disable + ash-91 [000] d..2. 762.013603: plic_toggle.constprop.0 <-plic_irq_disable + + /sys/kernel/tracing # + /sys/kernel/tracing # plictool + Source Pend Prio C0(7) C1(0) + 1 yes 1 - - + 2 - 1 - - + 3 - 1 - - + 4 yes 1 - - + /sys/kernel/tracing # plictool -C 1 + 0 + /sys/kernel/tracing # plictool -C 1 -w 4 + /sys/kernel/tracing # plictool -C 1 + 0 + /sys/kernel/tracing # plictool + Source Pend Prio C0(7) C1(0) + 1 yes 1 - - + 2 - 1 - - + 3 - 1 - - + 4 yes 1 - - + /sys/kernel/tracing # devmem 0x40014000 32 0xffffffff + /sys/kernel/tracing # plictool + Source Pend Prio C0(7) C1(0) + 1 yes 1 - - + 2 - 1 - - + 3 - 1 - - + 4 yes 1 - - + /sys/kernel/tracing # plictool -C 1 -w 4 + /sys/kernel/tracing # plictool + Source Pend Prio C0(7) C1(0) + 1 yes 1 - - + 2 - 1 - - + 3 - 1 - - + 4 yes 1 - - + /sys/kernel/tracing # plictool -E + plictool: option requires an argument: E + plictool v0.0.5 -- Rodrigo Arias Mallo + Usage: + plictool [-a addr] [-L] [-n nsrc] [-x nctx] # List (default) + plictool [-a addr] -C ctx [-w value] # Claim + plictool [-a addr] -T ctx [-w value] # Threshold + plictool [-a addr] -I src [-w value] # Priority + plictool [-a addr] -P src [-w value] # Pending + plictool [-a addr] -E src -c ctx [-w value] # Enabled + plictool -v # Version + /sys/kernel/tracing # plictool -E 4 -c 0 -w 1 + /sys/kernel/tracing # plictool + Source Pend Prio C0(7) C1(0) + 1 yes 1 - - + 2 - 1 - - + 3 - 1 - - + 4 yes 1 masked - + 36 - 0 masked - + /sys/kernel/tracing # plictool -C 0 + 4 + /sys/kernel/tracing # plictool -C 0 -w 4 + /sys/kernel/tracing # plictool + Source Pend Prio C0(7) C1(0) + 1 yes 1 - - + 2 - 1 - - + 3 - 1 - - + 4 - 1 masked - + 36 - 0 masked - + /sys/kernel/tracing # plictool -E 4 -c 0 -w 0 + /sys/kernel/tracing # plictool + Source Pend Prio C0(7) C1(0) + 1 yes 1 - - + 2 - 1 - - + 3 - 1 - - + 4 - 1 - - + /sys/kernel/tracing # cat /dev/ttyS1 & + /sys/kernel/tracing # cat-136 [000] d..2. 887.106509: plic_irq_enable <-irq_enable + cat-136 [000] d..2. 887.106631: plic_toggle.constprop.0 <-plic_irq_enable + cat-136 [000] d.h2. 887.106814: plic_handle_irq <-generic_handle_domain_irq + cat-136 [000] d.h3. 887.106997: plic_irq_eoi <-handle_fasteoi_irq + cat-136 [000] d..2. 887.107424: plic_irq_disable <-irq_disable + cat-136 [000] d..2. 887.107546: plic_toggle.constprop.0 <-plic_irq_disable + cat-136 [000] d..2. 887.107729: plic_irq_enable <-irq_enable + cat-136 [000] d..2. 887.107790: plic_toggle.constprop.0 <-plic_irq_enable + + /sys/kernel/tracing # plictool + Source Pend Prio C0(7) C1(0) + 1 yes 1 - firing + 2 - 1 - - + 3 - 1 - - + 4 - 1 - - + 33 - 0 - masked + +So, then an interrupt is fired, we disable the interrupts and then we enable +them again. Can this happen because the next interrupt fired when interrupts +were disabled? + +There are also other cases in which we already have an interrupt pending and we +enable the context 1. For example, by setting the enable bit, changing the prio, +or the threshold for that context or by removing the context 0 enable bit. + +I think I can do some tests with those. I'm not sure why the serial port is +firing an interrupt, but if this is the case, I can try to write a character in +the transmit register and wait for the pending bit, then try to claim the +interrupt for source 1. + +## 2024-10-03 + + /sys/kernel/tracing # plictool + Source Pend Prio C0(7) C1(0) + 1 yes 1 - - + 2 - 1 - - + 3 - 1 - - + 4 yes 1 - - + /sys/kernel/tracing # cat tracing_on + 1 + /sys/kernel/tracing # echo 0 > tracing_on + /sys/kernel/tracing # echo 1 > tracing_on + /sys/kernel/tracing # cat trace_pipe & + /sys/kernel/tracing # cat /dev/ttyS1 & + /sys/kernel/tracing # cat-122 [000] ...1. 2978.202587: serial8250_pm <-uart_port_startup + cat-122 [000] ...1. 2978.202648: serial8250_set_sleep <-serial8250_pm + cat-122 [000] ...1. 2978.202892: serial8250_startup <-uart_port_startup + cat-122 [000] ...1. 2978.203014: serial8250_do_startup <-serial8250_startup + cat-122 [000] d..2. 2978.203319: plic_irq_enable <-irq_enable + cat-122 [000] d..2. 2978.203380: plic_toggle.constprop.0 <-plic_irq_enable + cat-122 [000] d.h2. 2978.203563: plic_handle_irq <-generic_handle_domain_irq + cat-122 [000] d.h2. 2978.203685: serial8250_interrupt <-__handle_irq_event_percpu + cat-122 [000] d.h3. 2978.203746: serial8250_default_handle_irq <-serial8250_interrupt + cat-122 [000] d.h3. 2978.203807: serial8250_handle_irq <-serial8250_default_handle_irq + cat-122 [000] d.h3. 2978.203990: plic_irq_eoi <-handle_fasteoi_irq + cat-122 [000] d..2. 2978.204783: plic_irq_disable <-irq_disable + cat-122 [000] d..2. 2978.204844: plic_toggle.constprop.0 <-plic_irq_disable + cat-122 [000] d..2. 2978.205088: plic_irq_enable <-irq_enable + cat-122 [000] d..2. 2978.205149: plic_toggle.constprop.0 <-plic_irq_enable + cat-122 [000] d..2. 2978.205271: serial8250_do_set_mctrl <-serial8250_do_startup + cat-122 [000] ...1. 2978.205393: serial8250_set_termios <-uart_change_line_settings + cat-122 [000] ...1. 2978.206796: serial8250_do_set_termios <-serial8250_set_termios + cat-122 [000] .n.1. 2978.206918: serial8250_get_divisor <-serial8250_do_set_termios + cat-122 [000] dn.2. 2978.207040: serial8250_do_set_divisor <-serial8250_do_set_termios + cat-122 [000] dn.2. 2978.207101: serial8250_do_set_mctrl <-serial8250_do_set_termios + cat-122 [000] dn.2. 2978.207345: serial8250_set_mctrl <-uart_update_mctrl + cat-122 [000] dn.2. 2978.207406: serial8250_do_set_mctrl <-serial8250_set_mctrl + + /sys/kernel/tracing # plictool + Source Pend Prio C0(7) C1(0) + 1 yes 1 - firing + 2 - 1 - - + 3 - 1 - - + 4 yes 1 - - + 33 - 0 - masked + /sys/kernel/tracing # plictool -C 1 + 0 + +Another test tracing the UART serial too. + +It would be nice to see where those calls are coming from. + +Let's collect the commands I need to use here: + + mount -t tracefs nodev /sys/kernel/tracing/ + cd /sys/kernel/tracing/ + echo 'plic_*' >> set_ftrace_filter + echo 'serial8250_*' >> set_ftrace_filter + cat current_tracer + +Here is a trace with the stack trace printed: + + ~ # plictool + Source Pend Prio C0(7) C1(0) + 1 - 1 - - + 2 - 1 - - + 3 - 1 - - + 4 yes 1 - - + + ~ # + [...] + /sys/kernel/tracing # cat /dev/ttyS1 & + + cat-122 [000] ..... 591.693894: serial8250_pm <-uart_port_startup + cat-122 [000] ..... 591.694077: + => ftrace_call + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] ..... 591.694138: serial8250_set_sleep <-serial8250_pm + cat-122 [000] ..... 591.694260: + => ftrace_call + => uart_port_startup + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] ..... 591.694504: serial8250_startup <-uart_port_startup + cat-122 [000] ..... 591.694687: + => ftrace_call + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] ..... 591.694748: serial8250_do_startup <-serial8250_startup + cat-122 [000] ..... 591.694870: + => ftrace_call + => uart_port_startup + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] d..1. 591.695236: plic_irq_enable <-irq_enable + cat-122 [000] d..1. 591.695358: + => ftrace_call + => irq_startup + => __setup_irq + => request_threaded_irq + => univ8250_setup_irq + => serial8250_do_startup + => serial8250_startup + => uart_port_startup + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] d..1. 591.695480: plic_toggle.constprop.0 <-plic_irq_enable + cat-122 [000] d..1. 591.695602: + => ftrace_call + => irq_enable + => irq_startup + => __setup_irq + => request_threaded_irq + => univ8250_setup_irq + => serial8250_do_startup + => serial8250_startup + => uart_port_startup + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] d..1. 591.696456: plic_irq_disable <-irq_disable + cat-122 [000] d..1. 591.696578: + => ftrace_call + => __disable_irq_nosync + => disable_irq_nosync + => serial8250_do_startup + => serial8250_startup + => uart_port_startup + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] d..1. 591.696639: plic_toggle.constprop.0 <-plic_irq_disable + cat-122 [000] d..1. 591.696822: + => ftrace_call + => irq_disable + => __disable_irq_nosync + => disable_irq_nosync + => serial8250_do_startup + => serial8250_startup + => uart_port_startup + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] d..1. 591.697005: plic_irq_enable <-irq_enable + cat-122 [000] d..1. 591.697188: + => ftrace_call + => irq_startup + => __enable_irq + => enable_irq + => serial8250_do_startup + => serial8250_startup + => uart_port_startup + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] d..1. 591.697249: plic_toggle.constprop.0 <-plic_irq_enable + cat-122 [000] d..1. 591.697432: + => ftrace_call + => irq_enable + => irq_startup + => __enable_irq + => enable_irq + => serial8250_do_startup + => serial8250_startup + => uart_port_startup + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] d.h1. 591.697554: plic_handle_irq <-generic_handle_domain_irq + cat-122 [000] d.h1. 591.697676: + => ftrace_call + => riscv_intc_irq + => handle_riscv_irq + => call_on_irq_stack + => 0x1c003b9260 + cat-122 [000] d.h1. 591.697798: serial8250_interrupt <-__handle_irq_event_percpu + cat-122 [000] d.h1. 591.697920: + => ftrace_call + => handle_irq_event + => handle_fasteoi_irq + => generic_handle_domain_irq + => plic_handle_irq + => generic_handle_domain_irq + => riscv_intc_irq + => handle_riscv_irq + => call_on_irq_stack + => 0x1c001dc940 + cat-122 [000] d.h2. 591.697981: serial8250_default_handle_irq <-serial8250_interrupt + cat-122 [000] d.h2. 591.698103: + => ftrace_call + => __handle_irq_event_percpu + => handle_irq_event + => handle_fasteoi_irq + => generic_handle_domain_irq + => plic_handle_irq + => generic_handle_domain_irq + => riscv_intc_irq + => handle_riscv_irq + => call_on_irq_stack + => 0x1c001dc940 + cat-122 [000] d.h2. 591.698164: serial8250_handle_irq <-serial8250_default_handle_irq + cat-122 [000] d.h2. 591.698286: + => ftrace_call + => serial8250_interrupt + => __handle_irq_event_percpu + => handle_irq_event + => handle_fasteoi_irq + => generic_handle_domain_irq + => plic_handle_irq + => generic_handle_domain_irq + => riscv_intc_irq + => handle_riscv_irq + => call_on_irq_stack + => 0x1c00595ba0 + cat-122 [000] d.h2. 591.698469: plic_irq_eoi <-handle_fasteoi_irq + cat-122 [000] d.h2. 591.698530: + => ftrace_call + => generic_handle_domain_irq + => plic_handle_irq + => generic_handle_domain_irq + => riscv_intc_irq + => handle_riscv_irq + => call_on_irq_stack + => 0x1c00595ba0 + cat-122 [000] d..1. 591.698713: serial8250_do_set_mctrl <-serial8250_do_startup + cat-122 [000] d..1. 591.698835: + => ftrace_call + => serial8250_startup + => uart_port_startup + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] .n... 591.700177: serial8250_set_termios <-uart_change_line_settings + cat-122 [000] .n... 591.700299: + => ftrace_call + => uart_port_startup + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] .n... 591.700421: serial8250_do_set_termios <-serial8250_set_termios + cat-122 [000] .n... 591.700543: + => ftrace_call + => uart_change_line_settings + => uart_port_startup + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] .n... 591.700604: serial8250_get_divisor <-serial8250_do_set_termios + cat-122 [000] .n... 591.700787: + => ftrace_call + => serial8250_set_termios + => uart_change_line_settings + => uart_port_startup + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] dn.1. 591.700909: serial8250_do_set_divisor <-serial8250_do_set_termios + cat-122 [000] dn.1. 591.701031: + => ftrace_call + => serial8250_set_termios + => uart_change_line_settings + => uart_port_startup + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] dn.1. 591.701092: serial8250_do_set_mctrl <-serial8250_do_set_termios + cat-122 [000] dn.1. 591.701214: + => ftrace_call + => serial8250_set_termios + => uart_change_line_settings + => uart_port_startup + => uart_port_activate + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] dn.1. 591.701458: serial8250_set_mctrl <-uart_update_mctrl + cat-122 [000] dn.1. 591.701580: + => ftrace_call + => uart_dtr_rts + => tty_port_block_til_ready + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + cat-122 [000] dn.1. 591.701702: serial8250_do_set_mctrl <-serial8250_set_mctrl + cat-122 [000] dn.1. 591.701885: + => ftrace_call + => uart_update_mctrl + => uart_dtr_rts + => tty_port_block_til_ready + => tty_port_open + => uart_open + => tty_open + => chrdev_open + => do_dentry_open + => vfs_open + => path_openat + => do_filp_open + => do_sys_openat2 + => __riscv_sys_openat + => do_trap_ecall_u + => ret_from_exception + + /sys/kernel/tracing # plictool + Source Pend Prio C0(7) C1(0) + 1 yes 1 - firing + 2 - 1 - - + 3 - 1 - - + 4 yes 1 - - + 33 - 0 - masked + + /sys/kernel/tracing # plictool -C 1 + 0 + +## 2024-10-04 + +To trace which accesses are done to the PLIC, I can use the mmiotracer, which +will hopefully record how we configure the PLIC and lead to a reproducer. + + CONFIG_MMIOTRACE=y -- GitLab From 17d8e3c41b2f31eb0f9fe3d10a4d84717ddf8ceb Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 4 Oct 2024 09:03:09 +0200 Subject: [PATCH 280/310] Enable mmiotrace --- lagarto-ox.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index bf8bca6..ae1c560 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -145,6 +145,7 @@ TRACING y BOOTTIME_TRACING y STRICT_DEVMEM n + MMIOTRACE y '' # Disable SMP so we don't have IPI + '' -- GitLab From 16ee113cb879b6572dff15a1ae6c1ae0631326c1 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 4 Oct 2024 09:50:32 +0200 Subject: [PATCH 281/310] Add SPI variant --- dts/lagarto_ox.dts | 26 ++++++++++++++++++++++++++ dts/lagarto_ox.h | 3 +++ 2 files changed, 29 insertions(+) diff --git a/dts/lagarto_ox.dts b/dts/lagarto_ox.dts index 48a2442..1c0ea21 100644 --- a/dts/lagarto_ox.dts +++ b/dts/lagarto_ox.dts @@ -267,5 +267,31 @@ // xlnx,count-width = <0x20>; // xlnx,one-timer-only = <0x0>; //}; + + + #ifdef ENABLE_SPI + uart16750: serial@40005000 { + compatible = "ns16750"; + reg = <0x00000000 0x40005000 0x00000000 0x00001000>; + interrupt-parent = <&PLIC>; + interrupts = <5>; + clock-frequency = ; + current-speed = <0x0001c200>; + status = "okay"; + }; + spi0: spi@40007000 { + compatible = "ti,keystone-spi"; + reg = <0x00000000 0x40007000 0x00000000 0x00001000>; + #address-cells = <1>; + #size-cells = <0>; + interrupt-parent = <&PLIC>; + interrupt-names = "intvec0", "intvec1"; + interrupts = <6 0>, <0x00000007 0>; + ti,davinci-spi-intr-line = <0>; + spi-max-frequency = <24000000>; + loopback-mode = <1>; + status = "okay"; + }; + #endif }; }; diff --git a/dts/lagarto_ox.h b/dts/lagarto_ox.h index 1d5df29..9e83bb0 100644 --- a/dts/lagarto_ox.h +++ b/dts/lagarto_ox.h @@ -16,3 +16,6 @@ #define PLIC_ADDR 0x40800000 #define UART_ADDR 0x40001000 + +/* Toggles */ +#define ENABLE_SPI -- GitLab From 18a6e749d5ad8b5f8817d3bf1312391621411ebf Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 4 Oct 2024 10:52:02 +0200 Subject: [PATCH 282/310] Clean DTS --- dts/lagarto_ox.dts | 66 +++++++++++----------------------------------- 1 file changed, 16 insertions(+), 50 deletions(-) diff --git a/dts/lagarto_ox.dts b/dts/lagarto_ox.dts index 1c0ea21..df93255 100644 --- a/dts/lagarto_ox.dts +++ b/dts/lagarto_ox.dts @@ -195,25 +195,10 @@ interrupt-controller; /* Receives interrupts */ #address-cells = <0>; #interrupt-cells = <1>; - /* Sends interrupts to HART interrupt controllers */ - - /* - * From: linux-6.6.1/arch/riscv/include/asm/csr.h - * - * Interrupt causes (minus the high bit) - * #define IRQ_S_SOFT 1 - * #define IRQ_VS_SOFT 2 - * #define IRQ_M_SOFT 3 - * #define IRQ_S_TIMER 5 - * #define IRQ_VS_TIMER 6 - * #define IRQ_M_TIMER 7 - * #define IRQ_S_EXT 9 - * #define IRQ_VS_EXT 10 - * #define IRQ_M_EXT 11 - * #define IRQ_S_GEXT 12 - * #define IRQ_PMU_OVF 13 - * #define IRQ_LOCAL_MAX (IRQ_PMU_OVF + 1) - * #define IRQ_LOCAL_MASK GENMASK((IRQ_LOCAL_MAX - 1), 0) + /* Sends interrupts to HART interrupt controllers. + * Configures two output targets or contexts: + * - context 0: machine mode external interrupt (11) + * - context 1: supervisor mode external interrupt (9) */ interrupts-extended = <&HLIC0 11>, <&HLIC0 9>; reg = < 0x0 0x40800000 0x0 0x00400000>; @@ -236,38 +221,19 @@ interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; compatible = "riscv,clint0"; }; -// aux_timer: clint@40010000 { -// reg = <0x0 0x40010000 0x0 0x00010000>; -// reg-names = "control"; -// interrupts = <4>; /* PLIC input source 4 */ -// interrupt-parent = <&PLIC>; -// compatible = "riscv,clint0"; -// }; -// clint: clint@40002000 { -// /* MTIME and MTIMECMP address and size pairs */ -// reg = <0x0 0x40002000 0x0 0x8>, <0x0 0x40002008 0x0 0x8>; -// interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; -// /*<&CPU0 0x3>, <&CPU0 0x7>,*/ -// /*<&onic_pool 0x3>, <&onic_pool 0x7>,*/ -// /*<&SERIAL 0x3>, <&SERIAL 0x7>*/ -// -// /* Allows using the "generic" platform in OpenSBI. */ -// compatible = "riscv,aclint-mtimer"; -// }; - - /* Guesswork: There must be a timer at 0x40170000 as it is - * initialized in OpenSBI. It seems to drive the console. */ - //SERIAL_CLK: timer@40170000 { - // clock-frequency = <100000000>; - // clocks = <&clk_bus_0>; - // compatible = "xlnx,xps-timer-1.00.a"; - // interrupt-parent = <&axi_intc_1>; - // interrupts = <2 2>; - // reg = <0x41c00000 0x10000>; - // xlnx,count-width = <0x20>; - // xlnx,one-timer-only = <0x0>; - //}; + /* There is another auxiliar clint (timer) at 40010000 for + * tests, but we don't tell the kernel so we can use it for + * testing interrupts manually. */ + #if 0 + aux_timer: clint@40010000 { + reg = <0x0 0x40010000 0x0 0x00010000>; + reg-names = "control"; + interrupts = <4>; /* PLIC input source 4 */ + interrupt-parent = <&PLIC>; + compatible = "riscv,clint0"; + }; + #endif #ifdef ENABLE_SPI uart16750: serial@40005000 { -- GitLab From 6b262a4afd35eef52665dba59466397098597f06 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 4 Oct 2024 10:02:55 +0200 Subject: [PATCH 283/310] Add spi_davinci module --- lagarto-ox.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index ae1c560..318306f 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -151,6 +151,12 @@ + '' SMP n '' + # SPI driver + + '' + COMPILE_TEST y + SPI y + SPI_DAVINCI m + '' ; } ]; @@ -163,6 +169,9 @@ #"xilinx_dma" # Load the Ethernet module by default #"xxvnet_carv" + + # For SPI + "spi_davinci" ]; # Custom init script -- GitLab From 35eeddacda32554eb75cadc6bf313271c1a2e594 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 4 Oct 2024 13:17:31 +0200 Subject: [PATCH 284/310] Include spidev module --- lagarto-ox.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 318306f..e4c2ce7 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -172,6 +172,7 @@ # For SPI "spi_davinci" + "spidev" ]; # Custom init script -- GitLab From 056449243a8ea02a953caaa4875935907ec28d6f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 4 Oct 2024 13:17:44 +0200 Subject: [PATCH 285/310] Stop in stage1 too --- lagarto-ox.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index e4c2ce7..47645f6 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -523,7 +523,7 @@ in prev.runCommand "uboot.txt" {} '' cat > $out < Date: Fri, 4 Oct 2024 13:49:57 +0200 Subject: [PATCH 286/310] Disable SPI nodes in DT --- dts/lagarto_ox.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dts/lagarto_ox.h b/dts/lagarto_ox.h index 9e83bb0..6a92d95 100644 --- a/dts/lagarto_ox.h +++ b/dts/lagarto_ox.h @@ -18,4 +18,4 @@ #define UART_ADDR 0x40001000 /* Toggles */ -#define ENABLE_SPI +//#define ENABLE_SPI -- GitLab From a9350da4b8a8ebf4310691b326eb44fc04966c8e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 7 Oct 2024 09:23:28 +0200 Subject: [PATCH 287/310] Add SPEC CPU 2017 benchmarks --- overlay.nix | 4 + pkgs/spec-cpu/Makefile | 66 ++++++ pkgs/spec-cpu/Makefile.in | 80 +++++++ pkgs/spec-cpu/default.nix | 59 +++++ pkgs/spec-cpu/gcc-linux-x86.cfg | 391 ++++++++++++++++++++++++++++++++ pkgs/spec-cpu/tools.nix | 106 +++++++++ 6 files changed, 706 insertions(+) create mode 100644 pkgs/spec-cpu/Makefile create mode 100644 pkgs/spec-cpu/Makefile.in create mode 100644 pkgs/spec-cpu/default.nix create mode 100644 pkgs/spec-cpu/gcc-linux-x86.cfg create mode 100644 pkgs/spec-cpu/tools.nix diff --git a/overlay.nix b/overlay.nix index 8527ad1..5164c54 100644 --- a/overlay.nix +++ b/overlay.nix @@ -11,6 +11,10 @@ final: prev: rvb-clang = final.callPackage ./pkgs/rvb/default.nix { stdenv = final.stdenvClangEpi; }; stream = final.callPackage ./pkgs/stream/default.nix { }; + spec-cpu-tools = final.callPackage ./pkgs/spec-cpu/tools.nix { }; + spec-cpu = final.callPackage ./pkgs/spec-cpu/default.nix { }; + spec-cpu-clang = final.callPackage ./pkgs/spec-cpu/default.nix { stdenv = final.stdenvClangEpi; }; + blis = ((prev.blis.override { blas64 = true; withArchitecture = "generic"; diff --git a/pkgs/spec-cpu/Makefile b/pkgs/spec-cpu/Makefile new file mode 100644 index 0000000..5d6519e --- /dev/null +++ b/pkgs/spec-cpu/Makefile @@ -0,0 +1,66 @@ +include Makefile.in + +HPC_BENCHMARKS_DIRS=\ + axpy \ + jacobi-2d \ + somier \ +# Require submodules: +# fft \ +# fftp \ +# Require vehave: +# lulesh \ +# Missing compare_array_double: +# gemm \ +# Broken, not found: +# spmv + +DESKTOP_BENCHMARKS_DIRS=\ + blackscholes \ + canneal \ + particlefilter \ + pathfinder \ + streamcluster \ + swaptions + +MICRO_BENCHMARKS_DIRS=\ + BuffCopyUnit \ + BuffCopyStrided \ + BuffCopyIndexed \ + FpuMicroKernel \ + InstrNopBalance \ + MemArithBalance \ + LatencyVrgather + +.PHONY: default all clean $(HPC_BENCHMARKS_DIRS) $(MICRO_BENCHMARKS_DIRS) + +default: + @cd common; make; cd .. + @$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir);) + @$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir);) + @$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir);) + +all: + @cd common; make all; cd .. + @$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) all ;) + @$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) all ;) + @$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) all ;) + +fftp: + ${MAKE} -C third_party fftw + ${MAKE} -C hpc_benchmarks/fftp all + ${MAKE} -C hpc_benchmarks/fftp/test all + +spmv-ellpack: + rm -rf hpc_benchmarks/spmv-ellpack/spmv/build + mkdir -p hpc_benchmarks/spmv-ellpack/spmv/build + cd hpc_benchmarks/spmv-ellpack/spmv/build;\ + ../configure riscv;\ + INDEX64=1 EPI_EXT=07 PATH=${EPI_LLVM_HOME}/bin:${PATH} make + +clean: + @cd common; make clean; cd .. + @$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) clean ;) + @$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) clean ;) + @$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) clean ;) + @rm -rf hpc_benchmarks/spmv-ellpack/spmv/build + diff --git a/pkgs/spec-cpu/Makefile.in b/pkgs/spec-cpu/Makefile.in new file mode 100644 index 0000000..b828f6e --- /dev/null +++ b/pkgs/spec-cpu/Makefile.in @@ -0,0 +1,80 @@ +#Compile all benchmarks with individual settings defined in their Makefiles + +_default-target: default + +# RVB_ROOT defined as argument + +# TODO: RVB_COMMON_DIR should substitute COMMON_DIR +COMMON_DIR=$(RVB_ROOT)/common +RVB_COMMON_DIR=$(RVB_ROOT)/common + +CC=clang +CXX=clang++ + +# Needs EPI clang +#MEPI=-mepi +MEPI= + +VREPORT_FLAGS=-Rpass=loop-vectorize -Rpass-missed=loop-vectorize -Rpass-analysis=loop-vectorize +#Available CFLAGS conditional compilation: +# -DUSE_MALLOC_HP, redefines the use of malloc and free + +#Makefile: General compiler flags: CFLAGS, CFLAGS_, CFLAGS_, ... +#CFLAGS=-B ${LLVM_BIN} -Wall -Wextra -march=rv64g -O2 -I${RVB_COMMON_DIR} +CFLAGS=-B ${LLVM_BIN} -Wall -Wextra -O2 -I${RVB_COMMON_DIR} +LDFLAGS= + +#Only scalar instructions +CFLAGS_SCALAR=${CFLAGS} -DRVB_USE_SCALAR +LDFLAGS_SCALAR= + +#Vector instructions using intrinsics +CFLAGS_VECTORIAL=${CFLAGS} -fno-vectorize $(MEPI) +LDFLAGS_VECTORIAL= + +# TODO: RVV should substitute VECTORIAL (?) +#Vector instructions using intrinsics +CFLAGS_RVV=${CFLAGS} -DRVB_USE_RVV -fno-vectorize $(MEPI) +LDFLAGS_RVV=${LDFLAGS} + +#Vector instructions only when code is annotate +CFLAGS_EXPLICIT_AUTOVECTORIZATION=${CFLAGS} -fopenmp-simd -fno-vectorize $(MEPI) ${VREPORT_FLAGS} +LDFLAGS_EXPLICIT_AUTOVECTORIZATION= + +# TODO: OMP substitutes EXPLICIT_AUTOVECTORIZATION +#Vector instructions only when code is annotate +CFLAGS_OMP=${CFLAGS} -DRVB_USE_OMP -fopenmp-simd -fno-vectorize $(MEPI) ${VREPORT_FLAGS} +LDFLAGS_OMP=${LDFLAGS} + +#Vector instructions when compiler decides +CFLAGS_AUTOVECTORIZATION=${CFLAGS} -fopenmp-simd $(MEPI) ${VREPORT_FLAGS} +LDFLAGS_AUTOVECTORIZATION= + +# TODO: AUTOVECT shoud substitue AUTOVECTORIZATION (?) +#Vector instructions when compiler decides +CFLAGS_AUTOVECT=${CFLAGS} -DRVB_USE_AUTOVECT -fopenmp-simd $(MEPI) ${VREPORT_FLAGS} +LDFLAGS_AUTOVECT=${LDFLAGS} + +# CBLAS library compile and link flags +CFLAGS_CBLAS=${CFLAGS} -DRVB_USE_CBLAS -I${CBLAS_INC} -I${LLVM_INC} +LDFLAGS_CBLAS=${LDFLAGS} -L${CBLAS_LIB} -lblis -Wl,-rpath,${CBLAS_LIB} -fopenmp + +# BARE-METAL compile and link flags +# You may consider to disable OpenPiton Stream: -DDISABLE_OPSTREAM +CFLAGS_BMETAL=${CFLAGS} -DRVB_USE_BMETAL -B ${RVB_BMETAL_DIR} \ + --target=riscv64-unknown-elf \ + -I../ \ + -I/apps/riscv/llvm/EPI-0.7/cross/development/riscv64-unknown-linux-gnu/sysroot/usr/include \ + -I/usr/include/riscv64-linux-gnu \ + -I/usr/include \ + -I${RVB_BMETAL_DIR}/env \ + -I${RVB_BMETAL_DIR} \ + -DPREALLOCATE=1 -mcmodel=medany \ + -static -std=gnu99 \ + -ffast-math \ + -fno-common \ + -fno-builtin-printf \ + ${RVB_BMETAL_DIR}/syscalls.c \ + ${RVB_BMETAL_DIR}/crt.S \ + -static -nostdlib -T \ + ${RVB_BMETAL_DIR}/test.ld diff --git a/pkgs/spec-cpu/default.nix b/pkgs/spec-cpu/default.nix new file mode 100644 index 0000000..ef8df43 --- /dev/null +++ b/pkgs/spec-cpu/default.nix @@ -0,0 +1,59 @@ +{ + stdenv +, spec-cpu-tools +, libarchive +, xz +, gnutar +, gfortran +, coreutils +, requireFile +, autoPatchelfHook +, libxcrypt-legacy +, lib +}: + +stdenv.mkDerivation rec { + pname = "spec-cpu"; + version = "1.1.7"; + src = null; + unpackPhase = "true"; + # We need a working specxz binary + config = ./gcc-linux-x86.cfg; + buildPhase = '' + cp ${config} config.cfg + chmod +w config.cfg + export SPEC_NOCHECK=1 + mkdir out + runcpu \ + --verbose=80 \ + --tune=base \ + --output_root=$PWD/out \ + --config=$PWD/config.cfg \ + --define build_ncpus=$NIX_BUILD_CORES \ + --action=runsetup \ + all + ''; + # 519.lbm_r + # 502.gcc_r + # all + + # We only need the run directories + installPhase = '' + # Remove build/ and exe/ directories, we only need run/ + rm -rf out/benchspec/CPU/*/build/ + rm -rf out/benchspec/CPU/*/exe/ + + mkdir -p $out + cp -a out/benchspec $out + ''; + + #buildInputs = [ ]; + nativeBuildInputs = [ spec-cpu-tools gfortran ]; + enableParallelBuilding = false; + hardeningDisable = [ "all" ]; + dontStrip = true; + + meta.broken = (stdenv.buildPlatform.config != "x86_64-unknown-linux-gnu") || + (stdenv.hostPlatform.config != "riscv64-unknown-linux-gnu") || + (stdenv.targetPlatform.config != "riscv64-unknown-linux-gnu"); +} diff --git a/pkgs/spec-cpu/gcc-linux-x86.cfg b/pkgs/spec-cpu/gcc-linux-x86.cfg new file mode 100644 index 0000000..7a02182 --- /dev/null +++ b/pkgs/spec-cpu/gcc-linux-x86.cfg @@ -0,0 +1,391 @@ +#------------------------------------------------------------------------------ +# SPEC CPU(R) 2017 config for gcc/g++/gfortran on Linux x86 +#------------------------------------------------------------------------------ +# +# Usage: (1) Copy this to a new name +# cd $SPEC/config +# cp Example-x.cfg myname.cfg +# (2) Change items that are marked 'EDIT' (search for it) +# +# SPEC tested this config file with: +# Compiler version(s): Various. See note "Older GCC" below. +# Operating system(s): Oracle Linux Server 6, 7, 8 / +# Red Hat Enterprise Linux Server 6, 7, 8 +# SUSE Linux Enterprise Server 15 +# Ubuntu 19.04 +# Hardware: Xeon, EPYC +# +# If your system differs, this config file might not work. +# You might find a better config file at https://www.spec.org/cpu2017/results +# +# Note: Older GCC +# +# Please use the newest GCC that you can. The default version packaged with +# your operating system may be very old; look for alternate packages with a +# newer version. +# +# If you have no choice and must use an old version, here is what to expect: +# +# - "peak" tuning: Several benchmarks will fail at peak tuning if you use +# compilers older than GCC 7. +# In that case, please use base only. +# See: https://www.spec.org/cpu2017/Docs/overview.html#Q16 +# https://www.spec.org/cpu2017/Docs/config.html#tune +# Peak tuning is expected to work for all or nearly all +# benchmarks as of GCC 7 or later. +# Exception: +# - See topic "628.pop2_s basepeak", below. +# +# - "base" tuning: This config file is expected to work for base tuning with +# GCC 4.8.5 or later +# Exception: +# - Compilers vintage about 4.9 may need to turn off the +# tree vectorizer, by adding to the base OPTIMIZE flags: +# -fno-tree-loop-vectorize +# +# Unexpected errors? Try reducing the optimization level, or try removing: +# -march=native +# +# Compiler issues: Contact your compiler vendor, not SPEC. +# For SPEC help: https://www.spec.org/cpu2017/Docs/techsupport.html +#------------------------------------------------------------------------------ + + +#--------- Label -------------------------------------------------------------- +# Arbitrary string to tag binaries (no spaces allowed) +# Two Suggestions: # (1) EDIT this label as you try new ideas. +%ifndef %{label} +% define label "nix" # (2) Use a label meaningful to *you*. +%endif + + +#--------- Preprocessor ------------------------------------------------------- +%ifndef %{bits} # EDIT to control 32 or 64 bit compilation. Or, +% define bits 64 # you can set it on the command line using: +%endif # 'runcpu --define bits=nn' + +%ifndef %{build_ncpus} # EDIT to adjust number of simultaneous compiles. +% define build_ncpus 8 # Or, you can set it on the command line: +%endif # 'runcpu --define build_ncpus=nn' + +# Don't change this part. +%if %{bits} == 64 +% define model -m64 +%elif %{bits} == 32 +% define model -m32 +%else +% error Please define number of bits - see instructions in config file +%endif +%if %{label} =~ m/ / +% error Your label "%{label}" contains spaces. Please try underscores instead. +%endif +%if %{label} !~ m/^[a-zA-Z0-9._-]+$/ +% error Illegal character in label "%{label}". Please use only alphanumerics, underscore, hyphen, and period. +%endif + + +#--------- Global Settings ---------------------------------------------------- +# For info, see: +# https://www.spec.org/cpu2017/Docs/config.html#fieldname +# Example: https://www.spec.org/cpu2017/Docs/config.html#tune + +command_add_redirect = 1 +flagsurl = $[top]/config/flags/gcc.xml +ignore_errors = 1 +iterations = 1 +label = %{label}-m%{bits} +line_width = 1020 +log_line_width = 1020 +makeflags = --jobs=%{build_ncpus} +mean_anyway = 1 +output_format = txt,html,cfg,pdf,csv +preenv = 1 +reportable = 0 +tune = base,peak # EDIT if needed: set to "base" for old GCC. + # See note "Older GCC" above. + + +#--------- How Many CPUs? ----------------------------------------------------- +# Both SPECrate and SPECspeed can test multiple chips / cores / hw threads +# - For SPECrate, you set the number of copies. +# - For SPECspeed, you set the number of threads. +# See: https://www.spec.org/cpu2017/Docs/system-requirements.html#MultipleCPUs +# +# q. How many should I set? +# a. Unknown, you will have to try it and see! +# +# To get you started, some suggestions: +# +# copies - This config file defaults to testing only 1 copy. You might +# try changing it to match the number of cores on your system, +# or perhaps the number of virtual CPUs as reported by: +# grep -c processor /proc/cpuinfo +# Be sure you have enough memory. See: +# https://www.spec.org/cpu2017/Docs/system-requirements.html#memory +# +# threads - This config file sets a starting point. You could try raising +# it. A higher thread count is much more likely to be useful for +# fpspeed than for intspeed. +# +intrate,fprate: + copies = 1 # EDIT to change number of copies (see above) +intspeed,fpspeed: + threads = 4 # EDIT to change number of OpenMP threads (see above) + + +#------- Compilers ------------------------------------------------------------ +default: +# EDIT: The parent directory for your compiler. +# Do not include the trailing /bin/ +# Do not include a trailing slash +# Examples: +# 1 On a Red Hat system, you said: +# 'yum install devtoolset-9' +# Use: % define gcc_dir "/opt/rh/devtoolset-9/root/usr" +# +# 2 You built GCC in: /disk1/mybuild/gcc-10.1.0/bin/gcc +# Use: % define gcc_dir "/disk1/mybuild/gcc-10.1.0" +# +# 3 You want: /usr/bin/gcc +# Use: % define gcc_dir "/usr" +# WARNING: See section "Older GCC" above. +# +#%ifndef %{gcc_dir} +#% define gcc_dir "@GCC_DIR@" # EDIT (see above) +#%endif + +# EDIT: If your compiler version is 10 or greater, you must enable the next +# line to avoid compile errors for several FP benchmarks +# +%define GCCge10 # EDIT: remove the '#' from column 1 if using GCC 10 or later + +# EDIT if needed: the preENV line adds library directories to the runtime +# path. You can adjust it, or add lines for other environment variables. +# See: https://www.spec.org/cpu2017/Docs/config.html#preenv +# and: https://gcc.gnu.org/onlinedocs/gcc/Environment-Variables.html + #preENV_LD_LIBRARY_PATH = %{gcc_dir}/lib64/:%{gcc_dir}/lib/:/lib64 + #preENV_LD_LIBRARY_PATH = %{gcc_dir}/lib64/:%{gcc_dir}/lib/:/lib64:%{ENV_LD_LIBRARY_PATH} + #SPECLANG = %{gcc_dir}/bin/ + CC = %{ENV_CC} -std=c99 + CXX = %{ENV_CXX} -std=c++03 + FC = %{ENV_FC} + # How to say "Show me your version, please" + CC_VERSION_OPTION = --version + CXX_VERSION_OPTION = --version + FC_VERSION_OPTION = --version + +default: +%if %{bits} == 64 + sw_base_ptrsize = 64-bit + sw_peak_ptrsize = 64-bit +%else + sw_base_ptrsize = 32-bit + sw_peak_ptrsize = 32-bit +%endif + + +#--------- Portability -------------------------------------------------------- +default: # data model applies to all benchmarks +%if %{bits} == 32 + # Strongly recommended because at run-time, operations using modern file + # systems may fail spectacularly and frequently (or, worse, quietly and + # randomly) if a program does not accommodate 64-bit metadata. + EXTRA_PORTABILITY = -D_FILE_OFFSET_BITS=64 +%else + EXTRA_PORTABILITY = -DSPEC_LP64 +%endif + +# Benchmark-specific portability (ordered by last 2 digits of bmark number) + +500.perlbench_r,600.perlbench_s: #lang='C' +%if %{bits} == 32 +% define suffix IA32 +%else +% define suffix X64 +%endif + PORTABILITY = -DSPEC_LINUX_%{suffix} + +521.wrf_r,621.wrf_s: #lang='F,C' + CPORTABILITY = -DSPEC_CASE_FLAG + FPORTABILITY = -fconvert=big-endian + +523.xalancbmk_r,623.xalancbmk_s: #lang='CXX' + PORTABILITY = -DSPEC_LINUX + +526.blender_r: #lang='CXX,C' + PORTABILITY = -funsigned-char -DSPEC_LINUX + +527.cam4_r,627.cam4_s: #lang='F,C' + PORTABILITY = -DSPEC_CASE_FLAG + +628.pop2_s: #lang='F,C' + CPORTABILITY = -DSPEC_CASE_FLAG + FPORTABILITY = -fconvert=big-endian + +#---------------------------------------------------------------------- +# GCC workarounds that do not count as PORTABILITY +#---------------------------------------------------------------------- +# The workarounds in this section would not qualify under the SPEC CPU +# PORTABILITY rule. +# - In peak, they can be set as needed for individual benchmarks. +# - In base, individual settings are not allowed; set for whole suite. +# See: +# https://www.spec.org/cpu2017/Docs/runrules.html#portability +# https://www.spec.org/cpu2017/Docs/runrules.html#BaseFlags +# +# Integer workarounds - peak +# + 500.perlbench_r,600.perlbench_s=peak: # https://www.spec.org/cpu2017/Docs/benchmarks/500.perlbench_r.html + EXTRA_CFLAGS = -fno-strict-aliasing -fno-unsafe-math-optimizations -fno-finite-math-only + 502.gcc_r,602.gcc_s=peak: # https://www.spec.org/cpu2017/Docs/benchmarks/502.gcc_r.html + EXTRA_CFLAGS = -fno-strict-aliasing -fgnu89-inline + 505.mcf_r,605.mcf_s=peak: # https://www.spec.org/cpu2017/Docs/benchmarks/505.mcf_r.html + EXTRA_CFLAGS = -fno-strict-aliasing + 525.x264_r,625.x264_s=peak: # https://www.spec.org/cpu2017/Docs/benchmarks/525.x264_r.html + EXTRA_CFLAGS = -fcommon +# +# Integer workarounds - base - combine the above - https://www.spec.org/cpu2017/Docs/runrules.html#BaseFlags +# + intrate,intspeed=base: + EXTRA_CFLAGS = -fno-strict-aliasing -fno-unsafe-math-optimizations -fno-finite-math-only -fgnu89-inline -fcommon +# +# Floating Point workarounds - peak +# + 511.povray_r=peak: # https://www.spec.org/cpu2017/Docs/benchmarks/511.povray_r.html + EXTRA_CFLAGS = -fno-strict-aliasing + 521.wrf_r,621.wrf_s=peak: # https://www.spec.org/cpu2017/Docs/benchmarks/521.wrf_r.html +% ifdef %{GCCge10} # workaround for GCC v10 (and presumably later) + EXTRA_FFLAGS = -fallow-argument-mismatch +% endif + 527.cam4_r,627.cam4_s=peak: # https://www.spec.org/cpu2017/Docs/benchmarks/527.cam4_r.html + EXTRA_CFLAGS = -fno-strict-aliasing +% ifdef %{GCCge10} # workaround for GCC v10 (and presumably later) + EXTRA_FFLAGS = -fallow-argument-mismatch +% endif + # See also topic "628.pop2_s basepeak" below + 628.pop2_s=peak: # https://www.spec.org/cpu2017/Docs/benchmarks/628.pop2_s.html +% ifdef %{GCCge10} # workaround for GCC v10 (and presumably later) + EXTRA_FFLAGS = -fallow-argument-mismatch +% endif +# +# FP workarounds - base - combine the above - https://www.spec.org/cpu2017/Docs/runrules.html#BaseFlags +# + fprate,fpspeed=base: + EXTRA_CFLAGS = -fno-strict-aliasing +% ifdef %{GCCge10} # workaround for GCC v10 (and presumably later) + EXTRA_FFLAGS = -fallow-argument-mismatch +% endif + + +#-------- Tuning Flags common to Base and Peak -------------------------------- +# +# Speed (OpenMP and Autopar allowed) +# +%if %{bits} == 32 + intspeed,fpspeed: + # + # Many of the speed benchmarks (6nn.benchmark_s) do not fit in 32 bits + # If you wish to run SPECint2017_speed or SPECfp2017_speed, please use + # + # runcpu --define bits=64 + # + fail_build = 1 +%else + intspeed,fpspeed: + EXTRA_OPTIMIZE = -fopenmp -DSPEC_OPENMP + fpspeed: + # + # 627.cam4 needs a big stack; the preENV will apply it to all + # benchmarks in the set, as required by the rules. + # + preENV_OMP_STACKSIZE = 120M +%endif + +#-------- Base Tuning Flags ---------------------------------------------- +# EDIT if needed -- If you run into errors, you may need to adjust the +# optimization - for example you may need to remove +# the -march=native. See topic "Older GCC" above. +# +default=base: # flags for all base + OPTIMIZE = -g -O3 -march=native + + +#-------- Peak Tuning Flags ---------------------------------------------- +default=peak: + OPTIMIZE = -g -Ofast -march=native -flto + PASS1_FLAGS = -fprofile-generate + PASS2_FLAGS = -fprofile-use + +# 628.pop2_s basepeak: Depending on the interplay of several optimizations, +# 628.pop2_s might not validate with peak tuning. Use the base +# version instead. See: +# https:// www.spec.org/cpu2017/Docs/benchmarks/628.pop2_s.html +628.pop2_s=peak: + basepeak = yes + + +#------------------------------------------------------------------------------ +# Tester and System Descriptions - EDIT all sections below this point +#------------------------------------------------------------------------------ +# For info about any field, see +# https://www.spec.org/cpu2017/Docs/config.html#fieldname +# Example: https://www.spec.org/cpu2017/Docs/config.html#hw_memory +#------------------------------------------------------------------------------- + +#--------- EDIT to match your version ----------------------------------------- +default: + sw_compiler001 = C/C++/Fortran: Version 10.1.0 of GCC, the + sw_compiler002 = GNU Compiler Collection + +#--------- EDIT info about you ------------------------------------------------ +# To understand the difference between hw_vendor/sponsor/tester, see: +# https://www.spec.org/cpu2017/Docs/config.html#test_sponsor +intrate,intspeed,fprate,fpspeed: # Important: keep this line + hw_vendor = My Corporation + tester = My Corporation + test_sponsor = My Corporation + license_num = nnn (Your SPEC license number) +# prepared_by = # Ima Pseudonym # Whatever you like: is never output + + +#--------- EDIT system availability dates ------------------------------------- +intrate,intspeed,fprate,fpspeed: # Important: keep this line + # Example # Brief info about field + hw_avail = # Nov-2099 # Date of LAST hardware component to ship + sw_avail = # Nov-2099 # Date of LAST software component to ship + fw_bios = # Version Mumble released May-2099 # Firmware information + +#--------- EDIT system information -------------------------------------------- +intrate,intspeed,fprate,fpspeed: # Important: keep this line + # Example # Brief info about field +# hw_cpu_name = # Intel Xeon E9-9999 v9 # chip name + hw_cpu_nominal_mhz = # 9999 # Nominal chip frequency, in MHz + hw_cpu_max_mhz = # 9999 # Max chip frequency, in MHz +# hw_disk = # 9 x 9 TB SATA III 9999 RPM # Size, type, other perf-relevant info + hw_model = # TurboBlaster 3000 # system model name +# hw_nchips = # 99 # number chips enabled + hw_ncores = # 9999 # number cores enabled + hw_ncpuorder = # 1-9 chips # Ordering options + hw_nthreadspercore = # 9 # number threads enabled per core + hw_other = # TurboNUMA Router 10 Gb # Other perf-relevant hw, or "None" + +# hw_memory001 = # 999 GB (99 x 9 GB 2Rx4 PC4-2133P-R, # The 'PCn-etc' is from the JEDEC +# hw_memory002 = # running at 1600 MHz) # label on the DIMM. + + hw_pcache = # 99 KB I + 99 KB D on chip per core # Primary cache size, type, location + hw_scache = # 99 KB I+D on chip per 9 cores # Second cache or "None" + hw_tcache = # 9 MB I+D on chip per chip # Third cache or "None" + hw_ocache = # 9 GB I+D off chip per system board # Other cache or "None" + +# sw_file = # ext99 # File system +# sw_os001 = # Linux Sailboat # Operating system +# sw_os002 = # Distribution 7.2 SP1 # and version + sw_other = # TurboHeap Library V8.1 # Other perf-relevant sw, or "None" +# sw_state = # Run level 99 # Software state. + + power_management = # briefly summarize power settings + +# Note: Some commented-out fields above are automatically set to preliminary +# values by sysinfo +# https://www.spec.org/cpu2017/Docs/config.html#sysinfo +# Uncomment lines for which you already know a better answer than sysinfo diff --git a/pkgs/spec-cpu/tools.nix b/pkgs/spec-cpu/tools.nix new file mode 100644 index 0000000..e7cfd5f --- /dev/null +++ b/pkgs/spec-cpu/tools.nix @@ -0,0 +1,106 @@ +{ + stdenv +, libarchive +, xz +, gnutar +, gfortran +, coreutils +, requireFile +, autoPatchelfHook +, libxcrypt-legacy +, glibc +, lib +}: + +stdenv.mkDerivation rec { + pname = "spec-cpu-tools"; + version = "1.1.7"; + + src = requireFile { + name = "cpu2017-1.1.7.iso"; + sha256 = "02630819h64dyy57wkj33fhwwqgbw6mqc5awh1zm48pkvvl0l600"; + message = '' + Missing SPEC CPU 2017 1.1.7. + + Add it to the store with: + + $ nix-prefetch-url file:/path/to/cpu2017-1.1.7.iso + /nix/store/mk4hr8xwd62akp7iw5khq638ssba8qz0-cpu2017-1.1.7.iso + + Notice that the name must match exactly "cpu2017-1.1.7.iso". + ''; + }; + + unpackPhase = '' + set -x + mkdir iso + bsdtar -C iso -xf $src + chmod +w -R iso + + #for f in iso/install_archives/benchball/*; do + # bsdtar -xf $f + #done + #mkdir src + #bsdtar -C src -xf iso/install_archives/benchball/cpu2017-1.1.7.base.tar.xz + #bsdtar -C src -xf iso/install_archives/benchball/519.lbm_r-1.000503.tar.xz + + sourceRoot="$PWD/iso" + set +x + ''; + + # We need a working specxz binary + configurePhase = '' + patchShebangs install.sh + + set -x + + # Replace "spec*" tools by symlinks to working binaries + pushd tools/bin/linux-x86_64 + ln -fs ${xz}/bin/xz specxz + ln -fs ${gnutar}/bin/tar spectar + patchelf \ + --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" \ + specsha512sum + ls -l spec* + ./specxz -h + ./spectar --help + ldd ./specsha512sum + ./specsha512sum --help + popd + + # sha512sum requires -e, not provided by coreutils + + export SPEC="$sourceRoot" + + # Don't run the tests + sed -i 's/^.*shrc will.*$/exit 0/g' install.sh + + set +x + ''; + + installPhase = '' + bash -x install.sh -f -d $out -u linux-x86_64 + ''; + + preFixup = '' + # Fix temporary directory creation + sed -i '/^sub get_tmp_directory/a\ return tempdir(CLEANUP => 1);' \ + $out/bin/common/util_common.pl + + sed -i '/my $dir = jp($top, $config->resultdir, $subdir);/c\ my $dir = ::get_tmp_directory($config, 1);' \ + $out/bin/harness/log.pl + ''; + + # Missing libdb-4.7.so + autoPatchelfIgnoreMissingDeps = true; + + nativeBuildInputs = [ libxcrypt-legacy libarchive autoPatchelfHook ]; + #buildInputs = [ libxcrypt-legacy ]; + + enableParallelBuilding = false; + hardeningDisable = [ "all" ]; + dontStrip = true; + meta.broken = (stdenv.buildPlatform.config != "x86_64-unknown-linux-gnu") || + (stdenv.hostPlatform.config != "x86_64-unknown-linux-gnu") || + (stdenv.targetPlatform.config != "riscv64-unknown-linux-gnu"); +} -- GitLab From d7ef22936a1f57e4bc91ca69ca0ba773f53664cf Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 7 Oct 2024 13:33:53 +0200 Subject: [PATCH 288/310] Add SPEC CPU mini with a subset of benchmarks --- overlay.nix | 1 + pkgs/spec-cpu/default.nix | 2 ++ pkgs/spec-cpu/mini.nix | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 pkgs/spec-cpu/mini.nix diff --git a/overlay.nix b/overlay.nix index 5164c54..2840409 100644 --- a/overlay.nix +++ b/overlay.nix @@ -13,6 +13,7 @@ final: prev: spec-cpu-tools = final.callPackage ./pkgs/spec-cpu/tools.nix { }; spec-cpu = final.callPackage ./pkgs/spec-cpu/default.nix { }; + spec-cpu-mini = final.callPackage ./pkgs/spec-cpu/mini.nix { }; spec-cpu-clang = final.callPackage ./pkgs/spec-cpu/default.nix { stdenv = final.stdenvClangEpi; }; blis = ((prev.blis.override { diff --git a/pkgs/spec-cpu/default.nix b/pkgs/spec-cpu/default.nix index ef8df43..56cd2eb 100644 --- a/pkgs/spec-cpu/default.nix +++ b/pkgs/spec-cpu/default.nix @@ -10,6 +10,7 @@ , autoPatchelfHook , libxcrypt-legacy , lib +, benchSize ? "test" }: stdenv.mkDerivation rec { @@ -27,6 +28,7 @@ stdenv.mkDerivation rec { runcpu \ --verbose=80 \ --tune=base \ + --size=${benchSize} \ --output_root=$PWD/out \ --config=$PWD/config.cfg \ --define build_ncpus=$NIX_BUILD_CORES \ diff --git a/pkgs/spec-cpu/mini.nix b/pkgs/spec-cpu/mini.nix new file mode 100644 index 0000000..674e682 --- /dev/null +++ b/pkgs/spec-cpu/mini.nix @@ -0,0 +1,39 @@ +{ + stdenv +, spec-cpu +}: + +stdenv.mkDerivation rec { + pname = "spec-cpu-mini"; + version = spec-cpu.version; + src = null; + unpackPhase = "true"; + + # Select only a subset of the benchmarks: + # https://www.spec.org/cpu2017/Docs/#benchdocs + benchList = [ + # -- SPECspeed 2017 Integer -- + "600.perlbench_s" + "602.gcc_s" + "605.mcf_s" + "620.omnetpp_s" + #"623.xalancbmk_s" # Big + #"625.x264_s" # Big + "631.deepsjeng_s" + "641.leela_s" + "648.exchange2_s" + "657.xz_s" + ]; + + dontConfigure = true; + dontBuild = true; + installPhase = '' + mkdir -p $out/benchspec/CPU + for bench in $benchList; do + cp -a ${spec-cpu}/benchspec/CPU/$bench $out/benchspec/CPU/ + done + ''; + enableParallelBuilding = false; + hardeningDisable = [ "all" ]; + dontStrip = true; +} -- GitLab From dffebe4ad1ad26496398d761a5fee8b5e2b8e193 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 7 Oct 2024 13:34:52 +0200 Subject: [PATCH 289/310] Add spec-cpu-mini to rootfs --- configuration.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration.nix b/configuration.nix index f8f5644..e0794c5 100644 --- a/configuration.nix +++ b/configuration.nix @@ -42,7 +42,7 @@ #environment.systemPackages = with pkgs; [ vim gdb neofetch gcc bintools ]; environment.systemPackages = with pkgs; [ - rvb riscv-tools stream + rvb riscv-tools stream spec-cpu-mini config.boot.kernelPackages.perf ]; -- GitLab From bc3e8513db0ac20ff16e1d8e164edc352ce5f952 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 7 Oct 2024 14:16:47 +0200 Subject: [PATCH 290/310] Add binary to spec-cpu-mini --- pkgs/spec-cpu/mini.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkgs/spec-cpu/mini.nix b/pkgs/spec-cpu/mini.nix index 674e682..251a210 100644 --- a/pkgs/spec-cpu/mini.nix +++ b/pkgs/spec-cpu/mini.nix @@ -32,6 +32,9 @@ stdenv.mkDerivation rec { for bench in $benchList; do cp -a ${spec-cpu}/benchspec/CPU/$bench $out/benchspec/CPU/ done + mkdir -p $out/bin + echo -e "#!$SHELL\necho $out" > $out/bin/spec-cpu-mini + chmod +x $out/bin/spec-cpu-mini ''; enableParallelBuilding = false; hardeningDisable = [ "all" ]; -- GitLab From c2fcd40bc15f2476551aede50e786cf6be6e867c Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 7 Oct 2024 14:50:36 +0200 Subject: [PATCH 291/310] Prune spec-cpu-mini --- JOURNAL.md | 9 +++++++++ pkgs/spec-cpu/mini.nix | 21 +++++++++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index e7e8668..265cc2e 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -5140,3 +5140,12 @@ To trace which accesses are done to the PLIC, I can use the mmiotracer, which will hopefully record how we configure the PLIC and lead to a reproducer. CONFIG_MMIOTRACE=y + +## 2024-10-07 + +Let's move on to the SPEC CPU benchmarks. I compiled them again with the current +nixpkgs, but they are too large. I made a "mini" version with only the "speed" +and "integer" variants and removing a couple of large benchmarks. + +Still, the closure is gigantic, as they are collecting the environment during +the build phase and that makes the result depend on the build packages. diff --git a/pkgs/spec-cpu/mini.nix b/pkgs/spec-cpu/mini.nix index 251a210..8971a14 100644 --- a/pkgs/spec-cpu/mini.nix +++ b/pkgs/spec-cpu/mini.nix @@ -26,12 +26,25 @@ stdenv.mkDerivation rec { ]; dontConfigure = true; - dontBuild = true; - installPhase = '' - mkdir -p $out/benchspec/CPU + buildPhase = '' + pwd + mkdir -p benchspec/CPU for bench in $benchList; do - cp -a ${spec-cpu}/benchspec/CPU/$bench $out/benchspec/CPU/ + cp -r ${spec-cpu}/benchspec/CPU/$bench benchspec/CPU/ done + + # Make writable + chmod -R +w benchspec + + # Remove environment + find benchspec -name '*.cmd' | xargs sed -i '/^-E/d' + # Remove compare script as it refers to spec-cpu-tools + find benchspec -name 'compare.cmd' -delete + ''; + installPhase = '' + pwd + mkdir -p $out + cp -r benchspec/ $out mkdir -p $out/bin echo -e "#!$SHELL\necho $out" > $out/bin/spec-cpu-mini chmod +x $out/bin/spec-cpu-mini -- GitLab From a9e149a125f1050389c96766b2e8c823877f73e9 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 8 Oct 2024 09:27:32 +0200 Subject: [PATCH 292/310] Copy specinvoke in spec-cpu-mini --- pkgs/spec-cpu/mini.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkgs/spec-cpu/mini.nix b/pkgs/spec-cpu/mini.nix index 8971a14..b002f59 100644 --- a/pkgs/spec-cpu/mini.nix +++ b/pkgs/spec-cpu/mini.nix @@ -1,6 +1,7 @@ { stdenv , spec-cpu +, spec-cpu-tools }: stdenv.mkDerivation rec { @@ -48,7 +49,9 @@ stdenv.mkDerivation rec { mkdir -p $out/bin echo -e "#!$SHELL\necho $out" > $out/bin/spec-cpu-mini chmod +x $out/bin/spec-cpu-mini + cp $(command -v specinvoke) $out/bin ''; + nativeBuildInputs = [ spec-cpu-tools ]; enableParallelBuilding = false; hardeningDisable = [ "all" ]; dontStrip = true; -- GitLab From ce70726bf680ae708c1cd5c272629a7654caac9f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 8 Oct 2024 10:13:27 +0200 Subject: [PATCH 293/310] Build specinvoke from source The one contained in the CPU SPEC ISO is built for x86, and there is no binary for RISC-V. --- overlay.nix | 1 + pkgs/spec-cpu/mini.nix | 4 ++-- pkgs/spec-cpu/specinvoke.nix | 30 ++++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 pkgs/spec-cpu/specinvoke.nix diff --git a/overlay.nix b/overlay.nix index 2840409..2eeee5c 100644 --- a/overlay.nix +++ b/overlay.nix @@ -14,6 +14,7 @@ final: prev: spec-cpu-tools = final.callPackage ./pkgs/spec-cpu/tools.nix { }; spec-cpu = final.callPackage ./pkgs/spec-cpu/default.nix { }; spec-cpu-mini = final.callPackage ./pkgs/spec-cpu/mini.nix { }; + specinvoke = final.callPackage ./pkgs/spec-cpu/specinvoke.nix { }; spec-cpu-clang = final.callPackage ./pkgs/spec-cpu/default.nix { stdenv = final.stdenvClangEpi; }; blis = ((prev.blis.override { diff --git a/pkgs/spec-cpu/mini.nix b/pkgs/spec-cpu/mini.nix index b002f59..033d978 100644 --- a/pkgs/spec-cpu/mini.nix +++ b/pkgs/spec-cpu/mini.nix @@ -2,6 +2,7 @@ stdenv , spec-cpu , spec-cpu-tools +, specinvoke }: stdenv.mkDerivation rec { @@ -49,9 +50,8 @@ stdenv.mkDerivation rec { mkdir -p $out/bin echo -e "#!$SHELL\necho $out" > $out/bin/spec-cpu-mini chmod +x $out/bin/spec-cpu-mini - cp $(command -v specinvoke) $out/bin + cp ${specinvoke}/bin/specinvoke $out/bin ''; - nativeBuildInputs = [ spec-cpu-tools ]; enableParallelBuilding = false; hardeningDisable = [ "all" ]; dontStrip = true; diff --git a/pkgs/spec-cpu/specinvoke.nix b/pkgs/spec-cpu/specinvoke.nix new file mode 100644 index 0000000..61be2d0 --- /dev/null +++ b/pkgs/spec-cpu/specinvoke.nix @@ -0,0 +1,30 @@ +{ + stdenv +, libarchive +, spec-cpu-tools +, runCommandNoCC +}: + +let + version = spec-cpu-tools.version; + tar = runCommandNoCC "specinvoke-${version}.tar" { + src = spec-cpu-tools.src; + nativeBuildInputs = [ libarchive ]; + } '' + mkdir iso + bsdtar -C iso -xf $src + cp iso/install_archives/tools-src.tar $out + ''; +in stdenv.mkDerivation { + pname = "specinvoke"; + version = version; + src = tar; + sourceRoot = "tools/src/specinvoke/"; + # Almost no bugs + preInstall = '' + mkdir -p $out/bin + ''; + enableParallelBuilding = false; + hardeningDisable = [ "all" ]; + dontStrip = true; +} -- GitLab From 7e68d432f996a0a347cb10937ea2d9dc26b2dd18 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 8 Oct 2024 16:31:07 +0200 Subject: [PATCH 294/310] Patch specinvoke to report execve failure --- pkgs/spec-cpu/specinvoke-execve.patch | 15 +++++++++++++++ pkgs/spec-cpu/specinvoke.nix | 3 +++ 2 files changed, 18 insertions(+) create mode 100644 pkgs/spec-cpu/specinvoke-execve.patch diff --git a/pkgs/spec-cpu/specinvoke-execve.patch b/pkgs/spec-cpu/specinvoke-execve.patch new file mode 100644 index 0000000..a237b8a --- /dev/null +++ b/pkgs/spec-cpu/specinvoke-execve.patch @@ -0,0 +1,15 @@ +--- a/unix.c 2024-10-08 12:30:18.785111397 +0200 ++++ b/unix.c 2024-10-08 12:32:09.580923368 +0200 +@@ -165,7 +165,11 @@ pid_t invoke(copy_info_t *ui, command_in + /* We could redirect them here. This might be useful for VMS? */ + *(si->command_ptr) = cmd; + si->invoke_args[0] = si->shell; +- execve(si->shell, si->invoke_args, env); ++ if (execve(si->shell, si->invoke_args, env) != 0) { ++ fprintf (stderr, "Can't execute command: %s(%d)\n", ++ STRERROR(errno), errno); ++ specinvoke_exit (1, si); ++ } + } else { /* Parent */ + ui->pid = pid; + fprintf (si->outfp, diff --git a/pkgs/spec-cpu/specinvoke.nix b/pkgs/spec-cpu/specinvoke.nix index 61be2d0..e61525a 100644 --- a/pkgs/spec-cpu/specinvoke.nix +++ b/pkgs/spec-cpu/specinvoke.nix @@ -20,6 +20,9 @@ in stdenv.mkDerivation { version = version; src = tar; sourceRoot = "tools/src/specinvoke/"; + patches = [ + ./specinvoke-execve.patch + ]; # Almost no bugs preInstall = '' mkdir -p $out/bin -- GitLab From 3bcc4255dbdeb7054c1d91ef66b5c02a32acf7c0 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 9 Oct 2024 15:52:46 +0200 Subject: [PATCH 295/310] Add SPEC launcher --- JOURNAL.md | 24 +++++++++++++ overlay.nix | 1 + pkgs/spec-cpu/launcher.sh | 65 ++++++++++++++++++++++++++++++++++++ pkgs/spec-cpu/speclaunch.nix | 21 ++++++++++++ 4 files changed, 111 insertions(+) create mode 100644 pkgs/spec-cpu/launcher.sh create mode 100644 pkgs/spec-cpu/speclaunch.nix diff --git a/JOURNAL.md b/JOURNAL.md index 265cc2e..136d903 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -5149,3 +5149,27 @@ and "integer" variants and removing a couple of large benchmarks. Still, the closure is gigantic, as they are collecting the environment during the build phase and that makes the result depend on the build packages. + +## 2024-10-09 + +One of the problems with the `speccmds.cmd` file is that is assumes that it can +write the output of the benchmarks in the same place that the binaries are +located. + + hut% cat benchspec/CPU/602.gcc_s/run/run_base_test_nix-m64.0000/speccmds.cmd + -r + -N C + -C /build/out/benchspec/CPU/602.gcc_s/run/run_base_test_nix-m64.0000 + -o t1.opts-O3_-finline-limit_50000.out -e t1.opts-O3_-finline-limit_50000.err ../run_base_test_nix-m64.0000/sgcc_base.nix-m64 t1.c -O3 -finline-limit=50000 -o t1.opts-O3_-finline-limit_50000.s > t1.opts-O3_-finline-limit_50000.out 2>> t1.opts-O3_-finline-limit_50000.err + +We can address this problem by modifying the `-C ...` command and just use `-C +602.gcc_s` (not sure if it creates it directly). Then we need to modify the +../run... part to use the full path of the binary. + + hut% cat speccmds.cmd | sed '/^-C/d' + -r + -N C + -o t1.opts-O3_-finline-limit_50000.out -e t1.opts-O3_-finline-limit_50000.err ../run_base_test_nix-m64.0000/sgcc_base.nix-m64 t1.c -O3 -finline-limit=50000 -o t1.opts-O3_-finline-limit_50000.s > t1.opts-O3_-finline-limit_50000.out 2>> t1.opts-O3_-finline-limit_50000.err + +I can create a symlink to the benchmark directory, so it finds it at +`../run_base_test_nix-m64.0000`. diff --git a/overlay.nix b/overlay.nix index 2eeee5c..190d510 100644 --- a/overlay.nix +++ b/overlay.nix @@ -15,6 +15,7 @@ final: prev: spec-cpu = final.callPackage ./pkgs/spec-cpu/default.nix { }; spec-cpu-mini = final.callPackage ./pkgs/spec-cpu/mini.nix { }; specinvoke = final.callPackage ./pkgs/spec-cpu/specinvoke.nix { }; + speclaunch = final.callPackage ./pkgs/spec-cpu/speclaunch.nix { }; spec-cpu-clang = final.callPackage ./pkgs/spec-cpu/default.nix { stdenv = final.stdenvClangEpi; }; blis = ((prev.blis.override { diff --git a/pkgs/spec-cpu/launcher.sh b/pkgs/spec-cpu/launcher.sh new file mode 100644 index 0000000..4aa641d --- /dev/null +++ b/pkgs/spec-cpu/launcher.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +set -e + +if [ -z "$SPEC" ]; then + SPEC=$(spec-cpu-mini) +fi + +if [ -z "$SPEC" ]; then + echo "cannot find spec, set SPEC variable" + exit 1 +fi + +where=$TMPDIR +if [ -z "$where" ]; then + if [ -d /tmp ]; then + where=/tmp + else + where=$PWD + fi +fi + +cwd=$(readlink -f $where) +# Place the outcome here +wd="$cwd/spec" +mkdir -p "$wd" + +benchniter=1 +benchsize=test +benchtune=base + +echo "--- Placing output in $wd ---" + +printf 'benchmark\tsize\ttune\titer\ttime_s\n' > "$wd/time.csv" + +for srcbench in $SPEC/benchspec/CPU/*; do + name=$(basename $srcbench) + bench="$wd/$name" + rm -rf "$bench" + cp -r "$srcbench" "$bench" + chmod +w -R "$bench" + + rundir="$bench/run/run_${benchtune}_${benchsize}_nix-m64.0000" + sed -i '/^-C/d' "$rundir/speccmds.cmd" + echo "--- Running $name for $benchniter iterations ---" + ( + #set -x + cd $rundir + specinvoke -i $benchniter -E speccmds.cmd > /dev/null + #set +x + ) + # Print time + awk '/^run [0-9]* elapsed time/{printf \ + "%s\t%s\t%s\t%s\t%s\n", \ + "'$name'","'$benchsize'","'$benchtune'",$2,$7}' \ + "$rundir/speccmds.out" > "$rundir/time.csv" + + cat "$rundir/time.csv" + + # Accumulate in main CSV + cat "$rundir/time.csv" >> "$wd/time.csv" +done + +echo "--- RESULTS in $wd/time.csv ---" +cat "$wd/time.csv" diff --git a/pkgs/spec-cpu/speclaunch.nix b/pkgs/spec-cpu/speclaunch.nix new file mode 100644 index 0000000..cbbb95b --- /dev/null +++ b/pkgs/spec-cpu/speclaunch.nix @@ -0,0 +1,21 @@ +{ + stdenv +, bash +}: + +stdenv.mkDerivation { + name = "speclaunch"; + src = ./launcher.sh; + dontUnpack = true; + dontConfigure = true; + dontBuild = true; + installPhase = '' + mkdir -p $out/bin + cp $src $out/bin/speclaunch + chmod +x $out/bin/speclaunch + ''; + buildInputs = [ bash ]; + enableParallelBuilding = false; + hardeningDisable = [ "all" ]; + dontStrip = true; +} -- GitLab From cb7b813389162eaa4f931dda6693b58bad095b81 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 9 Oct 2024 15:53:57 +0200 Subject: [PATCH 296/310] Add speclaunch to spec mini --- pkgs/spec-cpu/mini.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/spec-cpu/mini.nix b/pkgs/spec-cpu/mini.nix index 033d978..d2ae59b 100644 --- a/pkgs/spec-cpu/mini.nix +++ b/pkgs/spec-cpu/mini.nix @@ -3,6 +3,7 @@ , spec-cpu , spec-cpu-tools , specinvoke +, speclaunch }: stdenv.mkDerivation rec { @@ -51,6 +52,7 @@ stdenv.mkDerivation rec { echo -e "#!$SHELL\necho $out" > $out/bin/spec-cpu-mini chmod +x $out/bin/spec-cpu-mini cp ${specinvoke}/bin/specinvoke $out/bin + cp ${speclaunch}/bin/speclaunch $out/bin ''; enableParallelBuilding = false; hardeningDisable = [ "all" ]; -- GitLab From 9b1f6b4460200aced69930653aecad7e9eb21dd2 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Wed, 9 Oct 2024 21:33:21 +0200 Subject: [PATCH 297/310] Disable 631.deepsjeng_s test in spec mini --- pkgs/spec-cpu/mini.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/spec-cpu/mini.nix b/pkgs/spec-cpu/mini.nix index d2ae59b..ccc0924 100644 --- a/pkgs/spec-cpu/mini.nix +++ b/pkgs/spec-cpu/mini.nix @@ -22,7 +22,7 @@ stdenv.mkDerivation rec { "620.omnetpp_s" #"623.xalancbmk_s" # Big #"625.x264_s" # Big - "631.deepsjeng_s" + #"631.deepsjeng_s" # Requires 7 GiB of RAM "641.leela_s" "648.exchange2_s" "657.xz_s" -- GitLab From 78b8c7f98118e6355d29b11e921acc870c0ff484 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 10 Oct 2024 10:28:48 +0200 Subject: [PATCH 298/310] Prepare CI to run SPEC benchmarks --- JOURNAL.md | 106 ++++++++++++++++++++++++++++++++++++++ fpga/run-node.sh | 9 +++- lagarto-ox.nix | 17 +++++- patches/stage-2-init.sh | 4 ++ pkgs/spec-cpu/launcher.sh | 11 ++-- 5 files changed, 141 insertions(+), 6 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 136d903..329755a 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -5173,3 +5173,109 @@ We can address this problem by modifying the `-C ...` command and just use `-C I can create a symlink to the benchmark directory, so it finds it at `../run_base_test_nix-m64.0000`. + +## 2024-10-10 + +Managed to run some: + + + for srcbench in $SPEC/benchspec/CPU/* + ++ basename /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/620.omnetpp_s + + name=620.omnetpp_s + + bench=/tmp/spec/620.omnetpp_s + + rm -rf /tmp/spec/620.omnetpp_s + + cp -r /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/620.omnetpp_s /tmp/spec/620.omnetpp_s + + chmod +w -R /tmp/spec/620.omnetpp_s + + rundir=/tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000 + + sed -i '/^-C/d' /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/speccmds.cmd + + echo '--- Running 620.omnetpp_s for 1 iterations ---' + --- Running 620.omnetpp_s for 1 iterations --- + + cd /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000 + + specinvoke -i 1 -E speccmds.cmd + + awk '/^run [0-9]* elapsed time/{printf \ + "%s\t%s\t%s\t%s\t%s\n", \ + "620.omnetpp_s","test","base",$2,$7}' /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/speccmds.out + + cat /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/time.csv + 620.omnetpp_s test base 1 1080.495394000 + + cat /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/time.csv + + for srcbench in $SPEC/benchspec/CPU/* + ++ basename /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/631.deepsjeng_s + + name=631.deepsjeng_s + + bench=/tmp/spec/631.deepsjeng_s + + rm -rf /tmp/spec/631.deepsjeng_s + + cp -r /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/631.deepsjeng_s /tmp/spec/631.deepsjeng_s + + chmod +w -R /tmp/spec/631.deepsjeng_s + + rundir=/tmp/spec/631.deepsjeng_s/run/run_base_test_nix-m64.0000 + + sed -i '/^-C/d' /tmp/spec/631.deepsjeng_s/run/run_base_test_nix-m64.0000/speccmds.cmd + + echo '--- Running 631.deepsjeng_s for 1 iterations ---' + --- Running 631.deepsjeng_s for 1 iterations --- + + cd /tmp/spec/631.deepsjeng_s/run/run_base_test_nix-m64.0000 + + specinvoke -i 1 -E speccmds.cmd + [12274.985482] __vm_enough_memory: pid: 661, comm: deepsjeng_s_bas, bytes: 7200002048 not enough memory for the allocation + [12274.998109] __vm_enough_memory: pid: 661, comm: deepsjeng_s_bas, bytes: 7200071680 not enough memory for the allocation + [12275.010615] __vm_enough_memory: pid: 661, comm: deepsjeng_s_bas, bytes: 7200133120 not enough memory for the allocation + bash-5.2# cat /tmp/spec/time.csv + benchmark size tune iter time_s + 600.perlbench_s test base 1 5326.027877000 + 602.gcc_s test base 1 2.515220000 + 605.mcf_s test base 1 1923.514976000 + 620.omnetpp_s test base 1 1080.495394000 + +After commenting `631.deepsjeng_s`: + + [12736.162509] 196608 pages RAM + [12736.165986] 0 pages HighMem/MovableOnly + [12736.171049] 18239 pages reserved + [12736.174892] 4096 pages cma reserved + [12736.179650] Tasks state (memory values in pages): + [12736.184896] [ pid ] uid tgid total_vm rss rss_anon rss_file rss_shmem pgtables_bytes swapents oom_score_adj name + [12736.197219] [ 221] 0 221 1090 126 125 1 0 28672 0 0 bash + [12736.209480] [ 236] 0 236 1057 84 83 1 0 28672 0 0 bash + [12736.221558] [ 642] 0 642 1057 83 83 0 0 28672 0 0 bash + [12736.233637] [ 643] 0 643 572 25 24 1 0 28672 0 0 specinvoke + [12736.246203] [ 644] 0 644 1057 70 69 1 0 32768 0 0 sh + [12736.258037] [ 645] 0 645 131277 105312 105279 1 32 872448 0 0 xz_s_base.nix-m + [12736.271031] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),global_oom,task_memcg=/,task=xz_s_base.nix-m,pid=645,uid=0 + [12736.284939] Out of memory: Killed process 645 (xz_s_base.nix-m) total-vm:525108kB, anon-rss:421116kB, file-rss:4kB, shmem-rss:128kB, UID:0 pgtables:852kB oom_score_adj:0 + [12749.985238] oom_reaper: reaped process 645 (xz_s_base.nix-m), now anon-rss:0kB, file-rss:0kB, shmem-rss:0kB + bash-5.2# cat /tmp/spec/time.csv + benchmark size tune iter time_s + 600.perlbench_s test base 1 5338.291831000 + 602.gcc_s test base 1 2.520710000 + 605.mcf_s test base 1 1923.825657000 + 620.omnetpp_s test base 1 1091.905020000 + 641.leela_s test base 1 767.870615000 + 648.exchange2_s test base 1 2815.577807000 + +Running out of memory on `657.xz_s`: + + bash-5.2# free -h + total used free shared buff/cache available + Mem: 696Mi 19Mi 405Mi 255Mi 271Mi 411Mi + Swap: 0B 0B 0B + + bash-5.2# df -h + Filesystem Size Used Avail Use% Mounted on + devtmpfs 35M 0 35M 0% /dev + tmpfs 349M 0 349M 0% /dev/shm + tmpfs 175M 56K 175M 1% /run + tmpfs 349M 0 349M 0% /run/wrappers + /dev/disk/by-label/NIXOS_SD 2.3G 1.9G 275M 88% / + nodev 349M 256M 93M 74% /tmp + + bash-5.2# du -sh /tmp/spec + 256M /tmp/spec + +Let's comment it for now. + +We may want to start running the tests in the CI, so I can read the logs there. +There are some operations we need to do on the FS before running the tests: + + bash-5.2# mkdir /tmp + bash-5.2# mount -t tmpfs nodev /tmp + bash-5.2# mkdir /bin + bash-5.2# mkdir /root + bash-5.2# ln -s $(which sh) /bin/sh + bash-5.2# export TMPDIR=/tmp + bash-5.2# speclaunch + +So, let's prepare a script that performs the mini-init. diff --git a/fpga/run-node.sh b/fpga/run-node.sh index 98da578..05ba16d 100755 --- a/fpga/run-node.sh +++ b/fpga/run-node.sh @@ -20,8 +20,13 @@ set -x # Then perform the boot ./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img -timeout=$((30 * 60)) # Always stop after 30 min -timeout_silent=$((3 * 60)) # Stop if 3 min without output +# Normal timeouts +#timeout=$((30 * 60)) # Always stop after 30 min +#timeout_silent=$((3 * 60)) # Stop if 3 min without output + +# Timeouts for SPEC benchmarks +timeout=$((6 * 60 * 60)) # Always stop after 6 h +timeout_silent=$((2 * 60 * 60)) # Stop if 2 h without output (some benchmarks take 1.6h) # Set dead switch sleep $timeout && killall picocom & diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 47645f6..ca54c7c 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -66,6 +66,7 @@ src = ./patches/stage-2-init.sh; shellDebug = "${pkgs.bashInteractive}/bin/bash"; bashInteractive = "${pkgs.bashInteractive}"; + bench2 = "${pkgs.bench2}"; shell = "${pkgs.bash}/bin/bash"; inherit (config.boot) readOnlyNixStore systemdExecutable extraSystemdUnitPaths; inherit (config.system.nixos) distroName; @@ -400,6 +401,19 @@ # }; #})); + bench2 = final.writeShellScript "bench2" '' + # Performs minimal FS setup and runs the SPEC benchmark + mkdir /tmp /bin /root + mount -t tmpfs tmpfs /tmp + export TMPDIR=/tmp + # We need /bin/sh + ln -s $(which sh) /bin/sh + bash -x speclaunch + cat /tmp/spec/time.csv + # Give me a shell at the end + bash -l + ''; + ox-dtb = prev.stdenv.mkDerivation rec { name = "ox.dtb"; src = ./dts; @@ -512,6 +526,7 @@ # NixOS interesting options: # debug1 enable debug shell in stage 1 # debug2 enable debug shell in stage 2 (custom) + # bench2 run benchmark on stage 2 (custom) # boot.trace enable set -x in stage 1 # boot.tracedebug enable set -x in stage 2 # Ftrace interesting options: @@ -523,7 +538,7 @@ in prev.runCommand "uboot.txt" {} '' cat > $out </dev/hvc0 2>/dev/hvc0" ;; + bench2) + export PATH="@bashInteractive@/bin:@systemConfig@/sw/bin:$PATH" + setsid bash -c "exec @bench2@ /dev/hvc0 2>/dev/hvc0" + ;; esac done diff --git a/pkgs/spec-cpu/launcher.sh b/pkgs/spec-cpu/launcher.sh index 4aa641d..c43add8 100644 --- a/pkgs/spec-cpu/launcher.sh +++ b/pkgs/spec-cpu/launcher.sh @@ -36,6 +36,7 @@ printf 'benchmark\tsize\ttune\titer\ttime_s\n' > "$wd/time.csv" for srcbench in $SPEC/benchspec/CPU/*; do name=$(basename $srcbench) bench="$wd/$name" + bench_out="$wd/$name.csv" rm -rf "$bench" cp -r "$srcbench" "$bench" chmod +w -R "$bench" @@ -53,13 +54,17 @@ for srcbench in $SPEC/benchspec/CPU/*; do awk '/^run [0-9]* elapsed time/{printf \ "%s\t%s\t%s\t%s\t%s\n", \ "'$name'","'$benchsize'","'$benchtune'",$2,$7}' \ - "$rundir/speccmds.out" > "$rundir/time.csv" + "$rundir/speccmds.out" > "$bench_out" - cat "$rundir/time.csv" + cat "$bench_out" # Accumulate in main CSV - cat "$rundir/time.csv" >> "$wd/time.csv" + cat "$bench_out" >> "$wd/time.csv" + + # Erase intermediate files as they occupy RAM. + rm -rf "$bench" done echo "--- RESULTS in $wd/time.csv ---" cat "$wd/time.csv" +echo "---------------------------------------" -- GitLab From c4da8156dcb01ffca3fe0b76cbba5bbfa0743ee6 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 10 Oct 2024 10:47:13 +0200 Subject: [PATCH 299/310] Disable 657.xz_s in SPEC mini --- pkgs/spec-cpu/mini.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/spec-cpu/mini.nix b/pkgs/spec-cpu/mini.nix index ccc0924..008a3d0 100644 --- a/pkgs/spec-cpu/mini.nix +++ b/pkgs/spec-cpu/mini.nix @@ -25,7 +25,7 @@ stdenv.mkDerivation rec { #"631.deepsjeng_s" # Requires 7 GiB of RAM "641.leela_s" "648.exchange2_s" - "657.xz_s" + # "657.xz_s" # Runs out of memory with 700 MiB of RAM ]; dontConfigure = true; -- GitLab From 99c8d5e3dec9768e64c61bdd041be568dedf8d2e Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 10 Oct 2024 15:34:06 +0200 Subject: [PATCH 300/310] Increase RAM and improve DT --- JOURNAL.md | 14 ++++- dts/Makefile | 1 + dts/lagarto_ox.dts | 154 +++++++++++++++++++++------------------------ dts/lagarto_ox.h | 67 ++++++++++++++++---- fpga/env.sh | 2 +- 5 files changed, 142 insertions(+), 96 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 329755a..3db80a5 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -5278,4 +5278,16 @@ There are some operations we need to do on the FS before running the tests: bash-5.2# export TMPDIR=/tmp bash-5.2# speclaunch -So, let's prepare a script that performs the mini-init. +So, let's prepare a script that runs the SPEC mini. + +The first benchmark to run is `600.perlbench_s` which seems to take 5338 seconds +(1.5 h) to run. I configured the pipeline to stop as soon as we have 2 h of +silence, but after 150 minutes (2.5 h) of execution time it has not finished yet. +Not sure if something is wrong now. Maybe I can run vmstat a few times and see +the mount points to check everything is correct. + +I may want to also increase the RAM available, so we can potentially run other +benchmarks too. + +At some point we may want to be able to specify the bootcmd from fpgactl +directly. diff --git a/dts/Makefile b/dts/Makefile index 7f7d3be..4a87c1e 100644 --- a/dts/Makefile +++ b/dts/Makefile @@ -10,6 +10,7 @@ clean: %.pp.dts: %.dts *.h $(CC) $(CPPFLAGS) $< -o $@ + sed -i 's/@0x0*/@/' $@ %.dtb: %.pp.dts dtc -O dtb -o $@ $^ diff --git a/dts/lagarto_ox.dts b/dts/lagarto_ox.dts index df93255..d3881b3 100644 --- a/dts/lagarto_ox.dts +++ b/dts/lagarto_ox.dts @@ -7,21 +7,13 @@ compatible = "riscv,rv64i"; model = "Barcelona Supercomputing Center - Lagarto Ox (NixOS)"; - aliases { - serial0 = &uart_console; // ttyS0 -// serial1 = &uart_testing; // ttyS1 - }; - -// chosen { -// bootargs = "earlyprintk ignore_loglevel earlycon=sbi console=hvc0 root=/dev/pmem0p1 ro init=/bin/bash"; -// }; cpus { #address-cells = <1>; #size-cells = <0>; - timebase-frequency = ; + timebase-frequency = ; CPU0: cpu@0 { - clock-frequency = ; + clock-frequency = ; device_type = "cpu"; reg = <0>; status = "okay"; @@ -30,13 +22,6 @@ mmu-type = "riscv,sv39"; tlb-split; - // OpenPiton+Ariane Platform - // L1I Size / Assoc: 16 kB / 4 - // L1D Size / Assoc: 32 kB / 4 - // L15 Size / Assoc: 128 kB / 8 - // L2 Size / Assoc: 256 kB / 4 - // L15/L1D Cacheline size 64 - i-cache-block-size = <64>; // Guess i-cache-sets = <4>; i-cache-size = <16384>; @@ -49,7 +34,6 @@ d-tlb-sets = <1>; // Guess d-tlb-size = <32>; // Guess - phandle = <0x00000004>; /* Hart-Level Interrupt Controller: Every interrupt is * ultimately routed through a hart's HLIC before it * interrupts that hart. */ @@ -57,7 +41,6 @@ #interrupt-cells = <1>; interrupt-controller; /* Receives interrupts */ compatible = "riscv,cpu-intc"; - phandle = <0x5>; }; }; cpu-map { @@ -68,109 +51,111 @@ }; }; }; - /* Memory layout: - * - * [0x0_6000_0000, 0x0_7000_0000) -> DMA pool (256 MiB) - * [0x0_7000_0000, 0x0_8000_0000) -> DMA pool (256 MiB) - * [0x0_8000_0000, 0x0_b000_0000) -> RAM memory (768 MiB) - * [0x0_b000_0000, 0x0_c000_0000) -> Broken? (256 MiB) - * [0x0_c000_0000, 0x1_0000_0000) -> Empty (1024 MiB) - * [0x1_0000_0000, 0x1_c000_0000) -> PMEM (3072 MiB) - * [0x1_c000_0000, 0x2_8000_0000) -> Empty (3072 MiB) - */ - memory@80000000 { + memory@MEM_ADDR { device_type = "memory"; - reg = <0x0 0x80000000 0x0 0x30000000>; + reg = /bits/ 64 ; }; reserved-memory { #address-cells = <2>; /* Starting address and size */ #size-cells = <2>; /* 64 bits memory addresses */ ranges; - eth_pool: dma_pool@60000000 { - reg = <0x0 0x60000000 0x0 0x10000000>; + eth_pool: dma_pool@ETHPOOL_ADDR { + reg = /bits/ 64 ; compatible = "shared-dma-pool"; }; - onic_pool: dma_pool@70000000 { - reg = <0x0 0x70000000 0x0 0x10000000>; + onic_pool: dma_pool@ONICPOOL_ADDR { + reg = /bits/ 64 ; compatible = "shared-dma-pool"; }; }; - dma_clk: dma_clk { - compatible = "fixed-clock"; - #clock-cells = <0x00000000>; - clock-frequency = <0x09502f90>; - phandle = <0x00000002>; - }; - pmem@100000000 { + pmem@PMEM_ADDR { /* volatile; This property indicates that this region is * actually backed by non-persistent memory. This lets the OS * know that it may skip the cache flushes required to ensure * data is made persistent after a write. */ volatile; compatible = "pmem-region"; - reg = <0x1 0x00000000 0x0 0xc0000000>; + reg = /bits/ 64 ; }; soc { - #address-cells = <0x00000002>; - #size-cells = <0x00000002>; + #address-cells = <2>; + #size-cells = <2>; compatible = "BSC,Lagarto-ox-soc", "simple-bus"; ranges; /* For bitstream e97dd7b2-397f-11ef-abe0-bbd201a5a630 with two * consoles */ +#ifdef ENABLE_UART0 /* The serial for the kernel console */ - uart_console: serial@UART0_ADDR_HEX { + uart_console: serial@UART0_ADDR { compatible = "ns16550"; - reg = <0x0 UART0_ADDR 0x0 0x1000>; + reg = /bits/ 64 ; reg-shift = <2>; /* No interrupts for this UART, use console=hvc0 */ /* This clock is the SERIAL_CLK */ - clock-frequency = ; - current-speed = ; + clock-frequency = ; + current-speed = ; status = "okay"; }; +#endif /* ENABLE_UART0 */ + +#ifdef ENABLE_UART1 /* The serial for interrupt tests */ - uart_testing: serial@40003000 { + uart_testing: serial@UART1_ADDR { compatible = "ns16550"; - reg = <0x0 0x40003000 0x0 0x1000>; + reg = /bits/ 64 ; reg-shift = <2>; /* Output interrupt 1 (the first one) */ interrupts = <1>; interrupt-parent = <&PLIC>; - clock-frequency = ; - current-speed = ; + clock-frequency = ; + current-speed = ; status = "okay"; }; +#endif /* ENABLE_UART1 */ +#ifdef ENABLE_ETHERNET ethernet0 { - xlnx,rxmem = <0x000005f2>; - carv,mtu = <0x000005dc>; - carv,no-mac; - device_type = "network"; - // 02:$node:00:01:00:$fpga -> 02:05:00:01:00:02 - // 10.5.1.$N/16 -> 10.5.1.184/16 - // N = 150 + ($node - 1) * 8 + $fpga - local-mac-address = [00 00 00 00 00 00]; - axistream-connected = <&axi_dma>; - compatible = "xlnx,xxv-ethernet-1.0-carv"; - memory-region = <ð_pool>; + xlnx,rxmem = <1522>; + carv,mtu = <1500>; + carv,no-mac; + device_type = "network"; + // 02:$node:00:01:00:$fpga -> 02:05:00:01:00:02 + // 10.5.1.$N/16 -> 10.5.1.184/16 + // N = 150 + ($node - 1) * 8 + $fpga + local-mac-address = [00 00 00 00 00 00]; + axistream-connected = <&axi_dma>; + compatible = "xlnx,xxv-ethernet-1.0-carv"; + memory-region = <ð_pool>; }; +#endif /* ENABLE_ETHERNET */ - axi_dma: dma@40400000 { +#ifdef ENABLE_AXIDMA + dma_clk: dma_clk { + compatible = "fixed-clock"; + #clock-cells = <0x0>; + clock-frequency = ; + }; + axi_dma: dma@AXIDMA_ADDR { + reg = /bits/ 64 ; + reg-shift = <2>; + #address-cells = <2>; + #size-cells = <2>; xlnx,include-dre; - #dma-cells = <0x00000001>; + #dma-cells = <0x1>; compatible = "xlnx,axi-dma-1.00.a"; - clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk", "m_axi_s2mm_aclk", "m_axi_sg_aclk"; + clock-names = "s_axi_lite_aclk", "m_axi_mm2s_aclk", + "m_axi_s2mm_aclk", "m_axi_sg_aclk"; clocks = <&dma_clk>, <&dma_clk>, <&dma_clk>, <&dma_clk>; - reg = <0x00000000 0x40400000 0x00000000 0x00400000>; interrupt-names = "mm2s_introut", "s2mm_introut"; interrupt-parent = <&PLIC>; interrupts = <2 3>; xlnx,addrwidth = <0x28>; xlnx,include-sg; xlnx,sg-length-width = <0x17>; - dma-channel@40400000 { + dma-channel@AXIDMA_CH0 { + reg = /bits/ 64 ; compatible = "xlnx,axi-dma-mm2s-channel"; dma-channels = <0>; interrupts = <2>; @@ -178,7 +163,8 @@ xlnx,device-id = <0x0>; xlnx,include-dre; }; - dma-channel@40400030 { + dma-channel@AXIDMA_CH1 { + reg = /bits/ 64 ; compatible = "xlnx,axi-dma-s2mm-channel"; dma-channels = <1>; interrupts = <3>; @@ -187,10 +173,13 @@ xlnx,include-dre; }; }; +#endif /* ENABLE_AXIDMA */ +#ifdef ENABLE_PLIC /* Platform-Level Interrupt Controller: Delivers interrupts to * HARTs. */ - PLIC: plic@40800000 { + PLIC: plic@PLIC_ADDR { + reg = /bits/ 64 ; compatible = "riscv,plic0"; interrupt-controller; /* Receives interrupts */ #address-cells = <0>; @@ -201,11 +190,12 @@ * - context 1: supervisor mode external interrupt (9) */ interrupts-extended = <&HLIC0 11>, <&HLIC0 9>; - reg = < 0x0 0x40800000 0x0 0x00400000>; - riscv,ndev = <4>; + riscv,ndev = ; //riscv,max-priority = <0x7>; - phandle = <0x3>; }; +#endif /* ENABLE_PLIC */ + +#ifdef ENABLE_CLINT /* Core Local Interruptor: It directly connects to the timer and * inter-processor interrupt lines of various HARTs (or CPUs) so * RISC-V per-HART (or per-CPU) local interrupt controller is @@ -216,32 +206,33 @@ * Documentation/devicetree/bindings/riscv/cpus.yaml */ clint: clint@40100000 { - reg = <0x0 0x40100000 0x0 0x00010000>; + reg = /bits/ 64 ; reg-names = "control"; interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; compatible = "riscv,clint0"; }; +#endif /* ENABLE_CLINT */ +#if 0 /* There is another auxiliar clint (timer) at 40010000 for * tests, but we don't tell the kernel so we can use it for * testing interrupts manually. */ - #if 0 aux_timer: clint@40010000 { - reg = <0x0 0x40010000 0x0 0x00010000>; + reg = /bits/ 64 <0x0 0x40010000 0x0 0x00010000>; reg-names = "control"; interrupts = <4>; /* PLIC input source 4 */ interrupt-parent = <&PLIC>; compatible = "riscv,clint0"; }; - #endif +#endif - #ifdef ENABLE_SPI +#ifdef ENABLE_SPI uart16750: serial@40005000 { compatible = "ns16750"; reg = <0x00000000 0x40005000 0x00000000 0x00001000>; interrupt-parent = <&PLIC>; interrupts = <5>; - clock-frequency = ; + clock-frequency = ; current-speed = <0x0001c200>; status = "okay"; }; @@ -258,6 +249,7 @@ loopback-mode = <1>; status = "okay"; }; - #endif +#endif /* ENABLE_SPI */ + }; }; diff --git a/dts/lagarto_ox.h b/dts/lagarto_ox.h index 6a92d95..f7e8f0e 100644 --- a/dts/lagarto_ox.h +++ b/dts/lagarto_ox.h @@ -1,21 +1,62 @@ -/* CPU is at 50 MHz */ -#define CPU_CLOCK_FREQUENCY 50000000 - -/* The RTC timer is clocked at the CPU frequency / 1525, so - * around 32786.88 Hz */ -//#define RTC_CLOCK_FREQUENCY 32786 - +#define CPU_FREQ 50000000 /* 50 MHz */ /* FIXME: The real RTC frequency is around half that, as the divider was wrongly * configured. So for now lets use the real frequency: * 50e6 / (1525*2) = 16393.44262295082 -> 16393 Hz */ -#define RTC_CLOCK_FREQUENCY 16393 +#define RTC_FREQ 16393 + +/* Memory layout: + * + * [0x0_4000_0000, 0x0_6000_0000) -> IO (512 MiB) + * [0x0_6000_0000, 0x0_7000_0000) -> DMA pool (256 MiB) + * [0x0_7000_0000, 0x0_8000_0000) -> DMA pool (256 MiB) + * [0x0_8000_0000, 0x1_c000_0000) -> RAM memory (5 GiB) + * [0x1_c000_0000, 0x2_8000_0000) -> PMEM (3 GiB) + */ + +#define UART0_SPEED 115200 +#define UART0_ADDR 0x40001000 +#define UART0_SIZE 0x00001000 + +#define UART1_SPEED UART0_SPEED +#define UART1_ADDR 0x40003000 +#define UART1_SIZE 0x00001000 + +#define AUXTIMER_ADDR 0x40010000 +#define AUXTIMER_SIZE 0x00010000 + +#define CLINT_ADDR 0x40100000 +#define CLINT_SIZE 0x00010000 -#define UART_SPEED 115200 -#define UART0_ADDR 0x40001000 -#define UART0_ADDR_HEX 40001000 +#define AXIDMA_ADDR 0x40400000 +#define AXIDMA_SIZE 0x00400000 +#define AXIDMA_CH0 0x40400000 +#define AXIDMA_CH1 0x40400030 +#define AXIDMA_FREQ 156250000 -#define PLIC_ADDR 0x40800000 -#define UART_ADDR 0x40001000 +#define PLIC_ADDR 0x40800000 +#define PLIC_SIZE 0x00400000 +#define PLIC_NDEV 4 + +#define ETHPOOL_ADDR 0x60000000 +#define ETHPOOL_SIZE 0x10000000 + +#define ONICPOOL_ADDR 0x70000000 +#define ONICPOOL_SIZE 0x10000000 + +/* Notice addresses > 32 bits from here */ + +#define MEM_ADDR 0x080000000 +#define MEM_SIZE 0x140000000 + +#define PMEM_ADDR 0x1c0000000 +#define PMEM_SIZE 0x0c0000000 /* Toggles */ + +#define ENABLE_UART0 +#define ENABLE_UART1 +#define ENABLE_ETHERNET +#define ENABLE_AXIDMA +#define ENABLE_PLIC +#define ENABLE_CLINT //#define ENABLE_SPI diff --git a/fpga/env.sh b/fpga/env.sh index 7df5b3b..6a11fff 100644 --- a/fpga/env.sh +++ b/fpga/env.sh @@ -43,7 +43,7 @@ function setup_meep() export FPGACTL_BOOTLOADER_ADDR=$((0x80000000+$delta_addr)) export FPGACTL_KERNEL_ADDR=$((0x84000000+$delta_addr)) export FPGACTL_INITRD_ADDR=$((0x8c300000+$delta_addr)) - export FPGACTL_ROOTFS_ADDR=$((0x100000000+$delta_addr)) + export FPGACTL_ROOTFS_ADDR=$((0x1c0000000+$delta_addr)) export FPGACTL_BOOTROM_ADDR=$((0x00000100)) } -- GitLab From f19f5b19ae15b32ac5999345595aac0d68d23ab2 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 10 Oct 2024 15:46:12 +0200 Subject: [PATCH 301/310] Check CPU usage before running SPEC --- lagarto-ox.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index ca54c7c..015acf8 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -408,6 +408,8 @@ export TMPDIR=/tmp # We need /bin/sh ln -s $(which sh) /bin/sh + # Check CPU usage + vmstat 5 5 bash -x speclaunch cat /tmp/spec/time.csv # Give me a shell at the end -- GitLab From 7d3fe35dfbed7ad38eaf921578b7fbee0a6e3dbe Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 10 Oct 2024 16:09:44 +0200 Subject: [PATCH 302/310] Reduce RAM to 4 GiB It seems to be hanging now in LVM. --- dts/lagarto_ox.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dts/lagarto_ox.h b/dts/lagarto_ox.h index f7e8f0e..92de7d4 100644 --- a/dts/lagarto_ox.h +++ b/dts/lagarto_ox.h @@ -9,7 +9,8 @@ * [0x0_4000_0000, 0x0_6000_0000) -> IO (512 MiB) * [0x0_6000_0000, 0x0_7000_0000) -> DMA pool (256 MiB) * [0x0_7000_0000, 0x0_8000_0000) -> DMA pool (256 MiB) - * [0x0_8000_0000, 0x1_c000_0000) -> RAM memory (5 GiB) + * [0x0_8000_0000, 0x1_8000_0000) -> RAM memory (4 GiB) + * [0x1_8000_0000, 0x1_c000_0000) -> Unused (1 GiB) * [0x1_c000_0000, 0x2_8000_0000) -> PMEM (3 GiB) */ @@ -46,7 +47,7 @@ /* Notice addresses > 32 bits from here */ #define MEM_ADDR 0x080000000 -#define MEM_SIZE 0x140000000 +#define MEM_SIZE 0x100000000 #define PMEM_ADDR 0x1c0000000 #define PMEM_SIZE 0x0c0000000 -- GitLab From a9a8a0a40dad5857f21a05ddc5001571ec23f35f Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 10 Oct 2024 19:40:49 +0200 Subject: [PATCH 303/310] Increase timeout for SPEC benchmarks --- fpga/run-node.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fpga/run-node.sh b/fpga/run-node.sh index 05ba16d..13edf0c 100755 --- a/fpga/run-node.sh +++ b/fpga/run-node.sh @@ -25,8 +25,8 @@ set -x #timeout_silent=$((3 * 60)) # Stop if 3 min without output # Timeouts for SPEC benchmarks -timeout=$((6 * 60 * 60)) # Always stop after 6 h -timeout_silent=$((2 * 60 * 60)) # Stop if 2 h without output (some benchmarks take 1.6h) +timeout=$((12 * 60 * 60)) # Always stop after 12 h +timeout_silent=$((4 * 60 * 60)) # Stop if 4 h without output (some benchmarks take 1.6h) # Set dead switch sleep $timeout && killall picocom & -- GitLab From 2c23e3d554bca8bd07ddb0c54497228e7cd58e31 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 11 Oct 2024 09:34:18 +0200 Subject: [PATCH 304/310] Update README --- README.md | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 3fe8665..0728bf0 100644 --- a/README.md +++ b/README.md @@ -2,45 +2,49 @@ This repository contains NixOS configurations for different RISC-V machines. -## QEMU +## Lagarto Ox on FPGA Alveo U55C -To boot the system in QEMU, first enter the development shell: +To build the system and boot it on an FPGA of the MEEP cluster, you can run the +following: ``` -$ nix develop +$ nix develop -L '.#lagarto-ox' --command fpga/run-remotely.sh fpgalogin1:ci ``` -Then run the boot script: +To do it manually, you can first enter the development shell: ``` -$ ./boot.sh +$ nix develop -L '.#lagarto-ox' ``` -To prevent the GC from erasing the system: +Then upload the files to the target machine (fpgalogin1 by default): ``` -$ nix build .#devShells.x86_64-linux.default --out-link result-env +$ fpga/upload.sh ``` -## Lagarto Hun on FPGA Alveo U55C - -First build required dependencies: +Then connect to the fpgalogin1 machine, allocate a FPGA node and load the +environment there: ``` -$ nix develop '.#lagarto-hun' +$ cd nixos +$ . env.sh ``` -Then upload to destination: +Flash the images to the FPGA: ``` -$ fpga/upload.sh +$ ./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img ``` -And then boot the system there: +And monitor the serial line: ``` -cucu$ ./boot.sh -... +$ picocom -q -b 115200 $FPGACTL_UART ``` -In the U-Boot prompt, paste the commands of the `uboot.env` file. +It should boot without any user interaction. + +## Lagarto Hun + +WIP -- GitLab From 36fdfb93f59a9bb693e6331443eb1a4110436bc3 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 11 Oct 2024 09:40:40 +0200 Subject: [PATCH 305/310] Add SPEC mini result --- JOURNAL.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/JOURNAL.md b/JOURNAL.md index 3db80a5..10275de 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -5291,3 +5291,17 @@ benchmarks too. At some point we may want to be able to specify the bootcmd from fpgactl directly. + +## 2024-10-11 + +Another successful execution of SPEC mini: + + + + benchmark size tune iter time_s + 600.perlbench_s test base 1 5380.726590000 + 602.gcc_s test base 1 2.525468000 + 605.mcf_s test base 1 1927.921307000 + 620.omnetpp_s test base 1 1110.200756000 + 641.leela_s test base 1 779.333069000 + 648.exchange2_s test base 1 2916.464893000 -- GitLab From 4d1e8d3dea1577ce78154cc495d5429757d614a0 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 11 Oct 2024 10:35:12 +0200 Subject: [PATCH 306/310] Fix squeue script Now we can see jobs from other users. --- fpga/run-login.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fpga/run-login.sh b/fpga/run-login.sh index cc95b40..8720bf3 100755 --- a/fpga/run-login.sh +++ b/fpga/run-login.sh @@ -42,7 +42,7 @@ fi # If this point is reached there is one job running -host=$(squeue -h -o %N) +host=$(squeue --me -h -o %N) echo "Switching to $host" # Continue the execution there -- GitLab From 4333f8074e6db4113ae2bf22e4273b1893c4f913 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 11 Oct 2024 11:01:48 +0200 Subject: [PATCH 307/310] Configure SPI addresses in header We use the ENABLE_SPI toggle to also set the PLIC number of inputs. --- dts/lagarto_ox.dts | 18 +++++++++--------- dts/lagarto_ox.h | 35 ++++++++++++++++++++++++----------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/dts/lagarto_ox.dts b/dts/lagarto_ox.dts index d3881b3..d9c41ce 100644 --- a/dts/lagarto_ox.dts +++ b/dts/lagarto_ox.dts @@ -205,7 +205,7 @@ * property is described in * Documentation/devicetree/bindings/riscv/cpus.yaml */ - clint: clint@40100000 { + clint: clint@CLINT_ADDR { reg = /bits/ 64 ; reg-names = "control"; interrupts-extended = <&HLIC0 3>, <&HLIC0 7>; @@ -217,8 +217,8 @@ /* There is another auxiliar clint (timer) at 40010000 for * tests, but we don't tell the kernel so we can use it for * testing interrupts manually. */ - aux_timer: clint@40010000 { - reg = /bits/ 64 <0x0 0x40010000 0x0 0x00010000>; + aux_timer: clint@AUXTIMER_ADDR { + reg = /bits/ 64 ; reg-names = "control"; interrupts = <4>; /* PLIC input source 4 */ interrupt-parent = <&PLIC>; @@ -227,23 +227,23 @@ #endif #ifdef ENABLE_SPI - uart16750: serial@40005000 { + serial@UART2_ADDR { compatible = "ns16750"; - reg = <0x00000000 0x40005000 0x00000000 0x00001000>; + reg = /bits/ 64 ; interrupt-parent = <&PLIC>; interrupts = <5>; clock-frequency = ; - current-speed = <0x0001c200>; + current-speed = ; status = "okay"; }; - spi0: spi@40007000 { + spi@SPI_ADDR { compatible = "ti,keystone-spi"; - reg = <0x00000000 0x40007000 0x00000000 0x00001000>; + reg = /bits/ 64 ; #address-cells = <1>; #size-cells = <0>; interrupt-parent = <&PLIC>; interrupt-names = "intvec0", "intvec1"; - interrupts = <6 0>, <0x00000007 0>; + interrupts = <6 0>, <7 0>; ti,davinci-spi-intr-line = <0>; spi-max-frequency = <24000000>; loopback-mode = <1>; diff --git a/dts/lagarto_ox.h b/dts/lagarto_ox.h index 92de7d4..4892738 100644 --- a/dts/lagarto_ox.h +++ b/dts/lagarto_ox.h @@ -1,3 +1,13 @@ +/* Toggles */ + +#define ENABLE_UART0 +#define ENABLE_UART1 +#define ENABLE_ETHERNET +#define ENABLE_AXIDMA +#define ENABLE_PLIC +#define ENABLE_CLINT +//#define ENABLE_SPI + #define CPU_FREQ 50000000 /* 50 MHz */ /* FIXME: The real RTC frequency is around half that, as the divider was wrongly * configured. So for now lets use the real frequency: @@ -22,6 +32,14 @@ #define UART1_ADDR 0x40003000 #define UART1_SIZE 0x00001000 +/* UART2 via SPI */ +#define UART2_SPEED UART0_SPEED +#define UART2_ADDR 0x40005000 +#define UART2_SIZE 0x00001000 + +#define SPI_ADDR 0x40007000 +#define SPI_SIZE 0x00001000 + #define AUXTIMER_ADDR 0x40010000 #define AUXTIMER_SIZE 0x00010000 @@ -36,7 +54,12 @@ #define PLIC_ADDR 0x40800000 #define PLIC_SIZE 0x00400000 -#define PLIC_NDEV 4 + +#ifdef ENABLE_SPI +# define PLIC_NDEV 7 /* extra UART2 + 2 x SPI */ +#else +# define PLIC_NDEV 4 +#endif #define ETHPOOL_ADDR 0x60000000 #define ETHPOOL_SIZE 0x10000000 @@ -51,13 +74,3 @@ #define PMEM_ADDR 0x1c0000000 #define PMEM_SIZE 0x0c0000000 - -/* Toggles */ - -#define ENABLE_UART0 -#define ENABLE_UART1 -#define ENABLE_ETHERNET -#define ENABLE_AXIDMA -#define ENABLE_PLIC -#define ENABLE_CLINT -//#define ENABLE_SPI -- GitLab From ad5dd729283d8e221afcb5f56a27b7c5a9dae4fa Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 11 Oct 2024 11:05:45 +0200 Subject: [PATCH 308/310] Switch to debug2 instead of bench2 No need to run the SPEC benchmarks all the time. Stopping at the stage2 shell is enough to test most of the system. --- fpga/run-node.sh | 8 ++++---- lagarto-ox.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fpga/run-node.sh b/fpga/run-node.sh index 13edf0c..c234acb 100755 --- a/fpga/run-node.sh +++ b/fpga/run-node.sh @@ -21,12 +21,12 @@ set -x ./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img # Normal timeouts -#timeout=$((30 * 60)) # Always stop after 30 min -#timeout_silent=$((3 * 60)) # Stop if 3 min without output +timeout=$((30 * 60)) # Always stop after 30 min +timeout_silent=$((3 * 60)) # Stop if 3 min without output # Timeouts for SPEC benchmarks -timeout=$((12 * 60 * 60)) # Always stop after 12 h -timeout_silent=$((4 * 60 * 60)) # Stop if 4 h without output (some benchmarks take 1.6h) +#timeout=$((12 * 60 * 60)) # Always stop after 12 h +#timeout_silent=$((4 * 60 * 60)) # Stop if 4 h without output (some benchmarks take 1.6h) # Set dead switch sleep $timeout && killall picocom & diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 015acf8..a90b0a0 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -540,7 +540,7 @@ in prev.runCommand "uboot.txt" {} '' cat > $out < Date: Fri, 11 Oct 2024 11:06:36 +0200 Subject: [PATCH 309/310] Print a message before we drop to a shell This may be used by another script to determine where we managed to boot. --- patches/stage-2-init.sh | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/patches/stage-2-init.sh b/patches/stage-2-init.sh index 8dd7b47..66afd2d 100755 --- a/patches/stage-2-init.sh +++ b/patches/stage-2-init.sh @@ -1,8 +1,6 @@ #! @shell@ -set -x - -echo "--- starting stage 2 ---" +#set -x systemConfig=@systemConfig@ @@ -10,6 +8,11 @@ export HOME=/root PATH="@path@" if [ "${IN_NIXOS_SYSTEMD_STAGE1:-}" != true ]; then + # Print a greeting. + echo + echo -e "\e[1;32m<<< @distroName@ Stage 2 >>>\e[0m" + echo + # Process the kernel command line. for o in $(>>\e[0m" - echo - - # Normally, stage 1 mounts the root filesystem read/writable. # However, in some environments, stage 2 is executed directly, and the # root is read-only. So make it writable here. -- GitLab From 52101b0d9a238f7077eb7da2298af4951d1e2ce8 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 11 Oct 2024 11:11:51 +0200 Subject: [PATCH 310/310] Disable U-Boot verbose build --- lagarto-ox.nix | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lagarto-ox.nix b/lagarto-ox.nix index a90b0a0..571cc1c 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -453,16 +453,16 @@ preConfigure = '' cp ${final.uboot-env} board/emulation/qemu-riscv/environ.env ''; - postConfigure = '' - echo --------------------------- generated config: - cat .config - echo --------------------------- - ''; - postBuild = '' - echo --------------------------- generated env starts - cat include/generated/env.in - echo --------------------------- generated env ends - ''; + #postConfigure = '' + # echo --------------------------- generated config: + # cat .config + # echo --------------------------- + #''; + #postBuild = '' + # echo --------------------------- generated env starts + # cat include/generated/env.in + # echo --------------------------- generated env ends + #''; # # CONFIG_SERIAL_PRESENT=n # CONFIG_SYS_NS16550=n @@ -502,7 +502,7 @@ # '' ; extraMakeFlags = [ - "V=1" + #"V=1" #"KCPPFLAGS=-DLOG_DEBUG" #"EXT_DTB=${final.ox-dtb}/lagarto_ox.dtb" ]; -- GitLab