From 26ad3e49f7f1dd2e57af94e2fd3242882130b8dd Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Fri, 30 Oct 2020 15:21:23 +0100
Subject: [PATCH 01/16] fwi: add gitBranch and copy params

---
 garlic/apps/fwi/default.nix | 43 ++++++++++++++++++++++---------------
 1 file changed, 26 insertions(+), 17 deletions(-)

diff --git a/garlic/apps/fwi/default.nix b/garlic/apps/fwi/default.nix
index 594e73ec..96c2691b 100644
--- a/garlic/apps/fwi/default.nix
+++ b/garlic/apps/fwi/default.nix
@@ -1,30 +1,32 @@
 {
   stdenv
-, nanos6
-, mpi
-, tampi
-, mcxx
-, icc
+, mpi ? null
+, tampi ? null
+, mcxx ? null
+, cc
+, gitBranch
 }:
 
+with stdenv.lib;
+
+assert !(tampi != null && mcxx == null);
+
 stdenv.mkDerivation rec {
   name = "fwi";
-  variant = "oss+task";
 
   src = builtins.fetchGit {
     url = "https://gitlab.com/srodrb/BSC-FWI.git";
-    ref = "${variant}";
+    ref = "${gitBranch}";
   };
 
   enableParallelBuilding = true;
 
   buildInputs = [
-    nanos6
-    mpi
-    icc
-    tampi
-    mcxx
-  ];
+    cc
+  ]
+  ++ optional (mpi != null) mpi
+  ++ optional (tampi != null) tampi
+  ++ optional (mcxx != null) mcxx;
 
   # FIXME: This is an ugly hack.
   # When using _GNU_SOURCE or any other definition used in features.h, we need
@@ -45,15 +47,22 @@ stdenv.mkDerivation rec {
   '';
 
   makeFlags = [
-    "NZF=108"
-    "NXF=108"
-    "NYF=208"
-    "PRECISION=float"
+    "CC=${cc.cc.CC}"
   ];
 
+  postBuild = ''
+    make input
+  '';
+
+  #FIXME split the input in another derivation
   installPhase = ''
     mkdir -p $out/bin
     cp fwi $out/bin
     cp ModelGenerator $out/bin
+    mv InputModels $out/bin
+    mkdir -p $out/etc/fwi
+    cp SetupParams/{fwi_frequencies.txt,fwi_params.txt} $out/etc/fwi
   '';
+
+  programPath = "/bin/fwi";
 }
-- 
GitLab


From f10f8472acf888d506b009490f479bfae7ca325f Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Fri, 30 Oct 2020 15:58:03 +0100
Subject: [PATCH 02/16] fwi: add seq test experiment

---
 garlic/exp/fwi/test.nix | 66 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 garlic/exp/fwi/test.nix

diff --git a/garlic/exp/fwi/test.nix b/garlic/exp/fwi/test.nix
new file mode 100644
index 00000000..8fe6cb8c
--- /dev/null
+++ b/garlic/exp/fwi/test.nix
@@ -0,0 +1,66 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = {
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    # Options for creams
+    cc = icc;
+    gitBranch = "seq";
+
+    # Repeat the execution of each unit 30 times
+    loops = 1;
+
+    # Resources
+    qos = "debug";
+    nodes = 1;
+    time = "02:00:00";
+    ntasksPerNode = 1;
+    cpuBind = "rank,verbose";
+    jobName = "fwi-${gitBranch}";
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  # Custom stage to copy the FWI input
+  copyInput = {nextStage, conf, ...}:
+    let
+      input = bsc.garlic.apps.fwi;
+    in
+      stages.exec {
+        inherit nextStage;
+        env = ''
+          cp -r ${input}/bin/InputModels .
+          chmod +w -R .
+        '';
+        argv = [
+          "${input}/etc/fwi/fwi_params.txt"
+          "${input}/etc/fwi/fwi_frequencies.txt"
+        ];
+      };
+
+  # FWI program
+  program = {nextStage, conf, ...}: with conf;
+    bsc.garlic.apps.fwi.override {
+      inherit cc gitBranch;
+    };
+
+  pipeline = stdexp.stdPipeline ++ [ copyInput program ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
-- 
GitLab


From 9bea3cc26402ef4377c792a5033223837b98e449 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Fri, 30 Oct 2020 16:12:30 +0100
Subject: [PATCH 03/16] fwi: add oss experiment

---
 garlic/exp/fwi/oss.nix | 69 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 garlic/exp/fwi/oss.nix

diff --git a/garlic/exp/fwi/oss.nix b/garlic/exp/fwi/oss.nix
new file mode 100644
index 00000000..63a4752a
--- /dev/null
+++ b/garlic/exp/fwi/oss.nix
@@ -0,0 +1,69 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = {
+    blocksize = [ 1 2 4 8 16 ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    # Options for creams
+    cc = icc;
+    gitBranch = "oss";
+    inherit (c) blocksize;
+
+    # Repeat the execution of each unit 30 times
+    loops = 30;
+
+    # Resources
+    qos = "debug";
+    nodes = 1;
+    time = "02:00:00";
+    ntasksPerNode = 1;
+    cpuBind = "sockets,verbose";
+    jobName = "fwi-${gitBranch}";
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  # Custom stage to copy the FWI input
+  copyInput = {nextStage, conf, ...}:
+    let
+      input = bsc.garlic.apps.fwi;
+    in
+      stages.exec {
+        inherit nextStage;
+        env = ''
+          cp -r ${input}/bin/InputModels .
+          chmod +w -R .
+        '';
+        argv = [
+          "${input}/etc/fwi/fwi_params.txt"
+          "${input}/etc/fwi/fwi_frequencies.txt"
+          "${toString conf.blocksize}"
+        ];
+      };
+
+  # FWI program
+  program = {nextStage, conf, ...}: with conf;
+    bsc.garlic.apps.fwi.override {
+      inherit cc gitBranch;
+    };
+
+  pipeline = stdexp.stdPipeline ++ [ copyInput program ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
-- 
GitLab


From bfbbc294ae6c476b73b347105483bae4b921f9db Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Fri, 29 Jan 2021 15:33:12 +0100
Subject: [PATCH 04/16] fwi: split into input and solver

All branches compile with several hacks.
---
 garlic/apps/fwi/default.nix | 38 ++++++++++----------
 garlic/apps/fwi/input.nix   | 40 +++++++++++++++++++++
 garlic/apps/index.nix       |  7 +++-
 garlic/exp/fwi/test.nix     | 71 ++++++++++++++++++++++---------------
 garlic/exp/index.nix        |  4 +++
 5 files changed, 112 insertions(+), 48 deletions(-)
 create mode 100644 garlic/apps/fwi/input.nix

diff --git a/garlic/apps/fwi/default.nix b/garlic/apps/fwi/default.nix
index 96c2691b..94d1bdcb 100644
--- a/garlic/apps/fwi/default.nix
+++ b/garlic/apps/fwi/default.nix
@@ -4,7 +4,8 @@
 , tampi ? null
 , mcxx ? null
 , cc
-, gitBranch
+, gitBranch ? "garlic/tampi+send+oss+task"
+, fwiInput
 }:
 
 with stdenv.lib;
@@ -12,6 +13,7 @@ with stdenv.lib;
 assert !(tampi != null && mcxx == null);
 
 stdenv.mkDerivation rec {
+  inherit gitBranch;
   name = "fwi";
 
   src = builtins.fetchGit {
@@ -19,14 +21,20 @@ stdenv.mkDerivation rec {
     ref = "${gitBranch}";
   };
 
-  enableParallelBuilding = true;
+  enableParallelBuilding = false;
 
   buildInputs = [
     cc
   ]
-  ++ optional (mpi != null) mpi
+  ++ optional (mpi   != null) mpi
   ++ optional (tampi != null) tampi
-  ++ optional (mcxx != null) mcxx;
+  ++ optional (mcxx  != null) mcxx;
+
+  # FIXME: Correct this on the Makefile so we can just type "make fwi"
+  # FIXME: Allow multiple MPI implementations
+  postPatch = ''
+    sed -i 's/= OPENMPI$/= INTEL/g' Makefile
+  '';
 
   # FIXME: This is an ugly hack.
   # When using _GNU_SOURCE or any other definition used in features.h, we need
@@ -35,33 +43,25 @@ stdenv.mkDerivation rec {
   # below, reaches the command line of the preprocessing stage with gcc.
   preConfigure = ''
     export DEFINES=-D_GNU_SOURCE
-    export NANOS6_CONFIG_OVERRIDE=version.debug=true
+
+    make depend
+
+    cp ${fwiInput}/generated_model_params.h src/
   '';
   
   # We compile the ModelGenerator using gcc *only*, as otherwise it will
   # be compiled with nanos6, which requires access to /sys to determine
   # hardware capabilities. So it will fail in the nix-build environment,
   # as there is no /sys mounted.
-  preBuild = ''
-    make COMPILER=GNU ModelGenerator
-  '';
-
   makeFlags = [
-    "CC=${cc.cc.CC}"
+    #"COMPILER=GNU"
+    #"CC=${cc.cc.CC}"
+    "fwi"
   ];
 
-  postBuild = ''
-    make input
-  '';
-
-  #FIXME split the input in another derivation
   installPhase = ''
     mkdir -p $out/bin
     cp fwi $out/bin
-    cp ModelGenerator $out/bin
-    mv InputModels $out/bin
-    mkdir -p $out/etc/fwi
-    cp SetupParams/{fwi_frequencies.txt,fwi_params.txt} $out/etc/fwi
   '';
 
   programPath = "/bin/fwi";
diff --git a/garlic/apps/fwi/input.nix b/garlic/apps/fwi/input.nix
new file mode 100644
index 00000000..a71014f6
--- /dev/null
+++ b/garlic/apps/fwi/input.nix
@@ -0,0 +1,40 @@
+{
+  stdenv
+}:
+
+with stdenv.lib;
+
+stdenv.mkDerivation rec {
+  name = "fwi-header";
+
+  src = builtins.fetchGit {
+    url = "https://gitlab.com/srodrb/BSC-FWI.git";
+    ref = "garlic/seq";
+  };
+
+  enableParallelBuilding = false;
+
+  # FIXME: This is an ugly hack.
+  # When using _GNU_SOURCE or any other definition used in features.h, we need
+  # to define them before mcc includes nanos6.h from the command line. So the
+  # only chance is by setting it at the command line with -D. Using the DEFINES
+  # below, reaches the command line of the preprocessing stage with gcc.
+  preConfigure = ''
+    export DEFINES=-D_GNU_SOURCE
+  '';
+  
+  # We compile the ModelGenerator using gcc *only*, as otherwise it will
+  # be compiled with nanos6, which requires access to /sys to determine
+  # hardware capabilities. So it will fail in the nix-build environment,
+  # as there is no /sys mounted.
+  # Also, we need to compile it with the builder platform as target, as is going
+  # to be executed during the build to generate the src/generated_model_params.h
+  # header.
+  makeFlags = [ "COMPILER=GNU" "params" "input" ];
+
+  installPhase = ''
+    mkdir -p $out/
+    cp src/generated_model_params.h $out/
+    cp -r InputModels $out/
+  '';
+}
diff --git a/garlic/apps/index.nix b/garlic/apps/index.nix
index ac2ccfd8..41da2955 100644
--- a/garlic/apps/index.nix
+++ b/garlic/apps/index.nix
@@ -53,5 +53,10 @@
 
   hpccg = callPackage ./hpccg/default.nix { };
 
-  fwi = callPackage ./fwi/default.nix { };
+  fwi = rec {
+    input = callPackage ./fwi/input.nix { };
+    solver = callPackage ./fwi/default.nix {
+      fwiInput = input;
+    };
+  };
 }
diff --git a/garlic/exp/fwi/test.nix b/garlic/exp/fwi/test.nix
index 8fe6cb8c..26de7f4d 100644
--- a/garlic/exp/fwi/test.nix
+++ b/garlic/exp/fwi/test.nix
@@ -11,24 +11,38 @@ with stdenv.lib;
 let
   # Initial variable configuration
   varConf = {
+    gitBranch = [
+      "garlic/tampi+send+oss+task"
+      "garlic/mpi+send+omp+task"
+      "garlic/mpi+send+oss+task"
+      "garlic/mpi+send+seq"
+      "garlic/oss+task"
+      "garlic/omp+task"
+      "garlic/seq"
+    ];
   };
 
+  machineConfig = targetMachine.config;
+
   # Generate the complete configuration for each unit
   genConf = with bsc; c: targetMachine.config // rec {
-    # Options for creams
+    expName = "fwi";
+    unitName = "${expName}-test";
+    inherit (machineConfig) hw;
+
     cc = icc;
-    gitBranch = "seq";
+    gitBranch = c.gitBranch;
 
-    # Repeat the execution of each unit 30 times
-    loops = 1;
+    # Repeat the execution of each unit several times
+    loops = 10;
 
     # Resources
-    qos = "debug";
+    cpusPerTask = hw.cpusPerSocket;
+    ntasksPerNode = 1;
     nodes = 1;
+    qos = "debug";
     time = "02:00:00";
-    ntasksPerNode = 1;
-    cpuBind = "rank,verbose";
-    jobName = "fwi-${gitBranch}";
+    jobName = unitName;
   };
 
   # Compute the array of configurations
@@ -37,29 +51,30 @@ let
   };
 
   # Custom stage to copy the FWI input
-  copyInput = {nextStage, conf, ...}:
-    let
-      input = bsc.garlic.apps.fwi;
-    in
-      stages.exec {
-        inherit nextStage;
-        env = ''
-          cp -r ${input}/bin/InputModels .
-          chmod +w -R .
-        '';
-        argv = [
-          "${input}/etc/fwi/fwi_params.txt"
-          "${input}/etc/fwi/fwi_frequencies.txt"
-        ];
-      };
+  #copyInput = {nextStage, conf, ...}:
+  #  let
+  #    input = bsc.garlic.apps.fwi;
+  #  in
+  #    stages.exec {
+  #      inherit nextStage;
+  #      env = ''
+  #        cp -r ${input}/bin/InputModels .
+  #        chmod +w -R .
+  #      '';
+  #      argv = [
+  #        "${input}/etc/fwi/fwi_params.txt"
+  #        "${input}/etc/fwi/fwi_frequencies.txt"
+  #      ];
+  #    };
+
+  apps = bsc.garlic.apps;
 
   # FWI program
-  program = {nextStage, conf, ...}: with conf;
-    bsc.garlic.apps.fwi.override {
-      inherit cc gitBranch;
-    };
+  program = {nextStage, conf, ...}: apps.fwi.solver.override {
+    inherit (conf) cc gitBranch;
+  };
 
-  pipeline = stdexp.stdPipeline ++ [ copyInput program ];
+  pipeline = stdexp.stdPipeline ++ [ program ];
 
 in
  
diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix
index 3a007fa4..2b34e4a0 100644
--- a/garlic/exp/index.nix
+++ b/garlic/exp/index.nix
@@ -97,6 +97,10 @@
     test = callPackage ./lulesh/test.nix { };
   };
 
+  fwi = {
+    test = callPackage ./fwi/test.nix { };
+  };
+
   osu = rec {
     latency = callPackage ./osu/latency.nix { };
     latencyShm = latency.override { interNode = false; };
-- 
GitLab


From de175b2380a2ccaa6deed490788785e350d46172 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 4 Mar 2021 17:50:51 +0100
Subject: [PATCH 05/16] fwi: fix input name

---
 garlic/apps/fwi/input.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/garlic/apps/fwi/input.nix b/garlic/apps/fwi/input.nix
index a71014f6..a8ba92fc 100644
--- a/garlic/apps/fwi/input.nix
+++ b/garlic/apps/fwi/input.nix
@@ -5,7 +5,7 @@
 with stdenv.lib;
 
 stdenv.mkDerivation rec {
-  name = "fwi-header";
+  name = "fwi-input";
 
   src = builtins.fetchGit {
     url = "https://gitlab.com/srodrb/BSC-FWI.git";
-- 
GitLab


From fa0e9f591f754aca31f140bdb05dd063555cc1d4 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 4 Mar 2021 17:51:47 +0100
Subject: [PATCH 06/16] fwi: update repo url to PM server

---
 garlic/apps/fwi/default.nix | 2 +-
 garlic/apps/fwi/input.nix   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/garlic/apps/fwi/default.nix b/garlic/apps/fwi/default.nix
index 94d1bdcb..013fcef5 100644
--- a/garlic/apps/fwi/default.nix
+++ b/garlic/apps/fwi/default.nix
@@ -17,7 +17,7 @@ stdenv.mkDerivation rec {
   name = "fwi";
 
   src = builtins.fetchGit {
-    url = "https://gitlab.com/srodrb/BSC-FWI.git";
+    url = "ssh://git@bscpm03.bsc.es/garlic/apps/fwi.git";
     ref = "${gitBranch}";
   };
 
diff --git a/garlic/apps/fwi/input.nix b/garlic/apps/fwi/input.nix
index a8ba92fc..63073da1 100644
--- a/garlic/apps/fwi/input.nix
+++ b/garlic/apps/fwi/input.nix
@@ -8,7 +8,7 @@ stdenv.mkDerivation rec {
   name = "fwi-input";
 
   src = builtins.fetchGit {
-    url = "https://gitlab.com/srodrb/BSC-FWI.git";
+    url = "ssh://git@bscpm03.bsc.es/garlic/apps/fwi.git";
     ref = "garlic/seq";
   };
 
-- 
GitLab


From 485b9150e57a3bd43beed7f8c55017f48439140e Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 4 Mar 2021 17:53:07 +0100
Subject: [PATCH 07/16] fwi: add problem size parameters

---
 garlic/apps/fwi/input.nix | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/garlic/apps/fwi/input.nix b/garlic/apps/fwi/input.nix
index 63073da1..f591731f 100644
--- a/garlic/apps/fwi/input.nix
+++ b/garlic/apps/fwi/input.nix
@@ -1,8 +1,12 @@
 {
   stdenv
+, nz ? 200
+, nx ? 200
+, ny ? 500
 }:
 
 with stdenv.lib;
+with builtins;
 
 stdenv.mkDerivation rec {
   name = "fwi-input";
@@ -14,6 +18,13 @@ stdenv.mkDerivation rec {
 
   enableParallelBuilding = false;
 
+  # Set the input size with the weird order (nz,nx,ny).
+  postPatch = ''
+    sed -i 1c${toString nz} SetupParams/fwi_params.txt
+    sed -i 2c${toString nx} SetupParams/fwi_params.txt
+    sed -i 3c${toString ny} SetupParams/fwi_params.txt
+  '';
+
   # FIXME: This is an ugly hack.
   # When using _GNU_SOURCE or any other definition used in features.h, we need
   # to define them before mcc includes nanos6.h from the command line. So the
-- 
GitLab


From 3de7b5a0b6d93f64c1dfe26da38b96275144df90 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 4 Mar 2021 18:40:27 +0100
Subject: [PATCH 08/16] fwi: save the params and frequencies files

---
 garlic/apps/fwi/input.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/garlic/apps/fwi/input.nix b/garlic/apps/fwi/input.nix
index f591731f..848759f1 100644
--- a/garlic/apps/fwi/input.nix
+++ b/garlic/apps/fwi/input.nix
@@ -46,6 +46,8 @@ stdenv.mkDerivation rec {
   installPhase = ''
     mkdir -p $out/
     cp src/generated_model_params.h $out/
+    cp SetupParams/fwi_params.txt $out/
+    cp SetupParams/fwi_frequencies.txt $out/
     cp -r InputModels $out/
   '';
 }
-- 
GitLab


From 7a6cbd3a9e32fe88c93e514a85377c4c1e551e40 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 4 Mar 2021 18:41:24 +0100
Subject: [PATCH 09/16] fwi: update test experiment

---
 garlic/exp/fwi/test.nix | 57 ++++++++++++++++++++++++-----------------
 1 file changed, 33 insertions(+), 24 deletions(-)

diff --git a/garlic/exp/fwi/test.nix b/garlic/exp/fwi/test.nix
index 26de7f4d..9c42fea2 100644
--- a/garlic/exp/fwi/test.nix
+++ b/garlic/exp/fwi/test.nix
@@ -13,13 +13,15 @@ let
   varConf = {
     gitBranch = [
       "garlic/tampi+send+oss+task"
-      "garlic/mpi+send+omp+task"
-      "garlic/mpi+send+oss+task"
-      "garlic/mpi+send+seq"
-      "garlic/oss+task"
-      "garlic/omp+task"
-      "garlic/seq"
+#      "garlic/mpi+send+omp+task"
+#      "garlic/mpi+send+oss+task"
+#      "garlic/mpi+send+seq"
+#      "garlic/oss+task"
+#      "garlic/omp+task"
+#      "garlic/seq"
     ];
+
+    blocksize = [ 1 2 4 ];
   };
 
   machineConfig = targetMachine.config;
@@ -31,7 +33,11 @@ let
     inherit (machineConfig) hw;
 
     cc = icc;
-    gitBranch = c.gitBranch;
+    inherit (c) gitBranch blocksize;
+    n = 500;
+    nx = n;
+    ny = n;
+    nz = n;
 
     # Repeat the execution of each unit several times
     loops = 10;
@@ -50,22 +56,25 @@ let
     inherit varConf genConf;
   };
 
-  # Custom stage to copy the FWI input
-  #copyInput = {nextStage, conf, ...}:
-  #  let
-  #    input = bsc.garlic.apps.fwi;
-  #  in
-  #    stages.exec {
-  #      inherit nextStage;
-  #      env = ''
-  #        cp -r ${input}/bin/InputModels .
-  #        chmod +w -R .
-  #      '';
-  #      argv = [
-  #        "${input}/etc/fwi/fwi_params.txt"
-  #        "${input}/etc/fwi/fwi_frequencies.txt"
-  #      ];
-  #    };
+  exec = {nextStage, conf, ...}:
+  let
+    input = bsc.apps.fwi.input.override {
+      inherit (conf) nx ny nz;
+    };
+  in stages.exec {
+    inherit nextStage;
+    pre = ''
+      ln -fs ${input}/InputModels InputModels || true
+    '';
+    argv = [
+      "${input}/fwi_params.txt"
+      "${input}/fwi_frequencies.txt"
+      conf.blocksize
+      "-1" # Fordward steps
+      "-1" # Backward steps
+      "-1" # Write/read frequency
+    ];
+  };
 
   apps = bsc.garlic.apps;
 
@@ -74,7 +83,7 @@ let
     inherit (conf) cc gitBranch;
   };
 
-  pipeline = stdexp.stdPipeline ++ [ program ];
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
 
 in
  
-- 
GitLab


From a8477b1b05b95990f04b9ec23d1fd0487a8f5f6c Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 4 Mar 2021 18:41:45 +0100
Subject: [PATCH 10/16] fwi: add test figure with the time

---
 garlic/fig/fwi/test.R | 46 +++++++++++++++++++++++++++++++++++++++++++
 garlic/fig/index.nix  |  4 ++++
 2 files changed, 50 insertions(+)
 create mode 100644 garlic/fig/fwi/test.R

diff --git a/garlic/fig/fwi/test.R b/garlic/fig/fwi/test.R
new file mode 100644
index 00000000..ca79f0dd
--- /dev/null
+++ b/garlic/fig/fwi/test.R
@@ -0,0 +1,46 @@
+library(ggplot2)
+library(dplyr)
+library(scales)
+library(jsonlite)
+
+args=commandArgs(trailingOnly=TRUE)
+
+# Read the timetable from args[1]
+input_file = "input.json"
+if (length(args)>0) { input_file = args[1] }
+
+# Load the dataset in NDJSON format
+dataset = jsonlite::stream_in(file(input_file)) %>%
+	jsonlite::flatten()
+
+# We only need the nblocks and time
+df = select(dataset, config.blocksize, config.gitBranch, time) %>%
+	rename(blocksize=config.blocksize, gitBranch=config.gitBranch) %>%
+  group_by(blocksize, gitBranch) %>%
+  mutate(mtime = median(time)) %>%
+  ungroup()
+
+df$gitBranch = as.factor(df$gitBranch)
+df$blocksize = as.factor(df$blocksize)
+
+ppi=300
+h=5
+w=5
+
+png("time.png", width=w*ppi, height=h*ppi, res=ppi)
+#
+## Create the plot with the normalized time vs nblocks
+p = ggplot(df, aes(x=blocksize, y=time)) +
+	geom_point() +
+	geom_line(aes(y=mtime, group=gitBranch, color=gitBranch)) +
+	theme_bw() +
+	labs(x="Blocksize", y="Time (s)", title="FWI granularity",
+    subtitle=input_file) +
+	theme(plot.subtitle=element_text(size=8)) +
+	theme(legend.position = c(0.5, 0.88))
+
+# Render the plot
+print(p)
+
+# Save the png image
+dev.off()
diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix
index c36f0b44..dbe73e38 100644
--- a/garlic/fig/index.nix
+++ b/garlic/fig/index.nix
@@ -61,6 +61,10 @@ in
     big.granularity = stdPlot ./creams/granularity.R [ big.granularity ];
   };
 
+  fwi = with exp.fwi; {
+    test = stdPlot ./fwi/test.R [ test ];
+  };
+
   osu = with exp.osu; {
     latency       = customPlot ./osu/latency.R (ds.osu.latency latency.result);
     latencyShm    = customPlot ./osu/latency.R (ds.osu.latency latencyShm.result);
-- 
GitLab


From 11e400abb5973d203615458a692d63b6e545007a Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 4 Mar 2021 18:43:18 +0100
Subject: [PATCH 11/16] fwi: remove old experiment

---
 garlic/exp/fwi/oss.nix | 69 ------------------------------------------
 1 file changed, 69 deletions(-)
 delete mode 100644 garlic/exp/fwi/oss.nix

diff --git a/garlic/exp/fwi/oss.nix b/garlic/exp/fwi/oss.nix
deleted file mode 100644
index 63a4752a..00000000
--- a/garlic/exp/fwi/oss.nix
+++ /dev/null
@@ -1,69 +0,0 @@
-{
-  stdenv
-, stdexp
-, bsc
-, targetMachine
-, stages
-}:
-
-with stdenv.lib;
-
-let
-  # Initial variable configuration
-  varConf = {
-    blocksize = [ 1 2 4 8 16 ];
-  };
-
-  # Generate the complete configuration for each unit
-  genConf = with bsc; c: targetMachine.config // rec {
-    # Options for creams
-    cc = icc;
-    gitBranch = "oss";
-    inherit (c) blocksize;
-
-    # Repeat the execution of each unit 30 times
-    loops = 30;
-
-    # Resources
-    qos = "debug";
-    nodes = 1;
-    time = "02:00:00";
-    ntasksPerNode = 1;
-    cpuBind = "sockets,verbose";
-    jobName = "fwi-${gitBranch}";
-  };
-
-  # Compute the array of configurations
-  configs = stdexp.buildConfigs {
-    inherit varConf genConf;
-  };
-
-  # Custom stage to copy the FWI input
-  copyInput = {nextStage, conf, ...}:
-    let
-      input = bsc.garlic.apps.fwi;
-    in
-      stages.exec {
-        inherit nextStage;
-        env = ''
-          cp -r ${input}/bin/InputModels .
-          chmod +w -R .
-        '';
-        argv = [
-          "${input}/etc/fwi/fwi_params.txt"
-          "${input}/etc/fwi/fwi_frequencies.txt"
-          "${toString conf.blocksize}"
-        ];
-      };
-
-  # FWI program
-  program = {nextStage, conf, ...}: with conf;
-    bsc.garlic.apps.fwi.override {
-      inherit cc gitBranch;
-    };
-
-  pipeline = stdexp.stdPipeline ++ [ copyInput program ];
-
-in
- 
-  stdexp.genExperiment { inherit configs pipeline; }
-- 
GitLab


From 1d9a5c4721458e0414587dc5f01fb6a8831add26 Mon Sep 17 00:00:00 2001
From: Aleix Roca Nonell <aleix.rocanonell@bsc.es>
Date: Mon, 8 Mar 2021 19:16:24 +0100
Subject: [PATCH 12/16] fwi: fix input derivation

The fwiInput derivation must be the same used when compiled the fwi app
as the fwi-input used in the experiment.
---
 garlic/exp/fwi/test.nix | 42 +++++++++++++++++++++++++++--------------
 1 file changed, 28 insertions(+), 14 deletions(-)

diff --git a/garlic/exp/fwi/test.nix b/garlic/exp/fwi/test.nix
index 9c42fea2..371f19de 100644
--- a/garlic/exp/fwi/test.nix
+++ b/garlic/exp/fwi/test.nix
@@ -21,9 +21,20 @@ let
 #      "garlic/seq"
     ];
 
-    blocksize = [ 1 2 4 ];
+    blocksize = [ 1 2 ];
+
+    n = [
+    	{nx=500; ny=500; nz=500;}
+    ];
   };
 
+# The c value contains something like:
+# {
+#   n = { nx=500; ny=500; nz=500; }
+#   blocksize = 1;
+#   gitBranch = "garlic/tampi+send+oss+task";
+# }
+
   machineConfig = targetMachine.config;
 
   # Generate the complete configuration for each unit
@@ -34,10 +45,18 @@ let
 
     cc = icc;
     inherit (c) gitBranch blocksize;
+
     n = 500;
-    nx = n;
-    ny = n;
-    nz = n;
+    #nx = c.n.nx;
+    #ny = c.n.ny;
+    #nz = c.n.nz;
+
+    # Same but shorter:
+    inherit (c.n) nx ny nz;
+
+    fwiInput = bsc.apps.fwi.input.override {
+      inherit (c.n) nx ny nz;
+    };
 
     # Repeat the execution of each unit several times
     loops = 10;
@@ -56,19 +75,14 @@ let
     inherit varConf genConf;
   };
 
-  exec = {nextStage, conf, ...}:
-  let
-    input = bsc.apps.fwi.input.override {
-      inherit (conf) nx ny nz;
-    };
-  in stages.exec {
+  exec = {nextStage, conf, ...}: stages.exec {
     inherit nextStage;
     pre = ''
-      ln -fs ${input}/InputModels InputModels || true
+      ln -fs ${conf.fwiInput}/InputModels InputModels || true
     '';
     argv = [
-      "${input}/fwi_params.txt"
-      "${input}/fwi_frequencies.txt"
+      "${conf.fwiInput}/fwi_params.txt"
+      "${conf.fwiInput}/fwi_frequencies.txt"
       conf.blocksize
       "-1" # Fordward steps
       "-1" # Backward steps
@@ -80,7 +94,7 @@ let
 
   # FWI program
   program = {nextStage, conf, ...}: apps.fwi.solver.override {
-    inherit (conf) cc gitBranch;
+    inherit (conf) cc gitBranch fwiInput;
   };
 
   pipeline = stdexp.stdPipeline ++ [ exec program ];
-- 
GitLab


From aadce016e14bbb861a21b9399a49c169d89e30e1 Mon Sep 17 00:00:00 2001
From: Aleix Roca Nonell <aleix.rocanonell@bsc.es>
Date: Wed, 24 Mar 2021 10:24:29 +0100
Subject: [PATCH 13/16] fwi: add granularity and data reuse experiments

The data reuse experiment shows the effect of poor data locality versus
task granularity.
---
 garlic/exp/fwi/data_reuse.nix  | 135 +++++++++++++++++++++++++++++++++
 garlic/exp/fwi/granularity.nix | 132 ++++++++++++++++++++++++++++++++
 garlic/exp/fwi/test.nix        |  46 ++++++++---
 3 files changed, 304 insertions(+), 9 deletions(-)
 create mode 100644 garlic/exp/fwi/data_reuse.nix
 create mode 100644 garlic/exp/fwi/granularity.nix

diff --git a/garlic/exp/fwi/data_reuse.nix b/garlic/exp/fwi/data_reuse.nix
new file mode 100644
index 00000000..623492a6
--- /dev/null
+++ b/garlic/exp/fwi/data_reuse.nix
@@ -0,0 +1,135 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+
+  inherit (targetMachine) fs;
+
+  # Initial variable configuration
+  varConf = {
+    gitBranch = [
+#      "garlic/tampi+send+oss+task"
+#      "garlic/mpi+send+omp+task"
+       "garlic/mpi+send+oss+task"
+       "garlic/mpi+send+oss+task+noreuse"
+#      "garlic/mpi+send+seq"
+#      "garlic/oss+task"
+#      "garlic/omp+task"
+#      "garlic/seq"
+    ];
+
+    blocksize = [ 1 2 4 8 ];
+    #blocksize = [ 1 2 ];
+
+    n = [
+#   	{nx=50;  ny=4000; nz=50;}
+#   	{nx=20;  ny=4000; nz=20;}
+#    	{nx=300; ny=8000; nz=300;} # half node, /
+#    	{nx=300; ny=1000; nz=300;} # half node, /
+#    	{nx=200; ny=1000; nz=200;} # half node, not enough tasks
+#    	{nx=200; ny=4000; nz=200;} # --/ half node
+#    	{nx=250; ny=2000; nz=250;} # / half node
+    	{nx=300; ny=2000; nz=300;} # / half node
+#    	{nx=100; ny=2000; nz=100;} # \-// half node
+#    	{nx=150; ny=2000; nz=150;} # \-/ half node
+#	{nx=200; ny=64000; nz=200;} # --/ 16 nodes
+#    	{nx=200; ny=4000; nz=200;} # --/ half node
+#    	{nx=200; ny=8000; nz=200;} # --/ 1 node
+#    	{nx=100; ny=8000; nz=100;} # --/ half node
+    ];
+  };
+
+# The c value contains something like:
+# {
+#   n = { nx=500; ny=500; nz=500; }
+#   blocksize = 1;
+#   gitBranch = "garlic/tampi+send+oss+task";
+# }
+
+  machineConfig = targetMachine.config;
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "fwi";
+    unitName = "${expName}-test";
+    inherit (machineConfig) hw;
+
+    cc = icc;
+    inherit (c) gitBranch blocksize;
+
+    #nx = c.n.nx;
+    #ny = c.n.ny;
+    #nz = c.n.nz;
+
+    # Same but shorter:
+    inherit (c.n) nx ny nz;
+
+    fwiInput = bsc.apps.fwi.input.override {
+      inherit (c.n) nx ny nz;
+    };
+
+    # Repeat the execution of each unit several times
+    loops = 10;
+    #loops = 1;
+
+    # Resources
+    cpusPerTask = hw.cpusPerSocket;
+    ntasksPerNode = 1;
+    nodes = 1;
+    qos = "debug";
+    time = "02:00:00";
+    jobName = unitName;
+
+    # Enable permissions to write in the local storage
+    extraMounts = [ fs.local.temp ];
+
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: stages.exec {
+    inherit nextStage;
+    pre = ''
+      #CDIR=$PWD
+      #export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf"
+      EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"
+      mkdir -p $EXECDIR
+      cd $EXECDIR
+      ln -fs ${conf.fwiInput}/InputModels InputModels || true
+    '';
+    argv = [
+      "${conf.fwiInput}/fwi_params.txt"
+      "${conf.fwiInput}/fwi_frequencies.txt"
+      conf.blocksize
+      "-1" # Fordward steps
+      "-1" # Backward steps
+      "-1" # Write/read frequency
+    ];
+    post = ''
+      rm -rf Results || true
+      #mv trace_* $CDIR
+    '';
+  };
+
+  apps = bsc.garlic.apps;
+
+  # FWI program
+  program = {nextStage, conf, ...}: apps.fwi.solver.override {
+    inherit (conf) cc gitBranch fwiInput;
+  };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/fwi/granularity.nix b/garlic/exp/fwi/granularity.nix
new file mode 100644
index 00000000..6aa48c27
--- /dev/null
+++ b/garlic/exp/fwi/granularity.nix
@@ -0,0 +1,132 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+
+  inherit (targetMachine) fs;
+
+  # Initial variable configuration
+  varConf = {
+    gitBranch = [
+#      "garlic/tampi+send+oss+task"
+#      "garlic/mpi+send+omp+task"
+       "garlic/mpi+send+oss+task"
+#      "garlic/mpi+send+seq"
+#      "garlic/oss+task"
+#      "garlic/omp+task"
+#      "garlic/seq"
+    ];
+
+    blocksize = [ 1 2 4 8 16 32 ];
+    #blocksize = [ 1 2 4 8 ];
+
+    n = [
+    	#{nx=500; nz=500; ny=1000; ntpn=1; nn=1;}
+    	{nx=500; nz=500; ny=2000; ntpn=2; nn=1;}
+    ];
+
+  };
+
+# The c value contains something like:
+# {
+#   n = { nx=500; ny=500; nz=500; }
+#   blocksize = 1;
+#   gitBranch = "garlic/tampi+send+oss+task";
+# }
+
+  machineConfig = targetMachine.config;
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "fwi";
+    unitName = "${expName}-test";
+    inherit (machineConfig) hw;
+
+    cc = icc;
+    inherit (c) gitBranch blocksize;
+
+    #nx = c.n.nx;
+    #ny = c.n.ny;
+    #nz = c.n.nz;
+
+    # Same but shorter:
+    inherit (c.n) nx ny nz ntpn nn;
+
+    fwiInput = bsc.apps.fwi.input.override {
+      inherit (c.n) nx ny nz;
+    };
+
+    # Other FWI parameters
+    ioFreq = -1;
+
+    # Repeat the execution of each unit several times
+    loops = 10;
+    #loops = 1;
+
+    # Resources
+    cpusPerTask = hw.cpusPerSocket;
+    ntasksPerNode = ntpn;
+    nodes = nn;
+    qos = "debug";
+    time = "02:00:00";
+    jobName = unitName;
+
+    tracing = "no";
+
+    # Enable permissions to write in the local storage
+    extraMounts = [ fs.local.temp ];
+
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: stages.exec {
+    inherit nextStage;
+    pre = ''
+      CDIR=$PWD
+      if [[ "${conf.tracing}" == "yes" ]]; then
+          export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf"
+      fi
+      EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"
+      mkdir -p $EXECDIR
+      cd $EXECDIR
+      ln -fs ${conf.fwiInput}/InputModels InputModels || true
+    '';
+    argv = [
+      "${conf.fwiInput}/fwi_params.txt"
+      "${conf.fwiInput}/fwi_frequencies.txt"
+      conf.blocksize
+      "-1" # Fordward steps
+      "-1" # Backward steps
+      conf.ioFreq # Write/read frequency
+    ];
+    post = ''
+      rm -rf Results || true
+      if [[ "${conf.tracing}" == "yes" ]]; then
+          mv trace_* $CDIR
+      fi
+    '';
+  };
+
+  apps = bsc.garlic.apps;
+
+  # FWI program
+  program = {nextStage, conf, ...}: apps.fwi.solver.override {
+    inherit (conf) cc gitBranch fwiInput;
+  };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/fwi/test.nix b/garlic/exp/fwi/test.nix
index 371f19de..49afa8bf 100644
--- a/garlic/exp/fwi/test.nix
+++ b/garlic/exp/fwi/test.nix
@@ -9,23 +9,29 @@
 with stdenv.lib;
 
 let
+
+  inherit (targetMachine) fs;
+
   # Initial variable configuration
   varConf = {
     gitBranch = [
-      "garlic/tampi+send+oss+task"
+#      "garlic/tampi+send+oss+task"
 #      "garlic/mpi+send+omp+task"
-#      "garlic/mpi+send+oss+task"
+       "garlic/mpi+send+oss+task"
 #      "garlic/mpi+send+seq"
 #      "garlic/oss+task"
 #      "garlic/omp+task"
 #      "garlic/seq"
     ];
 
-    blocksize = [ 1 2 ];
+    #blocksize = [ 1 2 4 8 16 32 ];
+    blocksize = [ 1 2 4 8 ];
 
     n = [
-    	{nx=500; ny=500; nz=500;}
+    	#{nx=500; nz=500; ny=1000; ntpn=1; nn=1;}
+    	{nx=500; nz=500; ny=2000; ntpn=2; nn=1;}
     ];
+
   };
 
 # The c value contains something like:
@@ -46,28 +52,37 @@ let
     cc = icc;
     inherit (c) gitBranch blocksize;
 
-    n = 500;
     #nx = c.n.nx;
     #ny = c.n.ny;
     #nz = c.n.nz;
 
     # Same but shorter:
-    inherit (c.n) nx ny nz;
+    inherit (c.n) nx ny nz ntpn nn;
 
     fwiInput = bsc.apps.fwi.input.override {
       inherit (c.n) nx ny nz;
     };
 
+    # Other FWI parameters
+    ioFreq = -1;
+
     # Repeat the execution of each unit several times
     loops = 10;
+    #loops = 1;
 
     # Resources
     cpusPerTask = hw.cpusPerSocket;
-    ntasksPerNode = 1;
-    nodes = 1;
+    ntasksPerNode = ntpn;
+    nodes = nn;
     qos = "debug";
     time = "02:00:00";
     jobName = unitName;
+
+    tracing = "no";
+
+    # Enable permissions to write in the local storage
+    extraMounts = [ fs.local.temp ];
+
   };
 
   # Compute the array of configurations
@@ -78,6 +93,13 @@ let
   exec = {nextStage, conf, ...}: stages.exec {
     inherit nextStage;
     pre = ''
+      CDIR=$PWD
+      if [[ "${conf.tracing}" == "yes" ]]; then
+          export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf"
+      fi
+      EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"
+      mkdir -p $EXECDIR
+      cd $EXECDIR
       ln -fs ${conf.fwiInput}/InputModels InputModels || true
     '';
     argv = [
@@ -86,8 +108,14 @@ let
       conf.blocksize
       "-1" # Fordward steps
       "-1" # Backward steps
-      "-1" # Write/read frequency
+      conf.ioFreq # Write/read frequency
     ];
+    post = ''
+      rm -rf Results || true
+      if [[ "${conf.tracing}" == "yes" ]]; then
+          mv trace_* $CDIR
+      fi
+    '';
   };
 
   apps = bsc.garlic.apps;
-- 
GitLab


From 3ef4a505d39387768d2f4c00bbab7822bc93008f Mon Sep 17 00:00:00 2001
From: Aleix Roca Nonell <aleix.rocanonell@bsc.es>
Date: Fri, 26 Mar 2021 17:34:34 +0100
Subject: [PATCH 14/16] fwi: add strong scalability tests

---
 garlic/exp/fwi/granularity.nix             |   6 +-
 garlic/exp/fwi/memory_affinity.nix         | 138 +++++++++++++++++++++
 garlic/exp/fwi/strong_scaling_forkjoin.nix | 132 ++++++++++++++++++++
 garlic/exp/fwi/strong_scaling_io.nix       | 134 ++++++++++++++++++++
 garlic/exp/fwi/strong_scaling_mpionly.nix  | 133 ++++++++++++++++++++
 garlic/exp/fwi/strong_scaling_task.nix     | 132 ++++++++++++++++++++
 garlic/exp/fwi/test.nix                    |   8 +-
 garlic/exp/index.nix                       |   7 +-
 garlic/fig/fwi/granularity.R               |  70 +++++++++++
 garlic/fig/fwi/strong_scaling.R            | 120 ++++++++++++++++++
 garlic/fig/fwi/strong_scaling_io.R         | 122 ++++++++++++++++++
 garlic/fig/index.nix                       |   5 +-
 12 files changed, 997 insertions(+), 10 deletions(-)
 create mode 100644 garlic/exp/fwi/memory_affinity.nix
 create mode 100644 garlic/exp/fwi/strong_scaling_forkjoin.nix
 create mode 100644 garlic/exp/fwi/strong_scaling_io.nix
 create mode 100644 garlic/exp/fwi/strong_scaling_mpionly.nix
 create mode 100644 garlic/exp/fwi/strong_scaling_task.nix
 create mode 100644 garlic/fig/fwi/granularity.R
 create mode 100644 garlic/fig/fwi/strong_scaling.R
 create mode 100644 garlic/fig/fwi/strong_scaling_io.R

diff --git a/garlic/exp/fwi/granularity.nix b/garlic/exp/fwi/granularity.nix
index 6aa48c27..25f4fa53 100644
--- a/garlic/exp/fwi/granularity.nix
+++ b/garlic/exp/fwi/granularity.nix
@@ -15,8 +15,8 @@ let
   # Initial variable configuration
   varConf = {
     gitBranch = [
-#      "garlic/tampi+send+oss+task"
-#      "garlic/mpi+send+omp+task"
+       "garlic/tampi+send+oss+task"
+       "garlic/mpi+send+omp+task"
        "garlic/mpi+send+oss+task"
 #      "garlic/mpi+send+seq"
 #      "garlic/oss+task"
@@ -25,10 +25,8 @@ let
     ];
 
     blocksize = [ 1 2 4 8 16 32 ];
-    #blocksize = [ 1 2 4 8 ];
 
     n = [
-    	#{nx=500; nz=500; ny=1000; ntpn=1; nn=1;}
     	{nx=500; nz=500; ny=2000; ntpn=2; nn=1;}
     ];
 
diff --git a/garlic/exp/fwi/memory_affinity.nix b/garlic/exp/fwi/memory_affinity.nix
new file mode 100644
index 00000000..3df7226d
--- /dev/null
+++ b/garlic/exp/fwi/memory_affinity.nix
@@ -0,0 +1,138 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+
+  inherit (targetMachine) fs;
+
+  # Initial variable configuration
+  varConf = {
+    gitBranch = [
+#      "garlic/tampi+send+oss+task"
+#      "garlic/mpi+send+omp+task"
+       "garlic/mpi+send+oss+task"
+#      "garlic/mpi+send+seq"
+#      "garlic/oss+task"
+#      "garlic/omp+task"
+#      "garlic/seq"
+    ];
+
+    blocksize = [ 1 ];
+
+    n = [
+#       {nx=500; nz=500; ny=8000;}
+        {nx=500; nz=500; ny=2000;}
+    ];
+
+    nodes = [ 1 ]
+
+    numactl = [ true false ]
+
+  };
+
+# The c value contains something like:
+# {
+#   n = { nx=500; ny=500; nz=500; }
+#   blocksize = 1;
+#   gitBranch = "garlic/tampi+send+oss+task";
+# }
+
+  machineConfig = targetMachine.config;
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "fwi";
+    unitName = "${expName}-test";
+    inherit (machineConfig) hw;
+
+    cc = icc;
+    inherit (c) gitBranch blocksize;
+    useNumactl = c.numactl
+
+    #nx = c.n.nx;
+    #ny = c.n.ny;
+    #nz = c.n.nz;
+
+    # Same but shorter:
+    inherit (c.n) nx ny nz;
+
+    fwiInput = bsc.apps.fwi.input.override {
+      inherit (c.n) nx ny nz;
+    };
+
+    # Other FWI parameters
+    ioFreq = -1;
+
+    # Repeat the execution of each unit several times
+    loops = 10;
+    #loops = 1;
+
+    # Resources
+    cpusPerTask = if (useNumactl) then hw.cpusPerNode else hw.cpusPerSocket;
+    ntasksPerNode = hw.cpusPerNode / cpusPerTask;
+    nodes = c.nodes;
+    qos = "debug";
+    time = "02:00:00";
+    jobName = unitName;
+
+    tracing = "no";
+
+    # Enable permissions to write in the local storage
+    extraMounts = [ fs.local.temp ];
+
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: stages.exec ({
+    inherit nextStage;
+    pre = ''
+      CDIR=$PWD
+      if [[ "${conf.tracing}" == "yes" ]]; then
+          export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf"
+      fi
+      EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"
+      mkdir -p $EXECDIR
+      cd $EXECDIR
+      ln -fs ${conf.fwiInput}/InputModels InputModels || true
+    '';
+    argv = [
+      "${conf.fwiInput}/fwi_params.txt"
+      "${conf.fwiInput}/fwi_frequencies.txt"
+      conf.blocksize
+      "-1" # Fordward steps
+      "-1" # Backward steps
+      conf.ioFreq # Write/read frequency
+    ];
+    post = ''
+      rm -rf Results || true
+      if [[ "${conf.tracing}" == "yes" ]]; then
+          mv trace_* $CDIR
+      fi
+    '';
+  } // optionalAttrs (conf.useNumact) {
+    program = "${numactl}/bin/numactl --interleave=all ${stageProgram nextStage}";
+  });
+
+  apps = bsc.garlic.apps;
+
+  # FWI program
+  program = {nextStage, conf, ...}: apps.fwi.solver.override {
+    inherit (conf) cc gitBranch fwiInput;
+  };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/fwi/strong_scaling_forkjoin.nix b/garlic/exp/fwi/strong_scaling_forkjoin.nix
new file mode 100644
index 00000000..8ff84c21
--- /dev/null
+++ b/garlic/exp/fwi/strong_scaling_forkjoin.nix
@@ -0,0 +1,132 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+
+  inherit (targetMachine) fs;
+
+  # Initial variable configuration
+  varConf = {
+    gitBranch = [
+#      "garlic/tampi+send+oss+task"
+#      "garlic/mpi+send+omp+task"
+#      "garlic/mpi+send+oss+task"
+       "garlic/mpi+send+omp+fork"
+#      "garlic/mpi+send+seq"
+#      "garlic/oss+task"
+#      "garlic/omp+task"
+#      "garlic/seq"
+    ];
+
+    blocksize = [ 0 ];
+
+    n = [
+        {nx=500; nz=500; ny=16000;}
+    ];
+
+    nodes = [ 1 2 4 8 16 ];
+
+  };
+
+# The c value contains something like:
+# {
+#   n = { nx=500; ny=500; nz=500; }
+#   blocksize = 1;
+#   gitBranch = "garlic/tampi+send+oss+task";
+# }
+
+  machineConfig = targetMachine.config;
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "fwi";
+    unitName = "${expName}-test";
+    inherit (machineConfig) hw;
+
+    cc = icc;
+    inherit (c) gitBranch blocksize;
+
+    #nx = c.n.nx;
+    #ny = c.n.ny;
+    #nz = c.n.nz;
+
+    # Same but shorter:
+    inherit (c.n) nx ny nz;
+
+    fwiInput = bsc.apps.fwi.input.override {
+      inherit (c.n) nx ny nz;
+    };
+
+    # Other FWI parameters
+    ioFreq = -1;
+
+    # Repeat the execution of each unit several times
+    loops = 10;
+    #loops = 1;
+
+    # Resources
+    cpusPerTask = hw.cpusPerSocket;
+    ntasksPerNode = 2;
+    nodes = c.nodes;
+    qos = "debug";
+    time = "02:00:00";
+    jobName = unitName;
+
+    tracing = "no";
+
+    # Enable permissions to write in the local storage
+    extraMounts = [ fs.local.temp ];
+
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: stages.exec {
+    inherit nextStage;
+    pre = ''
+      CDIR=$PWD
+      if [[ "${conf.tracing}" == "yes" ]]; then
+          export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf"
+      fi
+      EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"
+      mkdir -p $EXECDIR
+      cd $EXECDIR
+      ln -fs ${conf.fwiInput}/InputModels InputModels || true
+    '';
+    argv = [
+      "${conf.fwiInput}/fwi_params.txt"
+      "${conf.fwiInput}/fwi_frequencies.txt"
+      "-1" # Fordward steps
+      "-1" # Backward steps
+      conf.ioFreq # Write/read frequency
+    ];
+    post = ''
+      rm -rf Results || true
+      if [[ "${conf.tracing}" == "yes" ]]; then
+          mv trace_* $CDIR
+      fi
+    '';
+  };
+
+  apps = bsc.garlic.apps;
+
+  # FWI program
+  program = {nextStage, conf, ...}: apps.fwi.solver.override {
+    inherit (conf) cc gitBranch fwiInput;
+  };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/fwi/strong_scaling_io.nix b/garlic/exp/fwi/strong_scaling_io.nix
new file mode 100644
index 00000000..30f0e75b
--- /dev/null
+++ b/garlic/exp/fwi/strong_scaling_io.nix
@@ -0,0 +1,134 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+
+  inherit (targetMachine) fs;
+
+  # Initial variable configuration
+  varConf = {
+    gitBranch = [
+       "garlic/tampi+send+oss+task"
+#      "garlic/mpi+send+omp+task"
+#      "garlic/mpi+send+oss+task"
+#      "garlic/mpi+send+seq"
+#      "garlic/oss+task"
+#      "garlic/omp+task"
+#      "garlic/seq"
+    ];
+
+    blocksize = [ 1 2 4 8 ];
+
+    n = [
+        {nx=500; nz=500; ny=16000;}
+    ];
+
+    nodes = [ 1 2 4 8 16 ];
+
+    ioFreq = [ 9999 (-1) ];
+
+  };
+
+# The c value contains something like:
+# {
+#   n = { nx=500; ny=500; nz=500; }
+#   blocksize = 1;
+#   gitBranch = "garlic/tampi+send+oss+task";
+# }
+
+  machineConfig = targetMachine.config;
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "fwi";
+    unitName = "${expName}-test";
+    inherit (machineConfig) hw;
+
+    cc = icc;
+    inherit (c) gitBranch blocksize;
+
+    #nx = c.n.nx;
+    #ny = c.n.ny;
+    #nz = c.n.nz;
+
+    # Same but shorter:
+    inherit (c.n) nx ny nz;
+
+    fwiInput = bsc.apps.fwi.input.override {
+      inherit (c.n) nx ny nz;
+    };
+
+    # Other FWI parameters
+    ioFreq = c.ioFreq;
+
+    # Repeat the execution of each unit several times
+    loops = 10;
+    #loops = 1;
+
+    # Resources
+    cpusPerTask = hw.cpusPerSocket;
+    ntasksPerNode = 2;
+    nodes = c.nodes;
+    qos = "debug";
+    time = "02:00:00";
+    jobName = unitName;
+
+    tracing = "no";
+
+    # Enable permissions to write in the local storage
+    extraMounts = [ fs.local.temp ];
+
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: stages.exec {
+    inherit nextStage;
+    pre = ''
+      CDIR=$PWD
+      if [[ "${conf.tracing}" == "yes" ]]; then
+          export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf"
+      fi
+      EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"
+      mkdir -p $EXECDIR
+      cd $EXECDIR
+      ln -fs ${conf.fwiInput}/InputModels InputModels || true
+    '';
+    argv = [
+      "${conf.fwiInput}/fwi_params.txt"
+      "${conf.fwiInput}/fwi_frequencies.txt"
+      conf.blocksize
+      "-1" # Fordward steps
+      "-1" # Backward steps
+      conf.ioFreq # Write/read frequency
+    ];
+    post = ''
+      rm -rf Results || true
+      if [[ "${conf.tracing}" == "yes" ]]; then
+          mv trace_* $CDIR
+      fi
+    '';
+  };
+
+  apps = bsc.garlic.apps;
+
+  # FWI program
+  program = {nextStage, conf, ...}: apps.fwi.solver.override {
+    inherit (conf) cc gitBranch fwiInput;
+  };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/fwi/strong_scaling_mpionly.nix b/garlic/exp/fwi/strong_scaling_mpionly.nix
new file mode 100644
index 00000000..48283cba
--- /dev/null
+++ b/garlic/exp/fwi/strong_scaling_mpionly.nix
@@ -0,0 +1,133 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+
+  inherit (targetMachine) fs;
+
+  # Initial variable configuration
+  varConf = {
+    gitBranch = [
+#      "garlic/tampi+send+oss+task"
+#      "garlic/mpi+send+omp+task"
+#      "garlic/mpi+send+oss+task"
+#      "garlic/mpi+send+omp+fork"
+       "garlic/mpi+send+seq"
+#      "garlic/oss+task"
+#      "garlic/omp+task"
+#      "garlic/seq"
+    ];
+
+    blocksize = [ 0 ];
+
+    n = [
+        {nx=500; nz=500; ny=16000;}
+    ];
+
+    # Not enough planes for 8 and 16 nodes
+    nodes = [ 1 2 4 ];
+
+  };
+
+# The c value contains something like:
+# {
+#   n = { nx=500; ny=500; nz=500; }
+#   blocksize = 1;
+#   gitBranch = "garlic/tampi+send+oss+task";
+# }
+
+  machineConfig = targetMachine.config;
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "fwi";
+    unitName = "${expName}-test";
+    inherit (machineConfig) hw;
+
+    cc = icc;
+    inherit (c) gitBranch blocksize;
+
+    #nx = c.n.nx;
+    #ny = c.n.ny;
+    #nz = c.n.nz;
+
+    # Same but shorter:
+    inherit (c.n) nx ny nz;
+
+    fwiInput = bsc.apps.fwi.input.override {
+      inherit (c.n) nx ny nz;
+    };
+
+    # Other FWI parameters
+    ioFreq = -1;
+
+    # Repeat the execution of each unit several times
+    loops = 10;
+    #loops = 1;
+
+    # Resources
+    cpusPerTask = 1;
+    ntasksPerNode = hw.cpusPerNode;
+    nodes = c.nodes;
+    qos = "debug";
+    time = "02:00:00";
+    jobName = unitName;
+
+    tracing = "no";
+
+    # Enable permissions to write in the local storage
+    extraMounts = [ fs.local.temp ];
+
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: stages.exec {
+    inherit nextStage;
+    pre = ''
+      CDIR=$PWD
+      if [[ "${conf.tracing}" == "yes" ]]; then
+          export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf"
+      fi
+      EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"
+      mkdir -p $EXECDIR
+      cd $EXECDIR
+      ln -fs ${conf.fwiInput}/InputModels InputModels || true
+    '';
+    argv = [
+      "${conf.fwiInput}/fwi_params.txt"
+      "${conf.fwiInput}/fwi_frequencies.txt"
+      "-1" # Fordward steps
+      "-1" # Backward steps
+      conf.ioFreq # Write/read frequency
+    ];
+    post = ''
+      rm -rf Results || true
+      if [[ "${conf.tracing}" == "yes" ]]; then
+          mv trace_* $CDIR
+      fi
+    '';
+  };
+
+  apps = bsc.garlic.apps;
+
+  # FWI program
+  program = {nextStage, conf, ...}: apps.fwi.solver.override {
+    inherit (conf) cc gitBranch fwiInput;
+  };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/fwi/strong_scaling_task.nix b/garlic/exp/fwi/strong_scaling_task.nix
new file mode 100644
index 00000000..487506ad
--- /dev/null
+++ b/garlic/exp/fwi/strong_scaling_task.nix
@@ -0,0 +1,132 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+
+  inherit (targetMachine) fs;
+
+  # Initial variable configuration
+  varConf = {
+    gitBranch = [
+       "garlic/tampi+send+oss+task"
+       "garlic/mpi+send+omp+task"
+       "garlic/mpi+send+oss+task"
+#      "garlic/mpi+send+seq"
+#      "garlic/oss+task"
+#      "garlic/omp+task"
+#      "garlic/seq"
+    ];
+
+    blocksize = [ 1 2 4 8 ];
+
+    n = [
+        {nx=500; nz=500; ny=16000;}
+    ];
+
+    nodes = [ 1 2 4 8 16 ];
+
+  };
+
+# The c value contains something like:
+# {
+#   n = { nx=500; ny=500; nz=500; }
+#   blocksize = 1;
+#   gitBranch = "garlic/tampi+send+oss+task";
+# }
+
+  machineConfig = targetMachine.config;
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "fwi";
+    unitName = "${expName}-test";
+    inherit (machineConfig) hw;
+
+    cc = icc;
+    inherit (c) gitBranch blocksize;
+
+    #nx = c.n.nx;
+    #ny = c.n.ny;
+    #nz = c.n.nz;
+
+    # Same but shorter:
+    inherit (c.n) nx ny nz;
+
+    fwiInput = bsc.apps.fwi.input.override {
+      inherit (c.n) nx ny nz;
+    };
+
+    # Other FWI parameters
+    ioFreq = -1;
+
+    # Repeat the execution of each unit several times
+    loops = 10;
+    #loops = 1;
+
+    # Resources
+    cpusPerTask = hw.cpusPerSocket;
+    ntasksPerNode = 2;
+    nodes = c.nodes;
+    qos = "debug";
+    time = "02:00:00";
+    jobName = unitName;
+
+    tracing = "no";
+
+    # Enable permissions to write in the local storage
+    extraMounts = [ fs.local.temp ];
+
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: stages.exec {
+    inherit nextStage;
+    pre = ''
+      CDIR=$PWD
+      if [[ "${conf.tracing}" == "yes" ]]; then
+          export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf"
+      fi
+      EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"
+      mkdir -p $EXECDIR
+      cd $EXECDIR
+      ln -fs ${conf.fwiInput}/InputModels InputModels || true
+    '';
+    argv = [
+      "${conf.fwiInput}/fwi_params.txt"
+      "${conf.fwiInput}/fwi_frequencies.txt"
+      conf.blocksize
+      "-1" # Fordward steps
+      "-1" # Backward steps
+      conf.ioFreq # Write/read frequency
+    ];
+    post = ''
+      rm -rf Results || true
+      if [[ "${conf.tracing}" == "yes" ]]; then
+          mv trace_* $CDIR
+      fi
+    '';
+  };
+
+  apps = bsc.garlic.apps;
+
+  # FWI program
+  program = {nextStage, conf, ...}: apps.fwi.solver.override {
+    inherit (conf) cc gitBranch fwiInput;
+  };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/fwi/test.nix b/garlic/exp/fwi/test.nix
index 49afa8bf..a9ea3129 100644
--- a/garlic/exp/fwi/test.nix
+++ b/garlic/exp/fwi/test.nix
@@ -16,16 +16,16 @@ let
   varConf = {
     gitBranch = [
 #      "garlic/tampi+send+oss+task"
-#      "garlic/mpi+send+omp+task"
-       "garlic/mpi+send+oss+task"
+       "garlic/mpi+send+omp+task"
+#      "garlic/mpi+send+oss+task"
 #      "garlic/mpi+send+seq"
 #      "garlic/oss+task"
 #      "garlic/omp+task"
 #      "garlic/seq"
     ];
 
-    #blocksize = [ 1 2 4 8 16 32 ];
-    blocksize = [ 1 2 4 8 ];
+    blocksize = [ 1 2 4 8 16 32 ];
+    #blocksize = [ 1 2 4 8 ];
 
     n = [
     	#{nx=500; nz=500; ny=1000; ntpn=1; nn=1;}
diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix
index 2b34e4a0..c50cdeaa 100644
--- a/garlic/exp/index.nix
+++ b/garlic/exp/index.nix
@@ -98,7 +98,12 @@
   };
 
   fwi = {
-    test = callPackage ./fwi/test.nix { };
+    test                    = callPackage ./fwi/test.nix { };
+    strong_scaling_task     = callPackage ./fwi/strong_scaling_task.nix { };
+    strong_scaling_forkjoin = callPackage ./fwi/strong_scaling_forkjoin.nix { };
+    strong_scaling_mpionly  = callPackage ./fwi/strong_scaling_mpionly.nix { };
+    strong_scaling_io       = callPackage ./fwi/strong_scaling_io.nix { };
+    granularity             = callPackage ./fwi/granularity.nix { };
   };
 
   osu = rec {
diff --git a/garlic/fig/fwi/granularity.R b/garlic/fig/fwi/granularity.R
new file mode 100644
index 00000000..a1fc99d4
--- /dev/null
+++ b/garlic/fig/fwi/granularity.R
@@ -0,0 +1,70 @@
+library(ggplot2)
+library(dplyr)
+library(scales)
+library(jsonlite)
+
+args=commandArgs(trailingOnly=TRUE)
+
+# Read the timetable from args[1]
+input_file = "input.json"
+if (length(args)>0) { input_file = args[1] }
+
+# Load the dataset in NDJSON format
+dataset = jsonlite::stream_in(file(input_file)) %>%
+	jsonlite::flatten()
+
+# We only need the nblocks and time
+df = select(dataset, config.blocksize, config.gitBranch, time) %>%
+	rename(blocksize=config.blocksize, gitBranch=config.gitBranch) %>%
+  group_by(blocksize, gitBranch) %>%
+  mutate(mtime = median(time)) %>%
+  ungroup()
+
+df$gitBranch = as.factor(df$gitBranch)
+df$blocksize = as.factor(df$blocksize)
+
+ppi=300
+h=5
+w=5
+
+####################################################################
+### Line Graph
+####################################################################
+png("time.png", width=w*ppi, height=h*ppi, res=ppi)
+
+## Create the plot with the normalized time vs nblocks
+p = ggplot(df, aes(x = blocksize, y=mtime, group=gitBranch, color=gitBranch)) +
+  geom_point() +
+  geom_line() +
+  theme_bw() +
+  labs(x="Blocksize", y="Median Time (s)", title="FWI granularity",
+    subtitle=input_file) +
+  theme(plot.subtitle=element_text(size=8)) +
+  theme(legend.position = c(0.5, 0.88))
+
+# Render the plot
+print(p)
+
+# Save the png image
+dev.off()
+
+####################################################################
+### Boxplot
+####################################################################
+png("box.png", width=w*ppi, height=h*ppi, res=ppi)
+# Create the plot with the normalized time vs nblocks
+p = ggplot(df, aes(x=blocksize, y=time, group=gitBranch, colour=gitBranch)) +
+        # Labels
+        labs(x="Blocksize", y="Normalized time",
+              title=sprintf("FWI Time"),
+              subtitle=input_file) +
+        # Draw boxplots
+        geom_boxplot() +
+        theme_bw() +
+        theme(plot.subtitle=element_text(size=8)) +
+        theme(legend.position = c(0.5, 0.88))
+# Render the plot
+print(p)
+## Save the png image
+dev.off()
+
diff --git a/garlic/fig/fwi/strong_scaling.R b/garlic/fig/fwi/strong_scaling.R
new file mode 100644
index 00000000..5dd4bb56
--- /dev/null
+++ b/garlic/fig/fwi/strong_scaling.R
@@ -0,0 +1,120 @@
+library(ggplot2)
+library(dplyr)
+library(scales)
+library(jsonlite)
+
+args=commandArgs(trailingOnly=TRUE)
+
+# Read the timetable from args[1]
+input_file = "input.json"
+if (length(args)>0) { input_file = args[1] }
+
+# Load the dataset in NDJSON format
+dataset = jsonlite::stream_in(file(input_file)) %>%
+  jsonlite::flatten()
+
+# Select block size to display
+useBlocksize = 1
+
+# We only need the nblocks and time
+df = select(dataset, config.blocksize, config.gitBranch, config.nodes, time) %>%
+  rename(
+     blocksize=config.blocksize,
+     gitBranch=config.gitBranch,
+     nodes=config.nodes
+  ) %>%
+  filter(blocksize == useBlocksize | blocksize == 0) %>%
+  group_by(nodes, gitBranch) %>%
+  mutate(mtime = median(time)) %>%
+  mutate(nxmtime = mtime * nodes) %>%
+  mutate(nxtime = time * nodes) %>%
+  ungroup()
+
+df$gitBranch = as.factor(df$gitBranch)
+df$blocksize = as.factor(df$blocksize)
+df$nodes     = as.factor(df$nodes)
+
+ppi=300
+h=5
+w=5
+
+####################################################################
+### Line plot (time)
+####################################################################
+png("time.png", width=w*ppi, height=h*ppi, res=ppi)
+
+p = ggplot(df, aes(x=nodes, y=time, group=gitBranch, color=gitBranch)) +
+  geom_point() +
+  geom_line() +
+  theme_bw() +
+  labs(x="Nodes", y="Time (s)", title="FWI strong scaling",
+    subtitle=input_file) +
+  theme(plot.subtitle=element_text(size=8)) +
+  theme(legend.position = c(0.6, 0.75))
+
+# Render the plot
+print(p)
+
+# Save the png image
+dev.off()
+
+####################################################################
+### Line plot (timei x nodes)
+####################################################################
+png("nxtime.png", width=w*ppi, height=h*ppi, res=ppi)
+
+p = ggplot(df, aes(x=nodes, y=nxtime, group=gitBranch, color=gitBranch)) +
+  geom_point() +
+  geom_line() +
+  theme_bw() +
+  labs(x="Nodes", y="Time * Nodes (s)", title="FWI strong scaling",
+    subtitle=input_file) +
+  theme(plot.subtitle=element_text(size=8)) +
+  theme(legend.position = c(0.15, 0.80)) +
+  theme(legend.text = element_text(size = 7))
+
+# Render the plot
+print(p)
+
+# Save the png image
+dev.off()
+
+####################################################################
+### Line plot (median time)
+####################################################################
+png("mediantime.png", width=w*ppi, height=h*ppi, res=ppi)
+
+p = ggplot(df, aes(x=nodes, y=mtime, group=gitBranch, color=gitBranch)) +
+  geom_point() +
+  geom_line() +
+  theme_bw() +
+  labs(x="Nodes", y="Median Time (s)", title="FWI strong scaling",
+    subtitle=input_file) +
+  theme(plot.subtitle=element_text(size=8)) +
+  theme(legend.position = c(0.5, 0.88))
+
+# Render the plot
+print(p)
+
+# Save the png image
+dev.off()
+
+####################################################################
+### Line plot (nodes x median time)
+####################################################################
+png("nxmtime.png", width=w*ppi, height=h*ppi, res=ppi)
+
+p = ggplot(df, aes(x=nodes, y=nxmtime, group=gitBranch, color=gitBranch)) +
+  geom_point() +
+  geom_line() +
+  theme_bw() +
+  labs(x="Nodes", y="Median Time * Nodes (s)", title="FWI strong scaling",
+    subtitle=input_file) +
+  theme(plot.subtitle=element_text(size=8)) +
+  theme(legend.position = c(0.5, 0.88))
+
+# Render the plot
+print(p)
+
+# Save the png image
+dev.off()
diff --git a/garlic/fig/fwi/strong_scaling_io.R b/garlic/fig/fwi/strong_scaling_io.R
new file mode 100644
index 00000000..415bafda
--- /dev/null
+++ b/garlic/fig/fwi/strong_scaling_io.R
@@ -0,0 +1,122 @@
+library(ggplot2)
+library(dplyr)
+library(scales)
+library(jsonlite)
+library(forcats)
+
+args=commandArgs(trailingOnly=TRUE)
+
+# Read the timetable from args[1]
+input_file = "input.json"
+if (length(args)>0) { input_file = args[1] }
+
+# Load the dataset in NDJSON format
+dataset = jsonlite::stream_in(file(input_file)) %>%
+  jsonlite::flatten()
+
+# We only need the nblocks and time
+df = select(dataset, config.blocksize, config.ioFreq, config.gitBranch, config.nodes, time) %>%
+  rename(
+     blocksize=config.blocksize,
+     io=config.ioFreq,
+     gitBranch=config.gitBranch,
+     nodes=config.nodes
+  ) %>%
+  filter(blocksize == 1) %>%
+  group_by(nodes, gitBranch, io) %>%
+  mutate(mtime = median(time)) %>%
+  mutate(nxmtime = mtime * nodes) %>%
+  mutate(nxtime = time * nodes) %>%
+  ungroup()
+
+df$gitBranch = as.factor(df$gitBranch)
+df$io        = as.factor(df$io)
+df$blocksize = as.factor(df$blocksize)
+df$nodes     = as.factor(df$nodes)
+
+df$io = fct_recode(df$io, enabled = "-1", disabled = "9999")
+
+
+ppi=300
+h=5
+w=5
+
+####################################################################
+### Line plot (time)
+####################################################################
+png("time.png", width=w*ppi, height=h*ppi, res=ppi)
+
+p = ggplot(df, aes(x=nodes, y=time, group=io, color=io)) +
+  geom_point() +
+  geom_line() +
+  theme_bw() +
+  labs(x="Nodes", y="Time (s)", title="FWI strong scaling for mpi+send+oss+task",
+    subtitle=input_file) +
+  theme(plot.subtitle=element_text(size=8)) +
+  theme(legend.position = c(0.5, 0.88))
+
+# Render the plot
+print(p)
+
+# Save the png image
+dev.off()
+
+####################################################################
+### Line plot (time x nodes)
+####################################################################
+png("nxtime.png", width=w*ppi, height=h*ppi, res=ppi)
+
+p = ggplot(df, aes(x=nodes, y=nxtime, group=io, color=io)) +
+  geom_point() +
+  geom_line() +
+  theme_bw() +
+  labs(x="Nodes", y="Time * Nodes (s)", title="FWI strong scaling for mpi+send+oss+task",
+    subtitle=input_file) +
+  theme(plot.subtitle=element_text(size=8)) +
+  theme(legend.position = c(0.5, 0.88))
+
+# Render the plot
+print(p)
+
+# Save the png image
+dev.off()
+
+#####################################################################
+#### Line plot (median time)
+#####################################################################
+#png("mediantime.png", width=w*ppi, height=h*ppi, res=ppi)
+#
+#p = ggplot(df, aes(x=nodes, y=mtime, group=gitBranch, color=gitBranch)) +
+#  geom_point() +
+#  geom_line() +
+#  theme_bw() +
+#  labs(x="Nodes", y="Median Time (s)", title="FWI strong scaling",
+#    subtitle=input_file) +
+#  theme(plot.subtitle=element_text(size=8)) +
+#  theme(legend.position = c(0.5, 0.88))
+#
+## Render the plot
+#print(p)
+#
+## Save the png image
+#dev.off()
+#
+#####################################################################
+#### Line plot (nodes x median time)
+#####################################################################
+#png("nxmtime.png", width=w*ppi, height=h*ppi, res=ppi)
+#
+#p = ggplot(df, aes(x=nodes, y=nxmtime, group=gitBranch, color=gitBranch)) +
+#  geom_point() +
+#  geom_line() +
+#  theme_bw() +
+#  labs(x="Nodes", y="Median Time * Nodes (s)", title="FWI strong scaling",
+#    subtitle=input_file) +
+#  theme(plot.subtitle=element_text(size=8)) +
+#  theme(legend.position = c(0.5, 0.88))
+#
+## Render the plot
+#print(p)
+#
+## Save the png image
+#dev.off()
diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix
index dbe73e38..5ca637f7 100644
--- a/garlic/fig/index.nix
+++ b/garlic/fig/index.nix
@@ -62,7 +62,10 @@ in
   };
 
   fwi = with exp.fwi; {
-    test = stdPlot ./fwi/test.R [ test ];
+    test              = stdPlot ./fwi/test.R [ test ];
+    strong_scaling    = stdPlot ./fwi/strong_scaling.R [ strong_scaling_task strong_scaling_forkjoin strong_scaling_mpionly ];
+    strong_scaling_io = stdPlot ./fwi/strong_scaling_io.R [ strong_scaling_io ];
+    granularity       = stdPlot ./fwi/granularity.R [ granularity ];
   };
 
   osu = with exp.osu; {
-- 
GitLab


From 3e5a56ebdb1e5e4f72433b86a5abb3bda4be870c Mon Sep 17 00:00:00 2001
From: Aleix Roca Nonell <aleix.rocanonell@bsc.es>
Date: Mon, 29 Mar 2021 18:17:47 +0200
Subject: [PATCH 15/16] fwi: add tampi non-blocking variant

---
 garlic/exp/fwi/granularity.nix         | 1 +
 garlic/exp/fwi/strong_scaling_task.nix | 1 +
 2 files changed, 2 insertions(+)

diff --git a/garlic/exp/fwi/granularity.nix b/garlic/exp/fwi/granularity.nix
index 25f4fa53..1c8cac4e 100644
--- a/garlic/exp/fwi/granularity.nix
+++ b/garlic/exp/fwi/granularity.nix
@@ -16,6 +16,7 @@ let
   varConf = {
     gitBranch = [
        "garlic/tampi+send+oss+task"
+       "garlic/tampi+isend+oss+task"
        "garlic/mpi+send+omp+task"
        "garlic/mpi+send+oss+task"
 #      "garlic/mpi+send+seq"
diff --git a/garlic/exp/fwi/strong_scaling_task.nix b/garlic/exp/fwi/strong_scaling_task.nix
index 487506ad..048fe7eb 100644
--- a/garlic/exp/fwi/strong_scaling_task.nix
+++ b/garlic/exp/fwi/strong_scaling_task.nix
@@ -16,6 +16,7 @@ let
   varConf = {
     gitBranch = [
        "garlic/tampi+send+oss+task"
+       "garlic/tampi+isend+oss+task"
        "garlic/mpi+send+omp+task"
        "garlic/mpi+send+oss+task"
 #      "garlic/mpi+send+seq"
-- 
GitLab


From 989f6ee018a9aee8928fc270e10094ae6f99b3ce Mon Sep 17 00:00:00 2001
From: Aleix Roca Nonell <aleix.rocanonell@bsc.es>
Date: Wed, 7 Apr 2021 12:35:44 +0200
Subject: [PATCH 16/16] fwi: adjust input size to meet timing constraints

The previous iniput size for both granularity and strong scaling tests
where too big to meet the timing constrains needed for garlic. This
patch sets a new, smaller, input size.

Also, a minor cleanup is applied to the rest of the fwi experiments
and figures.
---
 garlic/apps/fwi/default.nix                |   1 +
 garlic/exp/fwi/data_reuse.nix              |  42 +++----
 garlic/exp/fwi/granularity.nix             |  14 ++-
 garlic/exp/fwi/memory_affinity.nix         | 138 ---------------------
 garlic/exp/fwi/strong_scaling_forkjoin.nix |  12 +-
 garlic/exp/fwi/strong_scaling_io.nix       |   7 ++
 garlic/exp/fwi/strong_scaling_mpionly.nix  |  17 ++-
 garlic/exp/fwi/strong_scaling_task.nix     |  12 +-
 garlic/exp/fwi/{test.nix => sync_io.nix}   |  26 ++--
 garlic/exp/index.nix                       |   5 +-
 garlic/fig/fwi/granularity.R               |  31 ++---
 garlic/fig/fwi/strong_scaling.R            |   4 +-
 garlic/fig/fwi/test.R                      |  46 -------
 garlic/fig/index.nix                       |   8 +-
 14 files changed, 96 insertions(+), 267 deletions(-)
 delete mode 100644 garlic/exp/fwi/memory_affinity.nix
 rename garlic/exp/fwi/{test.nix => sync_io.nix} (84%)
 delete mode 100644 garlic/fig/fwi/test.R

diff --git a/garlic/apps/fwi/default.nix b/garlic/apps/fwi/default.nix
index 013fcef5..5b1a9d63 100644
--- a/garlic/apps/fwi/default.nix
+++ b/garlic/apps/fwi/default.nix
@@ -34,6 +34,7 @@ stdenv.mkDerivation rec {
   # FIXME: Allow multiple MPI implementations
   postPatch = ''
     sed -i 's/= OPENMPI$/= INTEL/g' Makefile
+    sed -i 's/USE_O_DIRECT ?= NO/USE_O_DIRECT ?= YES/g' Makefile || true
   '';
 
   # FIXME: This is an ugly hack.
diff --git a/garlic/exp/fwi/data_reuse.nix b/garlic/exp/fwi/data_reuse.nix
index 623492a6..84f0c4b8 100644
--- a/garlic/exp/fwi/data_reuse.nix
+++ b/garlic/exp/fwi/data_reuse.nix
@@ -1,3 +1,23 @@
+# This test compares a FWI version using poor data locality (+NOREUSE) versus
+# the optimized version (used for all other experiments). Follows a pseudocode
+# snippet illustrating the fundamental difference between version.
+#
+# NOREUSE
+# ----------------------
+# for (y) for (x) for (z)
+#   computA(v[y][x][z]);
+# for (y) for (x) for (z)
+#   computB(v[y][x][z]);
+# for (y) for (x) for (z)
+#   computC(v[y][x][z]);
+#
+# Optimized version
+# ----------------------
+# for (y) for (x) for (z)
+#   computA(v[y][x][z]);
+#   computB(v[y][x][z]);
+#   computC(v[y][x][z]);
+
 {
   stdenv
 , stdexp
@@ -15,34 +35,14 @@ let
   # Initial variable configuration
   varConf = {
     gitBranch = [
-#      "garlic/tampi+send+oss+task"
-#      "garlic/mpi+send+omp+task"
        "garlic/mpi+send+oss+task"
-       "garlic/mpi+send+oss+task+noreuse"
-#      "garlic/mpi+send+seq"
-#      "garlic/oss+task"
-#      "garlic/omp+task"
-#      "garlic/seq"
+       "garlic/mpi+send+oss+task+NOREUSE"
     ];
 
     blocksize = [ 1 2 4 8 ];
-    #blocksize = [ 1 2 ];
 
     n = [
-#   	{nx=50;  ny=4000; nz=50;}
-#   	{nx=20;  ny=4000; nz=20;}
-#    	{nx=300; ny=8000; nz=300;} # half node, /
-#    	{nx=300; ny=1000; nz=300;} # half node, /
-#    	{nx=200; ny=1000; nz=200;} # half node, not enough tasks
-#    	{nx=200; ny=4000; nz=200;} # --/ half node
-#    	{nx=250; ny=2000; nz=250;} # / half node
     	{nx=300; ny=2000; nz=300;} # / half node
-#    	{nx=100; ny=2000; nz=100;} # \-// half node
-#    	{nx=150; ny=2000; nz=150;} # \-/ half node
-#	{nx=200; ny=64000; nz=200;} # --/ 16 nodes
-#    	{nx=200; ny=4000; nz=200;} # --/ half node
-#    	{nx=200; ny=8000; nz=200;} # --/ 1 node
-#    	{nx=100; ny=8000; nz=100;} # --/ half node
     ];
   };
 
diff --git a/garlic/exp/fwi/granularity.nix b/garlic/exp/fwi/granularity.nix
index 1c8cac4e..7773b3b4 100644
--- a/garlic/exp/fwi/granularity.nix
+++ b/garlic/exp/fwi/granularity.nix
@@ -1,3 +1,5 @@
+# Regular granularity test for FWI
+
 {
   stdenv
 , stdexp
@@ -15,20 +17,20 @@ let
   # Initial variable configuration
   varConf = {
     gitBranch = [
-       "garlic/tampi+send+oss+task"
-       "garlic/tampi+isend+oss+task"
-       "garlic/mpi+send+omp+task"
-       "garlic/mpi+send+oss+task"
+#      "garlic/tampi+send+oss+task"
+      "garlic/tampi+isend+oss+task"
+#      "garlic/mpi+send+omp+task"
+#      "garlic/mpi+send+oss+task"
 #      "garlic/mpi+send+seq"
 #      "garlic/oss+task"
 #      "garlic/omp+task"
 #      "garlic/seq"
     ];
 
-    blocksize = [ 1 2 4 8 16 32 ];
+    blocksize = [ 1 2 4 8 16 32 64 128 256 ];
 
     n = [
-    	{nx=500; nz=500; ny=2000; ntpn=2; nn=1;}
+    	{nx=100; nz=100; ny=8000; ntpn=2; nn=1;}
     ];
 
   };
diff --git a/garlic/exp/fwi/memory_affinity.nix b/garlic/exp/fwi/memory_affinity.nix
deleted file mode 100644
index 3df7226d..00000000
--- a/garlic/exp/fwi/memory_affinity.nix
+++ /dev/null
@@ -1,138 +0,0 @@
-{
-  stdenv
-, stdexp
-, bsc
-, targetMachine
-, stages
-}:
-
-with stdenv.lib;
-
-let
-
-  inherit (targetMachine) fs;
-
-  # Initial variable configuration
-  varConf = {
-    gitBranch = [
-#      "garlic/tampi+send+oss+task"
-#      "garlic/mpi+send+omp+task"
-       "garlic/mpi+send+oss+task"
-#      "garlic/mpi+send+seq"
-#      "garlic/oss+task"
-#      "garlic/omp+task"
-#      "garlic/seq"
-    ];
-
-    blocksize = [ 1 ];
-
-    n = [
-#       {nx=500; nz=500; ny=8000;}
-        {nx=500; nz=500; ny=2000;}
-    ];
-
-    nodes = [ 1 ]
-
-    numactl = [ true false ]
-
-  };
-
-# The c value contains something like:
-# {
-#   n = { nx=500; ny=500; nz=500; }
-#   blocksize = 1;
-#   gitBranch = "garlic/tampi+send+oss+task";
-# }
-
-  machineConfig = targetMachine.config;
-
-  # Generate the complete configuration for each unit
-  genConf = with bsc; c: targetMachine.config // rec {
-    expName = "fwi";
-    unitName = "${expName}-test";
-    inherit (machineConfig) hw;
-
-    cc = icc;
-    inherit (c) gitBranch blocksize;
-    useNumactl = c.numactl
-
-    #nx = c.n.nx;
-    #ny = c.n.ny;
-    #nz = c.n.nz;
-
-    # Same but shorter:
-    inherit (c.n) nx ny nz;
-
-    fwiInput = bsc.apps.fwi.input.override {
-      inherit (c.n) nx ny nz;
-    };
-
-    # Other FWI parameters
-    ioFreq = -1;
-
-    # Repeat the execution of each unit several times
-    loops = 10;
-    #loops = 1;
-
-    # Resources
-    cpusPerTask = if (useNumactl) then hw.cpusPerNode else hw.cpusPerSocket;
-    ntasksPerNode = hw.cpusPerNode / cpusPerTask;
-    nodes = c.nodes;
-    qos = "debug";
-    time = "02:00:00";
-    jobName = unitName;
-
-    tracing = "no";
-
-    # Enable permissions to write in the local storage
-    extraMounts = [ fs.local.temp ];
-
-  };
-
-  # Compute the array of configurations
-  configs = stdexp.buildConfigs {
-    inherit varConf genConf;
-  };
-
-  exec = {nextStage, conf, ...}: stages.exec ({
-    inherit nextStage;
-    pre = ''
-      CDIR=$PWD
-      if [[ "${conf.tracing}" == "yes" ]]; then
-          export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf"
-      fi
-      EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"
-      mkdir -p $EXECDIR
-      cd $EXECDIR
-      ln -fs ${conf.fwiInput}/InputModels InputModels || true
-    '';
-    argv = [
-      "${conf.fwiInput}/fwi_params.txt"
-      "${conf.fwiInput}/fwi_frequencies.txt"
-      conf.blocksize
-      "-1" # Fordward steps
-      "-1" # Backward steps
-      conf.ioFreq # Write/read frequency
-    ];
-    post = ''
-      rm -rf Results || true
-      if [[ "${conf.tracing}" == "yes" ]]; then
-          mv trace_* $CDIR
-      fi
-    '';
-  } // optionalAttrs (conf.useNumact) {
-    program = "${numactl}/bin/numactl --interleave=all ${stageProgram nextStage}";
-  });
-
-  apps = bsc.garlic.apps;
-
-  # FWI program
-  program = {nextStage, conf, ...}: apps.fwi.solver.override {
-    inherit (conf) cc gitBranch fwiInput;
-  };
-
-  pipeline = stdexp.stdPipeline ++ [ exec program ];
-
-in
- 
-  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/fwi/strong_scaling_forkjoin.nix b/garlic/exp/fwi/strong_scaling_forkjoin.nix
index 8ff84c21..902eaf60 100644
--- a/garlic/exp/fwi/strong_scaling_forkjoin.nix
+++ b/garlic/exp/fwi/strong_scaling_forkjoin.nix
@@ -1,3 +1,6 @@
+# Strong scaling test for FWI variants based on forkjoint. This
+# experiment does not rely on block sizes.
+
 {
   stdenv
 , stdexp
@@ -15,20 +18,13 @@ let
   # Initial variable configuration
   varConf = {
     gitBranch = [
-#      "garlic/tampi+send+oss+task"
-#      "garlic/mpi+send+omp+task"
-#      "garlic/mpi+send+oss+task"
        "garlic/mpi+send+omp+fork"
-#      "garlic/mpi+send+seq"
-#      "garlic/oss+task"
-#      "garlic/omp+task"
-#      "garlic/seq"
     ];
 
     blocksize = [ 0 ];
 
     n = [
-        {nx=500; nz=500; ny=16000;}
+        {nx=100; nz=100; ny=8000;}
     ];
 
     nodes = [ 1 2 4 8 16 ];
diff --git a/garlic/exp/fwi/strong_scaling_io.nix b/garlic/exp/fwi/strong_scaling_io.nix
index 30f0e75b..c0636595 100644
--- a/garlic/exp/fwi/strong_scaling_io.nix
+++ b/garlic/exp/fwi/strong_scaling_io.nix
@@ -1,3 +1,10 @@
+# Strong scaling test for FWI variants based on tasks with and without I/O.
+# This experiment solves a computationally expensive input which brings the
+# storage devices to saturation when I/O is enabled.  the same input us run
+# without I/O for comparison purposes..  Also, the experiments are runt for a
+# range of block sizes deemed as efficient according to the granularity
+# experiment.
+
 {
   stdenv
 , stdexp
diff --git a/garlic/exp/fwi/strong_scaling_mpionly.nix b/garlic/exp/fwi/strong_scaling_mpionly.nix
index 48283cba..94ffd8d6 100644
--- a/garlic/exp/fwi/strong_scaling_mpionly.nix
+++ b/garlic/exp/fwi/strong_scaling_mpionly.nix
@@ -1,3 +1,7 @@
+# Strong scaling test for FWI variants based exclusively on MPI. This
+# experiment does not rely on block sizes. An MPI process is instantiated per
+# core.
+
 {
   stdenv
 , stdexp
@@ -15,24 +19,17 @@ let
   # Initial variable configuration
   varConf = {
     gitBranch = [
-#      "garlic/tampi+send+oss+task"
-#      "garlic/mpi+send+omp+task"
-#      "garlic/mpi+send+oss+task"
-#      "garlic/mpi+send+omp+fork"
        "garlic/mpi+send+seq"
-#      "garlic/oss+task"
-#      "garlic/omp+task"
-#      "garlic/seq"
     ];
 
     blocksize = [ 0 ];
 
     n = [
-        {nx=500; nz=500; ny=16000;}
+        {nx=100; nz=100; ny=8000;}
     ];
 
-    # Not enough planes for 8 and 16 nodes
-    nodes = [ 1 2 4 ];
+    # Not enough planes for 4, 8 and 16 nodes
+    nodes = [ 1 2 ];
 
   };
 
diff --git a/garlic/exp/fwi/strong_scaling_task.nix b/garlic/exp/fwi/strong_scaling_task.nix
index 048fe7eb..4ad78338 100644
--- a/garlic/exp/fwi/strong_scaling_task.nix
+++ b/garlic/exp/fwi/strong_scaling_task.nix
@@ -1,3 +1,7 @@
+# Strong scaling test for FWI variants based on tasks. This
+# experiment explores a range of block sizes deemed as efficient
+# according to the granularity experiment.
+
 {
   stdenv
 , stdexp
@@ -19,16 +23,12 @@ let
        "garlic/tampi+isend+oss+task"
        "garlic/mpi+send+omp+task"
        "garlic/mpi+send+oss+task"
-#      "garlic/mpi+send+seq"
-#      "garlic/oss+task"
-#      "garlic/omp+task"
-#      "garlic/seq"
     ];
 
-    blocksize = [ 1 2 4 8 ];
+    blocksize = [ 1 2 4 8 16 ];
 
     n = [
-        {nx=500; nz=500; ny=16000;}
+        {nx=100; nz=100; ny=8000;}
     ];
 
     nodes = [ 1 2 4 8 16 ];
diff --git a/garlic/exp/fwi/test.nix b/garlic/exp/fwi/sync_io.nix
similarity index 84%
rename from garlic/exp/fwi/test.nix
rename to garlic/exp/fwi/sync_io.nix
index a9ea3129..59e791da 100644
--- a/garlic/exp/fwi/test.nix
+++ b/garlic/exp/fwi/sync_io.nix
@@ -1,3 +1,7 @@
+# This experiment compares the effect of not using I/O versus using O_DIRECT |
+# O_DSYNC enabled I/O. This is a reduced version of the strong_scaling_io
+# experiment.
+
 {
   stdenv
 , stdexp
@@ -15,8 +19,8 @@ let
   # Initial variable configuration
   varConf = {
     gitBranch = [
-#      "garlic/tampi+send+oss+task"
-       "garlic/mpi+send+omp+task"
+       "garlic/tampi+send+oss+task"
+#      "garlic/mpi+send+omp+task"
 #      "garlic/mpi+send+oss+task"
 #      "garlic/mpi+send+seq"
 #      "garlic/oss+task"
@@ -24,14 +28,16 @@ let
 #      "garlic/seq"
     ];
 
-    blocksize = [ 1 2 4 8 16 32 ];
-    #blocksize = [ 1 2 4 8 ];
+    blocksize = [ 1 ];
 
     n = [
-    	#{nx=500; nz=500; ny=1000; ntpn=1; nn=1;}
-    	{nx=500; nz=500; ny=2000; ntpn=2; nn=1;}
+        {nx=500; nz=500; ny=16000;}
     ];
 
+    nodes = [ 4 ];
+
+    ioFreq = [ 9999 (-1) ];
+
   };
 
 # The c value contains something like:
@@ -57,14 +63,14 @@ let
     #nz = c.n.nz;
 
     # Same but shorter:
-    inherit (c.n) nx ny nz ntpn nn;
+    inherit (c.n) nx ny nz;
 
     fwiInput = bsc.apps.fwi.input.override {
       inherit (c.n) nx ny nz;
     };
 
     # Other FWI parameters
-    ioFreq = -1;
+    ioFreq = c.ioFreq;
 
     # Repeat the execution of each unit several times
     loops = 10;
@@ -72,8 +78,8 @@ let
 
     # Resources
     cpusPerTask = hw.cpusPerSocket;
-    ntasksPerNode = ntpn;
-    nodes = nn;
+    ntasksPerNode = 2;
+    nodes = c.nodes;
     qos = "debug";
     time = "02:00:00";
     jobName = unitName;
diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix
index c50cdeaa..f6a518df 100644
--- a/garlic/exp/index.nix
+++ b/garlic/exp/index.nix
@@ -98,12 +98,13 @@
   };
 
   fwi = {
-    test                    = callPackage ./fwi/test.nix { };
+    granularity             = callPackage ./fwi/granularity.nix { };
     strong_scaling_task     = callPackage ./fwi/strong_scaling_task.nix { };
     strong_scaling_forkjoin = callPackage ./fwi/strong_scaling_forkjoin.nix { };
     strong_scaling_mpionly  = callPackage ./fwi/strong_scaling_mpionly.nix { };
+    data_reuse              = callPackage ./fwi/data_reuse.nix { };
     strong_scaling_io       = callPackage ./fwi/strong_scaling_io.nix { };
-    granularity             = callPackage ./fwi/granularity.nix { };
+    sync_io                 = callPackage ./fwi/sync_io.nix { };
   };
 
   osu = rec {
diff --git a/garlic/fig/fwi/granularity.R b/garlic/fig/fwi/granularity.R
index a1fc99d4..40d6f750 100644
--- a/garlic/fig/fwi/granularity.R
+++ b/garlic/fig/fwi/granularity.R
@@ -30,7 +30,7 @@ w=5
 ####################################################################
 ### Line Graph
 ####################################################################
-png("time.png", width=w*ppi, height=h*ppi, res=ppi)
+png("mtime.png", width=w*ppi, height=h*ppi, res=ppi)
 
 ## Create the plot with the normalized time vs nblocks
 p = ggplot(df, aes(x = blocksize, y=mtime, group=gitBranch, color=gitBranch)) +
@@ -49,22 +49,23 @@ print(p)
 dev.off()
 
 ####################################################################
-### Boxplot
+### Line Graph
 ####################################################################
-png("box.png", width=w*ppi, height=h*ppi, res=ppi)
-# Create the plot with the normalized time vs nblocks
-p = ggplot(df, aes(x=blocksize, y=time, group=gitBranch, colour=gitBranch)) +
-        # Labels
-        labs(x="Blocksize", y="Normalized time",
-              title=sprintf("FWI Time"),
-              subtitle=input_file) +
-        # Draw boxplots
-        geom_boxplot() +
-        theme_bw() +
-        theme(plot.subtitle=element_text(size=8)) +
-        theme(legend.position = c(0.5, 0.88))
+png("time.png", width=w*ppi, height=h*ppi, res=ppi)
+
+## Create the plot with the normalized time vs nblocks
+p = ggplot(df, aes(x = blocksize, y=time, group=gitBranch, color=gitBranch)) +
+  geom_point() +
+  geom_line() +
+  theme_bw() +
+  labs(x="Blocksize", y="Time (s)", title="FWI granularity",
+    subtitle=input_file) +
+  theme(plot.subtitle=element_text(size=8)) +
+  theme(legend.position = c(0.5, 0.88))
+
 # Render the plot
 print(p)
-## Save the png image
+
+# Save the png image
 dev.off()
 
diff --git a/garlic/fig/fwi/strong_scaling.R b/garlic/fig/fwi/strong_scaling.R
index 5dd4bb56..89d79f15 100644
--- a/garlic/fig/fwi/strong_scaling.R
+++ b/garlic/fig/fwi/strong_scaling.R
@@ -14,7 +14,7 @@ dataset = jsonlite::stream_in(file(input_file)) %>%
   jsonlite::flatten()
 
 # Select block size to display
-useBlocksize = 1
+useBlocksize = 2
 
 # We only need the nblocks and time
 df = select(dataset, config.blocksize, config.gitBranch, config.nodes, time) %>%
@@ -59,7 +59,7 @@ print(p)
 dev.off()
 
 ####################################################################
-### Line plot (timei x nodes)
+### Line plot (time x nodes)
 ####################################################################
 png("nxtime.png", width=w*ppi, height=h*ppi, res=ppi)
 
diff --git a/garlic/fig/fwi/test.R b/garlic/fig/fwi/test.R
deleted file mode 100644
index ca79f0dd..00000000
--- a/garlic/fig/fwi/test.R
+++ /dev/null
@@ -1,46 +0,0 @@
-library(ggplot2)
-library(dplyr)
-library(scales)
-library(jsonlite)
-
-args=commandArgs(trailingOnly=TRUE)
-
-# Read the timetable from args[1]
-input_file = "input.json"
-if (length(args)>0) { input_file = args[1] }
-
-# Load the dataset in NDJSON format
-dataset = jsonlite::stream_in(file(input_file)) %>%
-	jsonlite::flatten()
-
-# We only need the nblocks and time
-df = select(dataset, config.blocksize, config.gitBranch, time) %>%
-	rename(blocksize=config.blocksize, gitBranch=config.gitBranch) %>%
-  group_by(blocksize, gitBranch) %>%
-  mutate(mtime = median(time)) %>%
-  ungroup()
-
-df$gitBranch = as.factor(df$gitBranch)
-df$blocksize = as.factor(df$blocksize)
-
-ppi=300
-h=5
-w=5
-
-png("time.png", width=w*ppi, height=h*ppi, res=ppi)
-#
-## Create the plot with the normalized time vs nblocks
-p = ggplot(df, aes(x=blocksize, y=time)) +
-	geom_point() +
-	geom_line(aes(y=mtime, group=gitBranch, color=gitBranch)) +
-	theme_bw() +
-	labs(x="Blocksize", y="Time (s)", title="FWI granularity",
-    subtitle=input_file) +
-	theme(plot.subtitle=element_text(size=8)) +
-	theme(legend.position = c(0.5, 0.88))
-
-# Render the plot
-print(p)
-
-# Save the png image
-dev.off()
diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix
index 5ca637f7..b3f77eae 100644
--- a/garlic/fig/index.nix
+++ b/garlic/fig/index.nix
@@ -62,10 +62,12 @@ in
   };
 
   fwi = with exp.fwi; {
-    test              = stdPlot ./fwi/test.R [ test ];
-    strong_scaling    = stdPlot ./fwi/strong_scaling.R [ strong_scaling_task strong_scaling_forkjoin strong_scaling_mpionly ];
-    strong_scaling_io = stdPlot ./fwi/strong_scaling_io.R [ strong_scaling_io ];
     granularity       = stdPlot ./fwi/granularity.R [ granularity ];
+    strong_scaling    = stdPlot ./fwi/strong_scaling.R [ strong_scaling_task strong_scaling_forkjoin ];
+    #strong_scaling    = stdPlot ./fwi/strong_scaling.R [ strong_scaling_task strong_scaling_forkjoin strong_scaling_mpionly ];
+    data_reuse        = stdPlot ./fwi/granularity.R [ data_reuse ];
+    strong_scaling_io = stdPlot ./fwi/strong_scaling_io.R [ strong_scaling_io ];
+    sync_io           = stdPlot ./fwi/strong_scaling_io.R [ sync_io ];
   };
 
   osu = with exp.osu; {
-- 
GitLab