From 6c0f4ec1b3e128b773cf04277cf911dbb4f1cad2 Mon Sep 17 00:00:00 2001
From: Pedro Martinez <pedro.martinez-ferrer@bsc.es>
Date: Mon, 22 Feb 2021 20:47:26 +0100
Subject: [PATCH 01/13] creams: add granularity experiments

---
 garlic/exp/creams/gran+node1.nix  | 100 ++++++++++++++++++++++++++++++
 garlic/exp/creams/gran+node16.nix |  98 +++++++++++++++++++++++++++++
 garlic/exp/index.nix              |   4 ++
 3 files changed, 202 insertions(+)
 create mode 100644 garlic/exp/creams/gran+node1.nix
 create mode 100644 garlic/exp/creams/gran+node16.nix

diff --git a/garlic/exp/creams/gran+node1.nix b/garlic/exp/creams/gran+node1.nix
new file mode 100644
index 00000000..dd4a0d57
--- /dev/null
+++ b/garlic/exp/creams/gran+node1.nix
@@ -0,0 +1,100 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = {
+    input = [
+      { nodes=1 ; nprocz=2 ; granul=256; time= "02:00:00"; }
+      { nodes=1 ; nprocz=2 ; granul=128; time= "02:00:00"; }
+      { nodes=1 ; nprocz=2 ; granul=64;  time= "02:00:00"; }
+      { nodes=1 ; nprocz=2 ; granul=37;  time= "02:00:00"; }
+      { nodes=1 ; nprocz=2 ; granul=32;  time= "02:00:00"; }
+      { nodes=1 ; nprocz=2 ; granul=16;  time= "02:00:00"; }
+      { nodes=1 ; nprocz=2 ; granul= 9;  time= "02:00:00"; }
+      { nodes=1 ; nprocz=2 ; granul= 5;  time= "02:00:00"; }
+      { nodes=1 ; nprocz=2 ; granul= 4;  time= "02:00:00"; }
+      { nodes=1 ; nprocz=2 ; granul= 2;  time= "02:00:00"; }
+      { nodes=1 ; nprocz=2 ; granul= 1;  time= "02:00:00"; }
+    ];
+
+    gitBranch = [
+      "garlic/mpi+send+omp+fork"
+      "garlic/mpi+send+oss+task"
+      "garlic/mpi+isend+oss+task"
+      "garlic/tampi+isend+oss+task"
+    ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "creams-gran-node1";
+    unitName = "${expName}-${toString nodes}-${gitBranch}";
+    inherit (targetMachine.config) hw;
+    # Options for creams
+    cc = icc;
+    mpi = impi;
+    inherit (c.input) granul;
+    inherit (c) gitBranch;
+    nprocz = ntasksPerNode * nodes;
+
+    # Repeat the execution of each unit 30 times
+    loops = 30;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = hw.socketsPerNode;
+    cpusPerTask = hw.cpusPerSocket;
+    inherit (c.input) time nodes;
+    jobName = unitName;
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  # Use nanos6 with regions
+  nanos6Env = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
+    '';
+  };
+
+  # Custom stage to copy the creams input dataset
+  copyInput = {nextStage, conf, ...}:
+    let
+      input = bsc.garlic.apps.creamsInput.override {
+        inherit (conf) gitBranch granul nprocz;
+      };
+    in
+      stages.exec {
+        inherit nextStage;
+        env = ''
+          cp -r ${input}/SodTubeBenchmark/* .
+          chmod +w -R .
+        '';
+      };
+
+  # Creams program
+  creams = {nextStage, conf, ...}: with conf;
+    let
+      customPkgs = stdexp.replaceMpi conf.mpi;
+    in
+      customPkgs.apps.creams.override {
+        inherit cc mpi gitBranch;
+      };
+
+  pipeline = stdexp.stdPipeline ++ [ nanos6Env copyInput creams ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/creams/gran+node16.nix b/garlic/exp/creams/gran+node16.nix
new file mode 100644
index 00000000..16d04717
--- /dev/null
+++ b/garlic/exp/creams/gran+node16.nix
@@ -0,0 +1,98 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = {
+    input = [
+      { nodes=16 ; nprocz=32 ; granul=128; time= "02:00:00"; }
+      { nodes=16 ; nprocz=32 ; granul=64;  time= "02:00:00"; }
+      { nodes=16 ; nprocz=32 ; granul=32;  time= "02:00:00"; }
+      { nodes=16 ; nprocz=32 ; granul=16;  time= "02:00:00"; }
+      { nodes=16 ; nprocz=32 ; granul= 9;  time= "02:00:00"; }
+      { nodes=16 ; nprocz=32 ; granul= 5;  time= "02:00:00"; }
+      { nodes=16 ; nprocz=32 ; granul= 4;  time= "02:00:00"; }
+      { nodes=16 ; nprocz=32 ; granul= 2;  time= "02:00:00"; }
+      { nodes=16 ; nprocz=32 ; granul= 1;  time= "02:00:00"; }
+    ];
+
+    gitBranch = [
+      "garlic/mpi+send+omp+fork"
+      "garlic/mpi+send+oss+task"
+      "garlic/mpi+isend+oss+task"
+      "garlic/tampi+isend+oss+task"
+    ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "creams-gran-node16";
+    unitName = "${expName}-${toString nodes}-${gitBranch}";
+    inherit (targetMachine.config) hw;
+    # Options for creams
+    cc = icc;
+    mpi = impi;
+    inherit (c.input) granul;
+    inherit (c) gitBranch;
+    nprocz = ntasksPerNode * nodes;
+
+    # Repeat the execution of each unit 30 times
+    loops = 30;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = hw.socketsPerNode;
+    cpusPerTask = hw.cpusPerSocket;
+    inherit (c.input) time nodes;
+    jobName = unitName;
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  # Use nanos6 with regions
+  nanos6Env = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
+    '';
+  };
+
+  # Custom stage to copy the creams input dataset
+  copyInput = {nextStage, conf, ...}:
+    let
+      input = bsc.garlic.apps.creamsInput.override {
+        inherit (conf) gitBranch granul nprocz;
+      };
+    in
+      stages.exec {
+        inherit nextStage;
+        env = ''
+          cp -r ${input}/SodTubeBenchmark/* .
+          chmod +w -R .
+        '';
+      };
+
+  # Creams program
+  creams = {nextStage, conf, ...}: with conf;
+    let
+      customPkgs = stdexp.replaceMpi conf.mpi;
+    in
+      customPkgs.apps.creams.override {
+        inherit cc mpi gitBranch;
+      };
+
+  pipeline = stdexp.stdPipeline ++ [ nanos6Env copyInput creams ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix
index a90daa02..6b93c92e 100644
--- a/garlic/exp/index.nix
+++ b/garlic/exp/index.nix
@@ -41,6 +41,10 @@
       pure = callPackage ./creams/ss+pure.nix { };
       hybrid = callPackage ./creams/ss+hybrid.nix { };
     };
+    gran = {
+      pure = callPackage ./creams/gran+node1.nix { };
+      hybrid = callPackage ./creams/gran+node16.nix { };
+    };
   };
 
   hpcg = rec {
-- 
GitLab


From 938246322f3327076cf3e366988b1abfa348f42c Mon Sep 17 00:00:00 2001
From: Pedro Martinez <pedro.martinez-ferrer@bsc.es>
Date: Wed, 24 Feb 2021 15:31:13 +0100
Subject: [PATCH 02/13] creams: add OpenMP branches

---
 garlic/exp/creams/gran+node1.nix  | 2 ++
 garlic/exp/creams/gran+node16.nix | 2 ++
 garlic/exp/creams/ss+hybrid.nix   | 2 ++
 3 files changed, 6 insertions(+)

diff --git a/garlic/exp/creams/gran+node1.nix b/garlic/exp/creams/gran+node1.nix
index dd4a0d57..f874f36a 100644
--- a/garlic/exp/creams/gran+node1.nix
+++ b/garlic/exp/creams/gran+node1.nix
@@ -27,7 +27,9 @@ let
 
     gitBranch = [
       "garlic/mpi+send+omp+fork"
+      "garlic/mpi+send+omp+task"
       "garlic/mpi+send+oss+task"
+      "garlic/mpi+isend+omp+task"
       "garlic/mpi+isend+oss+task"
       "garlic/tampi+isend+oss+task"
     ];
diff --git a/garlic/exp/creams/gran+node16.nix b/garlic/exp/creams/gran+node16.nix
index 16d04717..a0b794c4 100644
--- a/garlic/exp/creams/gran+node16.nix
+++ b/garlic/exp/creams/gran+node16.nix
@@ -25,7 +25,9 @@ let
 
     gitBranch = [
       "garlic/mpi+send+omp+fork"
+      "garlic/mpi+send+omp+task"
       "garlic/mpi+send+oss+task"
+      "garlic/mpi+isend+omp+task"
       "garlic/mpi+isend+oss+task"
       "garlic/tampi+isend+oss+task"
     ];
diff --git a/garlic/exp/creams/ss+hybrid.nix b/garlic/exp/creams/ss+hybrid.nix
index e10071e0..15fdb72c 100644
--- a/garlic/exp/creams/ss+hybrid.nix
+++ b/garlic/exp/creams/ss+hybrid.nix
@@ -21,7 +21,9 @@ let
 
     gitBranch = [
       "garlic/mpi+send+omp+fork"
+      "garlic/mpi+send+omp+task"
       "garlic/mpi+send+oss+task"
+      "garlic/mpi+isend+omp+task"
       "garlic/mpi+isend+oss+task"
       "garlic/tampi+isend+oss+task"
     ];
-- 
GitLab


From 1aa0e77157397374670de40462cf116e49e21693 Mon Sep 17 00:00:00 2001
From: Pedro Martinez <pedro.martinez-ferrer@bsc.es>
Date: Fri, 26 Feb 2021 09:10:26 +0100
Subject: [PATCH 03/13] creams: avoid race condition

Ensure only one Slurm process performs environment operations
---
 garlic/exp/creams/gran+node1.nix  | 7 +++++--
 garlic/exp/creams/gran+node16.nix | 7 +++++--
 garlic/exp/creams/ss+hybrid.nix   | 7 +++++--
 garlic/exp/creams/ss+pure.nix     | 7 +++++--
 4 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/garlic/exp/creams/gran+node1.nix b/garlic/exp/creams/gran+node1.nix
index f874f36a..075d6318 100644
--- a/garlic/exp/creams/gran+node1.nix
+++ b/garlic/exp/creams/gran+node1.nix
@@ -81,8 +81,11 @@ let
       stages.exec {
         inherit nextStage;
         env = ''
-          cp -r ${input}/SodTubeBenchmark/* .
-          chmod +w -R .
+          # Only the MPI rank 0 will copy the files
+          if [ $SLURM_PROCID == 0 ]; then
+            cp -fr ${input}/SodTubeBenchmark/* .
+            chmod +w -R .
+          fi
         '';
       };
 
diff --git a/garlic/exp/creams/gran+node16.nix b/garlic/exp/creams/gran+node16.nix
index a0b794c4..942b4741 100644
--- a/garlic/exp/creams/gran+node16.nix
+++ b/garlic/exp/creams/gran+node16.nix
@@ -79,8 +79,11 @@ let
       stages.exec {
         inherit nextStage;
         env = ''
-          cp -r ${input}/SodTubeBenchmark/* .
-          chmod +w -R .
+          # Only the MPI rank 0 will copy the files
+          if [ $SLURM_PROCID == 0 ]; then
+            cp -fr ${input}/SodTubeBenchmark/* .
+            chmod +w -R .
+          fi
         '';
       };
 
diff --git a/garlic/exp/creams/ss+hybrid.nix b/garlic/exp/creams/ss+hybrid.nix
index 15fdb72c..0b39d366 100644
--- a/garlic/exp/creams/ss+hybrid.nix
+++ b/garlic/exp/creams/ss+hybrid.nix
@@ -75,8 +75,11 @@ let
       stages.exec {
         inherit nextStage;
         env = ''
-          cp -r ${input}/SodTubeBenchmark/* .
-          chmod +w -R .
+          # Only the MPI rank 0 will copy the files
+          if [ $SLURM_PROCID == 0 ]; then
+            cp -fr ${input}/SodTubeBenchmark/* .
+            chmod +w -R .
+          fi
         '';
       };
 
diff --git a/garlic/exp/creams/ss+pure.nix b/garlic/exp/creams/ss+pure.nix
index 75bcf5fb..244ccd20 100644
--- a/garlic/exp/creams/ss+pure.nix
+++ b/garlic/exp/creams/ss+pure.nix
@@ -66,8 +66,11 @@ let
       stages.exec {
         inherit nextStage;
         env = ''
-          cp -r ${input}/SodTubeBenchmark/* .
-          chmod +w -R .
+          # Only the MPI rank 0 will copy the files
+          if [ $SLURM_PROCID == 0 ]; then
+            cp -fr ${input}/SodTubeBenchmark/* .
+            chmod +w -R .
+          fi
         '';
       };
 
-- 
GitLab


From 8445fb0928417aef7d978e3fb2ee1501152e9a15 Mon Sep 17 00:00:00 2001
From: Pedro Martinez <pedro.martinez-ferrer@bsc.es>
Date: Mon, 1 Mar 2021 09:39:57 +0100
Subject: [PATCH 04/13] creams: run the cp command in one process only

---
 garlic/exp/creams/gran+node1.nix  | 48 +++++++++++-----------
 garlic/exp/creams/gran+node16.nix | 48 +++++++++++-----------
 garlic/exp/creams/ss+hybrid.nix   | 52 ++++++++++++------------
 garlic/exp/creams/ss+pure.nix     | 66 ++++++++++++++++---------------
 4 files changed, 109 insertions(+), 105 deletions(-)

diff --git a/garlic/exp/creams/gran+node1.nix b/garlic/exp/creams/gran+node1.nix
index 075d6318..35a2c1cb 100644
--- a/garlic/exp/creams/gran+node1.nix
+++ b/garlic/exp/creams/gran+node1.nix
@@ -38,24 +38,24 @@ let
   # Generate the complete configuration for each unit
   genConf = with bsc; c: targetMachine.config // rec {
     expName = "creams-gran-node1";
-    unitName = "${expName}-${toString nodes}-${gitBranch}";
     inherit (targetMachine.config) hw;
     # Options for creams
     cc = icc;
     mpi = impi;
-    inherit (c.input) granul;
+    inherit (c.input) granul time nodes;
     inherit (c) gitBranch;
-    nprocz = ntasksPerNode * nodes;
+    unitName = "${expName}-${toString nodes}-${gitBranch}";
 
-    # Repeat the execution of each unit 30 times
-    loops = 30;
+    # Repeat the execution of each unit 10 times
+    loops = 10;
 
     # Resources
     qos = "debug";
     ntasksPerNode = hw.socketsPerNode;
     cpusPerTask = hw.cpusPerSocket;
-    inherit (c.input) time nodes;
     jobName = unitName;
+    
+    nprocz = ntasksPerNode * nodes;
   };
 
   # Compute the array of configurations
@@ -63,29 +63,24 @@ let
     inherit varConf genConf;
   };
 
-  # Use nanos6 with regions
-  nanos6Env = {nextStage, conf, ...}: with conf; stages.exec {
-    inherit nextStage;
-    env = ''
-      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
-    '';
-  };
-
-  # Custom stage to copy the creams input dataset
-  copyInput = {nextStage, conf, ...}:
+  # Custom srun stage to copy the creams input dataset
+  customSrun = {nextStage, conf, ...}:
     let
       input = bsc.garlic.apps.creamsInput.override {
         inherit (conf) gitBranch granul nprocz;
       };
     in
-      stages.exec {
+      stages.srun {
+        # These are part of the stdndard srun stage:
+        inherit (conf) nixPrefix;
         inherit nextStage;
-        env = ''
-          # Only the MPI rank 0 will copy the files
-          if [ $SLURM_PROCID == 0 ]; then
-            cp -fr ${input}/SodTubeBenchmark/* .
-            chmod +w -R .
-          fi
+        cpuBind = "cores,verbose";
+
+        # Now we add some commands to execute before calling srun. These will
+        # only run in one rank (the first in the list of allocated nodes)
+        preSrun = ''
+          cp -r ${input}/SodTubeBenchmark/* .
+          chmod +w -R .
         '';
       };
 
@@ -98,7 +93,12 @@ let
         inherit cc mpi gitBranch;
       };
 
-  pipeline = stdexp.stdPipeline ++ [ nanos6Env copyInput creams ];
+  pipeline = stdexp.stdPipelineOverride {
+    overrides = {
+      # Replace the stdandard srun stage with our own
+      srun = customSrun;
+    };
+  } ++ [ creams ];
 
 in
  
diff --git a/garlic/exp/creams/gran+node16.nix b/garlic/exp/creams/gran+node16.nix
index 942b4741..f295c4b4 100644
--- a/garlic/exp/creams/gran+node16.nix
+++ b/garlic/exp/creams/gran+node16.nix
@@ -36,24 +36,24 @@ let
   # Generate the complete configuration for each unit
   genConf = with bsc; c: targetMachine.config // rec {
     expName = "creams-gran-node16";
-    unitName = "${expName}-${toString nodes}-${gitBranch}";
     inherit (targetMachine.config) hw;
     # Options for creams
     cc = icc;
     mpi = impi;
-    inherit (c.input) granul;
+    inherit (c.input) granul time nodes;
     inherit (c) gitBranch;
-    nprocz = ntasksPerNode * nodes;
+    unitName = "${expName}-${toString nodes}-${gitBranch}";
 
-    # Repeat the execution of each unit 30 times
-    loops = 30;
+    # Repeat the execution of each unit 10 times
+    loops = 10;
 
     # Resources
     qos = "debug";
     ntasksPerNode = hw.socketsPerNode;
     cpusPerTask = hw.cpusPerSocket;
-    inherit (c.input) time nodes;
     jobName = unitName;
+    
+    nprocz = ntasksPerNode * nodes;
   };
 
   # Compute the array of configurations
@@ -61,29 +61,24 @@ let
     inherit varConf genConf;
   };
 
-  # Use nanos6 with regions
-  nanos6Env = {nextStage, conf, ...}: with conf; stages.exec {
-    inherit nextStage;
-    env = ''
-      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
-    '';
-  };
-
-  # Custom stage to copy the creams input dataset
-  copyInput = {nextStage, conf, ...}:
+  # Custom srun stage to copy the creams input dataset
+  customSrun = {nextStage, conf, ...}:
     let
       input = bsc.garlic.apps.creamsInput.override {
         inherit (conf) gitBranch granul nprocz;
       };
     in
-      stages.exec {
+      stages.srun {
+        # These are part of the stdndard srun stage:
+        inherit (conf) nixPrefix;
         inherit nextStage;
-        env = ''
-          # Only the MPI rank 0 will copy the files
-          if [ $SLURM_PROCID == 0 ]; then
-            cp -fr ${input}/SodTubeBenchmark/* .
-            chmod +w -R .
-          fi
+        cpuBind = "cores,verbose";
+
+        # Now we add some commands to execute before calling srun. These will
+        # only run in one rank (the first in the list of allocated nodes)
+        preSrun = ''
+          cp -r ${input}/SodTubeBenchmark/* .
+          chmod +w -R .
         '';
       };
 
@@ -96,7 +91,12 @@ let
         inherit cc mpi gitBranch;
       };
 
-  pipeline = stdexp.stdPipeline ++ [ nanos6Env copyInput creams ];
+  pipeline = stdexp.stdPipelineOverride {
+    overrides = {
+      # Replace the stdandard srun stage with our own
+      srun = customSrun;
+    };
+  } ++ [ creams ];
 
 in
  
diff --git a/garlic/exp/creams/ss+hybrid.nix b/garlic/exp/creams/ss+hybrid.nix
index 0b39d366..5b205310 100644
--- a/garlic/exp/creams/ss+hybrid.nix
+++ b/garlic/exp/creams/ss+hybrid.nix
@@ -16,7 +16,7 @@ let
       { nodes=2 ; nprocz=4 ; granul=19; time= "02:00:00"; }
       { nodes=4 ; nprocz=8 ; granul=10; time= "02:00:00"; }
       { nodes=8 ; nprocz=16; granul=9 ; time= "02:00:00"; }
-      { nodes=16; nprocz=32; granul=9 ; time= "02:00:00"; }
+      { nodes=16; nprocz=32; granul=9 ; time= "02:00:00"; }    
     ];
 
     gitBranch = [
@@ -31,25 +31,25 @@ let
 
   # Generate the complete configuration for each unit
   genConf = with bsc; c: targetMachine.config // rec {
-    expName = "creams-ss";
-    unitName = "${expName}-${toString nodes}-${gitBranch}";
+    expName = "creams-ss-hybrid";
     inherit (targetMachine.config) hw;
     # Options for creams
     cc = icc;
     mpi = impi;
-    inherit (c.input) granul;
+    inherit (c.input) granul time nodes;
     inherit (c) gitBranch;
-    nprocz = ntasksPerNode * nodes;
+    unitName = "${expName}-${toString nodes}-${gitBranch}";
 
-    # Repeat the execution of each unit 30 times
-    loops = 30;
+    # Repeat the execution of each unit 10 times
+    loops = 10;
 
     # Resources
     qos = "debug";
     ntasksPerNode = hw.socketsPerNode;
     cpusPerTask = hw.cpusPerSocket;
-    inherit (c.input) time nodes;
     jobName = unitName;
+    
+    nprocz = ntasksPerNode * nodes;
   };
 
   # Compute the array of configurations
@@ -57,29 +57,24 @@ let
     inherit varConf genConf;
   };
 
-  # Use nanos6 with regions
-  nanos6Env = {nextStage, conf, ...}: with conf; stages.exec {
-    inherit nextStage;
-    env = ''
-      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
-    '';
-  };
-
-  # Custom stage to copy the creams input dataset
-  copyInput = {nextStage, conf, ...}:
+  # Custom srun stage to copy the creams input dataset
+  customSrun = {nextStage, conf, ...}:
     let
       input = bsc.garlic.apps.creamsInput.override {
         inherit (conf) gitBranch granul nprocz;
       };
     in
-      stages.exec {
+      stages.srun {
+        # These are part of the stdndard srun stage:
+        inherit (conf) nixPrefix;
         inherit nextStage;
-        env = ''
-          # Only the MPI rank 0 will copy the files
-          if [ $SLURM_PROCID == 0 ]; then
-            cp -fr ${input}/SodTubeBenchmark/* .
-            chmod +w -R .
-          fi
+        cpuBind = "cores,verbose";
+
+        # Now we add some commands to execute before calling srun. These will
+        # only run in one rank (the first in the list of allocated nodes)
+        preSrun = ''
+          cp -r ${input}/SodTubeBenchmark/* .
+          chmod +w -R .
         '';
       };
 
@@ -92,7 +87,12 @@ let
         inherit cc mpi gitBranch;
       };
 
-  pipeline = stdexp.stdPipeline ++ [ nanos6Env copyInput creams ];
+  pipeline = stdexp.stdPipelineOverride {
+    overrides = {
+      # Replace the stdandard srun stage with our own
+      srun = customSrun;
+    };
+  } ++ [ creams ];
 
 in
  
diff --git a/garlic/exp/creams/ss+pure.nix b/garlic/exp/creams/ss+pure.nix
index 244ccd20..7585576a 100644
--- a/garlic/exp/creams/ss+pure.nix
+++ b/garlic/exp/creams/ss+pure.nix
@@ -12,35 +12,39 @@ let
   # Initial variable configuration
   varConf = {
     input = [
-      { time="02:00:00"; nodes=1;  }
-      { time="02:00:00"; nodes=2;  }
-      { time="02:00:00"; nodes=4;  }
-      { time="02:00:00"; nodes=8;  }
-      { time="02:00:00"; nodes=16; }
+      { nodes=1 ; nprocz=2 ; granul=999999; time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul=999999; time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul=999999; time= "02:00:00"; }
+      { nodes=8 ; nprocz=16; granul=999999; time= "02:00:00"; }
+      { nodes=16; nprocz=32; granul=999999; time= "02:00:00"; }    
+    ];
+
+    gitBranch = [
+      "garlic/mpi+send+seq"
     ];
   };
 
   # Generate the complete configuration for each unit
   genConf = with bsc; c: targetMachine.config // rec {
-    expName = "creams-ss";
-    unitName = "${expName}-${toString nodes}-${gitBranch}";
+    expName = "creams-ss-pure";
     inherit (targetMachine.config) hw;
     # Options for creams
     cc = icc;
     mpi = impi;
-    granul = 0;
-    gitBranch = "garlic/mpi+send+seq";
-    nprocz = ntasksPerNode * nodes;
+    inherit (c.input) granul time nodes;
+    inherit (c) gitBranch;
+    unitName = "${expName}-${toString nodes}-${gitBranch}";
 
-    # Repeat the execution of each unit 30 times
-    loops = 30;
+    # Repeat the execution of each unit 10 times
+    loops = 10;
 
     # Resources
     qos = "debug";
     ntasksPerNode = hw.cpusPerNode;
     cpusPerTask = 1;
-    inherit (c.input) time nodes;
     jobName = unitName;
+    
+    nprocz = ntasksPerNode * nodes;
   };
 
   # Compute the array of configurations
@@ -48,29 +52,24 @@ let
     inherit varConf genConf;
   };
 
-  # Use nanos6 with regions
-  nanos6Env = {nextStage, conf, ...}: with conf; stages.exec {
-    inherit nextStage;
-    env = ''
-      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
-    '';
-  };
-
-  # Custom stage to copy the creams input dataset
-  copyInput = {nextStage, conf, ...}:
+  # Custom srun stage to copy the creams input dataset
+  customSrun = {nextStage, conf, ...}:
     let
       input = bsc.garlic.apps.creamsInput.override {
         inherit (conf) gitBranch granul nprocz;
       };
     in
-      stages.exec {
+      stages.srun {
+        # These are part of the stdndard srun stage:
+        inherit (conf) nixPrefix;
         inherit nextStage;
-        env = ''
-          # Only the MPI rank 0 will copy the files
-          if [ $SLURM_PROCID == 0 ]; then
-            cp -fr ${input}/SodTubeBenchmark/* .
-            chmod +w -R .
-          fi
+        cpuBind = "cores,verbose";
+
+        # Now we add some commands to execute before calling srun. These will
+        # only run in one rank (the first in the list of allocated nodes)
+        preSrun = ''
+          cp -r ${input}/SodTubeBenchmark/* .
+          chmod +w -R .
         '';
       };
 
@@ -83,7 +82,12 @@ let
         inherit cc mpi gitBranch;
       };
 
-  pipeline = stdexp.stdPipeline ++ [ nanos6Env copyInput creams ];
+  pipeline = stdexp.stdPipelineOverride {
+    overrides = {
+      # Replace the stdandard srun stage with our own
+      srun = customSrun;
+    };
+  } ++ [ creams ];
 
 in
  
-- 
GitLab


From 6818b29d02f4d322ad0374e964a61d43122f9f84 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 4 Mar 2021 12:45:29 +0100
Subject: [PATCH 05/13] creams: fix outdated nanos6.toml

This temporal fix allows the experiment to ignore the nanos6.toml in the
git repository, and only set version.dependencies variable.
---
 garlic/exp/creams/gran+node16.nix | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/garlic/exp/creams/gran+node16.nix b/garlic/exp/creams/gran+node16.nix
index f295c4b4..73221755 100644
--- a/garlic/exp/creams/gran+node16.nix
+++ b/garlic/exp/creams/gran+node16.nix
@@ -79,9 +79,22 @@ let
         preSrun = ''
           cp -r ${input}/SodTubeBenchmark/* .
           chmod +w -R .
+          rm -f nanos6.toml
         '';
       };
 
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
+    '';
+
+    # Remove restarts as is not needed and is huge
+    post = ''
+      rm -rf restarts || true
+    '';
+  };
+
   # Creams program
   creams = {nextStage, conf, ...}: with conf;
     let
@@ -96,7 +109,7 @@ let
       # Replace the stdandard srun stage with our own
       srun = customSrun;
     };
-  } ++ [ creams ];
+  } ++ [ exec creams ];
 
 in
  
-- 
GitLab


From c59f298ae27781f9cbbe52cf65ab481cab2f4a00 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 4 Mar 2021 12:46:42 +0100
Subject: [PATCH 06/13] creams: reduce granularity experiment units

---
 garlic/exp/creams/gran+node16.nix | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/garlic/exp/creams/gran+node16.nix b/garlic/exp/creams/gran+node16.nix
index 73221755..4818dc82 100644
--- a/garlic/exp/creams/gran+node16.nix
+++ b/garlic/exp/creams/gran+node16.nix
@@ -12,23 +12,23 @@ let
   # Initial variable configuration
   varConf = {
     input = [
-      { nodes=16 ; nprocz=32 ; granul=128; time= "02:00:00"; }
+#      { nodes=16 ; nprocz=32 ; granul=128; time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul=64;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul=32;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul=16;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul= 9;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul= 5;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul= 4;  time= "02:00:00"; }
-      { nodes=16 ; nprocz=32 ; granul= 2;  time= "02:00:00"; }
-      { nodes=16 ; nprocz=32 ; granul= 1;  time= "02:00:00"; }
+#      { nodes=16 ; nprocz=32 ; granul= 2;  time= "02:00:00"; }
+#      { nodes=16 ; nprocz=32 ; granul= 1;  time= "02:00:00"; }
     ];
 
     gitBranch = [
-      "garlic/mpi+send+omp+fork"
-      "garlic/mpi+send+omp+task"
-      "garlic/mpi+send+oss+task"
-      "garlic/mpi+isend+omp+task"
-      "garlic/mpi+isend+oss+task"
+#      "garlic/mpi+send+omp+fork"
+#      "garlic/mpi+send+omp+task"
+#      "garlic/mpi+send+oss+task"
+#      "garlic/mpi+isend+omp+task"
+#      "garlic/mpi+isend+oss+task"
       "garlic/tampi+isend+oss+task"
     ];
   };
-- 
GitLab


From 8a81c6bfbac6d8b11c7626e7d2b1bec0c8097929 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 4 Mar 2021 12:53:07 +0100
Subject: [PATCH 07/13] creams: add granularity figure

Only the hybrid experiment is used by now
---
 garlic/fig/creams/gran.R | 71 ++++++++++++++++++++++++++++++++++++++++
 garlic/fig/index.nix     |  1 +
 2 files changed, 72 insertions(+)
 create mode 100644 garlic/fig/creams/gran.R

diff --git a/garlic/fig/creams/gran.R b/garlic/fig/creams/gran.R
new file mode 100644
index 00000000..d0d31498
--- /dev/null
+++ b/garlic/fig/creams/gran.R
@@ -0,0 +1,71 @@
+library(ggplot2)
+library(dplyr)
+library(scales)
+library(jsonlite)
+
+args=commandArgs(trailingOnly=TRUE)
+
+# Read the timetable from args[1]
+input_file = "input.json"
+if (length(args)>0) { input_file = args[1] }
+
+# Load the dataset in NDJSON format
+dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>%
+  jsonlite::flatten()
+
+# We only need some colums
+df = select(dataset, unit, config.nodes, config.gitBranch,
+    config.granul, time, total_time) %>%
+  rename(nodes=config.nodes, gitBranch=config.gitBranch,
+    granul=config.granul)
+
+df$unit = as.factor(df$unit)
+df$nnodes = df$nodes
+df$nodes = as.factor(df$nodes)
+df$gitBranch = as.factor(df$gitBranch)
+df$granul = as.factor(df$granul)
+
+# Remove the "garlic/" prefix from the gitBranch
+levels(df$gitBranch) <- substring((levels(df$gitBranch)), 8)
+
+# Compute new columns
+D=group_by(df, unit) %>%
+  mutate(tnorm = time / median(time) - 1) %>%
+  mutate(bad = ifelse(max(abs(tnorm)) >= 0.01, 1, 0)) %>%
+  mutate(variability = ifelse(bad > 0, "large", "ok")) %>%
+  mutate(mtime = median(time)) %>%
+  mutate(nmtime = mtime*nnodes) %>%
+  mutate(ntime = time*nnodes) %>%
+  ungroup() %>%
+  mutate(min_nmtime = min(nmtime)) %>%
+  mutate(rnmtime = nmtime / min_nmtime) %>%
+  mutate(rntime = ntime / min_nmtime) %>%
+  mutate(rmeff = 1.0 / rnmtime) %>%
+  mutate(reff = 1.0 / rntime) %>%
+  group_by(gitBranch) %>%
+  mutate(tmax = max(mtime)) %>%
+  mutate(speedup=tmax/time) %>%
+  mutate(eff=speedup/nnodes) %>%
+  mutate(mspeedup=tmax/mtime) %>%
+  mutate(meff=mspeedup/nnodes) %>%
+  ungroup()
+
+D$bad = as.factor(D$bad > 0)
+D$variability = as.factor(D$variability)
+
+ppi=300
+h=5
+w=5
+
+png("time.png", width=w*1.5*ppi, height=h*ppi, res=ppi)
+p = ggplot(D, aes(x=granul, y=mtime, linetype=gitBranch, shape=nodes)) +
+  geom_line(aes(group=interaction(nodes, gitBranch))) +
+  geom_point(aes(y=time)) +
+  scale_y_continuous(trans=log2_trans()) +
+  labs(x="Granularity", y="Time (s)",
+    title="Creams granularity",
+    subtitle=input_file) +
+  theme_bw() +
+  theme(plot.subtitle=element_text(size=8))
+print(p)
+dev.off()
diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix
index 1b370aed..c51d3281 100644
--- a/garlic/fig/index.nix
+++ b/garlic/fig/index.nix
@@ -53,6 +53,7 @@ in
 
   creams = with exp.creams; {
     ss = stdPlot ./creams/ss.R [ ss.hybrid ss.pure ];
+    gran = stdPlot ./creams/gran.R [ gran.hybrid ];
   };
 
   osu = with exp.osu; {
-- 
GitLab


From a55019c6ef4d5b1acb5d70f85e203604197cd86c Mon Sep 17 00:00:00 2001
From: Pedro Martinez <pedro.martinez-ferrer@bsc.es>
Date: Fri, 5 Mar 2021 14:46:48 +0100
Subject: [PATCH 08/13] creams: add more nodes for granularity experiments

---
 garlic/exp/creams/gran+node1.nix  |  17 ++++-
 garlic/exp/creams/gran+node16.nix |  16 ++---
 garlic/exp/creams/gran+node2.nix  | 116 ++++++++++++++++++++++++++++++
 garlic/exp/creams/gran+node4.nix  | 116 ++++++++++++++++++++++++++++++
 garlic/exp/creams/gran+node8.nix  | 116 ++++++++++++++++++++++++++++++
 garlic/exp/creams/ss+hybrid.nix   |  15 +++-
 garlic/exp/creams/ss+pure.nix     |  15 +++-
 garlic/exp/index.nix              |   7 +-
 garlic/fig/index.nix              |   2 +-
 9 files changed, 404 insertions(+), 16 deletions(-)
 create mode 100644 garlic/exp/creams/gran+node2.nix
 create mode 100644 garlic/exp/creams/gran+node4.nix
 create mode 100644 garlic/exp/creams/gran+node8.nix

diff --git a/garlic/exp/creams/gran+node1.nix b/garlic/exp/creams/gran+node1.nix
index 35a2c1cb..5bd27265 100644
--- a/garlic/exp/creams/gran+node1.nix
+++ b/garlic/exp/creams/gran+node1.nix
@@ -21,8 +21,6 @@ let
       { nodes=1 ; nprocz=2 ; granul= 9;  time= "02:00:00"; }
       { nodes=1 ; nprocz=2 ; granul= 5;  time= "02:00:00"; }
       { nodes=1 ; nprocz=2 ; granul= 4;  time= "02:00:00"; }
-      { nodes=1 ; nprocz=2 ; granul= 2;  time= "02:00:00"; }
-      { nodes=1 ; nprocz=2 ; granul= 1;  time= "02:00:00"; }
     ];
 
     gitBranch = [
@@ -81,9 +79,22 @@ let
         preSrun = ''
           cp -r ${input}/SodTubeBenchmark/* .
           chmod +w -R .
+          rm -f nanos6.toml
         '';
       };
 
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
+    '';
+
+    # Remove restarts as is not needed and is huge
+    post = ''
+      rm -rf restarts || true
+    '';
+  };
+
   # Creams program
   creams = {nextStage, conf, ...}: with conf;
     let
@@ -98,7 +109,7 @@ let
       # Replace the stdandard srun stage with our own
       srun = customSrun;
     };
-  } ++ [ creams ];
+  } ++ [ exec creams ];
 
 in
  
diff --git a/garlic/exp/creams/gran+node16.nix b/garlic/exp/creams/gran+node16.nix
index 4818dc82..73221755 100644
--- a/garlic/exp/creams/gran+node16.nix
+++ b/garlic/exp/creams/gran+node16.nix
@@ -12,23 +12,23 @@ let
   # Initial variable configuration
   varConf = {
     input = [
-#      { nodes=16 ; nprocz=32 ; granul=128; time= "02:00:00"; }
+      { nodes=16 ; nprocz=32 ; granul=128; time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul=64;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul=32;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul=16;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul= 9;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul= 5;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul= 4;  time= "02:00:00"; }
-#      { nodes=16 ; nprocz=32 ; granul= 2;  time= "02:00:00"; }
-#      { nodes=16 ; nprocz=32 ; granul= 1;  time= "02:00:00"; }
+      { nodes=16 ; nprocz=32 ; granul= 2;  time= "02:00:00"; }
+      { nodes=16 ; nprocz=32 ; granul= 1;  time= "02:00:00"; }
     ];
 
     gitBranch = [
-#      "garlic/mpi+send+omp+fork"
-#      "garlic/mpi+send+omp+task"
-#      "garlic/mpi+send+oss+task"
-#      "garlic/mpi+isend+omp+task"
-#      "garlic/mpi+isend+oss+task"
+      "garlic/mpi+send+omp+fork"
+      "garlic/mpi+send+omp+task"
+      "garlic/mpi+send+oss+task"
+      "garlic/mpi+isend+omp+task"
+      "garlic/mpi+isend+oss+task"
       "garlic/tampi+isend+oss+task"
     ];
   };
diff --git a/garlic/exp/creams/gran+node2.nix b/garlic/exp/creams/gran+node2.nix
new file mode 100644
index 00000000..0e022f07
--- /dev/null
+++ b/garlic/exp/creams/gran+node2.nix
@@ -0,0 +1,116 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = {
+    input = [
+      { nodes=2 ; nprocz=4 ; granul=256; time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul=128; time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul=64;  time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul=37;  time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul=32;  time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul=16;  time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul= 9;  time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul= 5;  time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul= 4;  time= "02:00:00"; }
+    ];
+
+    gitBranch = [
+      "garlic/mpi+send+omp+fork"
+      "garlic/mpi+send+omp+task"
+      "garlic/mpi+send+oss+task"
+      "garlic/mpi+isend+omp+task"
+      "garlic/mpi+isend+oss+task"
+      "garlic/tampi+isend+oss+task"
+    ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "creams-gran-node2";
+    inherit (targetMachine.config) hw;
+    # Options for creams
+    cc = icc;
+    mpi = impi;
+    inherit (c.input) granul time nodes;
+    inherit (c) gitBranch;
+    unitName = "${expName}-${toString nodes}-${gitBranch}";
+
+    # Repeat the execution of each unit 10 times
+    loops = 10;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = hw.socketsPerNode;
+    cpusPerTask = hw.cpusPerSocket;
+    jobName = unitName;
+    
+    nprocz = ntasksPerNode * nodes;
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  # Custom srun stage to copy the creams input dataset
+  customSrun = {nextStage, conf, ...}:
+    let
+      input = bsc.garlic.apps.creamsInput.override {
+        inherit (conf) gitBranch granul nprocz;
+      };
+    in
+      stages.srun {
+        # These are part of the stdndard srun stage:
+        inherit (conf) nixPrefix;
+        inherit nextStage;
+        cpuBind = "cores,verbose";
+
+        # Now we add some commands to execute before calling srun. These will
+        # only run in one rank (the first in the list of allocated nodes)
+        preSrun = ''
+          cp -r ${input}/SodTubeBenchmark/* .
+          chmod +w -R .
+          rm -f nanos6.toml
+        '';
+      };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
+    '';
+
+    # Remove restarts as is not needed and is huge
+    post = ''
+      rm -rf restarts || true
+    '';
+  };
+
+  # Creams program
+  creams = {nextStage, conf, ...}: with conf;
+    let
+      customPkgs = stdexp.replaceMpi conf.mpi;
+    in
+      customPkgs.apps.creams.override {
+        inherit cc mpi gitBranch;
+      };
+
+  pipeline = stdexp.stdPipelineOverride {
+    overrides = {
+      # Replace the stdandard srun stage with our own
+      srun = customSrun;
+    };
+  } ++ [ exec creams ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/creams/gran+node4.nix b/garlic/exp/creams/gran+node4.nix
new file mode 100644
index 00000000..628ae1d2
--- /dev/null
+++ b/garlic/exp/creams/gran+node4.nix
@@ -0,0 +1,116 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = {
+    input = [
+      { nodes=4 ; nprocz=8 ; granul=64;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul=37;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul=32;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul=16;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul= 9;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul= 5;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=4 ; granul= 4;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul= 2;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul= 1;  time= "02:00:00"; }
+    ];
+
+    gitBranch = [
+      "garlic/mpi+send+omp+fork"
+      "garlic/mpi+send+omp+task"
+      "garlic/mpi+send+oss+task"
+      "garlic/mpi+isend+omp+task"
+      "garlic/mpi+isend+oss+task"
+      "garlic/tampi+isend+oss+task"
+    ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "creams-gran-node4";
+    inherit (targetMachine.config) hw;
+    # Options for creams
+    cc = icc;
+    mpi = impi;
+    inherit (c.input) granul time nodes;
+    inherit (c) gitBranch;
+    unitName = "${expName}-${toString nodes}-${gitBranch}";
+
+    # Repeat the execution of each unit 10 times
+    loops = 10;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = hw.socketsPerNode;
+    cpusPerTask = hw.cpusPerSocket;
+    jobName = unitName;
+    
+    nprocz = ntasksPerNode * nodes;
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  # Custom srun stage to copy the creams input dataset
+  customSrun = {nextStage, conf, ...}:
+    let
+      input = bsc.garlic.apps.creamsInput.override {
+        inherit (conf) gitBranch granul nprocz;
+      };
+    in
+      stages.srun {
+        # These are part of the stdndard srun stage:
+        inherit (conf) nixPrefix;
+        inherit nextStage;
+        cpuBind = "cores,verbose";
+
+        # Now we add some commands to execute before calling srun. These will
+        # only run in one rank (the first in the list of allocated nodes)
+        preSrun = ''
+          cp -r ${input}/SodTubeBenchmark/* .
+          chmod +w -R .
+          rm -f nanos6.toml
+        '';
+      };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
+    '';
+
+    # Remove restarts as is not needed and is huge
+    post = ''
+      rm -rf restarts || true
+    '';
+  };
+
+  # Creams program
+  creams = {nextStage, conf, ...}: with conf;
+    let
+      customPkgs = stdexp.replaceMpi conf.mpi;
+    in
+      customPkgs.apps.creams.override {
+        inherit cc mpi gitBranch;
+      };
+
+  pipeline = stdexp.stdPipelineOverride {
+    overrides = {
+      # Replace the stdandard srun stage with our own
+      srun = customSrun;
+    };
+  } ++ [ exec creams ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/creams/gran+node8.nix b/garlic/exp/creams/gran+node8.nix
new file mode 100644
index 00000000..d0fec23e
--- /dev/null
+++ b/garlic/exp/creams/gran+node8.nix
@@ -0,0 +1,116 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = {
+    input = [
+      { nodes=8 ; nprocz=16 ; granul=128; time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul=64;  time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul=32;  time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul=16;  time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul= 9;  time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul= 5;  time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul= 4;  time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul= 2;  time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul= 1;  time= "02:00:00"; }
+    ];
+
+    gitBranch = [
+      "garlic/mpi+send+omp+fork"
+      "garlic/mpi+send+omp+task"
+      "garlic/mpi+send+oss+task"
+      "garlic/mpi+isend+omp+task"
+      "garlic/mpi+isend+oss+task"
+      "garlic/tampi+isend+oss+task"
+    ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "creams-gran-node8";
+    inherit (targetMachine.config) hw;
+    # Options for creams
+    cc = icc;
+    mpi = impi;
+    inherit (c.input) granul time nodes;
+    inherit (c) gitBranch;
+    unitName = "${expName}-${toString nodes}-${gitBranch}";
+
+    # Repeat the execution of each unit 10 times
+    loops = 10;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = hw.socketsPerNode;
+    cpusPerTask = hw.cpusPerSocket;
+    jobName = unitName;
+    
+    nprocz = ntasksPerNode * nodes;
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  # Custom srun stage to copy the creams input dataset
+  customSrun = {nextStage, conf, ...}:
+    let
+      input = bsc.garlic.apps.creamsInput.override {
+        inherit (conf) gitBranch granul nprocz;
+      };
+    in
+      stages.srun {
+        # These are part of the stdndard srun stage:
+        inherit (conf) nixPrefix;
+        inherit nextStage;
+        cpuBind = "cores,verbose";
+
+        # Now we add some commands to execute before calling srun. These will
+        # only run in one rank (the first in the list of allocated nodes)
+        preSrun = ''
+          cp -r ${input}/SodTubeBenchmark/* .
+          chmod +w -R .
+          rm -f nanos6.toml
+        '';
+      };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
+    '';
+
+    # Remove restarts as is not needed and is huge
+    post = ''
+      rm -rf restarts || true
+    '';
+  };
+
+  # Creams program
+  creams = {nextStage, conf, ...}: with conf;
+    let
+      customPkgs = stdexp.replaceMpi conf.mpi;
+    in
+      customPkgs.apps.creams.override {
+        inherit cc mpi gitBranch;
+      };
+
+  pipeline = stdexp.stdPipelineOverride {
+    overrides = {
+      # Replace the stdandard srun stage with our own
+      srun = customSrun;
+    };
+  } ++ [ exec creams ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/creams/ss+hybrid.nix b/garlic/exp/creams/ss+hybrid.nix
index 5b205310..c78376bb 100644
--- a/garlic/exp/creams/ss+hybrid.nix
+++ b/garlic/exp/creams/ss+hybrid.nix
@@ -75,9 +75,22 @@ let
         preSrun = ''
           cp -r ${input}/SodTubeBenchmark/* .
           chmod +w -R .
+          rm -f nanos6.toml
         '';
       };
 
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
+    '';
+
+    # Remove restarts as is not needed and is huge
+    post = ''
+      rm -rf restarts || true
+    '';
+  };
+
   # Creams program
   creams = {nextStage, conf, ...}: with conf;
     let
@@ -92,7 +105,7 @@ let
       # Replace the stdandard srun stage with our own
       srun = customSrun;
     };
-  } ++ [ creams ];
+  } ++ [ exec creams ];
 
 in
  
diff --git a/garlic/exp/creams/ss+pure.nix b/garlic/exp/creams/ss+pure.nix
index 7585576a..5d36e5ee 100644
--- a/garlic/exp/creams/ss+pure.nix
+++ b/garlic/exp/creams/ss+pure.nix
@@ -70,9 +70,22 @@ let
         preSrun = ''
           cp -r ${input}/SodTubeBenchmark/* .
           chmod +w -R .
+          rm -f nanos6.toml
         '';
       };
 
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
+    '';
+
+    # Remove restarts as is not needed and is huge
+    post = ''
+      rm -rf restarts || true
+    '';
+  };
+
   # Creams program
   creams = {nextStage, conf, ...}: with conf;
     let
@@ -87,7 +100,7 @@ let
       # Replace the stdandard srun stage with our own
       srun = customSrun;
     };
-  } ++ [ creams ];
+  } ++ [ exec creams ];
 
 in
  
diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix
index 6b93c92e..ec903634 100644
--- a/garlic/exp/index.nix
+++ b/garlic/exp/index.nix
@@ -42,8 +42,11 @@
       hybrid = callPackage ./creams/ss+hybrid.nix { };
     };
     gran = {
-      pure = callPackage ./creams/gran+node1.nix { };
-      hybrid = callPackage ./creams/gran+node16.nix { };
+      node1 = callPackage ./creams/gran+node1.nix { };
+      node2 = callPackage ./creams/gran+node2.nix { };
+      node4 = callPackage ./creams/gran+node4.nix { };
+      node8 = callPackage ./creams/gran+node8.nix { };
+      node16 = callPackage ./creams/gran+node16.nix { };
     };
   };
 
diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix
index c51d3281..d2e4d5b7 100644
--- a/garlic/fig/index.nix
+++ b/garlic/fig/index.nix
@@ -53,7 +53,7 @@ in
 
   creams = with exp.creams; {
     ss = stdPlot ./creams/ss.R [ ss.hybrid ss.pure ];
-    gran = stdPlot ./creams/gran.R [ gran.hybrid ];
+    gran = stdPlot ./creams/gran.R [ gran.node1 gran.node2 gran.node4 gran.node8 gran.node16 ];
   };
 
   osu = with exp.osu; {
-- 
GitLab


From d27c6962592d46cccf792cbf2fe5657ef7a5751f Mon Sep 17 00:00:00 2001
From: Pedro Martinez <pedro.martinez-ferrer@bsc.es>
Date: Tue, 16 Mar 2021 20:19:45 +0100
Subject: [PATCH 09/13] creams: reduce granularity combinations to 8

---
 garlic/exp/creams/gran+node1.nix  | 1 -
 garlic/exp/creams/gran+node16.nix | 1 -
 garlic/exp/creams/gran+node2.nix  | 1 -
 garlic/exp/creams/gran+node4.nix  | 1 -
 garlic/exp/creams/gran+node8.nix  | 1 -
 5 files changed, 5 deletions(-)

diff --git a/garlic/exp/creams/gran+node1.nix b/garlic/exp/creams/gran+node1.nix
index 5bd27265..c4c836f2 100644
--- a/garlic/exp/creams/gran+node1.nix
+++ b/garlic/exp/creams/gran+node1.nix
@@ -12,7 +12,6 @@ let
   # Initial variable configuration
   varConf = {
     input = [
-      { nodes=1 ; nprocz=2 ; granul=256; time= "02:00:00"; }
       { nodes=1 ; nprocz=2 ; granul=128; time= "02:00:00"; }
       { nodes=1 ; nprocz=2 ; granul=64;  time= "02:00:00"; }
       { nodes=1 ; nprocz=2 ; granul=37;  time= "02:00:00"; }
diff --git a/garlic/exp/creams/gran+node16.nix b/garlic/exp/creams/gran+node16.nix
index 73221755..5b41d6a4 100644
--- a/garlic/exp/creams/gran+node16.nix
+++ b/garlic/exp/creams/gran+node16.nix
@@ -20,7 +20,6 @@ let
       { nodes=16 ; nprocz=32 ; granul= 5;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul= 4;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul= 2;  time= "02:00:00"; }
-      { nodes=16 ; nprocz=32 ; granul= 1;  time= "02:00:00"; }
     ];
 
     gitBranch = [
diff --git a/garlic/exp/creams/gran+node2.nix b/garlic/exp/creams/gran+node2.nix
index 0e022f07..e5724473 100644
--- a/garlic/exp/creams/gran+node2.nix
+++ b/garlic/exp/creams/gran+node2.nix
@@ -12,7 +12,6 @@ let
   # Initial variable configuration
   varConf = {
     input = [
-      { nodes=2 ; nprocz=4 ; granul=256; time= "02:00:00"; }
       { nodes=2 ; nprocz=4 ; granul=128; time= "02:00:00"; }
       { nodes=2 ; nprocz=4 ; granul=64;  time= "02:00:00"; }
       { nodes=2 ; nprocz=4 ; granul=37;  time= "02:00:00"; }
diff --git a/garlic/exp/creams/gran+node4.nix b/garlic/exp/creams/gran+node4.nix
index 628ae1d2..71dc9749 100644
--- a/garlic/exp/creams/gran+node4.nix
+++ b/garlic/exp/creams/gran+node4.nix
@@ -20,7 +20,6 @@ let
       { nodes=4 ; nprocz=8 ; granul= 5;  time= "02:00:00"; }
       { nodes=4 ; nprocz=4 ; granul= 4;  time= "02:00:00"; }
       { nodes=4 ; nprocz=8 ; granul= 2;  time= "02:00:00"; }
-      { nodes=4 ; nprocz=8 ; granul= 1;  time= "02:00:00"; }
     ];
 
     gitBranch = [
diff --git a/garlic/exp/creams/gran+node8.nix b/garlic/exp/creams/gran+node8.nix
index d0fec23e..01b66fa6 100644
--- a/garlic/exp/creams/gran+node8.nix
+++ b/garlic/exp/creams/gran+node8.nix
@@ -20,7 +20,6 @@ let
       { nodes=8 ; nprocz=16 ; granul= 5;  time= "02:00:00"; }
       { nodes=8 ; nprocz=16 ; granul= 4;  time= "02:00:00"; }
       { nodes=8 ; nprocz=16 ; granul= 2;  time= "02:00:00"; }
-      { nodes=8 ; nprocz=16 ; granul= 1;  time= "02:00:00"; }
     ];
 
     gitBranch = [
-- 
GitLab


From cb4d27aefb8e7db77ea5d04e421240d6ef87d85a Mon Sep 17 00:00:00 2001
From: Pedro Martinez <pedro.martinez-ferrer@bsc.es>
Date: Mon, 22 Mar 2021 12:34:49 +0100
Subject: [PATCH 10/13] creams: bugfix in granularity values

---
 garlic/exp/creams/gran+node1.nix | 2 +-
 garlic/exp/creams/gran+node2.nix | 2 +-
 garlic/exp/creams/gran+node4.nix | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/garlic/exp/creams/gran+node1.nix b/garlic/exp/creams/gran+node1.nix
index c4c836f2..10be85db 100644
--- a/garlic/exp/creams/gran+node1.nix
+++ b/garlic/exp/creams/gran+node1.nix
@@ -12,9 +12,9 @@ let
   # Initial variable configuration
   varConf = {
     input = [
+      { nodes=1 ; nprocz=2 ; granul=256; time= "02:00:00"; }
       { nodes=1 ; nprocz=2 ; granul=128; time= "02:00:00"; }
       { nodes=1 ; nprocz=2 ; granul=64;  time= "02:00:00"; }
-      { nodes=1 ; nprocz=2 ; granul=37;  time= "02:00:00"; }
       { nodes=1 ; nprocz=2 ; granul=32;  time= "02:00:00"; }
       { nodes=1 ; nprocz=2 ; granul=16;  time= "02:00:00"; }
       { nodes=1 ; nprocz=2 ; granul= 9;  time= "02:00:00"; }
diff --git a/garlic/exp/creams/gran+node2.nix b/garlic/exp/creams/gran+node2.nix
index e5724473..d7937e30 100644
--- a/garlic/exp/creams/gran+node2.nix
+++ b/garlic/exp/creams/gran+node2.nix
@@ -12,9 +12,9 @@ let
   # Initial variable configuration
   varConf = {
     input = [
+      { nodes=2 ; nprocz=4 ; granul=256; time= "02:00:00"; }
       { nodes=2 ; nprocz=4 ; granul=128; time= "02:00:00"; }
       { nodes=2 ; nprocz=4 ; granul=64;  time= "02:00:00"; }
-      { nodes=2 ; nprocz=4 ; granul=37;  time= "02:00:00"; }
       { nodes=2 ; nprocz=4 ; granul=32;  time= "02:00:00"; }
       { nodes=2 ; nprocz=4 ; granul=16;  time= "02:00:00"; }
       { nodes=2 ; nprocz=4 ; granul= 9;  time= "02:00:00"; }
diff --git a/garlic/exp/creams/gran+node4.nix b/garlic/exp/creams/gran+node4.nix
index 71dc9749..6e0ef6a0 100644
--- a/garlic/exp/creams/gran+node4.nix
+++ b/garlic/exp/creams/gran+node4.nix
@@ -12,8 +12,8 @@ let
   # Initial variable configuration
   varConf = {
     input = [
+      { nodes=4 ; nprocz=8 ; granul=128; time= "02:00:00"; }
       { nodes=4 ; nprocz=8 ; granul=64;  time= "02:00:00"; }
-      { nodes=4 ; nprocz=8 ; granul=37;  time= "02:00:00"; }
       { nodes=4 ; nprocz=8 ; granul=32;  time= "02:00:00"; }
       { nodes=4 ; nprocz=8 ; granul=16;  time= "02:00:00"; }
       { nodes=4 ; nprocz=8 ; granul= 9;  time= "02:00:00"; }
-- 
GitLab


From bfc32ef4b7c879a9a0c20c8d62b1f568900af15a Mon Sep 17 00:00:00 2001
From: Pedro Martinez <pedro.martinez-ferrer@bsc.es>
Date: Mon, 22 Mar 2021 12:37:42 +0100
Subject: [PATCH 11/13] creams: readjust granularity for strong scalability

---
 garlic/exp/creams/ss+hybrid.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/garlic/exp/creams/ss+hybrid.nix b/garlic/exp/creams/ss+hybrid.nix
index c78376bb..26e1293e 100644
--- a/garlic/exp/creams/ss+hybrid.nix
+++ b/garlic/exp/creams/ss+hybrid.nix
@@ -12,9 +12,9 @@ let
   # Initial variable configuration
   varConf = {
     input = [
-      { nodes=1 ; nprocz=2 ; granul=37; time= "02:00:00"; }
-      { nodes=2 ; nprocz=4 ; granul=19; time= "02:00:00"; }
-      { nodes=4 ; nprocz=8 ; granul=10; time= "02:00:00"; }
+      { nodes=1 ; nprocz=2 ; granul=16; time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul=16; time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul=9 ; time= "02:00:00"; }
       { nodes=8 ; nprocz=16; granul=9 ; time= "02:00:00"; }
       { nodes=16; nprocz=32; granul=9 ; time= "02:00:00"; }    
     ];
-- 
GitLab


From 5cd9894636c4eb0b472296f404cefb71ab9dc08b Mon Sep 17 00:00:00 2001
From: Pedro Martinez <pedro.martinez-ferrer@bsc.es>
Date: Tue, 23 Mar 2021 11:21:01 +0100
Subject: [PATCH 12/13] creams: redefine granularity experiments

---
 garlic/exp/creams/gran+node1.nix  | 1 -
 garlic/exp/creams/gran+node16.nix | 3 +--
 garlic/exp/creams/gran+node2.nix  | 1 -
 garlic/exp/creams/gran+node4.nix  | 3 +--
 garlic/exp/creams/gran+node8.nix  | 3 +--
 5 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/garlic/exp/creams/gran+node1.nix b/garlic/exp/creams/gran+node1.nix
index 10be85db..beb23cf7 100644
--- a/garlic/exp/creams/gran+node1.nix
+++ b/garlic/exp/creams/gran+node1.nix
@@ -23,7 +23,6 @@ let
     ];
 
     gitBranch = [
-      "garlic/mpi+send+omp+fork"
       "garlic/mpi+send+omp+task"
       "garlic/mpi+send+oss+task"
       "garlic/mpi+isend+omp+task"
diff --git a/garlic/exp/creams/gran+node16.nix b/garlic/exp/creams/gran+node16.nix
index 5b41d6a4..54a1ff23 100644
--- a/garlic/exp/creams/gran+node16.nix
+++ b/garlic/exp/creams/gran+node16.nix
@@ -12,7 +12,6 @@ let
   # Initial variable configuration
   varConf = {
     input = [
-      { nodes=16 ; nprocz=32 ; granul=128; time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul=64;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul=32;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul=16;  time= "02:00:00"; }
@@ -20,10 +19,10 @@ let
       { nodes=16 ; nprocz=32 ; granul= 5;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul= 4;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul= 2;  time= "02:00:00"; }
+      { nodes=16 ; nprocz=32 ; granul= 1;  time= "02:00:00"; }
     ];
 
     gitBranch = [
-      "garlic/mpi+send+omp+fork"
       "garlic/mpi+send+omp+task"
       "garlic/mpi+send+oss+task"
       "garlic/mpi+isend+omp+task"
diff --git a/garlic/exp/creams/gran+node2.nix b/garlic/exp/creams/gran+node2.nix
index d7937e30..4e05306d 100644
--- a/garlic/exp/creams/gran+node2.nix
+++ b/garlic/exp/creams/gran+node2.nix
@@ -23,7 +23,6 @@ let
     ];
 
     gitBranch = [
-      "garlic/mpi+send+omp+fork"
       "garlic/mpi+send+omp+task"
       "garlic/mpi+send+oss+task"
       "garlic/mpi+isend+omp+task"
diff --git a/garlic/exp/creams/gran+node4.nix b/garlic/exp/creams/gran+node4.nix
index 6e0ef6a0..5fc46cad 100644
--- a/garlic/exp/creams/gran+node4.nix
+++ b/garlic/exp/creams/gran+node4.nix
@@ -12,7 +12,6 @@ let
   # Initial variable configuration
   varConf = {
     input = [
-      { nodes=4 ; nprocz=8 ; granul=128; time= "02:00:00"; }
       { nodes=4 ; nprocz=8 ; granul=64;  time= "02:00:00"; }
       { nodes=4 ; nprocz=8 ; granul=32;  time= "02:00:00"; }
       { nodes=4 ; nprocz=8 ; granul=16;  time= "02:00:00"; }
@@ -20,10 +19,10 @@ let
       { nodes=4 ; nprocz=8 ; granul= 5;  time= "02:00:00"; }
       { nodes=4 ; nprocz=4 ; granul= 4;  time= "02:00:00"; }
       { nodes=4 ; nprocz=8 ; granul= 2;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul= 1;  time= "02:00:00"; }
     ];
 
     gitBranch = [
-      "garlic/mpi+send+omp+fork"
       "garlic/mpi+send+omp+task"
       "garlic/mpi+send+oss+task"
       "garlic/mpi+isend+omp+task"
diff --git a/garlic/exp/creams/gran+node8.nix b/garlic/exp/creams/gran+node8.nix
index 01b66fa6..c6ad00c6 100644
--- a/garlic/exp/creams/gran+node8.nix
+++ b/garlic/exp/creams/gran+node8.nix
@@ -12,7 +12,6 @@ let
   # Initial variable configuration
   varConf = {
     input = [
-      { nodes=8 ; nprocz=16 ; granul=128; time= "02:00:00"; }
       { nodes=8 ; nprocz=16 ; granul=64;  time= "02:00:00"; }
       { nodes=8 ; nprocz=16 ; granul=32;  time= "02:00:00"; }
       { nodes=8 ; nprocz=16 ; granul=16;  time= "02:00:00"; }
@@ -20,10 +19,10 @@ let
       { nodes=8 ; nprocz=16 ; granul= 5;  time= "02:00:00"; }
       { nodes=8 ; nprocz=16 ; granul= 4;  time= "02:00:00"; }
       { nodes=8 ; nprocz=16 ; granul= 2;  time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul= 1;  time= "02:00:00"; }
     ];
 
     gitBranch = [
-      "garlic/mpi+send+omp+fork"
       "garlic/mpi+send+omp+task"
       "garlic/mpi+send+oss+task"
       "garlic/mpi+isend+omp+task"
-- 
GitLab


From 617ef21d384cfc32565f8fcc6e0b5946b7370e36 Mon Sep 17 00:00:00 2001
From: Pedro Martinez <pedro.martinez-ferrer@bsc.es>
Date: Wed, 24 Mar 2021 13:51:11 +0100
Subject: [PATCH 13/13] creams: redefine granularity figures

---
 garlic/fig/index.nix | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix
index d2e4d5b7..137c6290 100644
--- a/garlic/fig/index.nix
+++ b/garlic/fig/index.nix
@@ -53,7 +53,11 @@ in
 
   creams = with exp.creams; {
     ss = stdPlot ./creams/ss.R [ ss.hybrid ss.pure ];
-    gran = stdPlot ./creams/gran.R [ gran.node1 gran.node2 gran.node4 gran.node8 gran.node16 ];
+    gran1 = stdPlot ./creams/gran.R [ gran.node1 ];
+    gran2 = stdPlot ./creams/gran.R [ gran.node2 ];
+    gran4 = stdPlot ./creams/gran.R [ gran.node4 ];
+    gran8 = stdPlot ./creams/gran.R [ gran.node8 ];
+    gran16 = stdPlot ./creams/gran.R [ gran.node16 ];
   };
 
   osu = with exp.osu; {
-- 
GitLab