/*
* Copyright (c) 2017, BSC (Barcelona Supercomputing Center)
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*     * Redistributions of source code must retain the above copyright
*       notice, this list of conditions and the following disclaimer.
*     * Redistributions in binary form must reproduce the above copyright
*       notice, this list of conditions and the following disclaimer in the
*       documentation and/or other materials provided with the distribution.
*     * Neither the name of the <organization> nor the
*       names of its contributors may be used to endorse or promote products
*       derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY BSC ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL <copyright holder> BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#include <stdlib.h>
#include <unistd.h>
#include <string.h>

// General definitions
#include "matmul.h"
// General definitions which are required to build the FPGA accelerators
#include "matmul.fpga.h"

#pragma omp task device(smp) no_copy_deps
#pragma omp task out([BSIZE*BSIZE]v)
void setBlock (elem_t* v, const elem_t val) {
   for (unsigned int i = 0; i < BSIZE*BSIZE; ++i) {
      v[i] = val;
   }
}

#pragma omp task device(smp) no_copy_deps
#pragma omp task in([BSIZE*BSIZE]v)
void checkBlock (unsigned int * check_ok, elem_t* v, const elem_t val, const float threshold )
{
   const elem_t maxv = val * (1.0 + (val < 0 ? -threshold : threshold));
   const elem_t minv = val * (1.0 - (val < 0 ? -threshold : threshold));
   for (unsigned int i = 0; i < BSIZE*BSIZE && ( *check_ok ); ++i) {
      elem_t tmp = v[i];
      if (tmp > maxv || tmp < minv) {
         *check_ok = FALSE;
         fprintf(stderr, "ERROR:\t Expected a %lf but found %lf.\n", (double)val, (double)tmp);
      }
   }
}

#pragma omp target device(fpga) copy_deps no_localmem_copies num_instances(1)
#pragma omp task in([BSIZE*BSIZE]a, [BSIZE*BSIZE]b) inout([BSIZE*BSIZE]c)
void matmulBlock(elem_t *a, elem_t *b, elem_t *c) {
   unsigned int i, j, k, l;

   elem_t aa[BSIZE][BSIZE];
   elem_t bb[BSIZE][BSIZE];//< Note assuming that content of B is transposed
   elem_t cc[BSIZE][BSIZE];
   elem_t r0[BSIZE];
   elem_t r1[BSIZE/2];
   elem_t r2[BSIZE/4];
   elem_t r3[BSIZE/8];
   elem_t r4[BSIZE/16];

#pragma HLS DATAFLOW
//#pragma HLS resource variable=aa core=ram_2p_lutram latency=1
//#pragma HLS resource variable=bb core=ram_2p_lutram latency=1
#pragma HLS array_partition variable=aa block factor=BSIZE/4 dim=2
#pragma HLS array_partition variable=bb block factor=BSIZE/4 dim=2
#pragma HLS array_partition variable=r0 block factor=BSIZE/4
#pragma HLS array_partition variable=r1 block factor=BSIZE/8
#pragma HLS array_partition variable=r2 block factor=BSIZE/16
#pragma HLS array_partition variable=r3 block factor=BSIZE/32
loop_memcpy_in:
   for (i = 0; i < BSIZE; i++) {
       nanos_instrument_point_event(DATA_COPY, i);
      #pragma HLS pipeline II=BSIZE
      memcpy(&aa[i][0], &a[i*BSIZE], BSIZE*sizeof(elem_t));
      memcpy(&bb[i][0], &b[i*BSIZE], BSIZE*sizeof(elem_t));
      memcpy(&cc[i][0], &c[i*BSIZE], BSIZE*sizeof(elem_t));
   }
loop_matrix_i:
   for (i = 0; i < BSIZE; i++) {
loop_matrix_j:
      for (j = 0; j < BSIZE; j++) {
         nanos_instrument_point_event(MULT, i*100 + j);
#pragma HLS pipeline II=4
loop_matrix_k_r0:
         for (k = 0; k < BSIZE/2; k++) {
            r0[k] = aa[i][k] * bb[j][k];
            r0[k+BSIZE/2] = aa[i][k+BSIZE/2] * bb[j][k+BSIZE/2];
         }

         //Reduction
         nanos_instrument_point_event(REDUCTION, i*100 + j);
loop_matrix_k_r1:
         for (k = 0; k < BSIZE/2;   k++) r1[k] = r0[k] + r0[BSIZE/2+k];
loop_matrix_k_r2:
         for (k = 0; k < BSIZE/4;   k++) r2[k] = r1[k] + r1[BSIZE/4+k];
loop_matrix_k_r3:
         for (k = 0; k < BSIZE/8;   k++) r3[k] = r2[k] + r2[BSIZE/8+k];
loop_matrix_k_r4:
         for (k = 0; k < BSIZE/16;  k++) r4[k] = r3[k] + r3[BSIZE/16+k];
         cc[i][j] += r4[0] + r4[1];
      }
   }
loop_memcpy_out:
   for (i = 0; i < BSIZE; i++) {
       nanos_instrument_point_event(DATA_COPY_OUT, i);
      #pragma HLS pipeline II=BSIZE
      memcpy(&c[i*BSIZE], &cc[i][0], BSIZE*sizeof(elem_t));
   }
}


int main(int argc, char** argv) {
   if (argc < 2) {
      usage(argv[0]);
      exit(1);
   }

   unsigned int const b2size = BSIZE*BSIZE;
   unsigned int const msize = atoi(argv[1]);
   unsigned int const m2size = msize*msize;
   unsigned char const check = argc > 2 ? atoi(argv[2]) : 1;
   if (msize%BSIZE != 0) {
      fprintf(stderr, "ERROR:\t<matrix size> must be multiple of <block size>\n");
      usage(argv[0]);
      exit(1);
   }

   size_t s = m2size*sizeof(elem_t);
   elem_t* a = (elem_t *)(malloc(s));
   elem_t* b = (elem_t *)(malloc(s));
   elem_t* c = (elem_t *)(malloc(s));

   if (a == NULL || b == NULL || c == NULL) {
      fprintf(stderr, "ERROR:\tCannot allocate memory for the matrices\n");
      exit(1);
   }

   nanos_instrument_register_key_with_key(DATA_COPY, "data_copy_in", "Data copy in", true);
   printf("r1\n");
   nanos_instrument_register_key_with_key(DATA_COPY_OUT, "data_copy_out", "Data copy out", true);
   printf("r2\n");
   nanos_instrument_register_key_with_key(MULT, "mult", "Multiply operations", true);
   printf("r3\n");
   nanos_instrument_register_key_with_key(REDUCTION, "mm_reduction", "Reduction start", true);
   printf("r4\n");

   for (unsigned int i = 0; i < m2size/b2size; i++) {
      setBlock(&a[i*b2size], (elem_t)VAL_A + i);
      setBlock(&b[i*b2size], (elem_t)VAL_B - i);
      setBlock(&c[i*b2size], (elem_t)VAL_C);
   }

   #pragma omp taskwait
   double t_start = wall_time();

   for (unsigned int i = 0; i < msize/BSIZE; i++) {
//NOTE: Assuming that the following task will be executed in a shared memory environment.
//      Otherwise, it must define the input and output data of child tasks.
//#pragma omp task firstprivate(i)
//{
      for (unsigned int j = 0; j < msize/BSIZE; j++) {
         unsigned int const ci = j*b2size + i*BSIZE*msize;
         for (unsigned int k = 0; k < msize/BSIZE; k++) {
            unsigned int const ai = k*b2size + i*BSIZE*msize;
            unsigned int const bi = j*b2size + k*BSIZE*msize;
            matmulBlock(&a[ai], &b[bi], &c[ci]);
         }
      }
//      #pragma omp taskwait
//}
   }

   #pragma omp taskwait
   double t_end = wall_time();

   unsigned int check_ok = TRUE;
   if (check) {
      printf( "=================== CHECKING ===================== \n" );
      for (unsigned int i = 0; i < msize/BSIZE && check_ok; i++) {
         for (unsigned int j = 0; j < msize/BSIZE && check_ok; j++) {
            elem_t val = VAL_C;
            for (unsigned int k = 0; k < msize/BSIZE; k++) {
               unsigned int const ai = k*b2size + i*BSIZE*msize;
               unsigned int const bi = j*b2size + k*BSIZE*msize;
               val += a[ai]*b[bi]*BSIZE;
            }
            unsigned int const ci = j*b2size + i*BSIZE*msize;
            checkBlock(&check_ok, &c[ci], val, THRESHOLD);
         }
      }
   }

   #pragma omp taskwait

   if (check) {
      if (check_ok) {
         printf( "Output matrix is OK!\n" );
      } else {
         printf( "Output matrix is Wrong!\n" );
      }
      printf( "================================================== \n" );
   }

   free(a);
   free(b);
   free(c);

   printf( "==================== RESULTS ===================== \n" );
   printf( "  Benchmark: %s (%s)\n", "Matmul", "OmpSs" );
   printf( "  Elements type: %s\n", ELEM_T_STR );
   printf( "  Execution time (secs): %f\n", t_end - t_start );
   printf( "================================================== \n" );

}

