#include "../include/lass.h"

/**
 *
 * 	@file kdnpgesv.c
 *
 * 	@brief LASs-DDSs kdnpgesv routine.
 *
 * 	LASs-DDSs is a software package provided by:
 * 	Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
 *
 * 	@author Pedro Valero-Lara pedro.valero@bsc.es
 * 	@date 2018-07-26
 * 	@reviewer 
 * 	@modified 
 *
 **/

/**
 *  
 *	@ingroup DDSS
 *   
 *	Solves a system of linear equations A \times X = B, where A is a N-by-N
*	general matrix and X and B are N-by-NRHS matrices. The matrix A is 
 *	factorized using the LU descomposition without pivoting.
 *	The matrix A is descomposed as:
 *
 *		A = L * U
 *
 *	where L is a lower triangular matrix with unit diagonal elements and 
 *	U is an upper triangular matrix.
 *
**/

/**
 *	
 *	@param[in]
 *	N       int.
 *          N specifies the order of the square matrix A. N >= 0.
 *	
 *	NRHS    int.
 *          NRHS specifies the number of right-hand-sides (number 
 *			of columns of B). NRHS >= 0.
 *
 *	@param[in,out]
 *  A  		double *.
 *	    	A is a pointer to a regular matrix of dimension N-by-LDA.
 *          On exit, if return value is Success, the matrix A is overwriten by 
 *          the factors L and U. The unit diagonal elements of L are not stored.
 *   
 *	@param[in]
 *  LDA     int.
 * 	    	LDA specifies the number of columns of A ( row-major order ).
 * 	    	LDA must be at least max( 1, N ).
 * 	    	
 *	@param[in,out]
 *  B  		double *.
 *	    	B is a pointer to a matrix of dimension N by NRHS, which stores the 
 *			right-hand-sides of the systems of linear equations.  
 *			(row-major order). On exit, if return value is Success, the matrix B
 * 			is overwriten by the solution matrix X.
 *   
 *	@param[in]
 *  LDB     int.
 * 	    	LDB specifies the number of columns of B ( row-major order ).
 * 	    	LDB must be at least max( 1, NRHS ).
 *
 **/

/**
 *
 *	@retval Success successful exit
 *	@retval NoSuccess unsuccessful exit
 *
 **/

/**
 *
 *	@sa dnpgesv
 *	@sa ddss_tile
 *	@sa ddss_flat2tiled
 *	@sa ddss_tiled2flat
 *
 **/ 

enum LASS_RETURN kdnpgesv( int N, int NRHS, 
		double *A, int LDA, 
		double *B, int LDB )
{

	// Local variables
	int nt, nrhst;
	int mi, mmi, ki, ni, nni;
	int tile_size_m;
	int tile_size_mm;
	int tile_size_n;
	int tile_size_nn;
	int tile_size_k;

	// Number of tiles
	if ( N % TILE_SIZE == 0 )
	{
		nt = N / TILE_SIZE;
	}
	else
	{
		nt = ( N / TILE_SIZE ) + 1;
	}

	if ( NRHS % TILE_SIZE == 0 )
	{
		nrhst = NRHS / TILE_SIZE;
	}
	else
	{
		nrhst = ( NRHS / TILE_SIZE ) + 1;
	}

	/***************************
	--Tile matrices allocation--
	***************************/
		
	double (*TILE_A)[nt][TILE_SIZE * TILE_SIZE] = malloc ( nt * nt * 
		TILE_SIZE * TILE_SIZE * sizeof( double ) );
	
	if ( TILE_A == NULL )
	{
		fprintf( stderr, "Failure in kdnpgesv for matrix TILE_A\n" );
		return NoSuccess;
	}
	
	double ( *TILE_B )[nrhst][TILE_SIZE * TILE_SIZE] = malloc(
			 nt * nrhst * TILE_SIZE * TILE_SIZE * sizeof( double ) );

	if ( TILE_B == NULL ) 
	{
		fprintf( stderr, "Failure in kdnpgesv for matrix TILE_B\n" );
		return NoSuccess;
	}
	
	/************************************************
	// --From flat data layout to tiled data layout--	
	************************************************/

	// From flat matrix A to tile matrix TILE_A
	ddss_dflat2tiled( N, N, A, LDA, nt, nt, TILE_A );
	
	// From flat matrix B to tile matrix TILE_B 
	ddss_dflat2tiled( N, NRHS, B, LDB, nt, nrhst, TILE_B );

	/****************
	--DNPGESV tile--
	****************/	

	// LU descomposition without pivoting
	for ( ki = 0; ki < nt; ki++ )
	{
		tile_size_k = ddss_tile_size( N, ki );
		
		#if defined(LASs_WITH_MKL)
		
		#pragma oss task inout( TILE_A[ki][ki] ) \
                shared( TILE_A ) \
                firstprivate( ki ) \
                label( mkl_dgetrfnpi )
        LAPACKE_mkl_dgetrfnpi( CblasRowMajor,
                               tile_size_k, tile_size_k,
                               tile_size_k,
                               TILE_A[ki][ki], tile_size_k );

		#else

		#pragma oss task inout( TILE_A[ki][ki] ) \
				shared( TILE_A ) \
				firstprivate( ki ) \
				label( dnpgetrf )
		dnpgetrf( tile_size_k, tile_size_k,
				  TILE_A[ki][ki],
				  tile_size_k );

		#endif
			
		for ( mi = ki + 1; mi < nt; mi++ )
		{
			tile_size_m = ddss_tile_size( N, mi );
			
			#pragma oss task in( TILE_A[ki][ki] ) \
					inout(TILE_A[mi][ki]) \
					shared( TILE_A ) \
					firstprivate( mi, ki ) \
					label( dnpgetrf_dtrsm_below )
			cblas_dtrsm( CblasRowMajor, 
				     	 ( CBLAS_SIDE ) Right, ( CBLAS_UPLO ) Upper, 
				     	 ( CBLAS_TRANSPOSE ) NoTrans, ( CBLAS_DIAG ) NonUnit, 
					 	 tile_size_m, tile_size_k, 
					 	 1.0, TILE_A[ki][ki], tile_size_k, 
						  	  TILE_A[mi][ki], tile_size_k );
		}
		for ( ni = ki + 1; ni < nt; ni++ )
		{
			tile_size_n = ddss_tile_size( N, ni );
			
			#pragma oss task in( TILE_A[ki][ki] ) \
					inout(TILE_A[ki][ni]) \
					shared( TILE_A ) \
					firstprivate( ni, ki ) \
					label( dnpgetrf_dtrsm_right )
			cblas_dtrsm( CblasRowMajor, 
				     	 ( CBLAS_SIDE ) Left, ( CBLAS_UPLO ) Lower, 
				     	 ( CBLAS_TRANSPOSE ) NoTrans, ( CBLAS_DIAG ) Unit, 
					 	 tile_size_k, tile_size_n, 
					 	 1.0, TILE_A[ki][ki], tile_size_k, 
						  	  TILE_A[ki][ni], tile_size_n );
			
			for ( mmi = ki + 1; mmi < nt; mmi++ )
			{
				tile_size_mm = ddss_tile_size( N, mmi );
								
				#pragma oss task in( TILE_A[mmi][ki] ) \
						in( TILE_A[ki][ni] ) \
						inout( TILE_A[mmi][ni] ) \
						shared( TILE_A ) \
						firstprivate( ni, mmi, ki ) \
						label( dnpgetrf_dgemm )
				cblas_dgemm( CblasRowMajor, 
							 ( CBLAS_TRANSPOSE ) NoTrans,
							 ( CBLAS_TRANSPOSE ) NoTrans,
							 tile_size_mm, 
							 tile_size_n, 
							 TILE_SIZE,
							 -1.0, TILE_A[mmi][ki], tile_size_k,
							 	   TILE_A[ki][ni],  tile_size_n,
							  1.0, TILE_A[mmi][ni], tile_size_n );
			}
		}
	}
	
	/*********************************************
	--From tiled data layout to flat data layout--	
	*********************************************/
	ddss_dtiled2flat_nb( N, N, A, LDA, nt, nt, TILE_A );

	// Triandular solve
	// --SIDE = Left & UPLO = Lower & TRANS_A = NoTrans & Unit-- 
	for ( ki = 0; ki < nt; ki++ )
	{
		tile_size_k = ddss_tile_size( N, ki );

		for ( ni = 0; ni < nrhst; ni++ )
		{
			tile_size_n = ddss_tile_size( NRHS, ni );
						
			#pragma oss task in( TILE_A[ki][ki] ) \
				inout(  TILE_B[ki][ni] ) \
				shared( TILE_A, TILE_B ) \
				firstprivate( ki, ni ) \
				label( dtrsm_dtrsm1 )
			cblas_dtrsm( CblasRowMajor,
						 ( CBLAS_SIDE ) Left, ( CBLAS_UPLO ) Lower,
						 ( CBLAS_TRANSPOSE ) NoTrans,
						 ( CBLAS_DIAG ) Unit,
						 tile_size_k,
						 tile_size_n,
						 1.0, TILE_A[ki][ki], tile_size_k,
							  TILE_B[ki][ni], tile_size_n );
		}
		for ( nni = 0; nni < nrhst; nni++ ) 
		{
			tile_size_nn = ddss_tile_size( NRHS, nni );
				
			for ( mi = ki + 1; mi < nt; mi++ )
			{
					
				#pragma oss task in( TILE_A[mi][ki] ) \
                    in( TILE_B[ki][nni] ) \
                    inout( TILE_B[mi][nni] ) \
                    shared( TILE_A, TILE_B ) \
                    firstprivate( ki, mi, nni ) \
                    label( dtrsm_dgemm1 )
				cblas_dgemm( CblasRowMajor,
							 CblasNoTrans, CblasNoTrans,
							 tile_size_k,
							 tile_size_nn,
							 tile_size_k,
							 -1.0,  TILE_A[mi][ki],  tile_size_k,
								   TILE_B[ki][nni], tile_size_nn,
							  1.0, TILE_B[mi][nni], tile_size_nn );
			}
		}
	}

	// Triandular solve
	// --SIDE = Left & UPLO = Upper & TRANS_A = NoTrans & NonUnit--
	for ( ki = nt - 1; ki >= 0; ki-- ) 
	{
		tile_size_k = ddss_tile_size( N, ki );
		
		for ( ni = 0; ni < nrhst; ni++ ) 
		{
			tile_size_n = ddss_tile_size( NRHS, ni );

			#pragma oss task in( TILE_A[ki][ki] ) \
				inout( TILE_B[ki][ni] ) \
				shared( TILE_A, TILE_B ) \
				firstprivate( ki, ni ) \
				label( dtrsmi_dtrsm2 )
			cblas_dtrsm( CblasRowMajor,
						 ( CBLAS_SIDE ) Left, ( CBLAS_UPLO ) Upper,
						 ( CBLAS_TRANSPOSE ) NoTrans,
						 ( CBLAS_DIAG ) NonUnit,
						 tile_size_k, 
						 tile_size_n,
						 1.0, TILE_A[ki][ki], tile_size_k,
							  TILE_B[ki][ni], tile_size_n );
		}

		for ( mi = ki - 1; mi >= 0; mi-- ) 
		{
			tile_size_m = ddss_tile_size( N, mi );
			
			for ( nni = 0; nni < nrhst; nni++ ) 
			{
				tile_size_nn = ddss_tile_size( NRHS, nni );

				#pragma oss task in( TILE_A[mi][ki] ) \
					in( TILE_B[ki][nni] ) \
					inout( TILE_B[mi][nni] ) \
					shared( TILE_A, TILE_B ) \
					firstprivate( ki, mi, nni ) \
					label( dtrsm_dgemm2 )
				cblas_dgemm( CblasRowMajor,
							 CblasNoTrans, CblasNoTrans,
							 tile_size_m, 
							 tile_size_nn, 
							 tile_size_k,
							 -1.0, TILE_A[mi][ki],  tile_size_k,
								   TILE_B[ki][nni], tile_size_nn,
							  1.0, TILE_B[mi][nni], tile_size_nn );
			}
		}
	}

	/*********************************************
	--From tiled data layout to flat data layout--	
	*********************************************/
	ddss_dtiled2flat( N, NRHS, B, LDB, nt, nrhst, TILE_B );
	
	// --Tile A and B matrices free--
	free( TILE_A );
	free( TILE_B );
	
	return Success;

}
