#!/bin/bash
RCSID='run_suite,v 0.3 2/12/2002 '
# runs nasa benchmark suite with various compilers and MPI's
# See README.plogic for more information

show_help() {
        echo ${RCSID}
        printf "Usage: $0 <OPTIONS>\n\n"
        printf "Options:\n"
        printf "  -v                  verbose output from make stage (default=make.log)\n"
	printf "  -c <compiler>       compiler (gnu/pgi/intel)\n"
	printf "  -n <processors>     number of processors\n"
	printf "  -t <test>           test size (A,B,C,S)\n"
	printf "  -m <mpi>            mpi version(lam,mpich,mpipro,dummy)\n"
        printf "  -o                  only build programs\n"
        printf "  -h                  show this help\n"
        printf "To run on a single CPU use: '-c gnu -n 1 -t S -m dummy'\n"
        printf "\n"
        exit 1
}

check_stuff() {
# check environment variables, only makes sense if not using
# modules
  if [ "$USINGMODULES" == "no" ];then
    case $MPI in 
	lam) 
	    if [ "$LAM_HOME" = "" ]
	    then
		echo "LAM_HOME is not defined"
		exit 2
	    else
		echo "LAM_HOME is defined as $LAM_HOME"
	    fi

	    if [ -e "$LAM_HOME/bin/mpirun" ]
	    then 
		echo "mpirun found for lam"
	    else
		echo "mpirun not found in $LAM_HOME/bin/mpirun"
		exit 3
	    fi
	    ;;


	mpich)
	    if [ -n "$MPICH_HOME" ]
	    then
                echo "MPICH_HOME is defined as $MPICH_HOME"
	    else
		echo "MPICH_HOME is not defined"
		exit 2
	    fi

	    if [ -e "$MPICH_HOME/bin/mpirun" ]
	    then 
		echo "mpirun found for mpich"
	    else
		echo "mpirun not found in $MPICH_HOME/bin/mpirun"
		exit 3
	    fi
	    ;;
    esac
  fi
# check if compilers are where they should be
    case $Compiler in
	gnu) 
            #this could be done better
	    if [ -e "/usr/bin/g77" ]
	    then 
		echo "GNU F77  Compiler found"
	    else
		echo "GNU F77 Compiler Not found : exiting"
		exit 3
	    fi
	    if [ -e "/usr/bin/gcc" ]
	    then 
		echo "GNU C Compiler found"
	    else
		echo "GNU Compiler Not found : exiting"
		exit 3
	    fi
	    ;;

	pgi)
            if [ "$USINGMODULES" == "yes" ];then
                module load  pgi
            fi
	    if [ -n "$PGI" ]
	    then
		echo "PGI is defined"
	    else
		echo "PGI is not defined"
		exit 2
	    fi

	    if [ -e "$PGI/linux86/bin/pgcc" ]
	    then 
		echo "PGI Compiler found"
	    else
		echo "PGI Compiler Not found : exiting"
		exit 2
	    fi
	    ;;


	abs)
	    if [ -n "$ABSOFT" ]
	    then
		echo "ABSOFT is defined"
	    else
		echo "ABSOFT is not defined"
		exit 2
	    fi

	    if [ -e "$ABSOFT/bin/f77" ]
	    then 
		echo "Absoft Compiler found"
	    else
		echo "Absoft Compiler Not found : exiting"
		exit 3
	    fi
	    ;;

	intel)
          if [ -n "$IA32ROOT" ]
            then
               echo "IA32ROOT is defined as $IA32ROOT"
            else
              #newer versions of IFC do not use IA32ROOT, but we will 
              #figure it out here.
              if [ "$USINGMODULES" == "yes" ];then
                  module load intel
              fi
              IFCPATH=`which ifc 2>/dev/null `
              if [ -z "$IFCPATH" ]
              then
                echo "IA32ROOT is not defined"
                exit 2
              fi
              IA32=`dirname $IFCPATH`
              IA32ROOT=`dirname $IA32`
             fi

            if [ -e "$IA32ROOT/bin/ifc" ]
            then
                echo "INTEL Compiler found at $IA32ROOT/bin"
            else
                echo "INTEL Compiler Not found : exiting"
                exit 2
            fi
	;;
    esac
}



clean_up() {
    cd -
    echo "cleaning up"
    if [ "$Report" = "" ]
    then
	$Report = "~/npbresults"
    fi

    mkdir $Report
    cp results/* $Report
}


#comment out next line to run local (w/o the tar file)
#move_to_temp
username=`whoami` 
while getopts "vc:n:t:m:oh" opt
do
        case $opt in
                v) verbose=1 ;;
                c) Compiler=$OPTARG ;;
                n) Processors=$OPTARG ;;
                t) Test=$OPTARG ;;
                m) MPI=$OPTARG ;;
		r) Report=$OPTARG ;;
                o) OnlyBuild=1 ;;
                h) show_help ;;
        esac
done
#Check the tests here, everthing else is checked in the script.
#Processors are checked when built, illegal CPU numbers will not
#wil not produce binaries. The legal numbers depends on the test
case $Test in
	A) ;;
	B) ;;
	C) ;;
	S) ;;
	*) 
		echo "Invalid Test: $Test"
		exit 1
		;;
esac

#if modules are used take care of modules
export USINGMODULES=no
rpm -qf /usr/local/Modules/default/bin/modulecmd >& /dev/null
if [ $? -eq 0 ]; then
  echo "Using Paralogic Modules"
  export USINGMODULES=yes
  . /etc/custom/profile.modules
  module rm mpi/lam
  module rm mpi/lam-intel
  module rm mpi/lam-pgi
  module rm mpi/mpich
  module rm mpi/mpich-intel
  module rm mpi/mpich-pgi
  module rm mpi/mpipro
  module rm mpi/intel
  module rm mpi/pgi
fi
#check soem of the path names etc.
check_stuff
# clean up everything
if [ -n "$verbose" ]
then
	make veryclean
else
	make veryclean >&/dev/null
fi

if [ "$MPI" = "lam" ]
then
	echo "Using LAM"
	sed -f ./config/$MPI.sed ./config/make.def.template> ./tmp/m1
	sed -f ./config/$MPI.run.sed ./run.template > ./tmp/r1
	sed s/fill_in_PROCS/$Processors/ ./tmp/r1 > ./run.$MPI
	if [ "$Compiler" = "gnu" ]
	then
		echo "Using g77 and gcc"
                if [ "$USINGMODULES" == "yes" ];then
                    module load mpi/lam
                fi
         	export LAMHF77=g77
		export LAMHCC=gcc
		sed -f ./config/gnu.sed ./tmp/m1> ./tmp/m2
	else 
		if [ "$Compiler" = "pgi" ]
        	then
			echo "Using pgf77 and pgcc"
                        if [ "$USINGMODULES" == "yes" ];then
                          module load mpi/lam-pgi
                          module load pgi
			fi
                	export LAMHF77=$PGI/linux86/bin/pgf77
                	export LAMHCC=$PGI/linux86/bin/pgcc
			sed -f ./config/pgi.lam.sed ./tmp/m1> ./tmp/m2
		else
		    if [ "$Compiler" = "abs" ]	   #absoft fortran compiler
		    then
			echo "Using Absoft f77 and gnu gcc"
			export LAMHF77=$ABSOFT/bin/f77
    			export LAMHCC=gcc
			sed -f ./config/absoft.sed ./tmp/m1> ./tmp/m2
		     else
		     if [ "$Compiler" = "intel" ]   #intel fortran compiler
		     then
			echo "Using Intel ifc and icc"
                        if [ "$USINGMODULES" == "yes" ];then
                                module load mpi/lam-intel
                                module load intel 
 			fi
                	export LAMHF77=$IA32ROOT/bin/ifc
                	export LAMHCC=$IA32ROOT/bin/icc
			sed -f ./config/intel.lam.sed ./tmp/m1> ./tmp/m2
		    else
			echo "Unsupported compiler: "$Compiler""
			exit 1
		    fi
		  fi
		fi
	fi
	sed 's/fill_in_dummy_library/ /' ./tmp/m2 >config/make.def
elif [ "$MPI" = "mpich" ]
then
	echo "Using MPICH"
	sed -f ./config/$MPI.sed ./config/make.def.template> ./tmp/m1
	sed -f ./config/$MPI.run.sed ./run.template > ./tmp/r1
	sed s/fill_in_PROCS/$Processors/ ./tmp/r1 > ./run.$MPI
	if [ "$Compiler" = "gnu" ]
	then
		echo "Using g77 and gcc"
		if [ "$USINGMODULES" == "yes" ];then
			module load mpi/mpich
		fi
               	export MPICH_F77=g77
               	export MPICH_F77LINKER=g77
               	export MPICH_CC=gcc
               	export MPICH_CLINKER=gcc
		sed -f ./config/gnu.sed ./tmp/m1> ./tmp/m2
		sed 's!fill_in_PGI! !' ./tmp/m2 >./tmp/m3
	else 
		if [ "$Compiler" = "pgi" ]
       		then
			echo "Using pgf77 and pgcc"
			if [ "$USINGMODULES" == "yes" ];then
				module load mpi/mpich-pgi
				module load pgi 
			fi
                	export MPICH_F77=$PGI/linux86/bin/pgf77
                	export MPICH_F77LINKER=$PGI/linux86/bin/pgf77
                	export MPICH_CC=$PGI/linux86/bin/pgcc
                	export MPICH_CLINKER=$PGI/linux86/bin/pgcc
			sed -f ./config/pgi.sed ./tmp/m1> ./tmp/m2
			cd pgi
			$PGI/linux86/bin/pgf77 -c farg.f -Msecond_underscore >&make.out
			cd ..
			sed 's!fill_in_PGI! ../pgi/farg.o !' ./tmp/m2 >./tmp/m3
		else
		    if [ "$Compiler" = "abs" ]	   #absoft fortran compiler
		    then
			echo "Using Absoft f77 and gnu gcc"
                	export MPICH_F77=$ABSOFT/bin/f77
                	export MPICH_F77LINKER=$ABSOFT/bin/f77
                	export MPICH_CC=gcc
                	export MPICH_CLINKER=gcc
			sed -f ./config/absoft.sed ./tmp/m1> ./tmp/m2
			sed 's!fill_in_PGI! !' ./tmp/m2 >./tmp/m3
		else
		    if [ "$Compiler" = "intel" ]   #intel fortran compiler
		    then
			echo "Using Intel ifc and icc"
			if [ "$USINGMODULES" == "yes" ];then
				module load mpi/mpich-intel
				module load intel
			fi
                	export MPICH_F77=$IA32ROOT/bin/ifc
                	export MPICH_F77LINKER=$IA32ROOT/bin/ifc
                	export MPICH_CC=$IA32ROOT/bin/icc
                	export MPICH_CLINKER=$IA32ROOT/bin/icc
			sed -f ./config/intel.sed ./tmp/m1> ./tmp/m2
                        cd intel
                        $IA32ROOT/bin/ifc -c -nus farg.f  >&make.out
                        cd ..
                	sed 's!fill_in_PGI! ../intel/farg.o !' ./tmp/m2 >./tmp/m3
		    else
			echo "Unsupported compiler: "$Compiler""
			exit 1
          fi
		    fi
		fi
	fi
	sed 's/fill_in_dummy_library/ /' ./tmp/m3 >config/make.def
elif [ "$MPI" = "mpipro" ]  
then
	echo "Using MPI-PRO"
        if [ "$USINGMODULES" == "yes" ];then
            module load mpi/mpipro
        fi
	if [ "$Compiler" = "gnu" ]
	then
               	export MPIPRO_F77=" "
               	export MPIPRO_CC="cc "
		echo "Using g77 and gcc"
		sed -f ./config/$MPI.sed ./config/make.def.template> ./tmp/m1
		sed -f ./config/$MPI.run.sed ./run.template > ./tmp/r1
		sed s/fill_in_PROCS/$Processors/ ./tmp/r1 > ./run.$MPI
		sed -f ./config/gnu.sed ./tmp/m1> ./tmp/m2
		sed 's!fill_in_PGI! !' ./tmp/m2 >./tmp/m3
	else 
		if [ "$Compiler" = "pgi" ]
       		then
			echo "Using pgf77 and pgcc"
                        if [ "$USINGMODULES" == "yes" ];then
                                module load pgi
                        fi
			export MPIPRO_F77="-pgf"
			export MPIPRO_CC="$PGI/linux86/bin/pgcc"
			sed -f ./config/$MPI.sed ./config/make.def.template> ./tmp/m1
			sed -f ./config/$MPI.run.sed ./run.template > ./tmp/r1
			sed s/fill_in_PROCS/$Processors/ ./tmp/r1 > ./run.$MPI
			sed -f ./config/pgi.sed ./tmp/m1> ./tmp/m2
#			cd pgi
#			$PGI/linux86/bin/pgf77 -c farg.f -Msecond_underscore
#			mpif77 -c farg.f -Msecond_underscore
#			cd ..
#			sed 's!fill_in_PGI! ../pgi/farg.o !' ./tmp/m2 >./tmp/m3
			sed 's!fill_in_PGI! !' ./tmp/m2 >./tmp/m3
          else
                    if [ "$Compiler" = "intel" ]
                    then
                        echo "Using ifc and icc"
                        if [ "$USINGMODULES" == "yes" ];then
                                module load intel 
                        fi
                        export MPIPRO_F77="-ifc"
                        export MPIPRO_CC="$IA32ROOT/bin/icc"

                        sed -f ./config/$MPI.sed ./config/make.def.template> ./tmp/m1
                        sed -f ./config/$MPI.run.sed ./run.template > ./tmp/r1
                        sed s/fill_in_PROCS/$Processors/ ./tmp/r1 > ./run.$MPI
                        sed -f ./config/intel.sed ./tmp/m1> ./tmp/m2
#                        cd intel
#                        mpif77 -c farg.f 
#                        cd ..
#                        sed 's!fill_in_PGI! ../pgi/farg.o !' ./tmp/m2 >./tmp/m3
                        sed 's!fill_in_PGI! !' ./tmp/m2 >./tmp/m3
           else
		    if [ "$Compiler" = "abs" ]	   #absoft fortran compiler
		    then
			echo "Using Absoft f77 and gnu gcc"
			export MPIPRO_F77=""
			export MPIPRO_CC=""
			sed -f ./config/$MPI.sed ./config/make.def.template> ./tmp/m1
			sed -f ./config/$MPI.run.sed ./run.template > ./tmp/r1
			sed s/fill_in_PROCS/$Processors/ ./tmp/r1 > ./run.$MPI
			sed -f ./config/absoft.sed ./tmp/m1> ./tmp/m2
			sed 's!fill_in_PGI! !' ./tmp/m2 >./tmp/m3
		    else
			echo "Unsupported compiler: "$Compiler""
			exit 1
		    fi
                  fi
		fi
	fi

	sed 's/fill_in_dummy_library/ /' ./tmp/m3 >config/make.def
elif [ "$MPI" = "dummy" ]
then
#	if [ "$Test" = "S" ]
#	then
        Processors="1"
        echo "Using dummy (single CPU) version"
        sed -f ./config/$MPI.sed ./config/make.def.template> ./tmp/m0
        sed -f ./config/$MPI.run.sed ./run.template > ./tmp/r1
        sed s/fill_in_PROCS/$Processors/ ./tmp/r1 > ./run.$MPI
	#Selecting the dummy compiler is a bit different becuase 
        #we are not using MPI wrappers around the compilers.
        if [ "$Compiler" = "gnu" ]
        then
                echo "Using g77 and gcc"
		sed -f ./config/dummy.gnu.sed ./tmp/m0> ./tmp/m1
		sed -f ./config/gnu.sed ./tmp/m1> ./tmp/m2
		cp ./MPI_dummy/Makefile.gnu ./MPI_dummy/Makefile
        else
                if [ "$Compiler" = "pgi" ]
                then
                        if [ "$USINGMODULES" == "yes" ];then
                                module load pgi 
                        fi
			echo "Using pgf77 and pgcc"
			sed -f ./config/dummy.pgi.sed ./tmp/m0> ./tmp/m1
			sed -f ./config/pgi.sed ./tmp/m1> ./tmp/m2
			cp ./MPI_dummy/Makefile.pgi ./MPI_dummy/Makefile
                else
			if [ "$Compiler" = "intel" ]
			then
                	        if [ "$USINGMODULES" == "yes" ];then
        	                        module load intel
	                        fi
				echo "Using ifc"
				sed -f ./config/dummy.intel.sed ./tmp/m0> ./tmp/m1
				sed -f ./config/intel.sed ./tmp/m1> ./tmp/m2
				cp ./MPI_dummy/Makefile.intel ./MPI_dummy/Makefile
			else
	                        echo "Unsupported compiler: $Compiler"
        	                exit 1
			fi
                fi
        fi
        sed 's!fill_in_dummy_library!include ../config/make.dummy!' ./tmp/m2 >config/make.def
        else
	echo "Unknown MPI version: $MPI"
	exit 1
fi


#now we make the suite.def file for the make
echo "Generating suite.def"
echo "#Generated by BPS">config/suite.def
echo "ft      $Test       $Processors">>config/suite.def
echo "mg      $Test       $Processors">>config/suite.def
echo "sp      $Test       $Processors">>config/suite.def
echo "lu      $Test       $Processors">>config/suite.def
echo "bt      $Test       $Processors">>config/suite.def
echo "is      $Test       $Processors">>config/suite.def
echo "ep      $Test       $Processors">>config/suite.def
echo "cg      $Test       $Processors">>config/suite.def
#make the benchmarks
echo "Making test programs"
if [ "$USINGMODULES" == "yes" ];then
  module list
fi
if [ -n "$verbose" ]
then
	make suite
else
	make suite >& ./make.log
fi

#run the tests
if [ -n "$OnlyBuild" ]
then
	echo "Compilation complete, you may run the programs generated by executing: sh ./run.$MPI $Processors $Test $Compiler $MPI"
else
	echo "Running NASA Parallel Suite Class $Test using:"
	echo "    $Compiler Compilers"
	echo "    $MPI MPI library" 
	echo "    $Processors Processors"
	sh run.$MPI $Processors $Test $Compiler $MPI
fi



#comment out next line to run w/o tar
#clean_up
exit 0
