#! /bin/bash

#PBS -N icesheet_all_varSS 
#PBS -o /home/evan/out/evan_out.$$ 
#PBS -e /home/evan/out/evan_err.$$ 
#PBS -j oe 
#PBS -l walltime=14:00:00 

PATH=${PATH}:/home/evan/bin
export PATH

# change these values if you change the amount of nodes used

#### now they are required to be input parameters!
#total_nodes=3
#total_cpus=12


# these are the parameters for the current analysis. Change for each run

#margin_model=7
#shear_stress_model=17

# in this version of the script, you must provide a shear stress file #####.csv, starting with 10000. It is an x-y file with the time in the first column, and the required shear stress model in the second column


log_file_extension=${shear_stress_model}_${margin_model}

# this earth model is used for calculating the ice sheet model. probably not necessary to change this

earth_model=ehgk 

current_folder=~/terrawulf_individual_run2

#rm ~/out/log_file*
if [ ! -e ${current_folder}/${shear_stress_model}.csv ]
then
	pwd
	echo "shear stress model ${shear_stress_model}.csv does not exist!"
	exit 0
fi


nodes=$(cat $PBS_NODEFILE)
echo nodes are $nodes >> ~/out/log_file.${log_file_extension}
nnodes=$(echo $nodes | wc -w)
echo Number of nodes = $nnodes >> ~/out/log_file.${log_file_extension}


# move GEBCO to each node



#for inode in $( seq 1 ${total_cpus} ${number_processors} )
for inode in $( seq 1 $nnodes )
do

	nodename=$(awk -v linenum=${inode} '{if(NR == linenum){print $1}}'  $PBS_NODEFILE)

	pbsdsh -h ${nodename} -v ${current_folder}/cp_gebco.sh # &> ~/out/cp_gebco_${inode}.log

	wait

done

wait

# initial ice model uses modern topography

inode=1
node_number=0

iteration=0

title="${iteration}_null_${shear_stress_model}_${margin_model}"
ice_file="naice_${title}"

if [ ! -e ~/imodels/${ice_file} ]
then
	mkdir ${current_folder}/${shear_stress_model}_${margin_model}_${iteration}

	job_number=0

#	for time_var in 160 140 130 120 110 90 80 70 60 50 40 30 28 26 24 22 20 18 17 16 15 14 13 12 11 10 9 8 7 6 4 0
	for time_var in 160 140 130 120 110 90 80 70 60 50 40 30 28 26 24 22 20 18 17 16 15 14 13.5 13 12.5 12 11.5 11 10.5 10 9.5 9 8.5 8 7.5 7 6.5 6 5 0
	do


	    	echo inode: ${inode} >> ~/out/log_file.${log_file_extension}

		echo "earth_model: " ${earth_model} >> ~/out/log_file.${log_file_extension}
		echo "PBS_Nodefile: " $PBS_NODEFILE >> ~/out/log_file.${log_file_extension}


		  nodename=$(awk -v linenum=${inode} '{if(NR == linenum){print $1}}'  $PBS_NODEFILE)

		  node_number=$( add_two ${node_number} 1 )

		  if [ "$node_number" -gt "${total_cpus}" ]
		  then
			node_number=1
			job_number=$( add_two ${job_number} 1 )
		  fi




		time_folder=$( multiply_two ${time_var} 1000 ) 

		echo "time:" ${time_folder} >> ~/out/log_file.${log_file_extension}

		echo ""  >> ~/out/log_file.${log_file_extension}

		  echo "what should be in pbsdsh " ${nodename}[${node_number}] >> ~/out/log_file.${log_file_extension}
		echo ${time_folder} ${margin_model}  ${inode} null null ${iteration} ${job_number} ${shear_stress_model} >> ~/out/log_file.${log_file_extension}


		pbsdsh -h ${nodename}  -v ${current_folder}/terra_init4.sh ${time_folder} ${margin_model}  ${inode} null null ${iteration} ${job_number} ${shear_stress_model} &
		sleep 5
 	   inode=$(add_two $inode 1)
	done

	wait
	echo "running merge3.sh" >> ~/out/log_file.${log_file_extension}
	date >> ~/out/log_file.${log_file_extension}
	sh ${current_folder}/merge3.sh ${shear_stress_model} ${margin_model} ${iteration} null

	wait
fi

# make sure the file is there

if [ -e imodels/${ice_file} ]
then

	echo "finished initial model" >> ~/out/log_file.${log_file_extension}
	date >> ~/out/log_file

else

	echo "did not calculate initial ice model for some reason" >> ~/out/log_file.${log_file_extension}
	date >> ~/out/log_file.${log_file_extension}
	exit 0
fi



# first iteration with topography deformed from the initial ice sheet model

inode=1
node_number=0

iteration=1

title="${iteration}_${earth_model}_${shear_stress_model}_${margin_model}"
ice_file="naice_${title}"

if [ ! -e ~/imodels/${ice_file} ]
then


	job_number=0

	# first calculate the deformation, everything is in the folder ~/def_mod

	if [ ! -e /home/evan/def_mod/${ice_file}.def ]
	then


		nodename=$(awk -v linenum=${inode} '{if(NR == linenum){print $1}}'  $PBS_NODEFILE)
		pbsdsh -h ${nodename}  -v ${current_folder}/run_calsea.sh ${earth_model} ${shear_stress_model} ${margin_model} ${inode} ${job_number} ${iteration} &

		wait

	fi



	if [ ! -e /home/evan/def_mod/${ice_file}.def ]
	then

		echo "calsea failed" >> ~/out/log_file.${log_file_extension}
		exit 0
	fi
	

	mkdir ${current_folder}/${shear_stress_model}_${margin_model}_${iteration}



#	for time_var in 160 140 130 120 110 90 80 70 60 50 40 30 28 26 24 22 20 18 17 16 15 14 13 12 11 10 9 8 7 6 4 0
	for time_var in 160 140 130 120 110 90 80 70 60 50 40 30 28 26 24 22 20 18 17 16 15 14 13.5 13 12.5 12 11.5 11 10.5 10 9.5 9 8.5 8 7.5 7 6.5 6 5 0
	do


	    echo inode: ${inode} >> ~/out/log_file.${log_file_extension}

	   echo "earth_model: " ${earth_model} >> ~/out/log_file.${log_file_extension}
	   echo "PBS_Nodefile: " $PBS_NODEFILE >> ~/out/log_file.${log_file_extension}


		  nodename=$(awk -v linenum=${inode} '{if(NR == linenum){print $1}}'  $PBS_NODEFILE)

		  node_number=$( add_two ${node_number} 1 )

		  if [ "$node_number" -gt "${total_cpus}" ]
		  then
	        	node_number=1
			job_number=$( add_two ${job_number} 1 )

		  fi

		  echo "what should be in pbsdsh " ${nodename}[${node_number}] >> ~/out/log_file.${log_file_extension}

		# find shear stress model for this particular time

		shear_stress_model_time=$( awk -v time_var=${time_var} '{if($1 == time_var) {print $2}}' ${current_folder}/${shear_stress_model}.csv )


		time_folder=$( multiply_two ${time_var} 1000 ) 

		echo ${time_folder} ${margin_model} ${shear_stress_model} ${inode} ${earth_model} ${ice_file} ${iteration} ${job_number} >> ~/out/log_file.${log_file_extension}

	#	pbsdsh -h ${nodename}  -v ${current_folder}/terra_init2.sh ${time_folder} ${margin_model} ${shear_stress_model} ${inode} ${earth_model} ${ice_file} ${iteration} ${job_number} &
		pbsdsh -h ${nodename}  -v ${current_folder}/terra_init4.sh ${time_folder} ${margin_model} ${shear_stress_model_time} ${inode} ${earth_model} ${ice_file} ${iteration} ${job_number} ${shear_stress_model} &
		sleep 5
   		 inode=$(add_two $inode 1)
	done

	wait
	echo "running merge3.sh" >> ~/out/log_file.${log_file_extension}
	date >> ~/out/log_file.${log_file_extension}
	sh ${current_folder}/merge3.sh ${shear_stress_model} ${margin_model} ${iteration} ${earth_model}
fi



if [ -e imodels/${ice_file} ]
then

	echo "finished first iteration of ice model" >> ~/out/log_file.${log_file_extension}
	date >> ~/out/log_file.${log_file_extension}
else

	echo "did not calculate first iteration of ice model for some reason" >> ~/out/log_file.${log_file_extension}
	date >> ~/out/log_file.${log_file_extension}
	exit 0

fi

# second iteration with topography deformed from the initial ice sheet model

inode=1
node_number=0

iteration=2

title="${iteration}_${earth_model}_${shear_stress_model}_${margin_model}"
ice_file="naice_${title}"

if [ ! -e ~/imodels/${ice_file} ]
then


	job_number=0

	# first calculate the deformation, everything is in the folder ~/def_mod

	if [ ! -e /home/evan/def_mod/${ice_file}.def ]
	then

		nodename=$(awk -v linenum=${inode} '{if(NR == linenum){print $1}}'  $PBS_NODEFILE)
		pbsdsh -h ${nodename}  -v ${current_folder}/run_calsea.sh ${earth_model} ${shear_stress_model} ${margin_model} ${inode} ${job_number} ${iteration} &

		wait
	fi
	
	if [ ! -e /home/evan/def_mod/${ice_file}.def ]
	then
		echo "did not calculate the deformation file in iteration 2"
		exit 0
	fi


	mkdir ${current_folder}/${shear_stress_model}_${margin_model}_${iteration}



#	for time_var in 160 140 130 120 110 90 80 70 60 50 40 30 28 26 24 22 20 18 17 16 15 14 13 12 11 10 9 8 7 6 4 0
	for time_var in 160 140 130 120 110 90 80 70 60 50 40 30 28 26 24 22 20 18 17 16 15 14 13.5 13 12.5 12 11.5 11 10.5 10 9.5 9 8.5 8 7.5 7 6.5 6 5 0
	do


	    echo inode: ${inode} >> ~/out/log_file.${log_file_extension}

	   echo "earth_model: " ${earth_model} >> ~/out/log_file.${log_file_extension}
	   echo "PBS_Nodefile: " $PBS_NODEFILE >> ~/out/log_file.${log_file_extension}


		  nodename=$(awk -v linenum=${inode} '{if(NR == linenum){print $1}}'  $PBS_NODEFILE)

		  node_number=$( add_two ${node_number} 1 )

		  if [ "$node_number" -gt "${total_cpus}" ]
		  then
	        	node_number=1
			job_number=$( add_two ${job_number} 1 )

		  fi

		  echo "what should be in pbsdsh " ${nodename}[${node_number}] >> ~/out/log_file.${log_file_extension}
		echo ${time_folder} ${margin_model} ${shear_stress_model} ${inode} ${earth_model} ${ice_file} ${iteration} ${job_number} >> ~/out/log_file.${log_file_extension}

		time_folder=$( multiply_two ${time_var} 1000 ) 

		# find shear stress model for this particular time

		shear_stress_model_time=$( awk -v time_var=${time_var} '{if($1 == time_var) {print $2}}' ${current_folder}/${shear_stress_model}.csv )

#		pbsdsh -h ${nodename}  -v ${current_folder}/terra_init2.sh ${time_folder} ${margin_model} ${shear_stress_model} ${inode} ${earth_model} ${ice_file} ${iteration} ${job_number} &
#		pbsdsh -h ${nodename}  -v ${current_folder}/terra_init3.sh ${time_folder} ${margin_model} ${shear_stress_model_time} ${inode} ${earth_model} ${ice_file} ${iteration} ${job_number}

		pbsdsh -h ${nodename}  -v ${current_folder}/terra_init4.sh ${time_folder} ${margin_model} ${shear_stress_model_time} ${inode} ${earth_model} ${ice_file} ${iteration} ${job_number} ${shear_stress_model} &
		sleep 5
 		   inode=$(add_two $inode 1)
	done

	wait
	echo "running merge3.sh" >> ~/out/log_file.${log_file_extension}
	date >> ~/out/log_file.${log_file_extension}
	sh ${current_folder}/merge3.sh ${shear_stress_model} ${margin_model} ${iteration} ${earth_model}
fi

if [ -e imodels/${ice_file} ]
then

	echo "finished second iteration of ice model" >> ~/out/log_file.${log_file_extension}
	date >> ~/out/log_file.${log_file_extension}
else

	echo "did not calculate second iteration of ice model for some reason" >> ~/out/log_file.${log_file_extension}
	date >> ~/out/log_file.${log_file_extension}

fi


# the final ice model should now be ready, time to calculate the lake and gps uplift rates

awk '{if(NR == 5){for (i=1;i<=NF;i++)printf("%8.3f",$i+.1); printf("\n")} else{print $0}}' ~/imodels/${ice_file} > ~/imodels/${ice_file}_p

number_processors=$( multiply_two ${total_nodes} ${total_cpus} )





# create directories to store results

mkdir /home/evan/lake_results/${ice_file}
mkdir /home/evan/uplift_results/${ice_file}

process_counter=0

inode=1
node_number=0
job_number=0

echo "calculating lake and gps uplift rates" >> ~/out/log_file.${log_file_extension}

for lith in  h #e f h j
do
	for upp_mant in a b c d e f g h i j A # a c e g i A #
	do
		for low_mant in A B C D E F G H I J k l m o p #A C E G I k m p #
		do

			earth_model=e${lith}${upp_mant}${low_mant}

			if [ ! -e /home/evan/lake_results/${ice_file}/${earth_model}.lake_sites -o ! -e /home/evan/uplift_results/${ice_file}/${earth_model}.gps_sites ]
			then
			process_counter=$( add_two ${process_counter} 1 )
			echo ${earth_model} ${process_counter}  >> ~/out/log_file.${log_file_extension}
			date  >> ~/out/log_file.${log_file_extension}

			nodename=$(awk -v linenum=${inode} '{if(NR == linenum){print $1}}'  $PBS_NODEFILE)

			node_number=$( add_two ${node_number} 1 )

			if [ "$node_number" -gt "${total_cpus}" ]
			then
				node_number=1
				job_number=$( add_two ${job_number} 1 )

			fi

			# only calculate if the files do not exist


				pbsdsh -h ${nodename} -v ${current_folder}/calculate_lake_gps.sh ${earth_model} ${ice_file} ${inode} ${job_number} &

				sleep 2

			fi

			    inode=$(add_two $inode 1)

			if [ "${process_counter}" -eq ${number_processors} -o "${earth_model}" = "ehAp"  ]
			then
				# hopefully this actually waits
				wait 

				# reset parameters
				process_counter=0

				inode=1
				node_number=0
				job_number=0

			fi


		done
	done
done

wait

# now calculate glacial lake sites

mkdir /home/evan/glacial_lakes/${ice_file}

process_counter=0

inode=1
node_number=0
job_number=0

echo "calculating glacial lakes" >> ~/out/log_file.${log_file_extension}

for lith in  h #e f h j
do
	for upp_mant in a b c d e f g h i j A # a c e g i A #
	do
		for low_mant in A B C D E F G H I J k l m o p #A C E G I k m p #
		do

			earth_model=e${lith}${upp_mant}${low_mant}

			if [ ! -e /home/evan/glacial_lakes/${ice_file}/${earth_model}.glacial_sites ]
			then
			process_counter=$( add_two ${process_counter} 1 )
			echo ${earth_model} ${process_counter}  >> ~/out/log_file.${log_file_extension}
			date  >> ~/out/log_file.${log_file_extension}

			nodename=$(awk -v linenum=${inode} '{if(NR == linenum){print $1}}'  $PBS_NODEFILE)

			node_number=$( add_two ${node_number} 1 )

			if [ "$node_number" -gt "${total_cpus}" ]
			then
				node_number=1
				job_number=$( add_two ${job_number} 1 )

			fi

			# only calculate if the files do not exist


				pbsdsh -h ${nodename} -v ${current_folder}/calculate_glacial_lakes.sh ${earth_model} ${ice_file} ${inode} ${job_number} &

				sleep 2

			fi

			    inode=$(add_two $inode 1)

			if [ "${process_counter}" -eq ${number_processors} -o "${earth_model}" = "ehAp"  ]
			then
				# hopefully this actually waits
				wait 

				# reset parameters
				process_counter=0

				inode=1
				node_number=0
				job_number=0

			fi


		done
	done
done

wait


# now calculate sea level sites

mkdir /home/evan/sea_level/${ice_file}

process_counter=0

inode=1
node_number=0
job_number=0

echo "calculating sea level" >> ~/out/log_file.${log_file_extension}

for lith in  h #e f h j
do
	for upp_mant in a b c d e f g h i j A # a c e g i A #
	do
		for low_mant in A B C D E F G H I J k l m o p #A C E G I k m p #
		do

			earth_model=e${lith}${upp_mant}${low_mant}

			if [ ! -e /home/evan/sea_level/${ice_file}/${earth_model}.sea_sites ]
			then
			process_counter=$( add_two ${process_counter} 1 )
			echo ${earth_model} ${process_counter}  >> ~/out/log_file.${log_file_extension}
			date  >> ~/out/log_file.${log_file_extension}

			nodename=$(awk -v linenum=${inode} '{if(NR == linenum){print $1}}'  $PBS_NODEFILE)

			node_number=$( add_two ${node_number} 1 )

			if [ "$node_number" -gt "${total_cpus}" ]
			then
				node_number=1
				job_number=$( add_two ${job_number} 1 )

			fi

			# only calculate if the files do not exist


				pbsdsh -h ${nodename} -v ${current_folder}/calculate_sea_level.sh ${earth_model} ${ice_file} ${inode} ${job_number} &

				sleep 2

			fi

			    inode=$(add_two $inode 1)

			if [ "${process_counter}" -eq ${number_processors} -o "${earth_model}" = "ehAp"  ]
			then
				# hopefully this actually waits
				wait 

				# reset parameters
				process_counter=0

				inode=1
				node_number=0
				job_number=0

			fi


		done
	done
done

wait

# remove temporary files

for inode in $( seq 1 ${total_cpus} ${number_processors} )
do

	nodename=$(awk -v linenum=${inode} '{if(NR == linenum){print $1}}'  $PBS_NODEFILE)

	pbsdsh -h ${nodename} -v ${current_folder}/rm_script.sh &



done

echo "finished make_full_icesheet" >> ~/out/log_file.${log_file_extension}
date >> ~/out/log_file.${log_file_extension}

