#
# Perl script used to run a CFX-5 Solver job through a PBS queue 'qsub' command
#
#  $Id$
#

# This script is intended to be used in conjunction with an Indirect Start 
# Method (e.g. Submit to PBS Queue), and should be copied to the 
# <CFX5ROOT>/config directory to enable the start method.
#
# It is expected that a system administrator will need to modify this script to
# accomodate the specific configuration of their queing system.

# This script does the following:
#  1. Parse command-line arguments to determine 
#     a. The machine to submit the qsub request on.
#     b. the number of requested cluster nodes
#     c. the desired CFX-5 solver
#     d. the location of the run set-up.
#
#  2. Generate PBS arguments for:
#      a. The node resource request
#      b. The job name
#      
#  3. Creates a job script that determines the cluster nodes that have 
#     been allocated to this run, and then executes the necessary cfx5solve 
#     command to run on those nodes. 
# 
#  4. Uses 'rsh' to remotely run the qsub command that submits the job script
#     to the queue. 

#
# The following system configuration variables are defined. 
#
#  $ppn = Number of processors on each cluster node
#  $qsub = Full path the PBS qsub command
#  $debug = Set to 1 for additional debug output

$ppn = 2;
$qsub = "/usr/local/bin/qsub"; 
$debug = 0;

#
# Parse the script arguments
#
use CFX5::CCLI;
use CFX5::Config qw(%Config);

use Getopt::Long;
$Getopt::Long::passthrough = 1;
$Getopt::Long::ignorecase = 0;

$options = {};

GetOptions($options, "--n=i","--solver=s","--fullpath=s","--drivemap=s","--master=s");

printDebug ("-n = " . $options->{n} . "\n");
printDebug ("-solver = " . $options->{solver} . "\n");
printDebug ("-fullpath = " . $options->{fullpath} . "\n");
printDebug ("-master = " . $options->{master} . "\n");
printDebug ("ARGS = @ARGV\n");

$fullpathOrig = $options->{fullpath}; 
$fullpath = $options->{fullpath};
$master = $options->{master};

#
# Define the qsub arguments
#

# Define number of nodes and processors per node
if ($options->{n}) {
    $numnodes = $options->{n};
    $pbsoptions = "-l nodes=" . $numnodes . "\\#ppn=" . $ppn;
}

# Define job name based on the name of the run directory 
@fullpathdirs = split(/[\/\\]/,$options->{fullpath});  #split the fullpath into directory names
$jobname = pop(@fullpathdirs);  # The job name is the last one
$pbsoptions = $pbsoptions . " -N " . substr($jobname, 0, 15);

# Define solver command
$cfx5solver = "cfx5solve";
if ($options->{solver}) {
    $cfx5solver = $options->{solver};
}

# Map Windows to Unix drive root if it has been defined
if ($options->{drivemap}) {
 $fullpath = drivemap($fullpath,$options->{drivemap});
 printDebug ("New fullpath is $fullpath\n");
}

# Put the fullpath back onto the arguments passed to the cfx5solve command 
unshift @ARGV,"-fullpath",$fullpath;

#
# Create the job script
#

$jobscript = <<"JOBSCRIPT";
#!/usr/bin/perl
#
#PBS -j oe
#PBS -m ae

#
# Get list of nodes provided by PBS from the PBS_NODEFILE environment variable
#
\$parallel = 0;
if (\$ENV{'PBS_NODEFILE'}) {
    open(OFH,\$ENV{'PBS_NODEFILE'});
    \@node_list = \<OFH\>;
    chomp(\@node_list);
    close(OFH);
    pop(\@node_list) if (!\$node_list[-1]);
    if (\@node_list > 1) {
	\$nodes = join(',', \@node_list); 
        \$numnodes = scalar(\@node_list);
        \$parallel = 1;
    }
}

#
# Change to working directory;
#
chdir(\$ENV{'PBS_O_WORKDIR'});

#
# Start CFX-5 Solver
#

if (\$parallel) {
        system("$cfx5solver @ARGV -part \$numnodes -par-dist \'\$nodes\' -start-method \'MPICH Distributed Parallel\' ");
} else {
        system("$cfx5solver @ARGV -start-method \'Serial\'");
}

JOBSCRIPT
#
# End creation of jobscript
#

#Write the jobscript file
$jobfileLocal = $fullpathOrig . ".dir/jobfile";
open(JOBFILE,">$jobfileLocal");
binmode JOBFILE;
print JOBFILE $jobscript;
close(JOBFILE);

$jobfileRemote = $fullpath . ".dir/jobfile";

#
#Run the qsub command on remote machine through rsh
#

  # Submission is slightly different if we're submitting from Windows to Unix vs. Unix to Unix
$OS = $Config{CFX5OS};
$rsh = $Config{CFX5RSH};
if ($OS eq 'winnt') {
  print "$rsh $master $qsub $pbsoptions $jobfileRemote\n";
  system ("$rsh $master $qsub $pbsoptions $jobfileRemote");
} else {
  print "$rsh $master \'$qsub $pbsoptions $jobfileRemote\'\n";
  system ("$rsh $master \'$qsub $pbsoptions $jobfileRemote\'");
}

#Clean up the jobscript file
#unlink($jobfile);

#
# End of qcfx5solve script
# 

#
# The following is the experimental 'drivemap' subroutine, which can
# be used to map from a windows drive letter to unix directory path.
# This can facilitate the submission of solver runs from a Windows machine on 
# a Unix/Linux machine.
#

sub drivemap {
 ($fullpath,$map) = @_;

 $fullpathOrig = $fullpath;
 printDebug ("Drivemap: $map\n");
 #Parse the drive map.  Assumed syntax is (e.g.) G:\from,/home/user/to
 ($fromRoot,$toRoot) = split(/\,/,$map);
 $fromRoot=~s/\\/\//g;  #change \ to /
 $toRoot=~s/\\/\//g;  #change \ to /
 $fullpath=~s/\\/\//g;  #change \ to /
 printDebug ("Mapping from $fromRoot to $toRoot in $fullpath\n");

 #change the full path
 $fullpath =~ s/$fromRoot/$toRoot/ig;

 # Change the mapping of the Definition File
 $cclapi = CFX5::CCLI::new_api_object();
 
   #Read the current file
 open(CCL,"<$fullpathOrig.dir/startup.ccl"); 
 @CCLfile = <CCL>;
 close(CCL);

   #Change the parameter setting 
 $cclapi->parseCCLData(\@CCLfile);
 $cclapi->getStringValue("SIMULATION CONTROL/EXECUTION CONTROL/RUN DEFINITION/Definition File",\$deffile);
 $deffile =~s/\\/\//g;  #change \ to /
 $deffile =~ s/$fromRoot/$toRoot/ig;
 $cclapi->setStringValue("SIMULATION CONTROL/EXECUTION CONTROL/RUN DEFINITION/Definition File",$deffile);
 $cclapi->getStringValue("SIMULATION CONTROL/EXECUTION CONTROL/RUN DEFINITION/Initial Values File",\$inifile);
 if (defined($inifile)) {
   $inifile =~s/\\/\//g;  #change \ to /
   $inifile =~ s/$fromRoot/$toRoot/ig;
   $cclapi->setStringValue("SIMULATION CONTROL/EXECUTION CONTROL/RUN DEFINITION/Initial Values File",$inifile);
}
 
   #write a new file
 $cclapi->getExpandedCCL(\@newCCLfile);
 open(CCL,">$fullpathOrig.dir/startup.ccl");
 foreach $line (@newCCLfile) {
  print CCL "$line\n";
 }
 close(CCL);

 return $fullpath;
}

sub printDebug {
 ($message) = @_;
 if ($debug) {
   print $message;
 }
}
