###################################################################################################
#
# Windows HPC job submission script
# =================================
#
# Default settings, typically overridden by options file
# ------------------------------------------------------
#
my $clusterhost=$ENV{CCP_SCHEDULER};
#
# Specific account to use
# --> Defaults to current user
my $account=$ENV{USERDOMAIN}.'\\'.$ENV{USERNAME};
#
# Process placement options
#
my $processorspersolver=1;
#
# Mapping of local paths to network paths
my %pathmap;
#
# Create cfxccs_options.txt in %USERPROFILE%\.cfx to override these settings e.g:
# ACCOUNT=MYDOMAIN\MyUser
# PATHMAP=C:\Users\MyUser;\\myheadnode\Users\MyUser
# PATHMAP=C:\Program Files\Ansys Inc;\\myheadnode\Ansys Inc
# PROCESSORSPERSOLVER=2
# CLUSTERHOST=myheadnode
#
# Command line options
# --------------------
#
# -part <number of partitions>
#    Number of partitions
#
# -indirect-startup-path <path>
#    Indirect startup path
#
# [-headnode <hostname>]
#    Host to which job will be submitted (overrides value in options file)
#
# [-node-group <value> -node-group <value2>]
#    Value(s) assigned to the Windows HPC job attribute 'NodeGroups'
#
# [-node-group-op <value>]
#    Value assigned to the Windows HPC job attribute 'NodeGroupOp'
#
# [-job-template <value>]
#    Name of job template to use when submitting job to Windows HPC cluster
#
# [-nowait ]
#    Don't wait for the job to complete
#
###################################################################################################

use strict;
use CFX5::Output qw(:all error_text);
use CFX5::Output::OutFile;
use CFX5::PathManip qw(join_file basename dirname normalize_filename is_unc);
use CFX5::CCLI;
use CFX5::Config qw(getconf);
use CFX5::FileOps qw(get_file_contents set_file_contents);
use CFX5::FileTests;
use Getopt::Long;

#
# Process command line options
#
$Getopt::Long::passthrough = 0;
$Getopt::Long::ignorecase = 0;
my $numpartitions = undef;
my $indirectpath = undef;
my $headnode = undef;
my @nodegroups = ();
my $nodegroupop = undef;
my $jobtemplate = undef;
my $nowait = undef;
GetOptions('-part=i'                  => \$numpartitions,
           '-indirect-startup-path=s' => \$indirectpath,
           '-headnode=s'              => \$headnode,
           '-node-group=s'            => \@nodegroups,
           '-node-group-op=s'         => \$nodegroupop,
           '-job-template=s'          => \$jobtemplate,
           '-nowait'                  => \$nowait,
           ) or _abort('Error in command line arguments.');

_abort('Number of partitions not specified.') unless defined $numpartitions;
_abort('Indirect path not specified.') unless defined $indirectpath;

#
# Make sure the solver output file exists and direct the output to it
#
my $outfile = $indirectpath.'.out';
CFX5::FileTests::datafile($outfile,'ANSYS CFX Solver Output File');
exit_if_error();
my $out = new CFX5::Output::OutFile($outfile);
CFX5::Output::replace_default_output($out);

# Read cfxccs_options.txt file, if it exists
# Look first in documented location: %USERPROFILE%\.cfx
# then (for backwards compatibility) in: %CFX_USER_CONFIG_DIR%
my $optionsfile;
CONFIGDIR:
for my $dir (getconf('USERPROFILE'), getconf('CFX_USER_CONFIG_DIR')) {
  for my $ext ('.txt', '.txt.txt') {
    $optionsfile = join_file($dir,'.cfx','cfxccs_options'.$ext);
    last CONFIGDIR if (-e $optionsfile);
  }
}

if (-e $optionsfile) {
  my @lines = split '[\n|\r]+', get_file_contents($optionsfile);
  my @lines_mesg;

  unless (@lines) {
    _abort("ERROR reading user CCS options file: $optionsfile");
  }

  for my $line (@lines) {
    # Remove leading/trailing whitespace
    for ($line) {
      s/^\s+//;
      s/\s+$//;
    }
    if ( $line=~m!^$! ) { # Ignore blank lines
      next;
    }
    elsif($line=~m!^#!) { # Echo comments to output
      push @lines_mesg, $line;
      next;
    }
    $line=~s!\s*([=;])\s*!$1!g; # Remove spaces around ";" and "=" separators
    my @fields = split(/=/, $line);
    if(scalar(@fields) > 1){
      $fields[0]=~tr/a-z/A-Z/; # Upper case
      $line=join("=",@fields);
    }
    if($line=~m!^PATHMAP=([^;]+)*;(.+)!){
      $pathmap{$1}=$2;
    }
    elsif($line=~m!^ACCOUNT=(.+)!){
      $account=$1;
    }
    elsif($line=~m!^CLUSTERHOST=(.+)!){
      $clusterhost=$1;
    }
    elsif($line=~m!^PROCESSORSPERSOLVER=([0-9]+)!){
      $processorspersolver=$1;
    }
    else {
      _abort("ERROR Processing user CCS options file $optionsfile: $line");
    }
    push @lines_mesg, $line;
  }

  $out->output('print',undef,"Processing user options from $optionsfile",map { " $_\n" } @lines_mesg);
}
else {
  my %h = (CLUSTERHOST         => $clusterhost,
           ACCOUNT             => $account,
           PROCESSORSPERSOLVER => $processorspersolver);

  $out->output('print',undef,"No user CCS options file has been specified. The following parameter settings are assumed:",
               map { sprintf(" %20s = %s\n", $_, defined $h{$_} ? $h{$_} : '<undefined>') } keys %h);
}

#
# Set the head node if specified on the command line
#
$clusterhost = $headnode if (defined $headnode);


# Set location of shared CFX Installation
my $sharedroot = dirname(dirname(`cfx5info -whereis cfx5info`));
if ( exists $ENV{CFX_SHAREDINST_DIR}) {
  $sharedroot = $ENV{CFX_SHAREDINST_DIR} ;
}

#
# Map paths (installation root, indirect path and any paths in indirect CCL)
#
my $pathmapper = CFX5::PathManip->pathmapper();
while (my ($loc_path, $unc_path) = each %pathmap) {
  $pathmapper->add_mapping($loc_path, $unc_path);
}

$sharedroot = $pathmapper->map_local_path($sharedroot, 'CFX installation directory');
unless(is_unc($sharedroot)) {
  _error("ERROR: Shared CFX installation is not a UNC path: $sharedroot");
}

$indirectpath = join_file($pathmapper->map_local_path(dirname($indirectpath), 'Working directory'), basename($indirectpath));
unless(is_unc(dirname($indirectpath))){
  _error("ERROR: Working directory is not a UNC path: $indirectpath");
}

exit_if_error();

my $cclfile = join_file($indirectpath.".dir","startup.ccl");
my $api=CFX5::CCLI::new_api_object();
my @ccl;
if ($api->readTextFile($cclfile, \@ccl)) {
  _abort("ERROR reading indirect ccl file: $cclfile: $!");
}
my $rv = $api->parseCCLData(\@ccl);
$pathmapper->map_ccl_paths($api,"");
$api->deleteObj('/SIMULATION CONTROL/EXECUTION CONTROL/PARALLEL HOST LIBRARY');
$rv=$api->getFormattedCCL("", 78, \@ccl);

unless (set_file_contents($cclfile => map { "$_\n" } @ccl)) {
  _abort("ERROR writing indirect ccl file: $cclfile: $!");
}

my @map_results = $pathmapper->mapping_info();
if (@map_results) {
  $out->output('print',undef,'Mapping Paths',
               map { sprintf("%s\n  Original: %s\n  Mapped  : %s\n", $_->{description}, $_->{original}, $_->{mapped}) } @map_results);
}

# Connect to cluster
use Win32::OLE;
use Win32::OLE::Variant;
Win32::OLE->Option(Warn => 0);

my $scheduler = Win32::OLE->new('Microsoft.HPC.Scheduler.Scheduler');
unless($scheduler) {
  _abort('ERROR: Could not create a scheduler OLE object');
}

my $isconsole = Variant(VT_BOOL, 0);
my $intptr = Variant(VT_EMPTY);
$scheduler->SetInterfaceMode($isconsole,\$intptr);

$scheduler->Connect($clusterhost);
if (my $error = Win32::OLE->LastError()) {
  _abort("ERROR: Connection to cluster $clusterhost failed:", $error,
         "\n\nNote:\n\n",
         "Check the host $clusterhost exists and can be reached.\n",
         "Ensure the Windows HPC client and server software versions match.");
}

# Create job
my $job = $scheduler->CreateJob();
unless ($job) {
  _abort('ERROR: Could not create scheduler job');
}

# Set job attributes
my $numprocessors = $numpartitions * $processorspersolver;
$job->{Name} = sprintf "ANSYS CFX - %s", basename($indirectpath);
$job->{IsExclusive} = 'false';
$job->{MinimumNumberOfCores} = $numprocessors;
$job->{MaximumNumberOfCores} = $numprocessors;

# Set node groups, if specified
if (@nodegroups) {
  my $nodegrouplist = $scheduler->GetNodeGroupList();
  if (my $error = Win32::OLE->LastError()) {
    _abort("ERROR: Could not get node group list: $error");
  }

  for my $nodegroup (@nodegroups) {
    unless ($nodegrouplist->Contains($nodegroup)) {
      _error("ERROR: Not a valid node group: $nodegroup");
    }
  }

  if (error_pending()) {
    my $valid = Win32::OLE::Enum->new($nodegrouplist);
    $out->output('print',undef,"Valid node groups:",map { " $_\n" } $valid->All());
  }
  exit_if_error();

  $nodegrouplist->Clear();
  for my $nodegroup (@nodegroups) {
    $nodegrouplist->Add($nodegroup);
  }
  $job->{NodeGroups} = $nodegrouplist;
}

# Set node group operator, if specified
my %JobNodeGroupOps = (Intersect => 0,
                       Uniform   => 1,
                       Union     => 2);

if (defined $nodegroupop) {
  my $value = $JobNodeGroupOps{$nodegroupop};
  if (defined $value) {
    $job->{NodeGroupOp} = $value;
  } else {
    _abort("ERROR: Not a valid node group operator: $nodegroupop");
  }
}

# Set job template, if specified
if (defined $jobtemplate) {
  my $jobtemplatelist = $scheduler->GetJobTemplateList();
  if (my $error = Win32::OLE->LastError()) {
    _abort("ERROR: Could not get job template list: $error");
  }

  if ($jobtemplatelist->Contains($jobtemplate)) {
    $job->SetJobTemplate($jobtemplate);
  } else {
    _error("ERROR: Not a valid job template: $jobtemplate");
  }

  if (error_pending()) {
    my $valid = Win32::OLE::Enum->new($jobtemplatelist);
    $out->output('print',undef,"Valid job templates:",map { " $_\n" } $valid->All());
  }
  exit_if_error();
}

# Create task
my $task = $job->CreateTask();
unless ($task) {
  _abort('ERROR: Could not create scheduler task');
}

# Assemble command line
my $cmd = join_file($sharedroot,'bin','cfx5solve');
my @cmdline_list = (qq("$cmd"), '-start-method', 'MSMPI',
                                '-part', $numpartitions);

if ($nowait) {
  my $indirectccl = join_file($indirectpath.'.dir','startup.ccl');
  push @cmdline_list, '-ccl', qq("$indirectccl"),
                      '-fullname', qq("$indirectpath");
} else {
  push @cmdline_list, '-indirect-startup-path', qq("$indirectpath");
}

my $cmdline = join ' ', @cmdline_list;

# Set task attributes
my $stdoutfile = $indirectpath . '.stdout';
my $stderrfile = $indirectpath . '.stderr';
$task->{WorkDirectory}        = dirname($indirectpath);
$task->{CommandLine}          = $cmdline;
$task->{IsRerunnable}         = 'false';
$task->{StdOutFilePath}       = $stdoutfile;
$task->{StdErrFilePath}       = $stderrfile;
$task->{Name}                 = 'ANSYS CFX Solver';
$task->{MinimumNumberOfCores} = $numprocessors;
$task->{MaximumNumberOfCores} = $numprocessors;

# Set environment variables
$task->SetEnvironmentVariable('CFX_DISABLE_REMOTE_CHECKS','1');
$task->SetEnvironmentVariable('CFX5_HOSTS_CCL','NUL');
$task->SetEnvironmentVariable('_CFX_PROC_PER_SOLVER',$processorspersolver);
$task->SetEnvironmentVariable('_CFX_ENABLE_MSMPI_SM','1');
$task->SetEnvironmentVariable('MPIEXEC_OPTIONS','-affinity');

# Add the task
$job->AddTask($task);

# Submit the job to the HPC cluster
$scheduler->SubmitJob($job, $account, "");
if (my $error = Win32::OLE->LastError()) {
  _abort("ERROR: Job submission to cluster failed: $error");
}

$out->output('print',undef,sprintf("Command (job %s) submitted to: %s",$job->{Id},$clusterhost),$cmdline);

# If we are not waiting, close the connection and stop
if ($nowait) {
  $scheduler->Close();
  exit 0;
}

# Wait for job to finish
my %JobStates = (Configuring        => 1,
                 Submitted          => 2,
                 Validating         => 4,
                 ExternalValidation => 8,
                 Queued             => 16,
                 Running            => 32,
                 Finishing          => 64,
                 Finished           => 128,
                 Failed             => 256,
                 Canceled           => 512,
                 Canceling          => 1024);
my %StateNames = reverse %JobStates;

while ($job->{State} < $JobStates{Finished}) {
  $job->Refresh();
  sleep(1);
}

# Wait for 10 seconds for any file cache updates to complete
sleep(10);

# Find final job state
$job->Refresh();
my $state = $job->{State};

# Close connection to cluster
$scheduler->Close();

# Remove empty stdout and stderr files
for my $file ($stdoutfile, $stderrfile) {
  next unless (-e $file);
  unlink $file unless (-s $file);
}

# Normal exit
exit 0 if ($state == $JobStates{Finished});

# Abnormal exit: Print state on exit print stdout and stderr if present
my $statename = exists($StateNames{$state}) ? $StateNames{$state} : 'Unknown';
_error("Abnormal exit. Job state on exit: $statename");

if(-e $stdoutfile) {
  my @lines = split '[\n|\r]+', get_file_contents($stdoutfile);
  _print('STDOUT:');
  for my $line (@lines) {
    _print($line);
  }
}
if(-e $stderrfile) {
  my @lines = split '[\n|\r]+', get_file_contents($stderrfile);
  _print('STDERR:');
  for my $line (@lines) {
    _print($line);
  }
}

exit 1;
