#! perl -w
#==============================================================================
# @(#) $Id$
#------------------------------------------------------------------------------
##
## Name:
##
## cfx5parallel
##
## Purpose:
##
## Update and modify the CFX-5 parallel host list information.
##
## Synopsis:
#S
#S cfx5parallel -setup [-hosts <list>|-local <processes>]
#S cfx5parallel -stop
#S cfx5parallel -pvm-console
#S
#S cfx5parallel -install-impi-service
#S cfx5parallel -uninstall-impi-service
#S cfx5parallel -start-impi-service
#S cfx5parallel -stop-impi-service
#S cfx5parallel -restart-impi-service
#S cfx5parallel -status-impi-service
#S cfx5parallel -register-impi-user
#S cfx5parallel -unregister-impi-user
#S
#S cfx5parallel -install-ibmmpi-service
#S cfx5parallel -uninstall-ibmmpi-service
#S cfx5parallel -start-ibmmpi-service
#S cfx5parallel -stop-ibmmpi-service
#S cfx5parallel -setportkey-ibmmpi-service <port number>
#S cfx5parallel -restart-ibmmpi-service
#S cfx5parallel -status-ibmmpi-service
#S cfx5parallel -register-ibmmpi-user
#S cfx5parallel -check-ibmmpi-user
#S cfx5parallel -unregister-ibmmpi-user
#S 
## []  denotes an optional argument.
## |   separates mutually exclusive arguments. 
## ... following an argument indicates that it may be repeated.
## <>  denotes that substitution of a suitable value is required. All other
##     arguments are keywords, some of which have a short form.
##
## Arguments:
##
#A -install-impi-service
#A -uninstall-impi-service
#A -start-impi-service
#A -stop-impi-service
#A -restart-impi-service
#A -status-impi-service
##   Install, uninstall, start, stop, restart or get status of the
##   Intel MPI remote launch service.  This service is required for Intel MPI
##   parallel runs on Windows machines.  These flags are not supported
##   on non-Windows platforms.
##
#A -install-ibmmpi-service
#A -uninstall-ibmmpi-service
#A -start-ibmmpi-service
#A -stop-ibmmpi-service
#A -restart-ibmmpi-service
#A -status-ibmmpi-service
##   Install, uninstall, start, stop, restart or get status of the
##   IBM MPI remote launch service.  This service must be installed
##   for distributed parallel runs using IBM MPI on windows.
##
#A -setportkey-ibmmpi-service <port number>
##   Specify the port number to be used by IBM MPI.  This is required
##   if the default number, 8636, is used by another service (e.g. the HP MPI
##   service used in older CFX versions).  The port number must be between 
##   1024 and 65535.  You should confirm what port numbers are still available
##   on your system.
##
#A -pvm-console
##   Start the PVM console interactively for debugging parallel
##   problems.
##
#A -register-impi-user
#A -unregister-impi-user
##   Allows you to register, or remove, a user name for use with Intel MPI
##   distributed parallel on Windows machines.  These flags are not
##   supported on UNIX.
##
#A -register-ibmmpi-user
#A -unregister-ibmmpi-user
#A -check-ibmmpi-user
##   The first two options allow you to register, or remove, a user 
##   name for use with IBM MPI distributed parallel on Windows machines.
##   The check option allows you to check if a user is registered.
##   These flags are not supported on UNIX.
##
#A -info-impi
##   Show where Intel MPI is obtained from and its version.
##
#A -info-ibmmpi
##   Show where IBM MPI is obtained from and its version.
##
#A -setup
##   Start the parallel environment.  You must give either -hosts or
##   -local.
##
#A -stop
##   Shut down the parallel environment.  All hosts with no remaining
##   jobs will be shutdown, and if no jobs remain, the daemon will be
##   stopped.
##
#A -hosts <list>
##   Include the listed hosts in the parallel environment.  Host list
##   is in the format <host>[*<count>][,<host2>[*<count>][,...]], as
##   for the solver.
##
#A -local <processes>
##   Start on the local host, with the stated number of processes.
##
#U ****************************************************************************
#U *                                                                          *
#U *     The following features are supported only within CFX and are not     *
#U *     shown by the -help option when the command is run elsewhere.         *
#U *                                                                          *
#U ****************************************************************************
#U -check-part-count <partitions>
#U   Used when you know how many partitions there should be, to save
#U   having to parse the host list in client programs.  If the number
#U   of partitions is not <partitions> after setting up, aborts with
#U   an error.
#U
#U -write-setup
#U   Write the state files necessary for CFX-5 codes to run directly.
#U
#U -exec <exec-template>
#U   Set the template for the executable to run, in the same way as
#U   used for the solver executable in cfx5solve.
#U
#------------------------------------------------------------------------------

# get the standard CFX-5 library directory
BEGIN {
 if ($0 =~ m!^([a-z]:)?[\\/]!i) {
    # it's an absolute path, use it as-is
    (my $nd=$0) =~ s!(perllib[\\/])?[^\\/]*$!!i;
    unshift @INC, "$nd/perllib";
  } else {
    # wrap this in an eval, and if it fails, pray ... for example,
    # this can fail on Windows if we're running off a share.
    eval {
      require FindBin;
      FindBin->import();
      unshift @INC, "$FindBin::Bin/perllib";
      my $dummy=$FindBin::Bin; # kill used only once warning
    };
    warn $@ if $@;
  }
  if (defined $ENV{CFX_LOCAL_PERLLIB}) {
     unshift @INC, $ENV{CFX_LOCAL_PERLLIB};
  }

}

use CFX5::Init;
CFX5::Init::init();

use vars qw($CVSVersion);
sub BEGIN {
  $CVSVersion='@(#) $Id$\n';
}

use strict;

use CFX5::Output qw(:all);
use CFX5::Config qw(%Config export_conf query_conf append_to_libpath);
#use CFX5::PrintConfig;

use CFX5::CmdLine qw(:util parse_cmdline);

use CFX5::PathManip qw(join_file); # split_file fullpath basename dirname 
#use CFX5::FileTests;
use CFX5::FileOps qw(get_file_contents);

use CFX5::PkgLoader qw(cfx5_require);

#
# read the configuration
#
CFX5::Config::read_config();

#
# parse the command line
#
my (@mode,$hostlist,$localpart,$writesetup,$exec,$ccl_file);
my ($part_check);
my ($ibmmpi_port);
my $ccl_root='/SIMULATION CONTROL/EXECUTION CONTROL/SOLVER STEP CONTROL/PARALLEL ENVIRONMENT';
my %args=(
          -setup => \@mode,
          -stop => \@mode,
          '-pvm-console' => \@mode,
          '-register-impi-user' => \@mode,
          '-unregister-impi-user' => \@mode,
          '-help-impi-service' => \@mode,
          '-install-impi-service' => \@mode,
          '-remove-impi-service' => \@mode,
          '-uninstall-impi-service' => \@mode,
          '-start-impi-service' => \@mode,
          '-stop-impi-service' => \@mode,
          '-restart-impi-service' => \@mode,
          '-status-impi-service' => \@mode,
          '-version-impi-service' => \@mode,
          '-getphrase-impi-service' => \@mode,
          '-help-impi-smpd-service' => \@mode,
          '-install-impi-smpd-service' => \@mode,
          '-remove-impi-smpd-service' => \@mode,
          '-uninstall-impi-smpd-service' => \@mode,
          '-start-impi-smpd-service' => \@mode,
          '-stop-impi-smpd-service' => \@mode,
          '-restart-impi-smpd-service' => \@mode,
          '-status-impi-smpd-service' => \@mode,
          '-version-impi-smpd-service' => \@mode,
          '-hosts-impi-smpd-service' => \@mode,
          '-sethosts-impi-smpd-service' => \@mode,
          '-getphrase-impi-smpd-service' => \@mode,
          '-help-ibmmpi-service' => \@mode,
          '-install-ibmmpi-service' => \@mode,
          '-uninstall-ibmmpi-service' => \@mode,
          '-start-ibmmpi-service' => \@mode,
          '-status-ibmmpi-service' => \@mode,
          '-removeeventkey-ibmmpi-service' => \@mode,
          '-removeportkey-ibmmpi-service' => \@mode,
          '-setportkey-ibmmpi-service+' => \$ibmmpi_port,
          '-restart-ibmmpi-service' => \@mode,
          '-stop-ibmmpi-service' => \@mode,
          '-register-ibmmpi-user' => \@mode,
          '-check-ibmmpi-user' => \@mode,
          '-unregister-ibmmpi-user' => \@mode,
          '-info-impi' => \@mode,
          '-info-ibmmpi' => \@mode,
          '-check-part-count+' => \$part_check,
          '-exec+' => \$exec,
          '-hosts+' => \$hostlist,
          '-local+' => \$localpart,
          '-write-setup' => \$writesetup,
          '-ccl-root+' => \$ccl_root,
          '-ccl+' => \$ccl_file,
          '-cfx_server_mode' => '-cfx-server-mode',
          '-cfx-server-mode' => sub {
            cfx5_require('CFX5::Output::MeshStyle');
            CFX5::Output::replace_default_output(CFX5::Output::MeshStyle->new(), undef);
          },
         );
my @nonopt=parse_cmdline
  ( \@ARGV,
    'INCLUDE_STD',
    %args,
  );

if (@nonopt) {
  cfx5_require('CFX5::Help');
  _error("Unexpected argument(s)\n", @nonopt, "\nwithout an associated option were found on the command line.", CFX5::Help::usage_msg_once());
}

if (defined $hostlist and defined $localpart) {
  cfx5_require('CFX5::Help');
  _error("You can only specify one of -hosts and -local.", CFX5::Help::usage_msg_once());
}

# pretend this is a mode for now
if (defined $part_check and not @mode) {
  push @mode, '-check-part-count';
}
if (defined $ibmmpi_port and not @mode) {
  push @mode, '-setportkey-ibmmpi-service';
}

unless (@mode) {
  cfx5_require('CFX5::Help');
  _error("You must give an action, either -setup or -stop.", CFX5::Help::usage_msg_once());
}
if (@mode > 1) {
  cfx5_require('CFX5::Help');
  _error("You can only give one of the", @mode, "switches.", CFX5::Help::usage_msg_once());
}

exit_if_error();

my $mode=shift @mode;

if ($mode eq '-setup' or $mode eq '-check-part-count') {
  #
  # check we've got what we need
  #
  unless (defined $hostlist or defined $localpart or defined $ccl_file) {
    cfx5_require('CFX5::Help');
    _error("You must specify one of -hosts, -local or -ccl with $mode.", CFX5::Help::usage_msg_once());
  }

  exit_if_error();
}

#
# Handle all the things which don't require the CCL database
#
if ($mode eq '-register-impi-user') {
  unless ($Config{windows_nt}) {
    _abort("The $mode switch is only supported on Windows hosts.");
  }

  _print "\n\nStarting the Intel MPI User Registration tool.  You will need to give a user account and password which can be used to log into the other machines in a distributed parallel run.  For network users, you can include the domain name and a backslash preceding the user, eg \"BEDROCK\\Fred\".  \n\n";

  my $exec=join_file($Config{CFX5_IMPI_DIR}, 'bin', 'mpiexec.exe');
  cfx5_require('CFX5::Commands');
  my $rv=CFX5::Commands::do_cmd($exec, "-register");
  # Windows processes always return junk, but see if it ran
  unless ($rv->ran()) {
    _error("The Intel MPI register command", $rv->errstr());
  }

  exit(0);
}
elsif ($mode eq '-unregister-impi-user') {
  unless ($Config{windows_nt}) {
    _abort("The $mode switch is only supported on Windows hosts.");
  }

  my $exec=join_file($Config{CFX5_IMPI_DIR}, 'bin', 'mpiexec.exe');
  cfx5_require('CFX5::Commands');
  my $rv=CFX5::Commands::do_cmd($exec, '-remove');
  # Windows processes always return junk, but see if it ran
  unless ($rv->ran()) {
    _error("The Intel MPI register command", $rv->errstr());
  }

  exit(0);
}
elsif ($mode =~ m!-(install|uninstall|remove|start|stop|hosts|sethosts|getphrase|restart|status|version|help)-(impi|impi-smpd)-service!) {
  my $action=$1;
  my $service=$2;

  unless ($Config{windows_nt}) {
    _abort("The $mode switch is only supported on Windows hosts.");
  }
  my $exec_name = $service eq 'impi-smpd' ? 'smpd' : 'hydra_service';
  my $exec=join_file($Config{CFX5_IMPI_DIR}, 'bin', $exec_name);
  cfx5_require('CFX5::Commands');
  my $rv=CFX5::Commands::do_cmd($exec, "-$action");
  unless ($rv->ran()) {
    _error("The Intel MPI $exec_name -$action command", $rv->errstr());
  }

  exit(0);
}
elsif ($mode eq '-register-ibmmpi-user') {
  unless ($Config{windows_nt}) {
    _abort("The $mode switch is only supported on Windows hosts.");
  }

  _print "\n\nStarting the IBM MPI User Registration tool.  You must be logged in as the domain user you want to register, and give the password for that user account.  This account is used to access shared resources in a distributed parallel run (eg: a shared CFX installation).  Note that the password is not echoed to the screen.\n\n";

  my $exec=join_file($Config{CFX5_IBMMPI_DIR}, 'bin', 'mpirun.exe');
  my $regExe=join_file($Config{CFX5EXECDIR}, 'ibmmpipass.exe');
  my $rv = system "\"$exec\" -cache -np 2 \"$regExe\"";
  exit(0);
}
elsif ($mode eq '-unregister-ibmmpi-user') {
  unless ($Config{windows_nt}) {
    _abort("The $mode switch is only supported on Windows hosts.");
  }

  my $exec=join_file($Config{CFX5_IBMMPI_DIR}, 'bin', 'mpirun.exe');
  cfx5_require('CFX5::Commands');
  my $rv=CFX5::Commands::do_cmd($exec, '-clearcache');
  # Windows processes always return junk, but see if it ran
  unless ($rv->ran()) {
    _error("The IBM MPI unregister command", $rv->errstr());
  }

  exit(0);
}
elsif ($mode eq '-check-ibmmpi-user') {
  unless ($Config{windows_nt}) {
    _abort("The $mode switch is only supported on Windows hosts.");
  }
  my $hostname=$ENV{COMPUTERNAME};
  my $exec=join_file($Config{CFX5_IBMMPI_DIR}, 'bin', 'mpidiag.exe');
  my $rv= system "\"$exec\" -s $hostname -iscached";
  exit(0);
}
elsif ($mode eq '-info-impi') {
  _print("\nVariable settings affecting Intel MPI:\n\n");
  for my $var (qw(I_MPI_ROOT CFX5_IMPI_DIR)) {
    my $value = defined $Config{$var} ? $Config{$var} : '<unset>';
    my $exists = -d $value ? '<directory exists>' : ' ';
    _print("$var = $value $exists");
  }
  
  _print("\nIntel MPI version information:\n\n");
  my $exec = join_file($Config{CFX5_IMPI_DIR}, 'bin', 'mpiexec');
  if ($Config{windows_nt}) { $exec .= '.exe'; }
  if (-e $exec) {
    cfx5_require('CFX5::Commands');
    my $rv = CFX5::Commands::do_output_cmd($exec, '-version');
    _print($rv->cmd_output());
  } else {
    _print("Unable to determine version. $exec cannot be located.");
  }

  exit(0);
}
elsif ($mode eq '-info-ibmmpi') {
  _print("\nVariable settings affecting IBM MPI:\n\n");
  for my $var (qw(MPI_ROOT CFX5_IBMMPI_DIR)) {
    my $value = defined $Config{$var} ? $Config{$var} : '<unset>';
    my $exists = -d $value ? '<directory exists>' : ' ';
    _print("$var = $value $exists");
  }
  
  _print("\nIBM MPI version information:\n\n");
  my $exec = join_file($Config{CFX5_IBMMPI_DIR}, 'bin', 'mpirun');
  if ($Config{windows_nt}) { $exec .= '.exe'; }
  if (-e $exec) {
    cfx5_require('CFX5::Commands');
    my $rv = CFX5::Commands::do_output_cmd($exec, '-version');
    _print($rv->cmd_output());
  } else {
    _print("Unable to determine version. $exec cannot be located.");
  }

  exit(0);
}
elsif ($mode =~ m!-(install|uninstall|start|stop|setportkey|restart|status|help|removeportkey|removeeventkey)-ibmmpi-service!) {
  my $action=$1;

  unless ($Config{windows_nt}) {
    _abort("The $mode switch is only supported on Windows hosts.");
  }

  my $exec=join_file($Config{CFX5_IBMMPI_DIR}, 'sbin','ibmmpiwin32service.exe');
  cfx5_require('CFX5::Commands');
  my $rv;
  if ($action eq 'setportkey') {
    $rv=CFX5::Commands::do_cmd($exec, "-$action", $ibmmpi_port);
  } else {
    $rv=CFX5::Commands::do_cmd($exec, "-$action");
  }

  # Windows processes always return junk, but see if it ran
  unless ($rv->ran()) {
    _error("The IBM MPI Remote Launch Service -$action command", $rv->errstr());
  }

  exit(0);
}

#
# put whatever information we have into a parallel object
#

cfx5_require('CFX5::WorkDir');

cfx5_require('CFX5::Parallel');
cfx5_require('CFX5::CCLI');
cfx5_require('CFX5::CCLI::Simple');

# need a ccl database object
my $api=CFX5::CCLI::new_api_object();

# read the central host library
my $hosts=$Config{CFX5_HOSTS_CCL};
if (defined $hosts) {
  my @lines=split("\n",get_file_contents($hosts));
  my $rv=$api->parseCCLData(\@lines, 1);
  if ($rv) {
    _error("Error reading CFX-5 hosts file $hosts:", 
           CFX5::CCLI::format_ccl_error($rv));
  }
}
# read the central host library
my $start_methods=$Config{CFX5_START_METHODS_CCL};
if (defined $start_methods) {
  my $lines=get_file_contents($start_methods);
  if (not defined $lines) {
    _error("Error reading start methods file $start_methods: $!");
  } else {
    my $rv=$api->parseCCLData([split("\n", $lines)], 1);
    if ($rv) {
      _error("Error reading start methods file $start_methods:", 
             CFX5::CCLI::format_ccl_error($rv));
    }
  }
}

if (defined $ccl_file) {
  cfx5_require('CFX5::FileTests');
  CFX5::FileTests::datafile($ccl_file, "CFX-5 Command Language input file $ccl_file")
      or exit(2);
  my @lines=split("\n",get_file_contents($ccl_file));
  my $rv=$api->parseCCLData(\@lines);
  if ($rv) {
    _error("Error reading CFX-5 Command Language input file $ccl_file:", 
           CFX5::CCLI::format_ccl_error($rv));
  }
}

my $db=CFX5::CCLI::Simple->new($api);
my $ppe=new CFX5::Parallel($db, $ccl_root);

exit_if_error();

# if command-line options were given, apply them; otherwise, assume
# the CCL is valid.
if (defined $hostlist) {
  $ppe->start_method('PVM Distributed Parallel');
  $ppe->host_list_text($hostlist);
} elsif (defined $localpart) {
  $ppe->start_method('PVM Local Parallel');
  $ppe->local_processes($localpart);
}

$ppe->put_master_first();

$ppe->mark_hosts();

if (defined $exec) {
  for my $h ($ppe->hosts()) {
    $h->executable($exec);
  }
}

#$db->dump();

# not really a working directory per se, but a useful way of
# referring to it nonetheless.
$ppe->workdir(CFX5::WorkDir->new('.'));

exit_if_error();

# now we have all the setup set up, as it were, make sure it's correct
# if we were asked to check it.
if (defined $part_check) {
  my $part_actual=$ppe->host_count();
  if ($part_actual != $part_check) {
    _error("Error in parallel setup: $part_actual partitions have been configured, but the problem setup requires $part_check partitions.");
  }
}

# in the solver, we needed to do these separately in case we were
# partitioning; here, we just do both.
$ppe->local_check();
$ppe->start_method()->local_check($ppe);

exit_if_error();

if ($mode eq '-setup') {
  _info("Setting up the CFX-5 parallel environment for host(s):", $ppe->host_list_text());
  $ppe->start();

  if ($writesetup) {
    # we always need this, it tells the application what partitions
    # are where
    $ppe->setup_req('phst')
      or return CFX5::Commands::Result->new_not_run('Setup of phst failed');

    # handle any prerequisites of the start method.  NB by
    # "prerequisites" we mean "things we need in order to run a command
    # using this start method," not "things we need to start this start
    # method".  XXX should we do this without $writesetup too?
    for my $req ($ppe->start_method()->prerequisites()) {
      $ppe->setup_req($req);
    }
  }

} elsif ($mode eq '-stop') {
  _info("Shutting down the CFX-5 parallel environment");
  $ppe->stop();
} elsif ($mode eq '-pvm-console') {
  # if PVM isn't already running, we'll need some sort of hosts
  # file.  If it is running, this will be ignored.
  $ppe->setup_req('PVMHosts');
  _print(qq(Starting the PVM console.  Type 'quit' to close the console and leave PVM running.\n\n));
  my $rv=CFX5::Parallel::Pvm->start_console();
  unless ($rv->is_success()) {
    _error("The PVM console", $rv->errstr());
  }
} elsif ($mode eq '-check-part-count') {
  # this can be a mode, or can be done with one of the others; in
  # either case, it's already happened!
} else {
  internal_error("Missing or unknown mode $mode");
}

exit_if_error();
exit(0);
