=head1 NAME

CFX5::Parallel - Parallel control routines for ANSYS CFX

=head1 SYNOPSIS

This module contains high-level control routines for parallel in ANSYS CFX,
including routines for reading F<hst> files, and starting and stopping
daemons. It uses the routines in L<CFX5::Parallel::Pvm> to interact with
the PVM daemon.

CFX5::Parallel (now) provides a class which represents a list of
hosts, and a parallel processing environment which uses this set of
machines.  It is based around a Command Language database which is
usually provided by a L<CFX5::Job::Settings> object or derivative
thereof, but can be any class providing the C<read_ccl_parm> and
C<set_ccl_parm> calls.

=cut

package CFX5::Parallel;

use strict;

use Net::Ping;

use vars qw($CVSVersion);
$CVSVersion='$Id$ ';

use CFX5::Output qw(:all);

use CFX5::Parallel::Pvm;
use CFX5::Parallel::Host;
use CFX5::Parallel::PvmHosts;

use CFX5::FileTests;

use CFX5::Config qw(%Config export_conf unexport_conf);

use CFX5::Commands qw(handle_rv get_cmd_output do_termerr_cmd);

use CFX5::CCLI::Defaulted;

use CFX5::HostAddr;

use CFX5::PkgLoader qw(cfx5_require);

use vars qw(@ISA);
@ISA=qw(CFX5::CCLI::Defaulted Exporter);

=head1 DESCRIPTION

=head2 CONSTRUCTOR

=head3 new

  my $ppe=CFX5::Parallel->new($job);

Returns a new object representing a parallel processing environment,
to which hosts may be added, and which may be used to run jobs.  The
first argument should be an object which can be used to access the CCL
database.

=cut

sub new {
  my $this=shift;
  my $class=ref($this) || $this;
  my $self={
            hostdb => {},
           };
  bless $self, $class;

  unless (@_) {
    internal_error("Not enough arguments in call to new");
  }

  my $config=shift;
  $self->CFX5::CCLI::Defaulted::init($config);
  $self->config_object($config);

  my $root=shift;
  unless (defined $root) {
    internal_error("Missing root parameter");
  }
  $self->{root}=$root;

  $self->{showMissingMasterWarning}=1;

  $self;
}

sub cclpath {
  my $self=shift;
  return $_[0];
}

sub root {
  my $self=shift;
  return $self->{root};
}

=head3 config_object

  $ppe->config_object();

Gets and sets the config object.

=cut
sub config_object
{
  my $self=shift;
  if (@_) {
    $self->{configObject} = shift;
  }
  return $self->{configObject};
}

=head2 MODE COMPATIBILITY / PARTIAL SPECIFICATION

Routines for handling old-style pvm-vs-mpi and local-vs-distributed
setups.

All this information is based on the start_method setting; when things
are set they modify that, and when they are queried, that is used to
give an answer.  The only exception to this is that we have to track
PVM vs MPI even in serial.



=head3 set_compat_option

Sets the PPE option, 'Serial', 'Local Parallel' or 'Distributed
Parallel'.

=cut

sub set_compat_option {
  my $self=shift;

  unless (@_) {
    internal_error("Missing arguments");
  }

  my $option=shift();
  unless ($option eq 'Serial' 
          or $option eq 'Local Parallel'
          or $option eq 'Distributed Parallel') {
    _error("Invalid PPE option $option");
    return '';
  }

  # Find the default parallel mode and start method for this platform
  # -> Now Intel MPI on all platforms unless overridden by CFX5_START_METHOD
  my $default_method = $Config{CFX5_START_METHOD};
  unless (defined $default_method)
  {
    my $mode = 'Intel MPI';
    $default_method = $option eq 'Serial' ? 'Serial' : "$mode $option";
  }

  # now fix the start method if necessary.  We can do this for the
  # hardwired ones, but if it's a user method we just ignore it.
  my $sm=$self->start_method();
  if ($sm and defined (my $smname=$sm->name())) {
    if ($smname eq 'Serial' or
        ($smname =~ m/^Intel MPI (?:Local|Distributed) Parallel$/) or
        $smname eq $default_method){
      # Default or legacy default for this platform. Use default method.
      _debug("Setting start method to $default_method for option $option");
      $self->start_method($default_method);
    } else {
      # Not the default for this platform, so preserve the existing
      _warn("The parallel start method specified in the input file: $smname, is not a default parallel start method for this platform and has therefore been preserved. A different start method may be specified using the -start-method option.");
    }
  } else {
    # we don't have a start method at all yet, so use the default method
    _debug("Setting start method to $default_method for option $option");
    $self->start_method($default_method);
  }

  1;
}

=head3 mode

  $mode=$ppe->mode();

Gets and sets the current PPE mode of the object.  This is based on
the CCL setting '/SIMULATION CONTROL/EXECUTION CONTROL/SOLVER STEP CONTROL/PARALLEL
ENVIRONMENT/Parallel Mode', or the C<CFX5_PARALLEL_MODE> config
record.

=cut

sub mode {
  my $self=shift;
  my $cclname=$self->root() . '/Parallel Mode';

  internal_error("XXX Shouldn't call mode any more");

  # set?
  if (@_) {
    my $v=shift;
    internal_error("Too many arguments in call to", (caller(0))[3])
      if @_;
    if (defined $v) {
      $v=uc($v);
      if ($v eq 'PVM' or $v eq 'MPI' or $v eq 'COMMAND') {
        $self->set_parm($cclname,$v);
      } else {
        _error("Invalid Parallel Mode $v");
      }
    } else {
      _error("Invalid Parallel Mode $v");
    }
  }

  my $v=$self->get_parm($cclname);

  # need to set or update default?
  if (!defined $v or $self->is_default($cclname)) {
    $v=$Config{CFX5_PARALLEL_MODE} || 'PVM';
    $v=uc($v);
    $self->set_default($cclname,$v);
  }

  $v=uc($v);
  return $v;
}

=head3 start_method

  $sm=$ppe->start_method();

Gets and sets the current start method used by the object.  This is
based on the CCL setting '/SIMULATION CONTROL/EXECUTION CONTROL/SOLVER STEP
CONTROL/PARALLEL ENVIRONMENT/Start Method', or the
C<CFX5_START_METHOD> config record.

The return value is a CFX5::Parallel::StartMethod object representing
the active start method.

=cut

sub start_method {
  my $self=shift;
  my $cclname=$self->root() . '/Start Method';

  # set?
  if (@_) {
    my $v=shift;
    internal_error("Too many arguments in call to", (caller(0))[3])
      if @_;
    if (defined $v) {
      # Legacy start methods: Use compatible start method
      # In CCL such changes should be applied by the CCL upgrade, but start methods 
      # specified from the command line aren't seen by the CCL upgrade.
      if ($v =~ s/(?:Platform|HP) MPI ((?:Local|Distributed)) (Parallel[\w\s]*)/IBM MPI $1 $2/) {
        _debug("Changing legacy start method to: $v");
      }
      if ($v =~ s/Submit to Windows CCS or HPC Queue/Submit to Windows HPC Queue/) {
        _debug("Changing legacy start method to: $v");
      }
      $self->set_parm($cclname,$v);
    } else {
      _error("Invalid Start Method $v");
    }
  }

  my $v=$self->get_parm($cclname);

  # need to set or update default?
  if (!defined $v or $self->is_default($cclname)) {
    if ($Config{CFX5_START_METHOD}) {
      $v=$Config{CFX5_START_METHOD};
    } else {
      $v='Serial';
    }
    $self->set_default($cclname,$v);
  }

  # we actually return an object to represent it, not the text
  # setting directly.  We cache these, so their state persists.
  unless ($self->{start_method_cache}{$v}) {
    cfx5_require('CFX5::Parallel::StartMethod')
      or return undef;
    $self->{start_method_cache}{$v}=CFX5::Parallel::StartMethod->new($self->get_ccldb_obj(), $v);
  }

  return $self->{start_method_cache}{$v};
}

=head3 mode_is

  if ($ppe->mode_is('pvm')) {
    # ...
  }

Because of the confusion between different capitalisations of PVM and
MPI, this is a simple call which just returns TRUE or FALSE if modes
match, regardless of case.

=cut

sub mode_is {
  my $self=shift;
  lc($self->mode()) eq lc($_[0]);
}

=head3 option

  $ppe->option('Local Parallel');

Sets the parallel option.  'Serial' is allowed here, although many of
the other routines may not work if this is set.

=cut

sub option {
  my $self=shift;
  my $cclname=$self->root() . '/Option';

  internal_error("Don't call option");

  # set?
  if (@_) {
    my $v=shift;
    internal_error("Too many arguments in call to", (caller(0))[3])
      if @_;

    _debug("Setting parallel option to $v");

    if (defined $v) {
      my %ok=(
              'Serial' => 1,
              'Local Parallel' => 1,
              'Distributed Parallel' => 1,
             );
      if ($ok{$v}) {
        $self->set_parm($cclname,$v);
      } else {
        _error("Invalid Parallel Option $v");
      }
    } else {
      _error("Invalid Parallel Option $v");
    }
  }

  my $v=$self->get_parm($cclname);

  # need to set or update default?
  if (!defined $v or $self->is_default($cclname)) {
    $v='Serial';
    $self->set_default($cclname,$v);
  }

  return $v;
}

=head3 explicit_host_list

  if ($ppe->explicit_host_list()) {
    ...
  }

Returns TRUE if the user has supplied (or should supply) a host list,
or FALSE if they just supplied a number of partitions.  

Currently, this is triggered by the option(), but this should probably
change. 

=cut

sub explicit_host_list {
  my $self=shift;

  if (@_) {
    internal_error("Cannot set this parameter");
  }

  #if ($self->option() eq 'Distributed Parallel') {
  if ($self->start_method()->use_host_list()) {
    return 1;
  } else {
    return '';
  }
}

=head3 workdir

  $ppe->workdir($workdir);

Allows you to specify a L<CFX5::WorkDir> object to use if creating any
temporary files.  This must be specified if any temporary files are to
be created, for example the MPI mode.

=cut

sub workdir {
  my $self=shift;
  if (@_) {
    $self->{workdir}=shift;
  }
  $self->{workdir};
}

=head2 HOST LIST MANIPULATION

A set of calls for manipulating host lists, using
L<CFX5::Parallel::Host> objects and text-based lists.  See L</LOCAL
PARALLEL SUPPORT> below for details on the support of Local Parallel
runs, which do not use a host list.

Text-based lists may be specified in the following formats:

  host1,host2,host1
  host1*2,host2

and may be comma- or space- or both-separated.  Order is important.

The normal method of use is for the host definitions to be read from a
hst file, CCL, or the CCL libraries.  If read from a hst file, this
also sets the initial host list.  The object will contain definitions
for all known hosts (for example, from the CFX central hosts file, or
if a hst file is read but not all hosts are used), but this may be
filtered by the currently-set host list.

There is also a set of routines in here for compatibility reasons.
The issue here is that the old interface would use a list of hosts to
run on, and a separate count of the number of partitions to use,
whereas the new interface allows you to directly specify the number of
partitions for each host in the list.  We therefore have two methods
of accessing the host list, and keep a flag internally saying which is
currently canonical - the get routines will switch between the two and
derive whichever has not been set.  This should be removed in a future
release once it is no longer necessary.

While the CCL database is used to get defaults for these settings, the
information is not actually stored in the database until one of the
query routines are used, as it is assumed that by then the settings
have stabilised.

=head3 host_list_text

  $ppe->host_list_text("host1 host2");
  print $ppe->host_list_text();

Sets or gets the text list of hosts.  This corresponds to the
'/SIMULATION CONTROL/EXECUTION CONTROL/SOLVER STEP CONTROL/PARALLEL ENVIRONMENT/Parallel
Host List' parameter in the CCL.

=cut

sub host_list_text {
  my $self=shift;
  my $cclname=$self->root() . '/Parallel Host List';

  if (@_) {
    my $new=shift;
    if ($self->use_count()) {
      # We are currently in old mode, where the count is explicit, so
      # we need to change the mode.  No data transformation is
      # necessary, as setting this will set all the information.
      $self->use_count(0);
    }
    $self->{host_list}=$new;
    if ($self->{ccl_active}) {
      $self->set_parm($cclname,$new);
    }
    _debug("host_list_text setting to $new");
  }

  if ($self->use_count()) {
    # the count is significant, so we may need to pad out the current
    # (old-style) host list.  This will set the relevant parameter in
    # the CCL.
    $self->pad_host_list_old();
  }

  my $v = $self->{host_list} || $self->get_parm($cclname);

  if ($self->{ccl_active} and $self->explicit_host_list()
     and defined $v) {
    # make sure there's something in the CCL
    $self->set_parm($cclname, $v);
  }

  if (defined $v) {
    _debug("Host list is $v");
  } else {
    _debug("Host list undefined!");
  }

  return $v;
}

sub split_host_list {
  my $self=shift;

  # split up and expand the host list into a list of names, with
  # multipliers expanded out.
  my @hl;
  my $list=shift;
  return () unless defined $list;
  for my $hn (split(/[,\s]+/,$list)) {
    if ($hn =~ m!^\s*(\S+)\s*\*\s*(\d+)\s*$!) {
      # host*num
      my $host=$1;
      my $num=$2;
      push @hl, ($host) x $num;
    } else {
      push @hl, $hn;
    }
  }
  @hl;
}

sub pad_host_list_old {
  my $self=shift;
  my @hl=$self->split_host_list($self->host_list_text_old());
  my $c=$self->host_count_old() || 0;
  if (@hl == $c) {
    # we're OK
    1;
  } elsif (@hl > $c) {
    # erm ... we need to remove some hosts!  We don't warn, as
    # depending on the order in which command-line arguments are
    # passed, we often end up doing this translation with a zero
    # host-count which will be filled in later.
    #warn "Removing hosts from host list during translation";
    @hl=@hl[0 .. ($c-1)];
  } elsif (!@hl) {
    # no hosts yet ... so I guess we ignore it!
    _debug("pad_host_list_old called with no hosts, ignoring");
  } else {
    # need to add some more.  This I can cope with!  We just increase
    # in a round-robin fashion until we have enough.
    my @from=@hl;
    my $count=$self->host_count_old();
    while (@hl < $count) {
      my $h=shift @from;
      push @hl, $h;
      push @from, $h;
    }
  }

  if ($self->use_count() and @hl) {
    # if we're in old mode (which is the only time it's sensible to
    # call this function, really), store this answer back into the
    # right place.
    $self->{host_list} = join(', ', @hl);

    _debug("pad_host_list_old in use_count mode: host list is $self->{host_list}");
  }

  @hl;
}

=head3 host_count

  my $parts=$ppe->host_count();

This routine returns the number of hosts in the current standard-style
parallel host list.  It cannot be used to set this data.

=cut

sub host_count {
  my $self=shift;

  if (@_) {
    internal_error("Cannot set the host count directly!");
  }

  my $l=$self->host_list_text();
  if (defined $l) {
    my @hl=$self->split_host_list($l);
    _debug("returning hosts counted from $l");
    return scalar(@hl);
  }
  _debug("host count undefined");
  undef;
}

=head3 host_list_text_old

  $ppe->host_list_text_old("host3,host4");
  my $l=$ppe->host_list_text_old();

This routine manipulates the old-style host list.

=cut

sub host_list_text_old {
  my $self=shift;

  if (@_) {
    my $new=shift;
    unless ($self->use_count()) {
      # We are currently in standard mode, where the count is
      # implicit, so we need to store the count as canonical and
      # change the mode.
      $self->{old_count}=$self->host_count();
      $self->use_count(1);
    }
    $self->{old_host_list}=$new;
    # we need to make sure the CCL is kept up-to-date, even though we
    # don't use the result of this.
    $self->host_list_text();
  }

  if ($self->use_count()) {
    # old mode
    return $self->{old_host_list};
  } else {
    # because the count is irrelevant, we can just return this list
    # as-is, I think.  We maybe need to uniqify it first.
    return $self->host_list_text();
  }
}

=head3 host_count_old

  $ppe->host_count_old(5);
  my $parts=$ppe->host_count_old();

This routine manipulates the old-style host count.  If it is used
before any information is available to set it from the other settings,
it will look in the CCL for an input partition file and use the
information from that.

=cut

sub host_count_old {
  my $self=shift;

  if (@_) {
    my $new=shift;
    unless ($self->use_count()) {
      # We are currently in standard mode, where the count is
      # implicit, so we need to store the current host list as
      # canonical and set the mode.
      $self->{old_host_list}=$self->host_list_text();
      $self->use_count(1);
    }
    $self->{old_count}=$new;
    # we need to make sure the CCL is kept up-to-date, even though we
    # don't use the result of this.
    $self->host_list_text();
    # and finally, in case we are/end up in local mode, set the
    # internal local process count
    $self->local_processes($new);
  }

  my $rv;
  if ($self->use_count()) {
    # old mode
    $rv=$self->{old_count};
  } else {
    # standard mode
    $rv=$self->host_count();
  }
  return $rv if defined $rv;

  # failure: before we die completely, let's see if we have an input
  # partition file we can interrogate.  If we do, and we're in old
  # mode already, that's fine.  If we're in standard mode, actually
  # setting it will re-set us to old mode, which will change the host
  # list to fit and therefore the error reporting from the solver
  # script; XXX we should be able to return it as a guess, though, at
  # least?
  return undef unless $self->use_count();

  my $parfile_in=$self->get_parm('/SIMULATION CONTROL/EXECUTION CONTROL/RUN DEFINITION/Initial Partition File');
  if (defined $parfile_in) {
    my $par=CFX5::Files::Partition->new($parfile_in);
    if (defined $par) {
      $rv=$par->num_parts();
    }
  }

  if (defined $rv) {
    $self->host_count_old($rv);
  }

  $rv;
}

=head3 use_count

I<internal>

Sets or gets the current compatibility mode.

=cut

sub use_count {
  my $self=shift;
  if (@_) {
    my $new=shift;
    $self->{use_count}=$new;
  }
  $self->{use_count};
}

=head2 LOCAL PARALLEL SUPPORT

The routines in this section are used to support the "Local Parallel"
mode of ANSYS CFX, which runs only on the local host.  This is a "standard
style" mode, but does not have a host name list, and uses a partition
count.

=head3 local_processes

  print "Using ", $ppe->local_processes(), ' processes';

  $ppe->local_processes(3);

Sets and gets the number of processes to use in local mode.  This
corresponds to the "Number of Processes" parameter in CCL.  If it is
read before being set, and the C<host_count_old> call has been used to
set an old-style host count, that will be used as a default (and placed
into the CCL).

=cut

sub local_processes {
  my $self=shift;
  my $cclname=$self->root() . '/Number of Processes';

  if (@_) {
    my $new=shift;
    _debug("Setting local process count to $new");
    $self->{local_procs}=$new;
    if ($self->{ccl_active}) {
      $self->set_parm($cclname, $new);
    }
  }

  my $v = defined $self->{local_procs}
    ? $self->{local_procs}
      : $self->get_parm($cclname);

  if (!defined $v or $self->is_default($cclname)) {
    my $newv=$self->host_count_old();

    # if that didn't help, let's see if we can figure it out from
    # elsewhere in the problem setup
    unless (defined $newv) {
      my $parfile_in=$self->get_parm('/SIMULATION CONTROL/EXECUTION CONTROL/RUN DEFINITION/Initial Partition File');
      if (defined $parfile_in and -e $parfile_in) {
        my $par=CFX5::Files::Partition->new($parfile_in);
        if (defined $par) {
          $newv=$par->num_parts();
          _debug("Got part count $newv from partition file, CCL is ", 
                 ($self->{ccl_active} ? () : "not"), "active");
        }
      }
    }

    # if we came up with a new value, by whatever means, we'll use
    # that.  Otherwise, we'll stick with the previous default, as we
    # may now be in a different directory and unable to access the
    # partition file, for example.
    if (defined $newv) {
      # re-insert this into the CCL if it's active
      if ($self->{ccl_active}) {
        $self->set_default($cclname, $newv);
      }

      $v=$newv;
    }

    # if all else fails, make sure we at least return something useful
    $v=1 unless defined $v;

  } elsif ($self->{ccl_active}) {
    # make sure there's something in the CCL
    $self->set_parm($cclname, $v);
  }

  _debug("Local process count is", (defined $v ? $v : '(undef)'));

  $v;
}

=head3 local_host

  my $host=$ppe->local_host();

Returns a host object for the local host.

=cut

sub local_host {
  my $self=shift;
  return $self->host($Config{CFX_HOST});
}

=head2 HOST LIST QUERYING

The calls in this section use the parallel mode, parallel option, and
host list calls to return programmer-friendly representations of the
data.

They also make sure that the values in the CCL are up-to-date, and
mark the CCL as active so that the values are kept up-to-date from now
on.

=head3 partitions

Returns a host object for each partition in the run, in order.
Objects for hosts with multiple partitions are returned multiple
times.

In a scalar context, returns the number of partitions.

=cut

sub partitions {
  my $self=shift;

  if (@_) {
    internal_error("Cannot set the partition list directly!");
  }

  # mark the CCL as active
  $self->{ccl_active}=1;

  # if we're not in distributed parallel mode, it's just the count we
  # have.  This includes both Local Parallel and Serial (for the
  # partitioning run).
  if (!$self->explicit_host_list()) {
    if (wantarray) {
      return ($self->master_host()) x $self->local_processes();
    } else {
      return $self->local_processes();
    }
  }

  # distributed parallel
  my @hl=$self->split_host_list($self->host_list_text());

  # Find host assignment specified in start method
  my $sm = $self->start_method();
  my $ha = $sm->host_assignment();
  
  if ($ha eq 'Block') {
    # Block assignment (sort the host list)
    @hl = sort(@hl);
  } elsif ($ha eq 'Round Robin') {
    # Round robin assignment
    my $count = scalar(@hl);
    @hl = ();
    my @hostnames;
    my @hosts = $self->hosts();
    for my $h (@hosts) {
      if ($self->is_master($h)) {
        unshift @hostnames, $h->name();
      } else {
        push @hostnames, $h->name();
      }
    }
    while (@hl < $count) {
      my $h = shift @hostnames;
      push @hl, $h;
      push @hostnames, $h;
    }
  }

  unless (wantarray) {
    # we only need to count the hosts
    return scalar(@hl);
  }

  my @rv;
  for my $hn (@hl) {
    push @rv, $self->host($hn);
  }

  @rv;
}

=head3 hosts

  for my $h ($ppe->hosts()) {
    # ...
  }

  print "There are ", scalar($ppe->hosts()), " hosts in the list.\n";

Reads the current host list, and returns the L<CFX5::Parallel::Host>
objects for the currently-assigned list of hosts.  Objects for hosts
which appear multiple times will B<not> be repeated.

If called in a scalar context, returns the number of distinct hosts in
the current host list.

=cut

sub hosts {
  my $self=shift;

  if (@_) {
    internal_error("Cannot set the partition list directly!");
  }

  # mark the CCL as active
  $self->{ccl_active}=1;

  if (!$self->explicit_host_list()) {
    # make sure this is in the CCL for consistency with the
    # Distributed option
    $self->local_processes();
    if (wantarray) {
      return $self->master_host();
    } else {
      return 1;
    }
  }

  my @hl=$self->split_host_list($self->host_list_text());

  {
    my %seen;
    my @newhl;
    for my $hn (@hl) {
      push @newhl, $hn unless $seen{$hn}++;
    }
    @hl=@newhl;
  }

  unless (wantarray) {
    # we only need to count the hosts
    return scalar(@hl);
  }

  my @rv;
  for my $hn (@hl) {
    push @rv, $self->host($hn);
  }

  @rv;
}

=head3 consolidated_partitions

  my @ch=$ppe->consolidated_partitions();
  for my $p (@ch) {
    print $p->[0], " partitions on host ", $p->[1]->descname(), "\n";
  }

Returns a list of listrefs, containing pairs of C<[I<partition-count>,
I<host-object>]>, made by consolidating the host list into as few
entries as possible.  If the same host appears multiple times in the
list with other hosts between, the entries will not be consolidated,
in order to preserve the ordering of any non-uniform partition
sizes. For example, we have the following mappings from the CCL:

   hydra*3                             [3, hydra]
   hydra,hydra,helios                  [2, hydra], [1, helios]
   hydra,helios,hydra                  [1, hydra], [1, helios], [1, hydra]

This function will use C<internal_error> to abort if called in a
scalar context.

=cut

sub consolidated_partitions {
  my $self=shift;

  if (@_) {
    internal_error("Cannot set the partition list directly!");
  }

  # guard against this, it currently (by definition) makes no sense.
  unless (wantarray) {
    internal_error("consolidated_partitions called in non-array context");
  }

  # mark the CCL as active
  $self->{ccl_active}=1;

  # if we're not in distributed parallel mode, it's just the count we
  # have.  This includes both Local Parallel and Serial (for the
  # partitioning run).
  if (!$self->explicit_host_list()) {
    return [$self->local_processes(), $self->master_host()];
  }

  # distributed parallel - get the full partition list, and
  # consolidate
  my @hl=$self->partitions();
  my @rv;

  for my $h (@hl) {
    # get the host object for the last host in the current list, if
    # any
    my $last;
    if (defined $rv[$#rv]) {
      $last=$rv[$#rv]->[1];
    }

    if (defined $last and $h->same_as($last)) {
      # same host, increment the count
      $rv[$#rv]->[0]++;
    } else {
      # different host, make a new record
      push @rv, [1, $h];
    }
  }

  @rv;
}

=head3 is_master

  if ($ppe->is_master($host)) {
    ...
  }

Returns TRUE if the supplied host is the master for this run, or FALSE
if not.

=cut

sub is_master {
  my $self=shift;
  my $host=shift;

  my $master=$self->master_host();

  if ($host->same_as($master)) {
    return 1;
  } else {
    return '';
  }
}

=head3 master_host

Returns the CFX5::Parallel::Host object for the master host in the
run.

=cut

sub master_host {
  my $self=shift;
  my $cclname=$self->root() . '/Master Host';

  # set?
  if (@_) {
    my $v=shift;
    internal_error("Too many arguments in call to", (caller(0))[3])
      if @_;
    if (defined $v) {
      $self->set_parm($cclname,$v);
    } else {
      _error("Invalid Master Host $v");
    }
  }

  my $v=$self->get_parm($cclname);

  # there's no default for this parameter -- if it's not specified,
  # we'll use the local host, but we don't record it so a run on a
  # different host will use the new local host
  if (!defined $v or $self->is_default($cclname)) {
    return $self->local_host();
  } else {
    return $self->host($v);
  }
}

=head3 partitions_master_first

  my @ch=$ppe->partitions_master_first(@weights);
  for my $p (@ch) {
    print $p->[0], " partitions on host ", $p->[1]->descname(), "\n";
  }

Returns a list of listrefs, containing pairs of C<[I<partition-count>,
I<host-object>]>, in the same way as C<consolidated_partitions>.  In
addition, it will locate the master host in the list, and ensure that
at least one partition on this host is the first in the list.

Exactly how this is done is still an open question, because of the use
of non-uniform partitionings.  Ideally, we should re-order the
partitions too, but this is Hard.  For now, in order to retain optimal
performance under MPI in the common (uniform) case, we will simply
exchange the record for the master host with that for the first in the
list.

If an argument is supplied, it is expected to be a reference to an
array of information, one entry per partition (not per host!), and it
will be reordered to reflect the new ordering of partitions.

If the master (ie, current) host cannot be found in the list, the
function will use _error() to signal an error, but still return a
valid list.

This function will use C<internal_error> to abort if called in a
scalar context.

=cut

sub partitions_master_first {
  my $self=shift;
  my $reorder=shift;

  if (@_) {
    internal_error("Cannot set the partition list directly!");
  }

  # guard against this, it currently (by definition) makes no sense.
  unless (wantarray) {
    internal_error("partitions_master_first called in non-array context");
  }

  # mark the CCL as active
  $self->{ccl_active}=1;

  # if we're not in distributed parallel mode, it's just the count we
  # have.  This includes both Local Parallel and Serial (for the
  # partitioning run).  The master *is* the local host, and therefore
  # is first.
  if (!$self->explicit_host_list()) {
    return [$self->local_processes(), $self->master_host()];
  }

  my @hl=$self->consolidated_partitions();

  if ($self->is_master($hl[0]->[1])) {
    # this is correct already
    return @hl;
  }

  # find the master host in the list
  my $pos=0;
  for (my $i=0; $i < @hl; $i++) {
    if ($self->is_master($hl[$i]->[1])) {
      # this is It (or at least one occurrence, which will do).  Swap.
      my $tmp=$hl[0];
      $hl[0]=$hl[$i];
      $hl[$i]=$tmp;

      # do we need to reorder any other lists?
      if (defined $reorder) {
        my $num=$hl[0]->[0]; # number of partitions on the master (now first)
        my $num2=$hl[$i]->[0]; # number of partitions for which we're
                               # swapping them, which are currently at
                               # the beginning

        # chop out $num elements from $pos, and put them at the
        # beginning, replacing the existing $num2 elements which move
        # back to where the original lot came from.
        my @tmp=splice(@$reorder,$pos,$num);
        my @tmp2=splice(@$reorder, 0, $num2, @tmp);
        splice(@$reorder, $pos-$num2+$num, 0, @tmp2);
      }

      return @hl;
    }
    # keep track of the number of partitions we've passed
    $pos += $hl[$i]->[0];
  }

  # if we get here, we've failed
  my $config = $self->config_object();
  if ( !$config || !$config->parent()->check_for_gui() ) {
    _error("Master host could not be found in host list");
  }

  # but return something useful anyway
  @hl;
}

=head3 put_master_first

  $ppe->put_master_first(\@weights);

Sets the host list up such that the master host is first, and writes
the new information back to CCL.  If the optional argument is passed,
it is passed to C<partitions_master_first>.

=cut

sub put_master_first {
  my $self=shift;
  my @parts=$self->partitions_master_first(@_);
  my @txt;
  for my $p (@parts) {
    if ($p->[0] == 1) {
      push @txt, $p->[1]->name();
    } else {
      push @txt, $p->[1]->name() . '*' . $p->[0];
    }
  }
  $self->host_list_text(join(',',@txt));
}

=head3 host

  my $h=$ppe->host('XENON');

Returns the L<CFX5::Parallel::Host> object corresponding to the named
host.  Host names ARE case-sensitive, as the information is stored in
CCL.  If the object does not yet exist, it is created.

=cut

sub host {
  my $self=shift;
  my $name=shift;

  unless (exists $self->{hostdb}{$name}) {
    $self->{hostdb}{$name}=CFX5::Parallel::Host
      ->new($self->get_ccldb_obj(), $name);
  }

  return $self->{hostdb}{$name};
}

=head3 all_known_hosts

  for my $h ($ppe->all_known_hosts()) {
    # ...
  }

Returns the host names of all hosts known in the hostinfo.ccl file, NOT
objects.

=cut

sub all_known_hosts {
  my $self=shift;

  if (@_) {
    internal_error("Cannot set the known hosts list directly!");
  }

  my $haref;
  my $rv = $self->get_ccldb_obj()->ccl_api_obj()->getChildInfoList('/SIMULATION CONTROL/EXECUTION CONTROL/PARALLEL HOST LIBRARY', \$haref);
  if ($rv) {
    _error("Error retrieving full host information list:", CFX5::CCLI::format_ccl_error($rv));
    return ();
  }

  my @rv;
  for my $hn (@$haref) {
    next unless $hn->{type} eq 'HOST DEFINITION';
    push @rv, $hn->{name};
  }

  @rv;
}

=head3 all_valid_hosts

  for my $h ($ppe->all_valid_hosts()) {
    # ...
  }

Returns the host names of all valid hosts in the ccl (can be pinged), NOT
objects. Returns all known hosts if remote checks are disabled.

=cut

sub all_valid_hosts {
  my $self=shift;

  return $self->all_known_hosts() if $Config{CFX_SOLVE_DISABLE_REMOTE_CHECKS};

  my @known_hosts=$self->hosts();
  my @valid_hosts;
  my $p = Net::Ping->new();
  foreach my $i (@known_hosts){
    if($p->ping($i->hostname())){
      push @valid_hosts, $i->name();
    } else {
      _warn "Could not ping host ". $i->hostname() . ", removing from parallel environment";
    } 
  }
  $p->close();
  @valid_hosts;
}

=head2 PPE Object Manipulation

The functions in this section start and stop the PVM daemons on the
lists of hosts, in the same way as the shell scripts F<cfx5pvmstart>
and F<cfx5pvmstop>.  They work on an object (of type CFX5::Parallel),
obtained by the C<new> call.

=head3 mark_hosts

  $ppe->mark_hosts();

This should be called when the host list is finalised.  It ensures
that the hosts included in the run are marked as user CCL in the
database, so they will be included in the output CCL.

=cut

sub mark_hosts {
  my $self=shift;

  my @hosts=$self->hosts();

  for my $h (@hosts) {
    $h->mark_non_library();
  }

  1;
}

=head3 setup

  $ppe->setup();
  exit_if_error();

Set the PVM environment variables, and make sure things are in place.
This should be called before C<show_vars_by_verbosity>; see the
C<setup> function in L<CFX5::Parallel::Pvm> (which this calls) for
more details.

The return value is TRUE on success or FALSE on failure.

=cut

sub setup {
  my $self=shift;

  unless ($self->{checked_userid}) {
    if (length($Config{CFX_USER}) > 15) {
      if ($Config{CFX5_NOCHECK_PPEID}) {
        _note("User ID $Config{CFX_USER} contains more than 15 characters, but CFX5_NOCHECK_PPEID is set, so continuing.");
      } else {
        _warn("Your user ID, $Config{CFX_USER}, appears to be longer than 15 characters.  This will cause rsh, and therefore the ANSYS CFX parallel code, to fail on many platforms.  Attempting to continue anyway, but if the parallel code fails to start correctly this may be the cause.

If it works, you can suppress this message by setting the variable CFX5_NOCHECK_PPEID, either in your cfx5rc file or the environment.");
      }
    }
    $self->{checked_userid}=1;
  }

  if ($self->start_method()->do_pvm_custom()) {
    return CFX5::Parallel::Pvm::setup();
  } else {
    return 1;
  }
}

=head3 setup_req

  $ppe->setup_req('pvmhosts') or die;

Instantiates the named prerequisite, if necessary, and calls its setup
function.  The return value will be the object representing the
prerequisite, or FALSE on failure.

If the call is repeated, no new object will be created, but the setup
call will be re-made.

=cut

sub setup_req {
  my $self=shift;
  my $orig_req=shift;

  # map to a standard package name
  my $req=lc($orig_req);
  $req =~ s![^a-z0-9_]!_!g;
  $req =~ s!_+!_!g;
  $req =~ s!^_|_$!!g;

  #packageall CFX5::Parallel::Reqs
  my $pkg="CFX5::Parallel::Reqs::$req";

  unless ($self->{reqs}{$pkg}) {
    _trace "About to invoke $pkg";

    eval "require $pkg;";
    if ($@) {
      (my $msg=$@) =~ s!^! !mg;
      _error("Error loading package $pkg for parallel prerequisite $orig_req.  The error given was:\n\n$msg");
      return '';
    }

    $self->{reqs}{$pkg}=$pkg->new($self);
  }

  # we need a working directory
  unless (defined $self->workdir()) {
    internal_error("setup_req called with no working directory");
  }

  my $obj=$self->{reqs}{$pkg};
  $obj->setup($self->workdir())
    or return undef;

  $obj;
}

=head3 local_check

  $ppe->local_check();

Does some quick local checks to make sure everything looks OK.

=cut

sub local_check {
  my $self=shift;

  my @hl=$self->hosts();

  # find the master host in the list
  my $found_master='';
  for my $h (@hl) {
    if ($self->is_master($h)) {
      $found_master++;
    }
  }
  unless ($found_master) {
    my $msg = "Unable to find the master host\n".$self->master_host()->descname()."\nin the host list: at least one partition must be assigned to the master host.";
    _debug("Host list is", map { $_->descname() } @hl);
    my $config = $self->config_object();
    if ( defined $config && $config->parent()->check_for_gui() ) {
      # Continue to check CCL if this is just extracting the CCL for the
      # SM to display
      # Only warn if the warning hasn't already been shown
      if ( $self->{showMissingMasterWarning} ) {
        _error($msg);
        reset_error_indicator();
        $self->{showMissingMasterWarning} = 0;
      }
      return 1;
    } else {
      _error($msg);
      return 0;
    }
  }

  # check local IP addresses of the master host, if we've got more
  # than one host in the list, and we're in PVM mode (MPI doesn't
  # care)
  if (@hl > 1 and $self->start_method()->need_master_ip_check()) {
    my $ip=CFX5::HostAddr->get_local_obj()->ip();
    my $ok;
    if (defined $ip) {
      if (join('.',unpack('C4',$ip)) eq '127.0.0.1') {
        $ok=0;
      } else {
        $ok=1;
      }
    }
    if (defined $ok and !$ok) {
      _error("The IP address of the master host \"$Config{CFX_HOST}\", according to the gethostbyname() system call, is the local loopback address 127.0.0.1.  This may prevent distributed parallel runs from working.  The IP address reported for this host should be its externally-visible network IP address, and can be checked using Show>System in the CFX $Config{CFXREVISION} Launcher or the command\n\n  cfx5info -host-addr $Config{CFX_HOST}");
      return 0;
    }
  }

  1;
}

=head3 check_root

  $ppe->check_root();

Check installation root

=cut

use vars qw ($WarnRootDir);
$WarnRootDir = 0;

sub check_root {
  my $self = shift;

  unless ($WarnRootDir) {
    my $localhost = $self->local_host();
    my $root_local = $localhost->root();
    my $config = $self->config_object();
    my $outobj = $config->outobj();

    # Warn if the local host's installation root is non-default
    unless ($localhost->is_default_root()) {
      my $root_default = $Config{CFX5ROOT};
      $config->outobj()->message("The Installation Root parameter specified in the EXECUTION CONTROL Command Language object is inconsistent with the current default.\n\nSpecified: $root_local\n\nDefault:   $root_default\n\nExecutables from the specified installation root will be used.");
      $WarnRootDir = 1;
    }

    # For each host, check if root is different from root of local host
    for my $host ($self->hosts()) {
      my $root = $host->root();
      next if ($host->is_local() or
               CFX5::PathManip::is_same_dir($root,$root_local));
      my $name = $host->name();
      $config->outobj()->message("Host $name has different Installation Root: $root");
      $WarnRootDir = 1;
    }
  }
}

=head3 interrogate

  $ppe->interrogate();
  exit_if_error();

Get the details of all hosts in the run, but do not actually start
anything on them.

=cut

sub interrogate {
  my $self=shift;

  _trace("Interrogating hosts for PPE");

  # this makes sure the correct environment variables are set and
  # exported.
  $self->setup() or return 0;

  # in PVM mode, we want to build the PVMhosts file before starting
  # PVM, which means we need the architectures of at least the hosts
  # involved in the run.  In any case, it's useful to have them in the
  # CCL.  If we calculate them now they will not be re-calculated.
  {
    my $ok=1;

    my $lh = $self->local_host();

    for my $h ($self->hosts()) {

      # On Windows only and if remote checks are disabled, for any
      # remote hosts not specified in the hostinfo.ccl file
      # apply the same specifications as the local host.
      if ($lh->is_win32() and $h->disable_remote_checks() and !$h->is_local())
      {
        my @host_params = ('Remote User Name',
                           'Installation Root',
                           'Solver Executable',
                           'Relative Speed',
                           'Number of Processors',
                           'Host Architecture String');

        # Check if any parameters are set for the remote host
        my $host_set = 0;
        for my $cclname (@host_params)
        {
          my $v = $h->get_parm($cclname);
          unless (!defined $v or $h->is_default($cclname))
          {
            $host_set = 1;
          }
        }

        # If nothing is set for remote host, apply any parameters set for
        # the local host to the remote host also.
        unless ($host_set)
        {
          for my $cclname (@host_params)
          {
            my $v = $lh->get_parm($cclname);
            unless (!defined $v or $lh->is_default($cclname))
            {
              $h->set_parm($cclname,$v);
            }
          }
        }
      }

      # if we don't get an architecture, we want to stop once we've gone
      # through all the hosts.  detect_host_arch will signal an error.
      unless ($h->detect_host_arch()) {
        $ok='';
      }
      # we also make sure the executable is filled in, if it's going to
      # be
      $h->executable();
    }

    return $ok unless $ok;
  }

  # now we know the OSes, we can check they all work together too
  unless ($self->start_method()->os_checks($self)) {
    return '';
  }

  1;
}

=head3 start

  $ppe->start();
  exit_if_error();

Starts the PPE B<environment> on the desired host(s), assuming the
current host to be the master.  This does not start the actual run,
and not actually have any permanent effect for some environments.
Errors are signalled by C<_error>, and the return value is TRUE on
success or FALSE on failure.

=cut

sub start {
  my $self=shift;

  _trace("Starting PPE type", $self->start_method()->name());

  # this makes sure the correct environment variables are set and
  # exported.
  $self->setup() or return 0;

  # in PVM mode, we want to build the PVMhosts file before starting
  # PVM, which means we need the architectures of at least the hosts
  # involved in the run.  If we calculate them now they will not be
  # re-calculated.  They have probably already been done if we're
  # running under cfx5solve, but not necessarily if we're under
  # cfx5parallel.  It doesn't matter, anyway!
  if ($self->start_method()->do_pvm_custom()) {
    for my $h ($self->hosts()) {
      my $arch=$h->detect_host_arch();
      #_debug("Queried arch for", $h->name() . ":", $arch);
    }
    $self->setup_req('PVMhosts');
  }

  my (@ok_hosts, @bad_hosts);

  for my $h ($self->hosts()) {
    unless ($self->add_host($h)) {
      push @bad_hosts, $h;
    }
  }

  # this is for debugging only; we will re-write it when we come to
  # run any commands anyway, but if we only do it then it's never
  # around when we want to look at it!
  if (defined $self->workdir()) {
    #$self->setup_req('p4.pg');
    #$self->setup_req('phst');
    #$self->setup_req('hst');
  }

  if (scalar(@bad_hosts)) {
    my $errstr;
    $errstr='The parallel environment could not be started because of problems on the following hosts:'."\n  ";
    $errstr .= join(", ", map {$_->name()} @bad_hosts);
    #$errstr .= "\n\nThe preceding error messages should give more detail.\nTypical problems include:\n";
    #$errstr .= " - The specified host does not exist.\n";
    #$errstr .= " - You have no account on the specified host.\n";
    #$errstr .= " - PVM is already running on the specified host.\n";
    #$errstr .= " - The specified ANSYS CFX binary does not exist.\n";
    #$errstr .= " - The specified ANSYS CFX binary is not readable by the user\n";
    #$errstr .= "   running the \"rshd\" service (Windows NT only)";
    #_error($errstr);
    # we died, so shut everything down again cleanly.
    $self->stop();
    return 0;
  } else {
    return 1;
  }
}

=head3 add_host

  @probs = $ppe->add_host($h);

Adds a further host to the PVM environment. Should not normally be
needed.  Return value is TRUE on success or FALSE on failure; errors
are signalled with C<_error>.

=cut

sub add_host {
  my $self=shift;
  my $h=shift;

  _info("Adding host", $h->descname(), "(".$h->architecture().") to the parallel environment.");

  my @probs;
  # check for the solver executable
  my $rv;
  my $exec=$h->actual_executable($self->start_method());
  $rv=$h->check_for_executable($exec);
  unless ($rv) {
    if (defined $rv) {
      # no error was signalled by check_for_executable, because the
      # check worked and returned success or failure
      my $onhost='';
      unless ($h->is_local()) {
        $onhost=" on host " . $h->descname();
      }
      my $add='';
      if ($h->is_win32() and not $h->is_local()) {
        # this is not necessarily obvious at first try
        $add='  Check that the executable is readable by the user running the rsh service.';
      }
      _error("The solver executable $exec$onhost could not be found, or is not readable.$add\n");
    }
    return 0;
  } else {
    # OK, add it to the parallel environment
    #
    # first, give a warning message if rsh gives extra stuff
    $h->check_link() or return 0;
    # and now do the add, even if we warned.  (the add call will
    # signal its own errors if necessary)
    if ($self->start_method()->do_pvm_custom()) {
      unless (CFX5::Parallel::Pvm::add($h)) {
        return 0;
      }
    }
  }

  return 1;
}

=head3 do_cmd

  my $rv=$ppe->do_cmd(@cmd);
  unless ($rv->is_success()) { ... }

This call is used to run a command inside the parallel environment,
and return the results.  In some environments, such as PVM, this
simply runs the command directly.  In others, for example MPI, it may
add additional command line arguments, or even use a separate command
(eg C<mpirun>) to fire it off.

The return value is a L<CFX5::Commands::Result> object, as returned by
C<parse_rv>.

The command is run with C<do_termerr_cmd>.

=cut

sub do_cmd {
  my $self=shift;
  my @args=@_;

  my $sm=$self->start_method();

  _trace("Starting solver with start method", $sm->name());

  for my $req ($sm->prerequisites()) {
    $self->setup_req($req)
      or return CFX5::Commands::Result->new_not_run("Setup of $req failed");
  }

  my $master=$self->master_host();

  $sm->use_solver_args(@args);

  my $command=$sm->start_command();

  $command=$sm->fill_shared_template($command, $self);
  $command=$sm->fill_host_template($command, $self, 1, $master, 'this is the master');

  $sm->assert_fully_replaced($command)
    or return CFX5::Commands::Result->new_not_run("start command template error");

  if (defined (my $shlib=$sm->shared_library_path())) {
    _debug("Adding $shlib to the library path");
    cfx5_require('CFX5::Config');
    $shlib=$sm->fill_shared_template($shlib);
    $shlib=$sm->fill_host_template($shlib, $self, 1, $master, 'this is the master');
    $sm->assert_fully_replaced($shlib)
      or return CFX5::Commands::Result->new_not_run("start command template error");
    my @shlib_paths = CFX5::PathManip::path_to_list($shlib);
    for my $shlib_path (@shlib_paths) {
      CFX5::Config::prepend_to_libpath($shlib_path);
    }
  }

  if (defined (my $exepath=$sm->execution_path())) {
    _debug("Adding $exepath to the execution path");
    cfx5_require('CFX5::Config');
    $exepath=$sm->fill_shared_template($exepath);
    $exepath=$sm->fill_host_template($exepath, $self, 1, $master, 'this is the master');
    $sm->assert_fully_replaced($exepath)
      or return CFX5::Commands::Result->new_not_run("start command template error");
    CFX5::Config::prepend_to_execpath($exepath);
  }

  my @cmd=CFX5::Commands::split_cmd($command);
  if (not @cmd) {
    # the command contains shell metacharacters, or was otherwise too
    # difficult to split up into its constituent pieces: we'd better
    # pass it through the shell.
    @cmd = $command;

    # However, we do want to see if the command being run exists;
    # since this will prevent do_cmd from making the check, we'll do
    # it ourselves here.
    my $exec=CFX5::Commands::split_cmd_exec($command);
    if (defined $exec) {
      CFX5::Commands::check_for_executable($exec)
          or return CFX5::Commands::Result->new_not_run("Executable $exec not found");
    } else {
      # we can't even figure out what the leading executable is!
      _debug("Complicated shell command $command, executable not checked");
    }
  }

  # Option to reset shared library path
  if ($sm->reset_shared_library_path()) {
    unexport_conf($Config{ld_path_var});
  }

  _debug("Running parallel solver:", @cmd);

  # we used to use do_termerr_cmd.  Host->do_batch_cmd uses
  # do_output_cmd on remote hosts.  We don't worry about this too much,
  # because the solver (which is the most common use here) redirects
  # its own stdout anyway!
  my $rv=$master->do_batch_cmd(@cmd);

  # Restore shared library path
  if ($sm->reset_shared_library_path()) {
    export_conf($Config{ld_path_var});
  }  

  # check for additional output.  We don't get a reasonable error code
  # from remote jobs, so do it anyway.
  #unless ($rv->is_success()) {
    my @errs=$rv->term_error();
    push @errs, $sm->gather_errors();
    $rv->set_term_error(@errs);
  #}

  $rv;
}

=head3 stop

  $ppe->stop();

Stops the PVM daemons on the listed host(s). Errors are signalled by
C<_error>.

In previous versions, it was necessary to pass the host list into the
stop function.  This is no longer required, and any arguments will be
ignored for now.  New calls should not pass any arguments, though, and
this may become an error in the future.

Return value is TRUE on success or FALSE on failure, and _error
is used to report problems (although if this is used as part of a
tidying up, it probably shouldn't be followed by C<exit_if_error>).

This routine goes to a fair bit of trouble to avoid killing other
parallel jobs, but this does sometimes mean that it leaves some daemons
lying around unkilled. Not often, but every now and then.

=cut '

sub stop {
  my $self=shift;
  my @hosts=$self->hosts();

  # this makes sure the correct environment variables are set and
  # exported.
  $self->setup() or return 0;

  if ($self->start_method()->do_pvm_custom()) {
    return CFX5::Parallel::Pvm::shutdown_pvm(@hosts);
  }

  # otherwise, it shuts itself down
  return 1;
}

=head3 read_hst

  @hosts = read_hst("./hst");
  exit_if_error();

Reads a ANSYS CFX style F<hst> file, and returns an array of the host
names from the file.  The information about each host is set in the
CCL database.

Note that this function does honour the number of hosts variable in
the first line of the file, so a file may be contracted by simply
changing this number. (I seem to recall this being useful for
something, but I have no idea what!)

The return value is the empty list on failure, and an error will be
signalled with _error. If fewer hosts are present than indicated, an
error is signalled and those which have been so far found are
returned.

=cut

sub read_hst {
  my $self=shift;
  my $file=shift;

  return '' unless CFX5::FileTests::datafile($file, "ANSYS CFX hosts file $file");

  local *HSTFILE;

  unless (open HSTFILE, $file) {
    _error("Unable to open ANSYS CFX hosts file $file: $!");
    return '';
  }

  my $nh = <HSTFILE>;
  chomp $nh;
  $nh =~ s/\s*(\w+)\s*/$1/g;
  
  if (!defined $nh) {
    _error("Error reading number of hosts from $file");
    return '';
  }

  unless ($nh > 0) {
    _error("Syntax error in hosts file: expected number of hosts, got\n\t$nh");
    return '';
  }

  # this is the first time I've ever used a C-style FOR loop in Perl ...
  my $errstring;
  my @rv;
  for (my $h=0; $h < $nh; $h++) {
    $errstring = "Unexpected EOF in hosts file reading host ".($nh+1);

    my $name=<HSTFILE>;
    goto eof_err unless defined $name;
    my $exec=<HSTFILE>;
    goto eof_err unless defined $exec;
    my $speed=<HSTFILE>;
    goto eof_err unless defined $speed;
    my $procs=<HSTFILE>;
    goto eof_err unless defined $procs;

    # now we've got the values, trim them up nicely
    for ($name, $exec, $speed, $procs) { s/^\s*//; s/\s*$//; }

    # and create a host object with them
    my $ho=$self->host($name);
    $ho->executable($exec);
    $ho->speed($speed);
    $ho->processors($procs);

    # it may be useful to be able to tell which hosts we found.
    push @rv, $name;
  }

  close HSTFILE;
  
  return @rv;

 eof_err:
  _error($errstring);
  close HSTFILE;
  return @rv;
}

=head3 read_cfxhosts

This function does a similar thing to the C<read_hst> call above, but
it will read from the CFXhosts file (as used by the Solver Manager)
instead.  The first argument is the location of the CFXhosts file to
read, often C<$Config{CFX5_HOSTS_FILE}>.

This call will auto-detect whether the file is a CCL-based hosts file
or an old-style Solver Manager host file by looking at the first line.
If it contains three words, ends in an ampersand, does not start with
a hash sign, and the second and third words are numeric, it assumes it
to be an old-style one.  Otherwise, it reads it as CCL.

The host definitions thus read are marked as being library data in the
CCL, so they are by default not included when the database contents
are dumped out.  Those which are used should be explicitly marked as
such, so they are included in the dump.

=cut

sub read_cfxhosts {
  my $self=shift;
  my $file=shift;

  return '' unless CFX5::FileTests::datafile($file, "ANSYS CFX hosts file $file");

  local *HSTFILE;

  unless (open HSTFILE, $file) {
    _error("Unable to open ANSYS CFX hosts file $file: $!");
    return '';
  }

  my @cont=<HSTFILE>;

  close HSTFILE;

  # Return normally for empty files (e.g. NUL for HPC runs)
  return 1 unless (defined $cont[0]);

  if ($cont[0] =~ m!^\s*\S+\s+\d+\s+[\d.]+\s*\&[\r\n]*$!) {
    # old-style - convert to CCL and then parse as normal
    @cont=$self->cfxhosts_old_to_ccl($file,@cont);
  }

  # We read it in directly into the database
  my $rv=1;
  $self->parse_ccl_data(\@cont,1) or $rv='';
  # now we have to re-mark the intervening objects as user, otherwise
  # it all gets skipped. :-(
  $self->parse_ccl_data([
    'SIMULATION CONTROL:',
      'EXECUTION CONTROL:', 
        'PARALLEL HOST LIBRARY:',
        'END',
      'END',
    'END'
  ],0) or $rv='';

  $rv;
}

sub cfxhosts_old_to_ccl {
  my $self=shift;
  my $file=shift;
  my @cont=@_;
  my @rv=('SIMULATION CONTROL:','EXECUTION CONTROL:','PARALLEL HOST LIBRARY:');

  my $errstring;
  while (@cont) {
    my $line1=shift @cont;
    last unless defined $line1;

    $line1 =~ s!^\s+!!;
    my ($name,$procs,$speed,$cont)=split(/\s+/,$line1,4);

    my $line2=shift @cont;
    unless (defined $line2) {
      _error("Unexpected EOF while reading ANSYS CFX Hosts file $file, host $name");
      close HSTFILE;
      return ();
    }

    # we write CCL directly, as it's much faster.  Maybe ideally we
    # should go through the object route.
    push @rv, "  HOST DEFINITION: $name";
    if ($speed and $speed != 1.00) {
      push @rv, "    Relative Speed = $speed";
    }
    if ($procs and $procs != 1) {
      push @rv, "    Number of Processors = $procs";
    }

    # what do we do with this?
    $line2 =~ s!^\s+|[\s\n\r]+$!!g;
    my $exec=$line2;
    my $rel=$Config{CFXBUILDID_MAJOR_MINOR};
    if (substr($line2,0,length($Config{CFX5BINDIR}))
        eq $Config{CFX5BINDIR}) {
      # it points into our own bin directory.
      push @rv, "    Installation Root = $Config{CFX5ROOT}";
      $line2=substr($line2,length($Config{CFX5BINDIR})+1);
    } elsif ($line2 =~ s!^(.*)[/\\]bin[/\\]${rel}[/\\]!!o) {
      # it points into somebody else's bin directory
      push @rv, "    Installation Root = $1";
    } else {
      push @rv, "    Solver Executable = $line2";
      undef $line2;
    }

    if (defined $line2) {
      # we've got an executable name left
      if ($line2 =~ m!^[^/\\]+[\\/]solver(?:-pvm|-mpi)?.exe!) {
        # it's a normal one
        1;
      } else {
        # not sure I understand this, so I'd better write it down just
        # in case
        push @rv, "    Solver Executable = $line2";
      }
    }

    push @rv, "  END";
  }

  push @rv, 'END','END';

  return @rv;
}

=head2 DEBUGGING SUPPORT

Support for debugging from CFX5::Job.

=head3 log_environment

  $ppe->log_environment($dbg_obj);

Writes the details of the current parallel environment to the trace
file for the supplied object.  This is kind of similar to the
host-file writing code, inasmuch as it tries only to record the actual
settings.

=cut

sub log_environment {
  my $self=shift;
  my $dbg=shift;

  my @hosts=$self->partitions();

  $dbg->log('parallel-environment',
            $self->start_method()->name(),
            scalar(@hosts));

  for my $h (@hosts) {
    $dbg->log('ppe-partition',
              $h->name(),
              $h->actual_executable($self->start_method()),
             );
  }
}

1;
