#! perl -w
#==============================================================================
# @(#) $Id$
#------------------------------------------------------------------------------
##
## Name:
##
## cfx5dfile
##
## Purpose:
##
## Access CFX-5 Data Files (Definition, Results and Commands)
##
## Synopsis:
#S
#S cfx5dfile <file> [-read-monitor] [-read-cmds]
#S                  [-compare-cmds <file2>] [-diff-cmds <file2>] 
#S                  [-list-profile-files] [-read-profile-file <filename>]
#S                  [-output <file>]
#S                  [-help] [-verbose]
#S
## []  denotes an optional argument.
## |   separates mutually exclusive arguments. 
## ... following an argument indicates that it may be repeated.
## <>  denotes that substitution of a suitable value is required. All other
##     arguments are keywords, some of which have a short form.
##
## Arguments:
##
#A <file>
##    Names a CFX-5 data file to work with.  This may be a Definition
##    (".def") or Results (".res") file, or a text file containing a
##    Command Language listing (".ccl").
##
#A -compare-cmds <file2>
##    Compares the Command Language in <file> and <file2> and sends a
##    list of differences to the standard output, in a Command
##    Language-like format.  Parameters which have changed are
##    listed, with the old value, an arrow "->", and the new value.
##
#A -diff-cmds <file2>
##    This option works exactly as -compare-cmds, except that if
##    differences are found it sets the return value to 1; if no
##    differences are found, the return value is 0.  This makes it
##    useful for scripting purposes.
##
#A -extract-all-profile-files
##    Reads all profile data, and saves it to a set of files in the
##    current directory.  If any of the files exist locally they will
##    not be overwritten.
##
#A -extract-profile-file <filename>
##    Reads the profile data contained in the named file, and saves it
##    to a file of the same name in the current directory.  This is
##    equivalent to using -read-profile-file, and using -output to set
##    the destination file name.
##
#A -list-profile-files
##    Gives a list of the profile files stored in a results file,
##    which can be extracted with the -read-profile-file switch.
##
#A -output <file>
##    Rather than sending output to the standard output, save it to
##    the given file.
##
#A -read-cmds
##    Reads the CFX Command Language from the file, and sends it to
##    the standard output (but see -output).
##
#A -read-pre-state
##    Reads the CFX Command Language representation of the saved
##    CFX-Pre state from the file, which should normally be a .cfx
##    file from CFX-Pre, and sends it to the standard output.
##
#A -read-profile-file <filename>
##    Reads the profile data contained in the named file, and sends it
##    to the standard output, or to the file specified with the
##    -output switch.
##
#A -read-monitor
##    Reads the most recent set of monitor point and residual data
##    from the results file (which will include the combined results
##    from all previous runs), and sends it to the standard output.
##    If the file contains no monitor point data, nothing will be
##    displayed.
##
#A -read-parinfo
##    Reads the parallel job information from the results file and
##    sends it to standard output.
##
#A -read-pre-state
##    Reads the stored CFX-Pre state from the file, and prints it to
##    the standard output.
##
#A -read-resinfo
##    Reads the results information (history) dataset from the file, and
##    prints it to the standard output.
##
#A -read-trninfo
##    Reads the transient results information (history) dataset from
##    the file and prints it to the standard output.
##
#A -remove-exec-control
##   Removes the SIMULATION CONTROL/EXECUTION CONTROL CCL
##
#A -write-monitor <file> 
##    Inserts the monitor point data from the specified file into the
##    results file.  To concatenate multiple files, repeat the
##    -write-monitor switch.
##
##    Note that the data is written to the correct dataset for the
##    final timestep in the supplied data.  If the results file
##    already contains monitor data for a different time step, that
##    data will not be overwritten, but may become inaccessible.  If
##    the existing data is from a later timestep than the new file
##    supplied, the new file will be hidden by the existing data.
##
##    If cfx5dfile is used to modify a file, details of this action
##    are recorded to a change log, which is stored in the file.
##
#U ****************************************************************************
#U *                                                                          *
#U *     The following features are supported only within CFX and are not     *
#U *     shown by the -help option when the command is run elsewhere.         *
#U *                                                                          *
#U ****************************************************************************
#U
#U -delete-all-monitor
#U    Deletes ALL monitor data in the specified file. Use with caution.
#U
#U -list-ds
#U    Lists the datasets in the definition file.  If the -what, -where
#U    or -when switches are given, they are used to filter the output.
#U
#U -read-and-upgrade-cmds
#U    Reads the command language from the file, upgrades it, and then
#U    prints it to stdout.
#U
#U -read-and-pre-upgrade-cmds
#U    Reads the command language from the file, upgrades it using the
#U    updatePreCCL call as using internally by CFX-Pre, and then
#U    prints it to stdout.
#U
#U -read-runinfo
#U    Reads the G/RUNINFO dataset, which contains the final gui-err.txt contents
#U    from the file.  This data is written to stdout or to the specified output
#U    file.
#U
#U -show-changelog
#U    Show the log of all changes made by cfx5dfile.
#U
#U -validate
#U    Takes the command language from the input file, and validates it
#U    against the standard solver physics rules.
#U
#U -validate-against <rules>
#U    Takes the command language from the input file, and validates it
#U    against the rules file(s) passed on the command line.
#U
#U -what <what>
#U -where <where>
#U -when <when>
#U    Filters the datasets returned by the -list-ds switch.  Pass a
#U    value or a wildcard.
#U
#U -write-resinfo
#U    Replaces the results information dataset (G/RES_INFO) with the contents
#U    of the specified file.
#U
#U -write-trninfo
#U    Replaces the transient results information dataset (G/TRN_INFO) with
#U    the contents of the specified file.
#U
#U -cat-trninfo
#U    Concatenate transient results information (G/TRN_INFO) data from all
#U    results files in a transient sequence into the final file of the sequence.
#U    The specified file must be the final results file in the required sequence
#U    and all intermediate results files must be available for cfxdfile to read.
#U    To make use of this modification to the results file, the transient files that
#U    are referenced by the modified results file must be copied into the modified
#U    results file's transient directory.
#Y
#------------------------------------------------------------------------------

# get the standard CFX-5 library directory
BEGIN {
 if ($0 =~ m!^([a-z]:)?[\\/]!i) {
    # it's an absolute path, use it as-is
    (my $nd=$0) =~ s!(perllib[\\/])?[^\\/]*$!!i;
    unshift @INC, "$nd/perllib";
  } else {
    # wrap this in an eval, and if it fails, pray ... for example,
    # this can fail on Windows if we're running off a share.
    eval {
      require FindBin;
      FindBin->import();
      unshift @INC, "$FindBin::Bin/perllib";
      my $dummy=$FindBin::Bin; # kill used only once warning
    };
    warn $@ if $@;
  }
  if (defined $ENV{CFX_LOCAL_PERLLIB}) {
     unshift @INC, $ENV{CFX_LOCAL_PERLLIB};
  }

}

use CFX5::Init;
CFX5::Init::init();

use vars qw($CVSVersion);
sub BEGIN {
  $CVSVersion='@(#) $Id$\n';
}

use strict;

use CFX5::Output qw(:all);
use CFX5::Config qw(%Config export_conf query_conf append_to_libpath);
use CFX5::PrintConfig;
use CFX5::ResInfo;
use CFX5::TrnInfo;

use CFX5::CmdLine qw(:util parse_cmdline);

use CFX5::PathManip qw(join_file split_file fullpath basename dirname);
use CFX5::FileTests;
use CFX5::FileOps;

use CFX5::Commands qw(do_cmd handle_rv);

use CFX5::MonFile;
use CFX5::DefFile;

use CFX5::CCLDiff;
use CFX5::CCLI;

use CFX5::Simulation::Util qw(GetDataSet); 

use CFX5::ChangeLog;

use CFX5::PkgLoader qw(cfx5_require);

#
# read the configuration
#
CFX5::Config::read_config();

my ($filename,@mode,$compfile,$OutFile,$rules,$profile_file,$resinfo_file,$trninfo_file,@write_mon_files);
my ($what, $where, $when)=('*','*','*');
my $log = 0;
my $nolog = 0;

my %args=(
          '-delete-all-monitor' => sub { push @mode, $_[0]; $log=1; },
          '-def+' => \$filename,
          '-definition+' => '-def',
          '-disable-changelog' => \$nolog,
          '-list-ds' => \@mode,
          '-read-ds' => \@mode,
          '-read-cmds' => \@mode,
          '-read-commands' => '-read-cmds',
          '-read-and-upgrade-cmds' => \@mode,
          '-read-and-upgrade-commands' => '-read-and-upgrade-cmds',
          '-read-and-pre-upgrade-cmds' => \@mode,
          '-read-and-upgrade-commands' => '-read-and-upgrade-cmds',
          '-read-monitor' => \@mode,
          '-read-mon' => '-read-monitor',
          '-read-parinfo' => \@mode,
          '-read-resinfo' => \@mode,
          '-write-resinfo+' => sub{
            push @mode, $_[0];
            $resinfo_file=$_[1];
            $log=1;
          },
          '-read-trninfo' => \@mode,
          '-write-trninfo+' => sub{
            push @mode, $_[0];
            $trninfo_file=$_[1];
            $log=1;
          },
          '-cat-trninfo' => sub { push @mode, $_[0]; $log=1; },
          '-read-runinfo' => \@mode, 
	        '-remove-exec-control' => \@mode,
          '-write-monitor+' => sub {
            push @mode, '-write-monitor' unless (scalar @write_mon_files);
            push @write_mon_files, $_[1];
            $log=1;
          },
          '-write-mon+' => '-write-monitor',
          '-read-profile-file+' => sub {
            push @mode, $_[0];
            $profile_file=$_[1];
          },
          '-extract-all-profile-files' => \@mode,
          '-extract-profile-file+' => sub {
            push @mode, $_[0];
            $profile_file=$_[1];
          },
          '-read-pre-state' => \@mode,
          '-compare+' => '-compare-cmds',
          '-compare-commands+' => '-compare-cmds',
          '-compare-cmds+'    => sub { push @mode, '-compare-cmds'; $compfile = $_[1]; },
          '-comp+' => '-compare',
          '-diff-cmds+'    => sub { push @mode, '-diff-cmds'; $compfile = $_[1]; },
          '-list-profile-files' => \@mode,
          '-list-pro-files' => \@mode,

          '-out+'     => \$OutFile,
          '-output+'  => '-out',
          '-show-changelog' => \@mode,
          '-validate-against+' => sub { push @mode, $_[0]; $rules=$_[1]; },
          '-validate-cmds' => '-validate',
          '-validate' => \@mode,
          '-what+' => \$what,
          '-where+' => \$where,
          '-when+' => \$when,
         );
my @nonopt=parse_cmdline
  ( \@ARGV,
    'INCLUDE_STD', 'ALLOW_NON_OPTION',
    %args,
  );

if (@nonopt and !defined $filename) {
  $filename=shift @nonopt;
}

$log = 0 if ($nolog);

if (defined $filename) {
  CFX5::FileTests::datafile($filename);
} else {
  cfx5_require('CFX5::Help');
  _error("You must specify a CFX-5 Definition or Results file to use.",
         CFX5::Help::usage_msg_once());
}

if (@nonopt) {
  cfx5_require('CFX5::Help');
  _error("Unexpected argument(s)\n", @nonopt, "\nwithout an associated option were found on the command line.", CFX5::Help::usage_msg_once());
}

exit_if_error();

#
# now create an object to represent the file
#
cfx5_require('CFX5::Files::Any');
my $file=CFX5::Files::Any->new($filename);

unless (@mode) {
  cfx5_require('CFX5::Help');
  _error("You must give something to do!", CFX5::Help::usage_msg_once());
}

if (defined $OutFile) {
  CFX5::FileTests::datafile_w($OutFile);
}

exit_if_error();

my $outputname = '-';
$outputname = $OutFile if (defined $OutFile);

MODE_ARG: for my $mode (@mode) {
  # de-reference any alias arguments
  while (exists $args{$mode} and !ref($args{$mode})) {
    $mode=$args{$mode};
  }
  # Monitor Data
  if ($mode eq '-delete-all-monitor') {
    CFX5::MonFile->delete_all_from_res($filename);
  } elsif ($mode eq '-read-monitor') {
    unless (CFX5::MonFile->extract_from_res($filename, $outputname)) {
      _warn("No monitor point data in file $filename");
    }
  } elsif ($mode eq '-write-monitor') {
    CFX5::MonFile->insert_into_res($filename, @write_mon_files);
  # Execution control
  } elsif ($mode eq '-remove-exec-control') {
    my @data=$file->extract_ccl();
    if(@data) {
      cfx5_require('CFX5::CCLI');
      my $api=CFX5::CCLI::new_api_object();
      my $rv=$api->parseCCLData(\@data);
      if ($rv) {
          _error("Errors processing the Command Language:\n",
                 CFX5::CCLI::format_ccl_error($rv));
      } else {
          my $rv=$api->deleteObj('SIMULATION CONTROL/EXECUTION CONTROL');
          if (defined $rv) {
            _error("Error deleting SIMULATION CONTROL/EXECUTION CONTROL from Command Language database:\n",
             CFX5::CCLI::format_ccl_error($rv));
          }
          $rv=$api->getExpandedCCL(\@data);
          if ($rv) {
            _error("Errors extracting the upgraded Command Language data:\n",
                   CFX5::CCLI::format_ccl_error($rv));
          }
      }
      @data = map {"$_\n"} @data;
      if (defined $OutFile) {
        CFX5::FileOps::safe_set_file_contents($OutFile, @data);
      } else {
        print(@data);
      }
    } else {
      _error("No CFX Command Language in file $filename");
    }
  # Command language
  } elsif ($mode =~ m/^-read-.*cmds$/) {
    my @data=$file->extract_ccl();
    if (@data) {
      if ($mode =~ m/-read-and-(pre-)?upgrade-cmds/) {
        my $dopre=$1;
        cfx5_require('CFX5::CCLI');
        my $api=CFX5::CCLI::new_api_object();
        my $rv=$api->parseCCLData(\@data);
        if ($rv) {
          _error("Errors processing the Command Language:\n",
                 CFX5::CCLI::format_ccl_error($rv));
        } else {
          if ($dopre) {
            $rv=$api->updatePreCCL();
          } else {
            $rv=$api->updateSolverCCL();
          }
          my @xtra;
          if ($rv) {
            # this means we did some upgrades.  Let's display them as
            # comments so it's still valid CCL
            $rv =~ s!^!\# !gm;
            push @xtra, $rv;
          }
          # even if we couldn't update, spit out the new stuff
          $rv=$api->getExpandedCCL(\@data);
          if ($rv) {
            _error("Errors extracting the upgraded Command Language data:\n",
                   CFX5::CCLI::format_ccl_error($rv));
          }
          # in case there were upgrades
          unshift @data, @xtra;
        }
      }

      @data = map {"$_\n"} @data;
      if (defined $OutFile) {
        CFX5::FileOps::safe_set_file_contents($OutFile, @data);
      } else {
        print(@data);
      }
    } else {
      _error("No CFX Command Language in file $filename");
    }
  # Pre state
  } elsif ($mode eq '-read-pre-state') {
    my $def=CFX5::DefFile->new($filename);
    unless (ref $def) {
      _error("Unable to open input file $filename");
      next MODE_ARG;
    }

    my $ds;
    my @dslist=$def->have_dataset(what => 'STATE', 
                                  when => 0);
    if(@dslist) {
      $ds=$def->read_dataset(what => 'STATE', 
                             when => 0);
    }
    else {
      @dslist=$def->have_dataset(what => 'G/PRESTATE', 
                                 when => 0);
      if (!@dslist) {
        _error("No CFX-Pre State Information in file $filename");
        next MODE_ARG;
      }
      else {
        $ds=$def->read_dataset(what => 'G/PRESTATE', 
                               when => 0);
      }
    }
    unless (ref $ds) {
      _error("No CFX-Pre State Information in file $filename");
      next MODE_ARG;
    }

    my @data=map{"$_\n"} $ds->data();
    if (@data) {
      if (defined $OutFile) {
        CFX5::FileOps::safe_set_file_contents($OutFile, @data);
      }else{
        print(@data);
      }
    }else{
      _error("No CFX-Pre State Information in file $filename");
    }
    
  # Read the G/RES_INFO dataset
  } elsif ($mode eq '-read-resinfo') {
    my $resinfo=new CFX5::ResInfo();
    my $haveresinfo=$resinfo->ReadFromFile($filename);
    _error("No RESINFO dataset exists in the file: $filename") unless $haveresinfo ; 
    exit_if_error();
    
    if(defined $OutFile)
    {
      $resinfo->WriteToASCII($OutFile);
    }
    else
    {
      $resinfo->Print();
    }
    
  # Write the G/RES_INFO dataset
  } elsif ($mode eq '-write-resinfo') {
    my $resinfo=new CFX5::ResInfo();
    $resinfo->ReadFromPrintFile($resinfo_file); 
    $resinfo->WriteToFile($filename);

  # Read the G/TRN_INFO dataset
  } elsif ($mode eq '-read-trninfo') {
    my $trninfo=new CFX5::TrnInfo();
    my $havetrninfo=$trninfo->ReadFromFile($filename);
    _error("No TRN_INFO dataset exists in the file: $filename") unless $havetrninfo;
    exit_if_error();
    if(defined $OutFile)
    {
      $trninfo->WriteToASCII($OutFile);
    }
    else
    {
      $trninfo->Print();      
    }
    
  # Write the G/TRN_INFO dataset
  } elsif ($mode eq '-write-trninfo') {
    my $trninfo=new CFX5::TrnInfo();
    $trninfo->ReadFromPrintFile($trninfo_file);
    $trninfo->WriteToFile($filename);

  # Concatenate G/TRN_INFO dataset from all res files to final res file
  } elsif ($mode eq '-cat-trninfo') {
    my $resinfo=new CFX5::ResInfo();
    my $haveresinfo=$resinfo->ReadFromFile($filename);
    _error("No RESINFO data exists in the file: $filename") unless $haveresinfo; 
    my $trninfo_full = new CFX5::TrnInfo();
    for (my $i=0; $i<$resinfo->GetCount(); $i++) {
      my %resentry = $resinfo->GetEntry($i);
      my $resfile = $resentry{FilePath};
      _print("Reading TRNINFO data from: $resfile");
      my $trninfo = new CFX5::TrnInfo();
      if ($trninfo->ReadFromFile($resfile)) {
        for (my $j=0; $j<$trninfo->GetCount(); $j++) {
          my %trnentry = $trninfo->GetEntry($j);
          $trninfo_full->AddEntry($trnentry{TimeStep},$trnentry{SimuTime},$trnentry{FilePath});
        }
      } else {
        _warn("No TRNINFO data exists in the file: $resfile");
      }
    }
    $trninfo_full->DeleteDuplTimestep();
    if ($trninfo_full->CheckDuplFilename() || $trninfo_full->CheckTimestepOrder()) {
      _error("Duplicate transient results file names found or transient results files are not in ascending time step order.");
    } else {
      _print("Writing the following TRN_INFO data to: $filename");
    }
    $trninfo_full->Print();
    exit_if_error();
    $trninfo_full->WriteToFile($filename);

  # Read the G/RUNINFO dataset
  } elsif ($mode eq '-read-runinfo') {
    my @ds = GetDataSet($filename,"G/RUNINFO");
    if (defined $ds[0]) {
      if (defined $OutFile) {
        CFX5::FileOps::safe_set_file_contents($OutFile, join("\n",@ds));
      } else {
        print (join("\n",@ds));
      }
    }
    else {
      _error("No RUNINFO dataset exists in the file: $filename") ;
    }

  # Read the parallel job info from the res file
  } elsif ($mode eq '-read-parinfo') {
    my $def=CFX5::DefFile->new($filename);
    if (ref $def) {
      my %ds_sig = (what => 'G/PAR_INFO', where => 'EVERY', when => 0);
      if ($def->have_dataset(%ds_sig)) {
        $def->extract_text_dataset($outputname, %ds_sig);
      } else {
        _warn("No parallel job information exists in the file: $filename");
      }
    } else {
      _error("Unable to open input file $filename");
    }
                                    
  # Compare the CCL in two DEF files
  } elsif ($mode eq '-compare-cmds' or $mode eq '-diff-cmds') {

    my $file2;
    if (CFX5::FileTests::datafile($compfile)) {
      $file2=CFX5::Files::Any->new($compfile);
    }
    exit_if_error();

    my @ccl1=$file->extract_ccl();
    my @ccl2=$file2->extract_ccl();

    my @data = CFX5::CCLDiff->compareCCL(\@ccl1,\@ccl2);
    if (@data) {
      if (defined $OutFile) {
        CFX5::FileOps::safe_set_file_contents($OutFile, @data);
      } else {
        print(@data);
      }
    }

    if ($mode eq '-diff-cmds' and @data) {
      exit(1);
    }

  } elsif ($mode eq '-validate' or $mode eq '-validate-against') {
    my @ccl1=$file->extract_ccl();
    cfx5_require('CFX5::CCLI');
    my $rf;
    if ($mode eq '-validate-against') {
      $rf=$rules;
    }
    my @data=map { "$_\n" } CFX5::CCLI->validate_ccl(\@ccl1, $rf);
    if (@data) {
      if (defined $OutFile) {
        CFX5::FileOps::safe_set_file_contents($OutFile, @data);
      } else {
        print(@data);
      }
      # if we had data ... that's a validation error
      exit(1);
    }

  } elsif ($mode eq '-list-ds') {
    my @dslist=$file->locate_dataset($what, $where, $when);
    if (@dslist) {
      my @data;
      for my $ds (@dslist) {
        push @data, join("\t", @$ds) . "\n";
      }
      if (defined $OutFile) {
        CFX5::FileOps::safe_set_file_contents($OutFile, @data);
      } else {
        print(@data);
      }
    } else {
      _error("No matching datasets in file $filename");
      # exit code 1 => no datasets; on error, we'll exit with retcode
      # 2
      exit(1);
    }

  } elsif ($mode eq '-list-profile-files') {
    cfx5_require('CFX5::DefFile::ProfileInterface');
    my $int=CFX5::DefFile::ProfileInterface->new($file);
    my %filelist=$int->file_list();
    if (keys %filelist) {
      my @data;
      for my $ds (keys %filelist) {
        push @data, $ds . "\n";
      }
      if (defined $OutFile) {
        CFX5::FileOps::safe_set_file_contents($OutFile, @data);
      } else {
        print(@data);
      }
    } else {
      _info("There is no profile data in file $filename");
      # exit code 1 => no datasets; on error, we'll exit with retcode
      # 2
      exit(1);
    }
  }

  elsif ($mode =~ m/-(read|extract)-profile-file/) {

    if ($1 eq 'extract' and not defined $OutFile) {
      # extract => read but to a file of the same basename
      $OutFile=basename($profile_file);
    }

    extract_profile($profile_file, $OutFile);

  } elsif ($mode eq '-extract-all-profile-files') {

    cfx5_require('CFX5::DefFile::ProfileInterface');
    my $int=CFX5::DefFile::ProfileInterface->new($file);
    my %filelist=$int->file_list();

    if (keys %filelist) {
      for my $ds (keys %filelist) {
        extract_profile($ds, $ds);
      }
    } else {
      _warn("File contains no profile information");
    }

  } elsif ($mode eq '-show-changelog') {

    my $changelog = CFX5::ChangeLog->new($filename,'DFILE/CHANGELOG');
    $changelog->show();

  } else {
    internal_error("Unhandled mode $mode");
  }
}

# Log any changes
if ($log)
{
  my $changelog = CFX5::ChangeLog->new($filename,'DFILE/CHANGELOG');
  $changelog->AddDatedEntry(sprintf "cfx5dfile %s",join(' ',@ARGV));
  $changelog->WriteToFile($filename); 
}

exit_if_error();

exit(0);

sub extract_profile {
  my $profile=shift;
  my $outfile=shift;

  if (defined $outfile) {
    if (-f $outfile) {
      _error("Output file $outfile already exists.  Please remove or rename this file and try again.");
      return '';
    }
  }

  _info("Extracting profile file $profile");

  cfx5_require('CFX5::DefFile::ProfileInterface');
  my $int=CFX5::DefFile::ProfileInterface->new($file);

  my $out=defined $outfile ? $outfile : '-';
  $int->extract_file($profile, $out);
}
