# Library functions for the batch_queue package
#
# File: batch_queue/lib/batch_queue.tcl
#
# $Revision$ on $Date$
# 
# Copyright (c) 1993-2003 ICEM CFD Engineering
#
#-----------------------------------------------------------------------------------

## Get the version of the batch_queue package

proc batch_queue_get_version {} {
    # cvs should update this string; it has for me, at any rate --jka
    set revstring "# \$Revision$ on \$Date$"
    return [lindex $revstring 2]
}


# (These procs are used in order that that "batch_queue" module
# *as well as* calling programs can reliably interpret results of the package.)


## This proc returns the string which indicates that the "batch_queue" command was a (smashing) SUCCESS.
## Procs like batch_queue::get_job_status might return such a string.

proc batch_queue_cmd_success {} { return "SUCCESS" }


## This proc returns the string which indicates that the "batch_queue" command was a (manque) FAILURE.
## Procs like batch_queue::get_job_status might return such a string.

proc batch_queue_cmd_failure {} { return "FAILURE" }


## This proc returns the string which indicates that the "batch_queue" job is RUNNING.
## Procs like batch_queue::get_job_status might return such a string.

proc batch_queue_job_running {}    { return "RUNNING"    }


## This proc returns the string which indicates that the "batch_queue" job is QUEUED.
## Procs like batch_queue::get_job_status might return such a string.

proc batch_queue_job_queued {}     { return "QUEUED"    }


## This proc returns the string which indicates that the "batch_queue" job was TERMINATED.
## Procs like batch_queue::get_job_status might return such a string.

proc batch_queue_job_terminated {} { return "TERMINATED" }


## This proc returns the string which indicates that the "batch_queue" job has FINISHED.
## Procs like batch_queue::get_job_status might return such a string.

proc batch_queue_job_finished {}   { return "FINISHED"   }


## This proc returns the string which indicates that the "batch_queue" job was a FAILURE.
## Procs like batch_queue::get_job_status might return such a string.

proc batch_queue_job_failure {}   { return "FAILURE"   }


## This proc returns the string which indicates that the "batch_queue" job is if UNKNOWN
## status. Procs like batch_queue::get_job_status might return such a string.

proc batch_queue_job_unknown {}    { return "UNKNOWN"    }


# ( ..not sure if these are used... --jka )
proc batch_queue_mode_remote {}     { return "REMOTE" }
proc batch_queue_mode_local {}      { return "LOCAL" }
proc batch_queue_mode_bqs   {}      { return "BATCH_QUEUE" }


#-----------------------------------------------------------------------------------

# The "batch_queue" package namespace
#
# NB: all batch_queue's share the same "jobman"... --jka
#     this makes it unnecessary to pass the bqid along with the job_token,
#     rather given a job_token, we can find the queue to which it belings.

namespace eval batch_queue {
    
    namespace export \
	init_queue queue_summary submit_job exec_job exec_queued_jobs \
	queue_is_sane joken_token_list get_job_status add_job_message \
	get_job_attribute add_job_attribute \
	is_job_running are_jobs_running num_jobs_running running_job_token_list \
	is_job_queued are_jobs_queued queued_job_token_list highest_priority_queued_job \
	delete_job delete_all delete_queue
    
    # queue serial id; incr'd with each queue init'd
    set v_queue_serial 0    
    
    # options
    set v_verbose      0
    set v_debug        0
}


#-----------------------------------------------------------------------------------
#------------------------------  P  U  B  L  I  C  ---------------------------------
#-----------------------------------------------------------------------------------


## Initialize a batch_queue, using the batch-queueing-system implementation <em>system</em>,
## which can be: < IcBQS | PBS | GridEngine >. Possible args are:
##
##    -servers <list of hosts>              ;; list of hosts to acess (esp. for IcBQS)
##                                             default is "localhost" for IcBQS, else whatever
##                                             configure for the given BQS.
##    -port    <port_number>                ;; if IcBQS, the port on which to set up sockets
##    -use_ssl <bool:0|1>                   ;; if IcBQS, the socket will use TSL & OpenSSL
##    -pfunc                                ;; proc to use for printing, e.g. "mess"
##                                             default is "puts"
## It will return the "handle" for this queue.

proc batch_queue::init_queue { system args } {
    variable v_queue_serial
    variable v_debug
    global icbqs_client_available_servers ;# this one works by $client_id
    
    set bqid $v_queue_serial
    incr v_queue_serial
    upvar ::batch_queue::batch_queue$bqid batch_queue

    #
    # --> init system & defaults
    #
    
    set bqs  [_verify_system $system]

    set batch_queue(job_tokens) ""
    
    set batch_queue(system)     $bqs
    set batch_queue(pfunc)      ""
    set batch_queue(port)       [batch_queue_icbqs_server_port_default]
    set batch_queue(servers)    ""
    
    #
    # --> parse args for options
    #
    
    set options { -servers -port -mode -pfunc -use_ssl }
    
    foreach {option value} $args {
	# this actually check the args ...
        if { [lsearch -exact $options $option] != "-1" } {
            if $v_debug {
                _msg $bqid "  $option = $value  \n"
            }
            set batch_queue([string range $option 1 end]) $value
	}
    }

    #set batch_queue(servers) [lreplace $batch_queue(servers) [info host] "localhost"]

    #
    # -->
    #
    
    switch -- $bqs {
	"icbqs" {
	    # the client_id is just the QUEUE id (bqid)
	    batch_queue_icbqs_client_setup $bqid -server_port $batch_queue(port) \
		-servers "$batch_queue(servers)"
	}
	default {
	    # nothing to do just yet for any other BQSen
	}
    }
    
    # this is the serial-handle for this namespace-hash
    # (add it to a hash of current queues --jka ?? fixme ?? )
    
    return $bqid
}


## Sumbit a job <em>command</em> to the batch_queue with id <em>bqid</em>. Possible args are:
##
##    -priority <int>        ;; priority of the job, "1" being highest and the default
##    -dispatch <bool>       ;; whether or not to exec_job immediately; "1" being default
##    -logfile  <full-path>  ;; full path to a logfile; default is a /tmp/one
##    -server   <hostname>   ;; hostname to exec on; if not specified, will choose most idle
##    -max_time <int> (sec)  ;; the maximum number of seconds for the execution
##
## This func will return the job_token.

proc batch_queue::submit_job { bqid command args } {
    upvar ::batch_queue::batch_queue$bqid batch_queue
    variable v_debug 
    
    #set options { -priority -dispatch }
    set priority 1
    set dispatch 1
    set logfile  "__NONE__"
    set server   ""
    set max_time 0
    
    set bqs $batch_queue(system)
    
    foreach {option value} $args {
	set [string range $option 1 end] $value
	if $v_debug {
	    puts " [string range $option 1 end] is [set [string range $option 1 end]]"
	}
    }
    
    set job_args "COMMAND \"$command\" LOGFILE \"$logfile\" MODE [batch_queue_mode_bqs] \
                  SYSTEM $bqs QUEUE $bqid PRIORITY $priority MAX_TIME $max_time"
    
    set job_token  [eval _register_job $bqid $job_args]
    #set job_status [get_job_status $job_token]
    
    #
    # --> check on immediate dispatch
    #
    
    switch -- $bqs {
	"icbqs" {
	    if {0 < $dispatch} {
		set job_status [eval exec_job $job_token $args]
		_set_job_datum $job_token STATUS $job_status
		if {[batch_queue_job_running] != $job_status} {
		    mess "\nWarning: may not have been able to start job $job_token, status is $job_status\n\n" red
		}
	    }
	}
	"pbs" {
	    if {0 == $dispatch} {
		set job_status [eval hold_job $job_token $args]
		_set_job_datum $job_token STATUS [batch_queue_job_queued]
	    }
	}
    }
    
    # this function has returned the list  list {job_token job_status}
    # but this is not too nice for an API. --jka
    
    return $job_token
}


## Execute the given job <em>job_token</em>. Possible args are:
##
##     -server  <host_name>   ;; the name of the host on which to exec the job
##                                default is the most idle server configured.

proc batch_queue::exec_job { job_token args } {
    
    set bqid [_get_job_datum $job_token QUEUE]
    upvar ::batch_queue::batch_queue$bqid batch_queue
    
    #
    # --> Check status of the job
    #
    
    set status [get_job_status $job_token]
    if { [batch_queue_job_queued] != $status } {
	mess "Warning: could not execute job that is not simply queued.\n"
	mess "The job \"$job_token\" is \"$status\"\n" red
	return $status
    }
    
    #
    # --> Parse args
    #
    
    set server "" ;# it is possible for the caller to specify "server" as
                   # in the case of app_ic_regression
    foreach {n v} $args {
	set [string range $n 1 end] $v
    }
    
    #
    # --> Execute it on the correct via the correct batch-queueing-system
    #
    set pid -1
    
    switch -- $batch_queue(system) {
	"icbqs" {
	    set command    [_get_job_datum $job_token COMMAND]
	    # the client_id is just the QUEUE id
	    set client_id [_get_job_datum $job_token QUEUE]
	    
	    # server might be "" as given, it will return whatever is actually used
	    set server_and_pid [batch_queue_icbqs_client_exec_job $client_id "$command" $server]
	    
	    if {[batch_queue_job_failure] == $server_and_pid} {
		
		# could not get the process started on the server...keep it in the queue
		set status [batch_queue_job_queued]
		set message "Failed to start the job"
		if {"" != $server} {
		    append message " on specified server $server"
		}
		append message "...leaving in queue"
		add_job_message $job_token $message
		
	    } else {
		set server [lindex $server_and_pid 0]
		set pid    [lindex $server_and_pid 1]
		
		#puts "server $server and pid $pid"
		_set_job_datum $job_token PID  $pid
		_set_job_datum $job_token HOST $server
		_set_job_datum $job_token START_TIME [clock seconds]
		
		set status [batch_queue_job_running]
	    }
	    #puts "debugA: for command \"$command\" got pid of \"$pid\" on server \"$server\""
	}
	default {
	    
	    # nothing yet -- any other BQS should probably be handling it implicitly
	}
    }
    
    _update_job_status $job_token $status
    return $status
}


## Exec each job registered as QUEUED in the queue <em>bqid</em> as servers become available.
## This function could--and will likely--return before those jobs are finished running.

proc batch_queue::exec_queued_jobs { bqid } {
    
    while { [are_jobs_queued $bqid] } {
	
	set job_token [highest_priority_queued_job $bqid]
	if {"" == $job_token} {
	    break
	}
	
	exec_job $job_token    
	
	after 1000 ;# this is "ms" ; i'm not sure if good val ; ?maybe base it on njobs? --fixme --jka
    }
}


## Print to mess info about each job in the batch_queue with id <em>bqid</em>.

proc batch_queue::queue_summary { bqid } {
    
    set job_tokens [job_token_list $bqid]
    
    if {0 >=  [llength $job_tokens]} {
	_msg $bqid "No known jobs to summarize in queue \"$bqid\"\n" red
	return 0
    }
    
    set color black
    
    set job_incr 1
    _msg $bqid "\nSummarizing known jobs in batch_queue \"$bqid\":\n" $color
    foreach job_token $job_tokens {
	
	_msg $bqid "\t--Job $job_incr: $job_token: [get_job_status $job_token]\n" $color
	foreach prop [_job_properties_list] {
	    set prop_val [_get_job_datum $job_token $prop]
	    if {"__NONE__" != $prop_val} {
		_msg $bqid "\t\t$prop: $prop_val\n" $color
	    }
	}
	
	incr job_incr
    }
    
    incr job_incr -1

    #_msg $bqid "Reported $job_incr jobs\n\n" $color
    
    return $job_incr
}


## Verify that the batch_queue with id <em>bqid</em> is sane.

proc batch_queue::queue_is_sane { bqid } {
    upvar ::batch_queue::batch_queue$bqid batch_queue
    
    switch -- $batch_queue(system) {
	"icbqs" {
	    return [batch_queue_icbqs_client_is_sane $bqid]
	}
    }
    
    return 1
}


## Get a list of all job tokens in the batch-queue <em>bqid</em>

proc batch_queue::job_token_list { bqid } {
    upvar ::batch_queue::batch_queue$bqid batch_queue
    
    if [info exists batch_queue(job_tokens)] {
	return $batch_queue(job_tokens)
    }
    
    return ""
}

## Get the job status of  the given <em>job_token</em>. For the return
## values, see the doc for <em>batch_queue::update_job_status</em>

proc batch_queue::get_job_status { job_token } {
    _update_job_status $job_token ""
    return [_get_job_datum $job_token STATUS]
}


## Add a (custom) the job attribute of the given <em>job_token</em>.

proc batch_queue::add_job_attribute { job_token attribute val } {
    
    if {-1 < [lsearch [batch_queue::_job_properties_list] $attribute]} {
	mess "Error: cannot add attribute \"$attribute\" since this key is reserved.\n" red
	return "__NONE__"
    }
    
    return [_set_job_datum $job_token $attribute $val]
}


## Get the job attribute of the given <em>job_token</em>. ?TMP-FIXME?

proc batch_queue::get_job_attribute { job_token attribute } {
    return [_get_job_datum $job_token $attribute]
}


## Get the elapsed time for the job indicated by the given <em>job_token</em>.

proc batch_queue::get_job_elapsed_time { job_token } {
    
    # *** do not call "is_job_running" here since it will call _update_job_status which calls this !! --jka
    # *** do not call "get_job_status" here since it will call _update_job_status which calls this !! --jka	
    
    set start_time [_get_job_datum $job_token START_TIME]
    if {"__NONE__" == $start_time} {
	if [is_job_running $job_token] {
	    mess "Warning: not sure what's up with job \"$job_token\"...start time is \"$start_time\"\n" red
	}
	return 0
    }
    
    set stop_time [_get_job_datum $job_token STOP_TIME]
    if {"__NONE__" == $stop_time} {
	
	catch { expr [clock seconds]  - $start_time } time_elapsed
	
    } else {
	
	if { [batch_queue_job_running] == [_get_job_datum $job_token STATUS] } {
	    mess "Warning: not sure what's up with job \"$job_token\"...it is running bust has stop_time\"$stop_time\"\n" red	    
	}
	
	catch { expr $stop_time - $start_time } time_elapsed    
    }
    
    #puts "job \"$job_token\" has been running for $time_elapsed seconds"

    return $time_elapsed
}


## Is the job given by <em>job_token</em> still running?
## This proc updates the job status!

proc batch_queue::is_job_running { job_token } {    
    return [string equal [batch_queue_job_running] [get_job_status $job_token]]
}


## Are jobs still running in the batch_queue with id <em>bqid</em>.
## This proc updates the job statuses!

proc batch_queue::are_jobs_running { bqid } {
    upvar ::batch_queue::batch_queue_jobman batch_queue_job_data
      
    foreach job_token [job_token_list $bqid] {
	if [is_job_running $job_token] {
	    return 1
	}
    }
    
    # no jobs are still running
    return 0
}


## Are jobs still running in the batch_queue with id <em>bqid</em>.
## This proc DOES NOT update the job statuses!

proc batch_queue::num_jobs_running { bqid } {
    upvar ::batch_queue::batch_queue_jobman batch_queue_job_data
    
    set njobs_running 0
    foreach job_token [job_token_list $bqid] {
	if { [batch_queue_job_running] == [_get_job_datum $job_token STATUS] } {
	    incr njobs_running
	}
    }
    
    return $njobs_running
}


## Get a list of all job_tokens whose jobs are running (have status [batch_queue_job_running]).

proc batch_queue::running_job_token_list { bqid } {
    
    set running_jobs ""
    
    foreach job_token [job_token_list $bqid] {
	if [is_job_running $job_token] {
	    lappend running_jobs $job_token
	}
    }
    
    return $running_jobs
}


## Is the job given by <em>job_token</em> still queued?

proc batch_queue::is_job_queued { job_token } {    
    return [string equal [batch_queue_job_queued] [get_job_status $job_token]]
}


## Are there are jobs queued (that is, not RUNNING, TERMINATED, FINISHED or UNKNOWN)
## in the batch_queue with id <em>bqid</em>.

proc batch_queue::are_jobs_queued { bqid } {
    
    foreach job_token [job_token_list $bqid] {
	if [string equal [get_job_status $job_token] [batch_queue_job_queued]] {
	    return 1
	}
    }
    
    return 0
}


## Get a list of all job_tokens whose jobs are QUEUED ([batch_queue_job_queued]).
## in the queue with id <em>bqid</em>.

proc batch_queue::queued_job_token_list { bqid } {
    
    set queued_jobs ""
    
    foreach job_token [job_token_list $bqid] {
	if [is_job_queued $job_token] {
	    lappend queued_jobs $job_token
	}
    }
    return $queued_jobs
}


## Get the highest priority job in the queue with id <em>bqid</em>.

proc batch_queue::highest_priority_queued_job { bqid } {
    
    set hipri_job_token ""
    set highest_priority -1 ;# priority is such that 1=highest
    
    foreach job_token [job_token_list $bqid] {
	
	if [string equal [get_job_status $job_token] [batch_queue_job_queued]] {
	    
	    set pri [_get_job_datum $job_token PRIORITY]
	    
	    if { ( ("" != $hipri_job_token) || ("-1" == $highest_priority) ) || \
		     ( ("" != $pri) && ($pri < $highest_priority) ) } {
		
		set hipri_job_token $job_token
		set highest_priority $pri
	    }
	}
    }
    
    return $hipri_job_token
}


## Hold the job associated with the given <em>job_token</em>. Not impl'd. FIXME

proc batch_queue::hold_job { job_token args } {
    
    mess "fixme: batch_queue::hold_job is not impl'd"    
    return [batch_queue_cmd_failure]
    
    # kill the process
    # set the job as [batch_queue_job_queued]
}


## Move the job with <em>job_token</em> to the queue with id <em>bqid_new</em>.

proc batch_queue::move_job { job_token bqid_new } {
    upvar ::batch_queue::batch_queue$bqid_new batch_queue_new
    
    set bqid_old [_get_job_datum $job_token QUEUE]
    if {"__NONE__" == $bqid} {
	mess "Error: could not get the queue id to which \"$job_token\" belongs\n" red
	return [batch_queue_cmd_failure]
    }
    
    upvar ::batch_queue::batch_queue$bqid_old batch_queue_old
    
    set status [_get_job_datum $job_token STATUS]
    
    if { ([batch_queue_job_finished]   == $status) || \
	 ([batch_queue_job_terminated] == $status) || \
	 ([batch_queue_job_failure]    == $status) } {
	# nothing to do, really
	mess "Warning: not reason to move the job since it is $status\n" red
	return [batch_queue_cmd_success]
    }
    
    switch -- $batch_queue(system) {
	"icbqs" {
	    # nothing to do
	}
	"pbs" {
	    # figure out destination -- fixme --jka
	    set stat [batch_queue_pbs_move_job $job_token $destination]
	}
    }
    
    #
    # --> Update our queue hashes, which is BQS independent.
    #
    
    set ind [lsearch $batch_queue_old(job_tokens) $job_token]
    if {-1 < $ind} {
	set batch_queue_old(job_tokens) [lreplace $batch_queue_old(job_tokens) $ind $ind]
    }

    lappend batch_queue_new(job_tokens) $job_token
    _set_job_datum $job_token QUEUE $bqid_new
    

    return [batch_queue_cmd_success]
}


## Get info about the job with <em>job_token</em>. a PBS synonym ; fixme -- not impl'd.

proc batch_queue::query_job { job_token } {
    upvar ::batch_queue::batch_queue$bqid batch_queue
    
    # fixme; this should just get remapped; the nomenclature is PBS'.
    mess "fixme: batch_queue::query_job is not impl'd"
    return [batch_queue_cmd_failure]
}


## Alter the job with <em>job_token</em>. a PBS synonym ; fixme -- not impl'd.

proc batch_queue::alter_job { job_token args } {
    upvar ::batch_queue::batch_queue$bqid batch_queue
    
    # fixme
    mess "fixme: batch_queue::alter_job is not impl'd"

    # check status of job
    
    # allow, for example, caller to add a -max_time    

    return [batch_queue_cmd_failure]
}


## Add a message <em>message</em> to a job <em>job_token</em>.

proc batch_queue::add_job_message { job_token message } {
    upvar ::batch_queue::batch_queue_jobman batch_queue_job_data
    
    
    set bqid [_get_job_datum $job_token QUEUE]
    if {"__NONE__" == $bqid} {
	mess "Error: could not get the queue id to which \"$job_token\" belongs\n" red
	return [batch_queue_cmd_failure]
    }
    
    upvar ::batch_queue::batch_queue$bqid batch_queue

    switch -- $batch_queue(system) {
	"pbs" {
	    # fixme for pbs--use qmsg...
	    batch_queue_pbs_message $job_token $message
	}
    }
    
    # --> Do this in the batch_queue hashes for any BQS
    
    lappend batch_queue_job_data($job_token,MESSAGES) $message
    
    return [batch_queue_cmd_success]
}


## fixme-doc; fixme-impl

proc batch_queue::resub_job { job_token args } {
    upvar ::batch_queue::batch_queue$bqid batch_queue
    
    # fixme
    mess "fixme: batch_queue::resub_job is not impl'd"
    return [batch_queue_cmd_failure]
}


## Delete the job given by <em>job_token</em>. <em>args</em> is not
## used just yet.

proc batch_queue::delete_job { job_token args } {
    
    set bqid [_get_job_datum $job_token QUEUE]
    if {"__NONE__" == $bqid} {
	mess "Error: could not get the queue id to which \"$job_token\" belongs\n" red
	return [batch_queue_cmd_failure]
    }
    
    upvar ::batch_queue::batch_queue$bqid batch_queue
    
    set status [_get_job_datum $job_token STATUS]
    if { ([batch_queue_job_finished]   == $status) || \
	 ([batch_queue_job_terminated] == $status) || \
	 ([batch_queue_job_failure]    == $status) } {
	#
	return [batch_queue_cmd_success]
    }
    
    #mess "deleting job with status \"$status\""
    
    switch -- $batch_queue(system) {
	"icbqs" {
	    if {[batch_queue_job_running] == $status} {
		set pid    [_get_job_datum $job_token PID]
		set server [_get_job_datum $job_token HOST]
		set stat [batch_queue_icbqs_client_kill_remote_process $pid $server]
		if {[batch_queue_cmd_success] == $stat} {
		    # mess "deleting job yes i am : $job_token\n" red
		    _set_job_datum $job_token STOP_TIME [clock seconds]
		    _set_job_datum $job_token STATUS    [batch_queue_job_terminated]
		} else {
		    mess "Warning: failed to kill remote process \"$pid\" on \"$server\": $stat\n" red
		}
	    } elseif {[batch_queue_job_queued] == [get_job_status $job_token]} {
		_update_job_status $job_token [batch_queue_job_terminated]
	    }
	}
	"pbs" {
	    if {[batch_queue_job_running] == $status} {
		set fixme [batch_queue_pbs_delete_job $job_token]
	    } elseif {[batch_queue_job_queued] == $status} {
		set fixme [batch_queue_pbs_release_job $job_token]
	    }
	}
	default {
	    # fixme
	    return [batch_queue_cmd_failure]
	}
    }
    
    # fixme: do more --jka

    _remove_job_traces $job_token

    return [batch_queue_cmd_success]
}


## Nuke all jobs and their associated files in the queue with id <em>bqid</em>.
## This should not delete the jobman info about these jobs.

proc batch_queue::delete_all { bqid } {
    
    foreach job_token [job_token_list $bqid] { delete_job $job_token }
    return [batch_queue_cmd_success]
}


## Completely kill the queue with id <em>bqid</em>, deleting jobs, and all
## info about them, shutting down the IcBQS client if necessary.

proc batch_queue::delete_queue { bqid } {
    upvar ::batch_queue::batch_queue$bqid batch_queue
    variable v_debug
    
    # --> Clear out all jobs
    
    delete_all $bqid
    set batch_queue(job_tokens) ""
    
    # --> Forget all sockets to servers
    
    switch $batch_queue(system) {
	"icbqs" {
	    foreach server $batch_queue(servers) {
		batch_queue_icbqs_client_forget_socket $server $batch_queue(port)
		if $v_debug {
		    _msg $bqid "forgetting socket on $server\n"
		}
	    }
	}
    }
    
    if $v_debug {
	_msg $bqid "terminated queue $bqid"
    }    
    
    return [batch_queue_cmd_success]
}



#-----------------------------------------------------------------------------------
#-----------------------------  P  R  I  V  A  T  E  -------------------------------
#-----------------------------------------------------------------------------------

## [Private] Put a message <em>text</em> to the proper place for the queue
## with id <em>bqid</em>

proc batch_queue::_msg { bqid text {color black} } {
    upvar ::batch_queue::batch_queue$bqid batch_queue
    
    if {"" != [info commands mess]} {
	mess $text $color
	return
    }

    # fixme -- this proc doesn't work just yet.
    puts -nonewline $text
    return

    if {[info exists batch_queue(pfunc)] && \
	    ($batch_queue(pfunc) == [info commands $batch_queue(pfunc)]) } {
	eval $batch_queue(pfunc) $text
	return
    }
    puts -nownewline $text
}


## [Private] Verify that the given string <em>bqs</em> is a system supported
## by this package. This function will return the string used internally for
## switch statements, etc.

proc batch_queue::_verify_system { bqs } {
    
    switch -- $bqs {
	PBS - pbs -
	icbqs - IcBQS - ICBQS - IcBqs -
	GridEngine - gridengine - GRIDENGINE -
	Ansys - AnSys - ANSYS - ansys {
	    return [string tolower $bqs]
	}
	default {
	    return -code error "Unrecognized batch-queueing-system: $bqs"
	}
    }
}


## [Private] Get the tmpfile hash-keys that we might have to clean up
## after the job is finished.

proc batch_queue::_job_tmpfiles_list {} {
    return [list LOGFILE COMMAND]
}


## [Private] Return a list of keys used in the hash for the job properties.
 
proc batch_queue::_job_properties_list {} {
    return [concat [_job_tmpfiles_list] [list HOST MODE PID STATUS PRIORITY QUEUE \
					     SYSTEM MAX_TIME START_TIME STOP_TIME MESSAGES]]
}

# NOT DOC -->
# So, given these properties, for a given job, job_token,
# the batch_queue_jobman might look like:
#
#   batch_queue_jobman(${job_token},PRIORITY)     1
#   batch_queue_jobman(${job_token},MODE)         REMOTE
#   batch_queue_jobman(${job_token},LOGFILE)      /full/path/to/logfile.log
#   batch_queue_jobman(${job_token},COMMAND)      /full/path/to/startup.csh     ;; maybe a Tcl proc name with args!!
#   batch_queue_jobman(${job_token},HOST)         maxi
#   batch_queue_jobman(${job_token},PID)          1234
#   batch_queue_jobman(${job_token},STATUS)       RUNNING
#   batch_queue_jobman(${job_token},QUEUE)        $bqid
#   batch_queue_jobman(${job_token},START_TIME)   23485793
#   batch_queue_jobman(${job_token},STOP_TIME)    23485797
#   batch_queue_jobman(${job_token},MESSAGES)      "Command was terminated due to time elapsed"
# <-- NOT DOC

## [Private] Get a unique job token, based on <em>base</em>.

proc batch_queue::_get_unique_job_token { {base ""} } {
    upvar ::batch_queue::batch_queue_jobman batch_queue_job_data
    
    for {set i 0} {[info exists batch_queue_job_data(${base}_${i},QUEUE)]} {incr i} { }
    set token ${base}_${i}
    
    return $token
}


## [Private] Register a job in the the queue with id <em>bqid</em>,
## specifying job-properties. Returns a unique job-token for the job.

proc batch_queue::_register_job { bqid args } {
    upvar ::batch_queue::batch_queue$bqid batch_queue
    variable v_verbose
    variable v_debug
    
    if $v_verbose {
	puts "Adding job add to queue \"$bqid\":"
    }
    
    #set options [_job_properties_list] ; fixme-check against options list?? --jka
    foreach {option value} $args {
	set $option $value
	if $v_debug {
	    puts "\t$option = $value"
	}
    }
    
    set token_base "${bqid}_job"
    #if [info exists COMMAND] {
    #  set token_base [file base [file root $SCRIPT]]
    #}
    set job_token [_get_unique_job_token $token_base]
    
    lappend batch_queue(job_tokens) $job_token
    
    foreach {option value} $args {
	_set_job_datum $job_token $option $value
    }
    
    _set_job_datum $job_token QUEUE  $bqid
    _set_job_datum $job_token STATUS [batch_queue_job_queued]
    
    return $job_token
}


## [Private] Set the datum <em>datum_name</em> for a <em>job_token</em> to a val.
## <em>datum_name</em> can be any property string of those returned by
## <em>batch_queue::_job_properties_list</em>

proc batch_queue::_set_job_datum { job_token datum_name val } {
    upvar ::batch_queue::batch_queue_jobman batch_queue_job_data
    set batch_queue_job_data($job_token,$datum_name) $val
    return $batch_queue_job_data($job_token,$datum_name)
}


## [Private] Get the datum <em>datum_name</em> for a <em>job_token</em>.
## <em>datum_name</em> can be any property string of those returned by
## <em>batch_queue::_job_properties_list</em>

proc batch_queue::_get_job_datum { job_token datum_name } {
    upvar ::batch_queue::batch_queue_jobman batch_queue_job_data
    
    if ![info exists batch_queue_job_data($job_token,$datum_name)] {
	#mess "Warning: cannot find datum \"$datum_name\" for job \"$job_token\"\n" red ??fixme??
	return "__NONE__"
    }
    return $batch_queue_job_data($job_token,$datum_name)
}


## [Private] Remove all tmpfiles (csh scripts, etc.) associated with the given
## <em>job_token</em>.

proc batch_queue::_remove_job_traces { job_token } {
    upvar ::batch_queue::batch_queue_jobman batch_queue_job_data
    
    set status [get_job_status $job_token]
    set bqid   [_get_job_datum $job_token QUEUE]
    
    if {[batch_queue_job_running] == $status} {
	
	_msg $bqid "Warning: cannot remove job-traces for \"$status\" job \"$job_token\"\n" red
	return [batch_queue_job_failure]
	
    } elseif {[batch_queue_job_unknown] == $status} {
	
	_msg $bqid "Warning: cannot remove job-traces for \"$status\" job \"$job_token\"\n" red
	return [batch_queue_job_failure]
    }
    
    foreach prop [_job_tmpfiles_list] {
	
	set tmpfile [_get_job_datum $job_token $prop]
	if [file isfile $tmpfile] {
	    # file delete -force $tmpfile fixme -- this commenting is for testing -- fixme --jka
	    _set_job_datum $job_token $prop "__NONE__"
	}
    }
}


## [Private] Change the status of the given <em>job_token</em> to <em>status</em>.
## <em>status</em> can be one of:
##     [batch_queue_job_running]
##     [batch_queue_job_terminated]
##     [batch_queue_job_finished]
##     [batch_queue_job_unknown]

proc batch_queue::_update_job_status { job_token status } {   
    
    # DO NOT call get_job_status from inside of here !!! (inf-loop)
    
    #
    # --> If status is not NULL, then set it
    #
    
    if {"" != $status} {
	
	_set_job_datum $job_token STATUS $status
	return
    }
    
    #
    # --> Check on things
    #
    
    if {[batch_queue_job_running] != [_get_job_datum $job_token STATUS]} {
	# then we really don't have anything to do, do we?
	return
    }
    
    set bqid [_get_job_datum $job_token QUEUE]
    if {"__NONE__" == $bqid} {
	mess "Error: could not get the queue id to which \"$job_token\" belongs\n" red
	return [batch_queue_cmd_failure]
    }    
    upvar ::batch_queue::batch_queue$bqid batch_queue
    
    switch $batch_queue(system) {	    
	"icbqs" {
	    
	    set pid    [_get_job_datum $job_token PID]
	    set server [_get_job_datum $job_token HOST]
	    
	    if { ("" != $pid) && [string is integer $pid]} {
		
		if ![batch_queue_icbqs_client_remote_process_exists $pid $server] {
		    
		    set stop_time [_set_job_datum $job_token STOP_TIME [clock seconds]]		
		    _set_job_datum $job_token STATUS [batch_queue_job_finished]
		    
		} else {
		    
		    #
		    # --> Check that the job has not exceeded its limit
		    #
		    set max_elapse [_get_job_datum $job_token MAX_TIME]
		    if {[string is integer $max_elapse] && (0 < $max_elapse)} {
			set job_time [get_job_elapsed_time $job_token]
			if {$max_elapse < $job_time} {
			    set stat [delete_job $job_token]
			    set message "Killed job \"$job_token\" because it has taken $job_time and the max_time is $max_elapse"
			    mess "Warning: $message\n" red
			    add_job_message $job_token $message
			}
		    }
		    
		    return ;# doing nothing
		}
	    }	    
	}
	"pbs" {
	    
	    set pbs_qstat [batch_queue_pbs_status $job_token]
	    # fixme -- parse this stat into our own language
	    # fixme
	}
	default {
	    # fixme
	}
    }
    
}    
