#
# $Revision$ on $Date$
# Copyright (c) 1993-2003 ICEM CFD Engineering
# Author: Jason Aftosmis (jason@icemcfd.com)
# Date:   November 2003
#
#-------------------------------------------------------------------------
#


## app_proc_init: this is a specially-names proc which ScribApp recognizes and calls upon startup

proc app_proc_init {} {
    #
    # --> 
    #
    app_ic_regression_setup_self 
    #
    # --> 
    #
    app_ic_regression_common_tree_data_init_hash [app_ic_regression_get_models_all]
    #
    # --> 
    #
    set modules [app_ic_regression_get_modules_all]
    app_ic_regression_module_tree_data_set_modules $modules    
    foreach m $modules {
	app_ic_regression_module_tree_data_set_module_models $m [app_ic_regression_get_models_for_modules $m]
    }
    #
    # --> 
    #
    set functions [app_ic_regression_get_functions_used_by_models]
    app_ic_regression_function_tree_data_set_modules $functions
    foreach f $functions {
	app_ic_regression_function_tree_data_set_function_models $f [app_ic_regression_get_models_using_function $f]
    }
    #
    # --> create the tk images from the tree icons
    #   
    set icons_dir "[file join [file dirname [info script]] .. icons]" ;# FIXME --jka
    foreach im {success failure active inactive unknown text} {
	image create bitmap ${im}_image -file $icons_dir/$im.xbm -foreground black -background ""
    }
}


## app_proc_fter_gui_configure: a specially-names proc that 
## ScribApp recognizes and calls upon startup. This is where
## one might print mouse-bindings for something.

proc app_proc_after_gui_configure {} {
    
    #app_ic_regression_print_mouse_bindings
}


# ----------------------------------------------------------------------------


## Get the persistent settings for the base_test_dir

proc app_ic_regression_setup_self {} {
    global proc_scrib_data ; scrib_app_data_get ;# tmp:fixme
    global regression_setup_self_is_done
    
    if [info exists regression_setup_self_is_done] {
	return 1
    }
    
    if ![info exists proc_scrib_data(base_test_dir)] {
	stdout "Warning: there is no value for the base test directory; assuming working directory.\n" red
	set proc_scrib_data(base_test_dir) [pwd]
    }
    
    set base_test_dir $proc_scrib_data(base_test_dir)
    
    if ![file isdir $base_test_dir] {
	stdout "Not a valid path for a base test directory: $base_test_dir\n" red
	return 0
    }
    #
    # fixme--this could be a little smarter so that it didn't have to be a particular filename
    #        rather just a TCL file with a certain string in a comment... --jka
    #    
    set setup_file [file join $base_test_dir "regression_setup.tcl"]
    if [file exists $setup_file] {
	if [catch {source $setup_file} res] {
	    stdout "Warning: could not source setup file: $setup_file" red	
	}
    } else {
	stdout "Warning: could not find setup file in $base_test_dir!" red
    }
    #
    # --> This is REALLY important !!
    #
    set stat [app_ic_regression_init_models_using_functions]
    if !$stat {
	stdout "Warning: there may have been an error in setting up the model-command mapping!\n" red
    }    
    #
    # --> Register the fact that this things have been set up.
    #
    set regression_setup_self_is_done 1
    
    return 1
}


## Create the hashtable of functions and models using functions

proc app_ic_regression_init_models_using_functions {} {
    global proc_scrib_data ; scrib_app_data_get ;# tmp:fixme    
    global regression_models_using_function
    global regression_models_using_function_initialized
    
    # we only want to do this once foreach function
    
    if [info exists regression_models_using_function_initialized] {
	return 1
    }
    
    set list_of_models ""
    
    if ![info exists proc_scrib_data(base_test_dir)] {
	stdout "Not a valid base test directory\n" red
	return ""
    }
    set base_test_dir $proc_scrib_data(base_test_dir)
    if ![file isdir $base_test_dir] {
	stdout "Not a valid path for a base test directory: $base_test_dir\n" red
	return ""
    }
    
    foreach module [glob -nocomplain -type d $base_test_dir/*] {
	
	set module_path [file join $base_test_dir $module]
	foreach test_model [glob -nocomplain -type d $module_path/*] {
	    #
	    # foreach module under $base_test_dir, check each test_model for
	    # a list of important functions.
	    #
	    set prefs_file [file join $test_model [scrib_lib_regression_preferences_file $test_model]]
	    set cmds [scrib_lib_regression_preferences_representative_cmds $prefs_file]
	    #
	    # else harvest cmds from the replay file
	    #
	    if {"" == $cmds} {
		set replay_file [file join $test_model [scrib_lib_regression_preferences_replay $prefs_file]]
		set cmds [scrib_lib_regression_get_ic_procs_from_replay $replay_file]
	    }	
	    #
	    # now add to the list
	    #
	    foreach function $cmds {
		if {"" != $function} {
		    lappend regression_models_using_function($function) $test_model
		}
	    }
	}
    }
    
    set regression_models_using_function_initialized 1
}


## Get all models using the given function name

proc app_ic_regression_get_models_using_function {function} {
    
    app_ic_regression_setup_self
    
    global regression_models_using_function
    if ![info exists regression_models_using_function($function)] {
	return ""
    }
    
    return $regression_models_using_function($function)    
}


## Get all commands used by models in the base_test_dir

proc app_ic_regression_get_functions_used_by_models {} {
    
    app_ic_regression_setup_self
    
    global regression_models_using_function
    if ![array exists regression_models_using_function] {
	return ""
    }
    
    return [lsort [array names regression_models_using_function]]
}


## Get all models in the base test dir

proc app_ic_regression_get_models_all {} {
    global proc_scrib_data ; scrib_app_data_get ;# tmp:fixme
    
    set list_of_models ""
    
    if ![info exists proc_scrib_data(base_test_dir)] {
	stdout "Not a valid base test directory\n" red
	return ""
    }
    set base_test_dir $proc_scrib_data(base_test_dir)
    if ![file isdir $base_test_dir] {
	stdout "Not a valid path for a base test directory: $base_test_dir\n" red
	return ""
    }
    
    set file_separator "//" ;# [file separator] not yet...need tcl_8.4 for that
    foreach module [glob -nocomplain -type d $base_test_dir/*] {
	
	set module_path [file join $base_test_dir $module]	
	foreach test_model [glob -nocomplain -type d $module_path/*] {	    
	    lappend list_of_models $test_model
	}
    }
    
    return [lsort $list_of_models]
}


## Get all models for the given module in the base_test_dir

proc app_ic_regression_get_models_for_module {module} {
    global proc_scrib_data ; scrib_app_data_get ;# tmp:fixme
    
    app_ic_regression_setup_self
    
    set list_of_models ""
    
    if ![info exists proc_scrib_data(base_test_dir)] {
	stdout "Not a valid base test directory\n" red
	return ""
    }
    set base_test_dir $proc_scrib_data(base_test_dir)
    if ![file isdir $base_test_dir] {
	stdout "Not a valid path for a base test directory: $base_test_dir\n" red
	return ""
    }
    set module_path [file join $base_test_dir $module]
    if ![file isdir $module_path] {
	stdout "Not a valid directory for a test module: $module_path\n" red
	return ""
    }
    
    set list_of_models [glob -nocomplain -type d $module_path/*]
    
    return $list_of_models
}


## Get all modules in the base_test_dir

proc app_ic_regression_get_modules_all {} {
    global proc_scrib_data ; scrib_app_data_get ;# tmp:fixme
        
    app_ic_regression_setup_self

    set list_of_modules ""
    
    if ![info exists proc_scrib_data(base_test_dir)] {
	stdout "Not a valid base test directory\n" red
	return ""
    }
    set base_test_dir $proc_scrib_data(base_test_dir)
    if ![file isdir $base_test_dir] {
	stdout "Not a valid path for a base test directory: $base_test_dir\n" red
	return ""
    }
    
    foreach module [glob -nocomplain -type d $base_test_dir/*] {
	
	lappend list_of_modules [file tail $module]
    }
    
    return $list_of_modules
}


##

proc app_ic_regression_get_models_for_modules {modules} {
    set models_for_module ""
    foreach module $modules {
	eval lappend models_for_module [app_ic_regression_get_models_for_module $module]
    }
    return $models_for_module
}


## Test all models in the given module

proc app_ic_regression_test_models_for_modules {modules} {
    
    app_ic_regression_setup_self
    
    set models_for_module [app_ic_regression_get_models_for_modules $modules]
    
    set stat [app_ic_regression_test_models $models_for_module]
    
    stdout "\nFinished testing modules: $modules\n" green
    
    return $stat
}


## Test all models using the given function

proc app_ic_regression_test_models_using_functions {functions} {
    
    app_ic_regression_setup_self
    
    set models_using_function ""
    foreach function $functions {
	eval lappend models_using_function [app_ic_regression_get_models_using_function $function]
    }
    
    set stat [app_ic_regression_test_models [lsort -unique $models_using_function]]
    
    stdout "\nFinished testing function: $functions\n" green
    
    return $stat
}


## Test all models in the base_test_dir

proc app_ic_regression_test_models_all {} {
    
    set models [app_ic_regression_get_models_all]
    
    set nmodels   [llength $models]
    
    if {50 < $nmodels} {
	if {1 == [tk_dialog .view_xml "Got some time on your hands?" \
		      "Are you sure you want to test $nmodels models?" "" 0 Yes No]} {
	    return ""
	}
    }
    
    return [app_ic_regression_test_models $models]
}


##

proc app_ic_regression_test_models_active {} {
    global common_tree_data
    
    set separator [scrib_app_tree_get_separator common_tree_data]
    set active_models ""
    
    foreach mtoken [lsort -unique $common_tree_data(__MODEL_KEYS__)] {
	
	if {[info exists common_tree_data($mtoken,__ACTIVE__)] && \
		(1 == $common_tree_data($mtoken,__ACTIVE__))} {
	    
	    lappend active_models $common_tree_data($mtoken,__PATH__)
	}
    }
    
    if {"" == $active_models} {
	mess "Error: there are no active models in the \"modules\" tree!\n" red
	return
    }
    
    app_ic_regression_test_models $active_models
}


## Test the given models

proc app_ic_regression_test_models {models} {
    global proc_scrib_data ; scrib_app_data_get ;# tmp:fixme
    
    set report_bugs 0
    if {[info exists proc_scrib_data(report_to_maintainer)] && \
	    (1 == $proc_scrib_data(report_to_maintainer))} {
	set report_bugs 1
    }
    
    set site "berkeley"
    if [info exists proc_scrib_data(test_site)] {
	set site $proc_scrib_data(test_site)
    }
    scrib_lib_regression_set_site $site
        
    set bqs "icbqs"
    if [info exists proc_scrib_data(test_bqs)] {
	set bqs $proc_scrib_data(test_bqs)
    }
    
    if {[info exists proc_scrib_data(plats_to_test)] && \
	    ("" != $proc_scrib_data(plats_to_test))} {
	set platforms $proc_scrib_data(plats_to_test)
    } else {
	set platforms [scrib_lib_regression_get_platforms $site]
    }
    
    if {[info exists proc_scrib_data(testing_icem_acn)] && \
	    ("" != $proc_scrib_data(testing_icem_acn))} {
	global env
	set env(ICEM_ACN) $proc_scrib_data(testing_icem_acn)
    }
    
    if [info exists proc_scrib_data(run_comparison)] {
	set comparison_type $proc_scrib_data(run_comparison)
    } else { set comparison_type "" }
    
    if [info exists proc_scrib_data(time_profiling)] {	
	set time_profiling $proc_scrib_data(time_profiling)
    } else { set time_profiling "" }
    
    set new_models ""
    foreach m $models {
	if ![file isfile $m] {
	    set m [file join $proc_scrib_data(base_test_dir) $m]
	}
	lappend new_models $m
    }
    set models $new_models
    #
    #
    #
    if {![file exists $proc_scrib_data(test_output_dir)] && ![file isdir $proc_scrib_data(test_output_dir)]} {
	file mkdir $proc_scrib_data(test_output_dir)
    }
    set xml_report "[scrib_lib_utils_logfile $proc_scrib_data(test_output_dir) RegressionGresults].xml"
    set html_report "[file root $xml_report].html"
    
    catch {
	TestBenchReport::createDocument "NAME \"$xml_report\" \
                                         TITLE \"Regression G Results\" \
                                         APPLICATION \"ANSYS/ICEM CFD\" \
                                         VERSION \"$proc_scrib_data(testing_icem_acn)\" \
                                         MACHINE \"$platforms\" \
                                         DATE \"[clock format [clock seconds]]\""
    } dom_handle
    
    mess "\nTesting models on platforms \"$platforms\"\n\n"
    
    #
    # --> Set up a batch-queue
    #
    
    set servers ""
    foreach plat $platforms {
	lappend servers [scrib_lib_regression_get_host $plat $site]
    }
    set queue_handle [batch_queue::init_queue $bqs -servers "$servers"]
    if {"" == $queue_handle} {
	mess "Error: could not init queue of type \"$bqs\" on servers \"$servers\"" red
	return
    }
    
    #
    # --> Check the queue for sanity, insofaras we are sane enough to judge
    #
    
    if ![batch_queue::queue_is_sane $queue_handle] {
	mess "Error: the queue is not sane...please make sure the servers are running on the selected hosts!\n" red
	return 0
    }
    
    set njobs_queued 0
    
    foreach model $models {
	
	set logfile [scrib_lib_utils_logfile $proc_scrib_data(test_output_dir) "[file tail $model]-regressionLog"]
	
	foreach plat $platforms {
	    
	    set job_token [scrib_lib_regression_model_submit_to_test_queue $queue_handle $model $plat $logfile $comparison_type]
	    batch_queue::add_job_attribute $job_token MODEL $model
	    incr njobs_queued
	    scrib_app_status_update "Have queued $njobs_queued jobs"
	}
    }
    
    #
    # --> Wait for the jobs to finish
    #
    
    while { [batch_queue::are_jobs_running $queue_handle] } {
	
	set njobs [batch_queue::num_jobs_running $queue_handle]
	if {1 == $njobs} {
	    set message "There is $njobs job running..."
	} else {
	    set message "There are $njobs jobs running..."
	}
	scrib_app_status_update $message
	
	after 5000 ;# fixme ?? watch ; make depend on njobs?? --jka
    }
    
    scrib_app_status_update "Jobs have completed..."
    set njobs [batch_queue::queue_summary $queue_handle]
    
    #
    # --> Get all info about the jobs...and, report bugs, and write reports.
    #
    
    set nfailed    0
    set nsucceeded 0
    set nunknown   0
    
    foreach job_token [batch_queue::job_token_list $queue_handle] {
	
	set status [batch_queue::get_job_status $job_token]
	
	switch -- $status {
	    FINISHED - SUCCESS {
		incr nsucceeded
		set status SUCCESS
	    }
	    TERMINATED - FAILURE {
		incr nfailed
		set status FAILURE
	    }
	    MISCONFIG - default { 
		incr nunknown
		set status MISCONFIG
	    }
	}
	
	set logfile    [batch_queue::get_job_attribute $job_token LOGFILE]	    		
	set start_time   [batch_queue::get_job_attribute $job_token START_TIME]
	if { ("__NONE__" != $start_time) && [string is integer $start_time] } {
	    set start_time [clock format $start_time]
	}
	set platform     [scrib_lib_regression_get_plat [batch_queue::get_job_attribute $job_token HOST]]
	set time_elapsed [batch_queue::get_job_elapsed_time $job_token]
	set model        [batch_queue::get_job_attribute $job_token MODEL]
	
	TestBenchReport::addTestcase $dom_handle [file tail $model] "START \"$start_time\" RESULT $status \
                                   TESTPATH $model ELAPSED $time_elapsed DUMP $logfile PLATFORM $platform DATE"
	
	app_ic_regression_update_tree_nodes $model $status $logfile
    }
    
    # this is the end of this queue
    batch_queue::delete_queue $queue_handle
    
    #
    # --> Finish up the report
    #
    
    if [catch {
	TestBenchReport::writeReport $dom_handle $xml_report "xml"
	TestBenchReport::writeReport $dom_handle $html_report "html"
	TestBenchReport::closeReport $dom_handle
    } res] {
	mess "Error: could not write dom $dom_handle as report $xml_report: $res\n" red
    } else {
	mess "\nWrote results to XML file: $xml_report\n\n"
	global last_xml_report
	set last_xml_report $xml_report
    }
    
    set result_string "$nsucceeded succeeded; $nfailed failed; not sure about $nunknown"
    #tk_messageBox -message "Results: $result_string" -title "Test results"
    scrib_app_status_update $result_string
    
    #
    # --> View results
    #
    
    set view_file [tk_dialog .view_xml "View XML reults?" \
		       "View XML results from this test set?" "" 0 None XML HTML]
    switch -- $view_file {
	0 {
	    mess "Not viewing results...you can also \"View XML results\" later with file: \"$xml_report\"\n"
	}
	1 {
	    app_ic_regression_view_results_ml_file $xml_report
	}
	2 {
	    app_ic_regression_view_results_ml_file $html_report
	}
    }    
}

## *ML: could be html or xml

proc app_ic_regression_view_results_ml_file {{xml_file ""}} {
    
    return [scrib_app_results_view $xml_file]
}


#---------------------------------------------------------------------------------------
#
#
# The following code relates to the TREE widgets and their hashtables:
#                 function_tree_data,
#                  module_tree_data
# 
# The most important thing to note is that the TreeWidget node maps to a
# hashtable key. So, in:
#         $function_tree_data(__TREE__) ... $token ...
# sits the value:
#         $function_tree_data($token)
#
# Tokens abides by rules then, so use the global var "tree_hashkey_separator" and
# the command "scrib_lib_utils_get_valid_token" and note the "eval list split"
# usage hanging around.
#


##

proc app_ic_regression_get_test_model_hash_key {path} {
    return [join [list [lindex [file split $path] end-1] [lindex [file split $path] end]] "_"]
}


## register a tree node

proc app_ic_regression_register_tree_node {model_path tree_name node} {
    global common_tree_data
    
    set key [app_ic_regression_get_test_model_hash_key $model_path]

    lappend common_tree_data(__MODEL_KEYS__) $key
    
    set common_tree_data(${key},${tree_name},node) $node
    
    if ![info exists common_tree_data(${key},__PATH__)] {
	set common_tree_data(${key},__PATH__) $model_path
    }
    if ![info exists common_tree_data(${key},__ACTIVE__)] {
	set common_tree_data(${key},__ACTIVE__) 0
    }
}


## fixme

proc app_ic_regression_update_tree_nodes {model_path status {logfile ""}} {
    global common_tree_data function_tree_data module_tree_data

    set key [app_ic_regression_get_test_model_hash_key $model_path]
    
    common_tree_set_model_activity $model_path 0
    
    foreach tree_name {function_tree_data module_tree_data} {
	
	set node $common_tree_data(${key},${tree_name},node)
	
	app_ic_regression_tree_refresh_node_status $tree_name $node $status
	
	# fixme for logfile	
	#set common_tree_data($token,CMD_VIEW_LOG) "View log"
	#set common_tree_data($token,CMD_VIEW_LOG,__IMAGE__) "text_image"
	#set common_tree_data($token,CMD_VIEW_LOG,__PATH__)  $logfile    
	#set common_tree_data($token,status) $status
	#set common_tree_data($token,status,__IMAGE__) "text_image"
    }        
}

##

proc app_ic_regression_function_tree_data_set_modules {functions} {
    global function_tree_data
    
    set separator [scrib_app_tree_get_separator function_tree_data]
    
    foreach f $functions {
	set nt [llength [app_ic_regression_get_models_using_function $f]]	
	set function_tree_data($f) "$f ($nt model[expr {($nt == 1) ? "" : "s"}])"
	set function_tree_data($f,__IMAGE__)  "inactive_image"
	set function_tree_data($f,__ACTIVE__) 0
    }
}

##

proc app_ic_regression_function_tree_data_set_function_models {functions test_model_paths} {
    global function_tree_data
    
    set separator [scrib_app_tree_get_separator function_tree_data]
    
    foreach f $functions {
	
	foreach test_path $test_model_paths {
	    
	    set tname [app_ic_regression_get_test_model_hash_key $test_path]
	    set token [scrib_lib_utils_get_valid_token [list $f $tname] $separator]
	    
	    set function_tree_data($token) $tname
	    
	    set function_tree_data($token,__PATH__) $test_path
	    set function_tree_data($token,__ACTIVE__) 0
	    set function_tree_data($token,__IMAGE__)  "inactive_image"
	    
	    #lappend function_tree_data(__MODEL_KEYS__) $token

	    app_ic_regression_register_tree_node $test_path "function_tree_data" $token
	}
    }
}

##

proc app_ic_regression_module_tree_data_set_modules {modules} {
    global module_tree_data
    
    set separator [scrib_app_tree_get_separator module_tree_data]
    
    foreach m $modules {
	
	set mtoken [scrib_lib_utils_get_valid_token [list $m] $separator]	
	set ntests_module [llength [app_ic_regression_get_models_for_module $m]]
	
	set module_tree_data($mtoken) "$m ($ntests_module models)"
	set module_tree_data($mtoken,__IMAGE__)  "inactive_image"
	set module_tree_data($mtoken,__ACTIVE__) 0
    }
}

##

proc app_ic_regression_module_tree_data_set_module_models {modules test_model_paths} {    
    global module_tree_data
    
    set separator [scrib_app_tree_get_separator module_tree_data]
    
    foreach m $modules {
	
	set mname [lindex [file split $m] end]
	
	foreach test_path $test_model_paths {
	    
	    set tname [app_ic_regression_get_test_model_hash_key $test_path]
	    set token [scrib_lib_utils_get_valid_token [list $mname $tname] $separator]
	    
	    set status "NOT RUN"
	    set module_tree_data($token) "$tname" ;# " (status: $status)"
	    set module_tree_data($token,__PATH__) $test_path
	    
	    set module_tree_data($token,__ACTIVE__) 0
	    set module_tree_data($token,__IMAGE__)  "inactive_image"
	    
	    #lappend module_tree_data(__MODEL_KEYS__) $token

	    app_ic_regression_register_tree_node $test_path "module_tree_data" $token
	}
    }
}

#
# ---------------------------------------------------------------------------------------------------
#

proc app_ic_regression_print_mouse_bindings {} {
    mess "Tree mouse bindings:\n\tFIXME\n" blue    
}

##

proc app_ic_regression_tree_refresh_node_status {tree_name token new_status} {
    set tree [scrib_app_tree_get_path $tree_name]
    app_ic_regression_tree_refresh_node_image $tree $token $new_status
}

##

proc app_ic_regression_tree_refresh_node_image {tree node status} {
    switch -- $status {
	SUCCESS {
	    return [scrib_app_tree_node_configure_image $tree $node success_image]
	}
	FAILURE {
	    return [scrib_app_tree_node_configure_image $tree $node failure_image]
	}
	INACTIVE {	    
	    return [scrib_app_tree_node_configure_image $tree $node inactive_image]
	}
	ACTIVE {	    
	    return [scrib_app_tree_node_configure_image $tree $node active_image]
	}
	MISCONFIG - UNKNOWN {
	    return [scrib_app_tree_node_configure_image $tree $node unknown_image]
	}
	default {
	    mess "Error: trying to change node $node status to $status\n"
	}
    }
    return ERROR
}

# ------------------------------------------------------------------------------------------------------
#
# These commands (*_action_mouse_button_*) are specified in the data_column.tcl file
# as arguments to the scrib_lib_data_tree_widget, and are bound to either text or image
# or the tree-node.
#
# Note again, that {TreeWidget $node == *_$hashkey in ${treename}_tree_data hash}
#


proc common_tree_set_model_activity {model_path {given_active ""}} {
    global common_tree_data function_tree_data module_tree_data
    #
    #
    #
    set key [app_ic_regression_get_test_model_hash_key $model_path]

    if ![info exists common_tree_data(${key},__ACTIVE__)] {
	mess "Error: could not get active flag for $key\n" red
	set common_tree_data(${key},__ACTIVE__) 0
    }
    if {"" == $given_active} {
	#
	# then toggle
	#
	set active $common_tree_data(${key},__ACTIVE__)
	if $active {
	    set common_tree_data(${key},__ACTIVE__) 0		    
	} else {
	    set common_tree_data(${key},__ACTIVE__) 1
	}		
    } else {
	set common_tree_data(${key},__ACTIVE__) $given_active
    }
    set active $common_tree_data(${key},__ACTIVE__)
    
    if $active {
	set flag ACTIVE
    } else {
	set flag INACTIVE
    }
    
    foreach tree_name {function_tree_data module_tree_data} {
	if [info exists common_tree_data($key,$tree_name,node)] {
	    set node $common_tree_data(${key},${tree_name},node)
	    app_ic_regression_tree_refresh_node_status $tree_name $node $flag
	}
    }
}


## 

proc module_tree_text_action_mouse_button_left {node {given_active ""}} {
    global module_tree_data
    #mess "module_tree_action_mouse_button_left $node: $module_tree_data($node)\n" blue
    #
    # decide what to do based on the node-order (==hashkey)
    #
    set separator [scrib_app_tree_get_separator module_tree_data]
    set order [llength [split $node $separator]]
    set tree [scrib_app_tree_get_path module_tree_data]
    switch -- $order {
	1 {
	    # this is a module, like "hexa" or "tetra"	    
	    # 
	    if ![info exists module_tree_data($node,__ACTIVE__)] {
		mess "Error: could not get active flag for $node\n" red
	    }
	    #
	    # ignore given_active
	    #
	    set active $module_tree_data($node,__ACTIVE__)
	    if $active {
		set module_tree_data($node,__ACTIVE__) 0
		app_ic_regression_tree_refresh_node_image $tree $node INACTIVE
	    } else {
		set module_tree_data($node,__ACTIVE__) 1
		app_ic_regression_tree_refresh_node_image $tree $node ACTIVE
	    }
	    set active $module_tree_data($node,__ACTIVE__)
	    #
	    # now, call this function re-entrantly foreach subnode
	    #
	    set sub_nodes [scrib_app_tree_get_nodes $tree $node]
	    foreach sn $sub_nodes {
		module_tree_text_action_mouse_button_left $sn $active
	    }
	}
	2 {
	    set model_path $module_tree_data($node,__PATH__)	    
	    common_tree_set_model_activity $model_path $given_active
	}
	3 {
	    # nothing for now
	}
	default {
	    mess "Error: not sure how we ended up with a zero-order node: $node: $module_tree_data($node)\n" red
	}
    }
}


## 

proc module_tree_image_action_mouse_button_left {node} {
    return [module_tree_text_action_mouse_button_left $node]
}


## 

proc module_tree_text_action_mouse_button_middle {node} {
    global module_tree_data
    global common_tree_data
    #
    # decide what to do based on the node-order (==hashkey)
    #
    set separator [scrib_app_tree_get_separator module_tree_data]
    set order [llength [split $node $separator]]
    switch -- $order {
	1 {
	    # this is a module node (hexa, tetra, etc.)
	    # test all models in this module
	    #
	    set models [app_ic_regression_get_models_for_modules $node]
mess "test_tree_action_mouse_button_middle: testing models: $models\n" red
	    #app_ic_regression_test_models $models
	}
	2 {
	    # this is a test-model node
	    # test this model only
	    #
	    set rnode [lindex [split $node $separator] end]
	    set path $module_tree_data($rnode,__PATH__)
mess "test_tree_action_mouse_button_middle:: $rnode $path\n" red
	    app_ic_regression_test_models $path
	}
	3 {
	    # nothing
	}
	default {
	    mess "Error: not sure how we ended up with a zero-order node: $node: $module_tree_data($node)\n" red
	}
    }        
}

## 

proc module_tree_image_action_mouse_button_middle {node} {
    return [module_tree_text_action_mouse_button_middle $node]
}

## 

proc module_tree_text_action_mouse_button_right {node} {
    global module_tree_data
    #
    # decide what to do based on the node-order (==hashkey)
    #
    set separator [scrib_app_tree_get_separator module_tree_data]
    set order [llength [split $node $separator]]
    switch -- $order {
	1 {
	    # this is a module node (hexa, tetra, etc.)
	    set nt [llength [app_ic_regression_get_models_for_module $node]]
	    tk_messageBox -message "Module \"$node\" has $nt model[expr {($nt == 1) ? "" : "s"}]" -title "Number of module test models"
	}
	2 {
	    # this is a test-model node
	    set name [lindex [split $node $separator] end]
	    set status [app_ic_regression_common_tree_get_status $name]
	    tk_messageBox -message "$name status: $status" -title "Model test status"
	}
	3 {
	    # this is a test-model attribute node
	    set name [lindex [split $node $separator] end]
	    tk_messageBox -message "$name: $module_tree_data($node)" -title "Node info"
	}
	default {
	    mess "Error: not sure how we ended up with a zero-order node: $node: $module_tree_data($node)\n" red
	}
    }
}

##

proc module_tree_image_action_mouse_button_right {node} {
    return [module_tree_text_action_mouse_button_right $node]
}

# -----------------------------------------------------------------------------------------

##

proc function_tree_text_action_mouse_button_left {node {given_active ""}} {
    global function_tree_data
    #mess "function_tree_action_mouse_button_left $node: $function_tree_data($node)\n" blue
    #
    # decide what to do based on the node-order (==hashkey)
    #
    set separator [scrib_app_tree_get_separator function_tree_data]
    set order [llength [split $node $separator]]
    set tree [scrib_app_tree_get_path function_tree_data]
    switch -- $order {
	1 {
	    # this is a module, like "hexa" or "tetra"	    
	    # 
	    if ![info exists function_tree_data($node,__ACTIVE__)] {
		mess "Error: could not get active flag for $node\n" red
	    }
	    #
	    # ignore given_active
	    #
	    set active $function_tree_data($node,__ACTIVE__)
	    if $active {
		set function_tree_data($node,__ACTIVE__) 0
		app_ic_regression_tree_refresh_node_image $tree $node INACTIVE
	    } else {
		set function_tree_data($node,__ACTIVE__) 1
		app_ic_regression_tree_refresh_node_image $tree $node ACTIVE
	    }
	    set active $function_tree_data($node,__ACTIVE__)
	    #
	    # now, call this function re-entrantly foreach subnode
	    #
	    set sub_nodes [scrib_app_tree_get_nodes $tree $node]
	    foreach sn $sub_nodes {
		function_tree_text_action_mouse_button_left $sn $active
	    }
	}
	2 {
	    set model_path $function_tree_data($node,__PATH__)
	    common_tree_set_model_activity $model_path $given_active
	}
	3 {
	    # nothing
	}
	default {
	    mess "Error: not sure how we ended up with a zero-order node: $node: $function_tree_data($node)\n" red
	}
    }
}

##

proc function_tree_image_action_mouse_button_left {node} {
    function_tree_text_action_mouse_button_left $node
}

##

proc function_tree_text_action_mouse_button_middle {node} {
    global function_tree_data
    #
    # decide what to do based on the node-order (==hashkey)
    #
    set separator [scrib_app_tree_get_separator function_tree_data]
    set order [llength [split $node $separator]]
    switch -- $order {
	1 {
	    # this is a module node (hexa, tetra, etc.)
	    # test all models in this module
	    set models [app_ic_regression_get_models_using_function $node]
mess "command_test_tree_action_mouse_button_middle: testing models: $models\n" red
	    app_ic_regression_test_models $models
	}
	2 {
	    # this is a test-model node
	    # test this model only
	    set path $function_tree_data($node,__PATH__)
mess "command_test_tree_action_mouse_button_middle:: $node $path\n" red
	    #app_ic_regression_test_models $path
	}
	3 {
	    # nothing
	}
	default {
	    mess "Error: not sure how we ended up with a zero-order node: $node: $function_tree_data($node)\n" red
	}
    }        
}

##

proc function_tree_text_action_mouse_button_right {node} {
    global function_tree_data
    #
    # decide what to do based on the node-order (==hashkey)
    #
    set separator [scrib_app_tree_get_separator function_tree_data]
    set order [llength [split $node $separator]]
    switch -- $order {
	1 {
	    # this is a module node (hexa, tetra, etc.)
	    set nm [llength [app_ic_regression_get_models_using_function $node]]
	    tk_messageBox -message "Command \"$node\" is used by $nm model[expr {($nm == 1) ? "" : "s"}]" -title "Number of test models using command"
	}
	2 {
	    # this is a test-model node
	    set name [lindex [split $node $separator] end]
	    set status [app_ic_regression_common_tree_get_status $name]
	    tk_messageBox -message "$name status: $status" -title "Model test status"
	}
	3 {
	    # this is a test-model attribute node
	    set name [lindex [split $node $separator] end]
	    tk_messageBox -message "$name: $function_tree_data($node)" -title "Node info"
	}
	default {
	    mess "Error: not sure how we ended up with a zero-order node: $node: $function_tree_data($node)\n" red
	}
    }
}


##

proc function_tree_image_action_mouse_button_middle {node} {
    mess "function_tree_image_action_mouse_button_middle: $node\n" blue
}


## 

proc function_tree_image_action_mouse_button_right {node} {
    mess "function_tree_image_action_mouse_button_right: $node\n" red
}


# -----------------------------------------------------------------------------------------

##

proc unused__app_ic_regression_common_tree_get_status {token} {
    global common_tree_data
    
    if ![info exists common_tree_data($token,status)] {
	mess "Error: result status for $token does not exist\n" red
	return ""
    }
    
    return $common_tree_data($token,status)    
}

proc unused__app_ic_regression_results_tree_update_node {model_path status logfile} {
    global common_tree_data
    
    set separator [scrib_app_tree_get_separator common_tree_data]
    set tname [app_ic_regression_get_test_model_hash_key $model_path]
    set token [scrib_lib_utils_get_valid_token [list $tname] $separator]
    
    app_ic_regression_tree_refresh_node_status common_tree_data $token $status
    
    set common_tree_data($token) "$tname" ;# " (status: $status)"
    
    set common_tree_data($token,CMD_VIEW_LOG) "View log"
    set common_tree_data($token,CMD_VIEW_LOG,__IMAGE__) "text_image"
    set common_tree_data($token,CMD_VIEW_LOG,__PATH__)  $logfile    
}

## Set or unset the __ACTIVE__ flag in the common_tree_data_hash

proc unused__common_tree_text_action_mouse_button_left {node} {
    global common_tree_data
    mess "model_test_tree_action_mouse_button_left $node: $common_tree_data($node)\n" blue
    return
    #
    # decide what to do based on the node-order (==hashkey)
    #
    set separator [scrib_app_tree_get_separator common_tree_data]
    set tree [scrib_app_tree_get_path common_tree_data]
    set order [llength [split $node $separator]]
    switch -- $order {
	1 {
	    
	}
	2 {
	    # 
	}
	default {
	    mess "Error: not sure how we ended up with $order-order node: $node: $common_tree_data($node)\n" red
	}
    }
}

## 

proc unused__common_tree_image_action_mouse_button_left {node} { 
    return [common_tree_text_action_mouse_button_left $node]
}

##

proc unused__common_tree_text_action_mouse_button_middle {node} {
    global common_tree_data
    #
    # decide what to do based on the node-order (==hashkey)
    #
    set separator [scrib_app_tree_get_separator common_tree_data]
    set order [llength [split $node $separator]]
    switch -- $order {
	1 {
	    set path $common_tree_data($node,__PATH__)
mess "model_test_tree_action_mouse_button_middle:: $node $path\n" red
	}
	2 {
mess "model_test_tree_action_mouse_button_middle: $node\n" red
	}
	default {
	    mess "Error: not sure how we ended up with a zero-order node: $node: $common_tree_data($node)\n" red
	}
    }        
}

##

proc unused__common_tree_image_action_mouse_button_middle {node} {
    return [common_tree_text_action_mouse_button_middle $node]
}

##

proc unused__common_tree_text_action_mouse_button_right {node} {
    global common_tree_data
    #
    # decide what to do based on the node-order (==hashkey)
    #
    set separator [scrib_app_tree_get_separator common_tree_data]
    set order [llength [split $node $separator]]
    switch -- $order {
	1 {
	    # fixme
	}
	2 {
	    # fixme
	}
	default {
	    mess "Error: not sure how we ended up with $order-order node: $node: $common_tree_data($node)\n" red
	}
    }
}

##

proc unused__common_tree_image_action_mouse_button_right {node} {
    return [common_tree_text_action_mouse_button_right $node]
}


#
# ------------------------------------------------------------------------------------------------
#

##

proc unused__app_ic_regression_common_tree_refresh {} {
   
    error "this is an inefficient approach: app_ic_regression_common_tree_refresh"
    
    set tree [scrib_app_tree_get_path common_tree_data]
    set models [scrib_app_tree_get_nodes $tree "root"]
    #mess "common_tree_refresh_data_in_gui:\n\ttree: $tree; commands: $models\n" black
    
    set status "FAILURE"
    
    foreach model $models {
	
	app_ic_regression_tree_refresh_node_image $tree $model $status
    }
}

## 

proc unused__app_ic_regression_module_tree_refresh {} {
    
    error "this is an inefficient approach: app_ic_regression_function_tree_refresh"
    
    set tree [scrib_app_tree_get_path module_tree_data]
    set modules [scrib_app_tree_get_nodes $tree "root"]
    mess "module_tree_refresh_data_in_gui:\n\ttree: $tree; modules: $modules\n" black
    
    set status "FAILURE"
    
    foreach module $modules {
	
	set models [scrib_app_tree_get_nodes $tree $module]
	#mess "\tfor module $module, found model-nodes: $models\n" black
	
	foreach model $models {
	    
	    app_ic_regression_module_tree_refresh_node_status $model $status
	}
    }
}

## 

proc unused__app_ic_regression_function_tree_refresh {} {

    error "this is an inefficient approach: app_ic_regression_function_tree_refresh"
   
    set tree [scrib_app_tree_get_path function_tree_data]
    set commands [scrib_app_tree_get_nodes $tree "root"]
    #mess "function_tree_refresh_data_in_gui:\n\ttree: $tree; commands: $commands\n" black
    
    set status "FAILURE"
    
    foreach command $commands {
	
	set models [scrib_app_tree_get_nodes $tree $command]
	mess "\tfor command $command, found model-nodes: $models\n" black
		
	foreach model $models {

	    app_ic_regression_tree_refresh_node_image $tree $model $status
	}
    }
}


##

proc unused_app_ic_regression_test_models_active {} {
    global module_tree_data common_tree_data
    
    set separator [scrib_app_tree_get_separator module_tree_data]
    set active_models ""
    
    foreach mtoken [lsort -unique $module_tree_data(__MODEL_KEYS__)] {
	
	if {[info exists module_tree_data($mtoken,__ACTIVE__)] && \
		(1 == $module_tree_data($mtoken,__ACTIVE__))} {
	    
	    set rnode [lindex [split $mtoken $separator] end]
	    set path $common_tree_data($rnode,path)
	    
	    lappend active_models $path
	}
    }
    
    if {"" == $active_models} {
	mess "Error: there are no active models in the \"modules\" tree!\n" red
	return
    }
    
    app_ic_regression_test_models $active_models
}


##

proc app_ic_regression_test_commands_active {} {
    global function_tree_data common_tree_data
    
    set separator [scrib_app_tree_get_separator function_tree_data]
    set active_models ""
    
    foreach mtoken $function_tree_data(__MODEL_KEYS__) {
	
	if {[info exists function_tree_data($mtoken,__ACTIVE__)] && \
		(1 == $function_tree_data($mtoken,__ACTIVE__))} {
	    
	    set rnode [lindex [split $mtoken $separator] end]
	    set path $common_tree_data($rnode,path)
	    
	    lappend active_models $path
	}
    }
    
    if {"" == $active_models} {
	mess "Error: there are no active models in the \"functions called\" tree!\n" red
	return
    }
    
    app_ic_regression_test_models $active_models
}

## 

proc unused_app_ic_regression_model_update_status {model new_status} {
    
    return ;# FIXME

    # for a given model, update the tree-elements related to it...wrt status, especially
    
    set mtree_token fixme ;# get the token for each tree
    app_ic_regression_tree_refresh_node_status $mtree_token $new_status
    
    set ctree_token fixme ;# get the token for each tree
    app_ic_regression_tree_refresh_node_status $ctree_token $new_status
    
    set rtree_token fixme ;# get the token for each tree
    app_ic_regression_tree_refresh_node_status $rtree_token $new_status
    scrib_app_tree_node_configure_text
}

##

proc app_ic_regression_common_tree_data_init_hash {model_paths} {
    global common_tree_data

    return ;# fixme (unused)

    set separator [scrib_app_tree_get_separator common_tree_data]
    set status    "NOT RUN"
    
    foreach test_path $model_paths {
	
	array set test_attributes [scrib_lib_regression_model_preferences_get $test_path]
	
	set tname [app_ic_regression_get_test_model_hash_key $test_path]
	set token [scrib_lib_utils_get_valid_token [list $tname] $separator]
	
	#set common_tree_data($token) "$tname (status: $status)"
	#set common_tree_data($token,__IMAGE__)  "unknown_image"
	#set common_tree_data($token,__ACTIVE__) 0
	set common_tree_data($token,__PATH__)   $test_path
	#set common_tree_data($token,path,__IMAGE__) "text_image"
	#set common_tree_data($token,status) $status
	#set common_tree_data($token,status,__IMAGE__) "text_image"
	#set common_tree_data($token,CMD_VIEW_LOG) "View log"
	#set common_tree_data($token,CMD_VIEW_LOG,__IMAGE__) "text_image"
	#
	# --> get the prefs for the model
	#
	#set hashkeys [array names test_attributes]
	#if {0 > [lsearch $hashkeys "__NOTHING__"]} {
	#    foreach a $hashkeys {
	#	set common_tree_data($token,$a) "$a: $test_attributes($a)"
	#	set common_tree_data($token,$a,__IMAGE__) "text_image"
	#    }
	#}
    }
    #parray common_tree_data
}

# -------------------------------------------------------------------------------------------------

proc OLD_NON_BQS_app_ic_regression_test_models {models} {
    global proc_scrib_data ; scrib_app_data_get ;# tmp:fixme
    
    #parray proc_scrib_data ;# tmp: fixme
    
    set report_bugs 0
    if {[info exists proc_scrib_data(report_to_maintainer)] && \
	    (1 == $proc_scrib_data(report_to_maintainer))} {
	set report_bugs 1
    }
    
    set site "berkeley"
    if [info exists proc_scrib_data(test_site)] {
	set site $proc_scrib_data(test_site)
    }
    scrib_lib_regression_set_site $site
        
    set bqs "icbqs"
    if [info exists proc_scrib_data(test_bqs)] {
	set bqs $proc_scrib_data(test_bqs)
    }
    
    if {[info exists proc_scrib_data(plats_to_test)] && \
	    ("" != $proc_scrib_data(plats_to_test))} {
	set platforms $proc_scrib_data(plats_to_test)
    } else {
	set platforms [scrib_lib_regression_get_platforms $site]
    }
    
    if {[info exists proc_scrib_data(testing_icem_acn)] && \
	    ("" != $proc_scrib_data(testing_icem_acn))} {
	global env
	set env(ICEM_ACN) $proc_scrib_data(testing_icem_acn)
    }
    
    if [info exists proc_scrib_data(run_comparison)] {
	set comparison_type $proc_scrib_data(run_comparison)
    } else { set comparison_type "" }
    
    if [info exists proc_scrib_data(time_profiling)] {	
	set time_profiling $proc_scrib_data(time_profiling)
    } else { set time_profiling "" }
    
    set new_models ""
    foreach m $models {
	if ![file isfile $m] {
	    set m [file join $proc_scrib_data(base_test_dir) $m]
	}
	lappend new_models $m
    }
    set models $new_models
    #
    #
    #
    if {![file exists $proc_scrib_data(test_output_dir)] && ![file isdir $proc_scrib_data(test_output_dir)]} {
	file mkdir $proc_scrib_data(test_output_dir)
    }
    set xml_report "[scrib_lib_utils_logfile $proc_scrib_data(test_output_dir) RegressionGresults].xml"
    set html_report "[file root $xml_report].html"
    
    if {"" != $xml_report} {
	catch {
	    TestBenchReport::createDocument "NAME \"$xml_report\" \
                                         TITLE \"Regression G Results\" \
                                         APPLICATION \"ANSYS/ICEM CFD\" \
                                         VERSION \"$proc_scrib_data(testing_icem_acn)\" \
                                         MACHINE \"$platforms\" \
                                         DATE \"[clock format [clock seconds]]\""
	} dom_handle
    }
    
    set old_dir [pwd] 
    
    #
    # --> Set up a batch-queue, if requested
    #
    if {"" != $bqs} {
	set servers ""
	foreach plat $platforms {
	    lappend servers [scrib_lib_regression_get_host $plat $site]
	}
	set queue_handle [batch_queue::init_queue $bqs -servers "$servers"]
	if {"" == $queue_handle} {
	    mess "Error: could not init queue of type \"$bqs\" on servers \"$servers\"" red
	    return
	}
    }
    
    mess "\nTesting models on platforms \"$platforms\"\n\n"
    
    set njobs_queued 0

    set nfailed    0
    set nsucceeded 0
    set nunknown    0
    
    foreach model $models {
	
	set logfile [scrib_lib_utils_logfile $proc_scrib_data(test_output_dir) "[file tail $model]-regressionLog"]
	
	foreach plat $platforms {
	    
	    if {"" == $bqs} {
		
		set status [scrib_lib_regression_model_test $model $plat $logfile $report_bugs $comparison_type "EXISTING_DOM"]
		switch -- $status {
		    SUCCESS {
			incr nsucceeded
		    }
		    FAILURE {
			incr nfailed
		    }
		    MISCONFIG - default {
			incr nunknown
		    }
		}
		#
		# --> update results_tree_data
		#
		app_ic_regression_update_tree_nodes $model $status $logfile
		
	    } else { ;# that is, we are batch-queueing
		
		set job_token [scrib_lib_regression_model_submit_to_test_queue $queue_handle $model $plat $logfile $comparison_type]
		batch_queue::add_job_attribute $job_token MODEL $model
		incr njobs_queued
		scrib_app_status_update "Have queued $njobs_queued jobs"
	    }
	}
    }
    
    if {"" != $bqs} {
	
	#batch_queue::queue_summary $queue_handle
	
	while { [batch_queue::are_jobs_running $queue_handle] } {
	    
	    set njobs [batch_queue::num_jobs_running $queue_handle]
	    if {1 == $njobs} {
		set message "There is $njobs job running..."
	    } else {
		set message "There are $njobs jobs running..."
	    }
	    scrib_app_status_update $message
	    
	    after 5000 ;# fixme ?? watch ; make depend on njobs?? --jka
	}
	
	scrib_app_status_update "Jobs have completed..."
	set njobs [batch_queue::queue_summary $queue_handle]

	#
	# --> get all info about the jobs...and, report bugs, and write reports.
	#
	
	foreach job_token [batch_queue::job_token_list $queue_handle] {
	    
	    set status [batch_queue::get_job_status $job_token]
	    
	    switch -- $status {
		FINISHED - SUCCESS {
		    incr nsucceeded
		    set status SUCCESS
		}
		TERMINATED - FAILURE {
		    incr nfailed
		    set status FAILURE
		}
		MISCONFIG - default { 
		    incr nunknown
		    set status MISCONFIG
		}
	    }
	    
	    set logfile    [batch_queue::get_job_attribute $job_token LOGFILE]	    
	    
	    app_ic_regression_update_tree_nodes $model $status $logfile
	    
	    set start_time   [batch_queue::get_job_attribute $job_token START_TIME]
	    if { ("__NONE__" != $start_time) && [string is integer $start_time] } {
		set start_time [clock format $start_time]
	    }
	    set platform     [scrib_lib_regression_get_plat [batch_queue::get_job_attribute $job_token HOST]]
	    set time_elapsed [batch_queue::get_job_elapsed_time $job_token]
	    set model        [batch_queue::get_job_attribute $job_token MODEL]
	    
	    TestBenchReport::addTestcase $dom_handle [file tail $model] "START \"$start_time\" RESULT $status \
                                   TESTPATH $model ELAPSED $time_elapsed DUMP $logfile PLATFORM $platform DATE"
	}
    }
    
    cd $old_dir
    
    if {"" != $xml_report} {
	if [catch {
	    TestBenchReport::writeReport $dom_handle $xml_report "xml"
	    TestBenchReport::writeReport $dom_handle $html_report "html"
	    TestBenchReport::closeReport $dom_handle
	} res] {
	    mess "Error: could not write dom $dom_handle as report $xml_report: $res\n" red
	} else {
	    mess "\nWrote results to XML file: $xml_report\n\n"
	    global last_xml_report
	    set last_xml_report $xml_report
	}
    }
    
    set result_string "$nsucceeded succeeded; $nfailed failed; not sure about $nunknown"
    #tk_messageBox -message "Results: $result_string" -title "Test results"
    scrib_app_status_update $result_string
    
    set view_file [tk_dialog .view_xml "View XML reults?" \
		       "View XML results from this test set?" "" 0 None XML HTML]    
    switch -- $view_file {
	0 {
	    mess "Not viewing results...you can also \"View XML results\" later with file: \"$xml_report\"\n"
	}
	1 {
	    app_ic_regression_view_results_ml_file $xml_report
	}
	2 {
	    app_ic_regression_view_results_ml_file $html_report
	}
    }
    
    if {"" != $bqs} {
	batch_queue::delete_queue $queue_handle
    }
}
