Changeset 6021 for trunk/lang/tcl/scripts/task.tcl

Timestamp:

Feb 18, 2016, 4:13:14 PM (9 years ago)

Author:

ldelgass

Message:

Merge UQ and fixes from 1.4 branch

Location:

trunk

Files:

• . (modified) (1 prop)
• lang/tcl/scripts/task.tcl (modified) (13 diffs)

trunk/lang/tcl/scripts/task.tcl

@@ -1 @@
-# -*- mode: tcl; indent-tabs-mode: nil -*- 
+# -*- mode: tcl; indent-tabs-mode: nil -*-
 # ----------------------------------------------------------------------
 #  COMPONENT: task - represents the executable part of a tool
@@ -29 +29 @@
 
     public method run {args}
+    public method get_uq {args}
     public method abort {}
     public method reset {}
@@ -37 +38 @@
     protected method _output {data}
     protected method _log {args}
+    protected method _build_submit_cmd {cmd tfile params_file}
+    protected method _get_params {varlist uq_type uq_args}
 
     private variable _xmlobj ""      ;# XML object with inputs/outputs
@@ -114 +117 @@
     }
     return ""
+}
+
+itcl::body Rappture::Task::get_uq {args} {
+    foreach {path val} $args {
+        if {$path == "-uq_type"} {
+            set uq_type $val
+        } elseif {$path == "-uq_args"} {
+            set uq_args $val
+        }
+    }
+    #set varlist [$_xmlobj uq_get_vars]
+    foreach {varlist num} [$_xmlobj uq_get_vars] break
+    return [Rappture::UQ ::#auto $varlist $num $uq_type $uq_args]
 }
 
@@ -156 +172 @@
     # if there are any args, use them to override parameters
     set _outputcb ""
+    set uq_type ""
     foreach {path val} $args {
         if {$path == "-output"} {
             set _outputcb $val
+        } elseif {$path == "-uq_type"} {
+            set uq_type $val
+        } elseif {$path == "-uq_args"} {
+            set uq_args $val
         } else {
             $_xmlobj put $path.current $val
@@ -165 +186 @@
 
     foreach item {control output error} { set job($item) "" }
-
-    # write out the driver.xml file for the tool
-    set file "driver[pid].xml"
-    set status [catch {
-        set fid [open $file w]
-        puts $fid "<?xml version=\"1.0\"?>"
-        puts $fid [$_xmlobj xml]
-        close $fid
-    } result]
 
     # Set limits for cpu time
@@ -187 +199 @@
             set limit 10;               # lower bound is 10 seconds.
         }
-    } 
-    Rappture::rlimit set cputime $limit  
+    }
+    Rappture::rlimit set cputime $limit
+
+    # write out the driver.xml file for the tool
+    set file "driver[pid].xml"
+    set status [catch {
+        set fid [open $file w]
+        puts $fid "<?xml version=\"1.0\"?>"
+        puts $fid [$_xmlobj xml]
+        close $fid
+    } result]
+
+    if {$uq_type != ""} {
+        # Copy xml into a new file
+        set tfile "template[pid].xml"
+        set fid [open $tfile w]
+        puts $fid "<?xml version=\"1.0\"?>"
+        puts $fid [$_xmlobj xml]
+        close $fid
+
+        # Return a list of the UQ variables and their PDFs.
+        # Also turns $tfile into a template file.
+        set uq_varlist [lindex [$_xmlobj uq_get_vars $tfile] 0]
+    }
+
+
     # execute the tool using the path from the tool description
     if {$status == 0} {
         set cmd [$_xmlobj get tool.command]
         regsub -all @tool $cmd $_installdir cmd
-        regsub -all @driver $cmd $file cmd
-        regsub -all {\\} $cmd {\\\\} cmd
         set cmd [string trimleft $cmd " "]
-        if { $cmd == "" } {
-            puts stderr "cmd is empty"
-            return [list 1 "Command is empty.\n\nThere is no command specified by\n\n <command>\n </command>\n\nin the tool.xml file."]
-        }
-
-        switch -glob -- [resources -jobprotocol] {
-            "submit*" {
-                # if job_protocol is "submit", then use use submit command
-                set cmd "submit --local $cmd"
-            }
-            "mx" {
-                # metachory submission
-                set cmd "mx $cmd"
-            }
-            "exec" {
-                # default -- nothing special
-            }
-        }
-        $_xmlobj put tool.execute $cmd
+
+        if { $cmd == "" } {
+            puts stderr "cmd is empty"
+            return [list 1 "Command is empty.\n\nThere is no command specified by\n\n <command>\n </command>\n\nin the tool.xml file."]
+        }
+
+        if {$uq_type == ""} {
+            regsub -all @driver $cmd $file cmd
+
+            switch -glob -- [resources -jobprotocol] {
+                "submit*" {
+                    # if job_protocol is "submit", then use use submit command
+                    set cmd "submit --local $cmd"
+                }
+                "mx" {
+                    # metachory submission
+                    set cmd "mx $cmd"
+                }
+                "exec" {
+                    # default -- nothing special
+                }
+            }
+        } else {
+            set params_file [_get_params $uq_varlist $uq_type $uq_args]
+            set cmd [_build_submit_cmd $cmd $tfile $params_file]
+            file delete -force puq
+        }
+
+        $_xmlobj put tool.execute $cmd
 
         # starting job...
@@ -223 +267 @@
         if {0 == [string compare -nocase -length 5 $cmd "ECHO "] } {
             set status 0;
-            set job(output) [string range $cmd 5 end] 
+            set job(output) [string range $cmd 5 end]
         } else {
             set status [catch {
-                set ::Rappture::Task::job(control) ""
-                eval blt::bgexec \
-                    ::Rappture::Task::job(control) \
-                    -keepnewline yes \
-                    -killsignal SIGTERM \
-                    -onoutput [list [itcl::code $this _output]] \
-                    -output ::Rappture::Task::job(output) \
-                    -error ::Rappture::Task::job(error) \
-                    $cmd
-            } result]
+                set ::Rappture::Task::job(control) ""
+                eval blt::bgexec \
+                ::Rappture::Task::job(control) \
+                -keepnewline yes \
+                -killsignal SIGTERM \
+                -onoutput [list [itcl::code $this _output]] \
+                -output ::Rappture::Task::job(output) \
+                -error ::Rappture::Task::job(error) \
+                $cmd
+            } result]
 
             if { $status != 0 } {
-                # We're here because the exec-ed program failed 
-                set logmesg $result
-                if { $::Rappture::Task::job(control) ne "" } {
-                    foreach { token pid code mesg } \
-                        $::Rappture::Task::job(control) break
-                    if { $token == "EXITED" } {
-                        # This means that the program exited normally but
-                        # returned a non-zero exitcode.  Consider this an
-                        # invalid result from the program.  Append the stderr
-                        # from the program to the message.
-                        set logmesg "Program finished: exit code is $code"
-                        set result "$logmesg\n\n$::Rappture::Task::job(error)"
-                    } elseif { $token == "abort" }  {
-                        # The user pressed the abort button.
+                # We're here because the exec-ed program failed
+                set logmesg $result
+                if { $::Rappture::Task::job(control) ne "" } {
+                    foreach { token pid code mesg } \
+                    $::Rappture::Task::job(control) break
+                    if { $token == "EXITED" } {
+                       # This means that the program exited normally but
+                       # returned a non-zero exitcode.  Consider this an
+                       # invalid result from the program.  Append the stderr
+                       # from the program to the message.
+                       set logmesg "Program finished: exit code is $code"
+                       set result "$logmesg\n\n$::Rappture::Task::job(error)"
+                    } elseif { $token == "abort" }  {
+                        # The user pressed the abort button.
                         set logmesg "Program terminated by user."
-                        set result "$logmesg\n\n$::Rappture::Task::job(output)"
-                    } else {
-                        # Abnormal termination
-                        set logmesg "Abnormal program termination: $mesg"
-                        set result "$logmesg\n\n$::Rappture::Task::job(output)"
-                    }
-                }
+                        set result "$logmesg\n\n$::Rappture::Task::job(output)"
+                    } else {
+                        # Abnormal termination
+                        set logmesg "Abnormal program termination: $mesg"
+                        set result "$logmesg\n\n$::Rappture::Task::job(output)"
+                    }
+                }
                 _log run failed [list $logmesg]
-                return [list $status $result]
+                return [list $status $result]
             }
         }
@@ -279 +323 @@
             if {[string length $jobstats] > 0} {
                 uplevel #0 $jobstats [list job [incr jobnum] \
-                    event $simulation start $times(start) \
-                    walltime $times(walltime) cputime $times(cputime) \
-                    status $status]
+                event $simulation start $times(start) \
+                walltime $times(walltime) cputime $times(cputime) \
+                status $status]
             }
 
@@ -340 +384 @@
     }
     if {$status == 0} {
-        file delete -force -- $file
+        # file delete -force -- $file
     }
 
@@ -353 +397 @@
     # a reference to the run.xml file containing results.
     #
+
     if {$status == 0} {
         set result [string trim $job(output)]
+
+        if {$uq_type != ""} {
+            # UQ. Collect data from all jobs and put it in one xml run file.
+            file delete -force -- run_uq.xml
+            if {[catch {exec puq_analyze.py puq_[pid].hdf5} res]} {
+                set fp [open "uq_debug.err" r]
+                set rdata [read $fp]
+                close $fp
+                puts "PUQ analysis failed: $res\n$rdata"
+                error "UQ analysis failed: $res\n$rdata"
+            } else {
+                append result "\n" $res
+            }
+        }
         if {[regexp {=RAPPTURE-RUN=>([^\n]+)} $result match file]} {
             set _lastrun $file
@@ -389 +448 @@
 
     return [list $status $result]
+}
+
+# ----------------------------------------------------------------------
+#  Turn the command string from tool.xml into the proper syntax to use
+#  with a submit parameter sweep with a temlate file.  Proper quoting
+# of the template file is necessary to prevent submit from being too smart
+# and converting it to a full pathname.
+# ----------------------------------------------------------------------
+itcl::body Rappture::Task::_build_submit_cmd {cmd tfile params_file} {
+    set quote_next 0
+    set newcmd "submit --progress submit --runName=puq -l -i @:$tfile -d $params_file"
+    set cmds [split $cmd " "]
+    for {set i 0} {$i < [llength $cmds]} {incr i} {
+        set arg [lindex $cmds $i]
+        if {$quote_next == 1} {
+            set nc [string range $arg 0 0]
+            if {$nc != "\""} {
+                set arg "\"\\\"$arg\\\"\""
+            }
+        }
+        if {$arg == "--eval"} {
+            set quote_next 1
+        } else {
+            set quote_next 0
+        }
+        if {$arg == "@driver"} {
+            set arg "\"\\\"$tfile\\\"\""
+        }
+        append newcmd " " $arg
+    }
+    regsub -all @driver $newcmd $tfile newcmd
+    return $newcmd
 }
 
@@ -550 +641 @@
     puts stderr $line
 }
+
+
+#
+# Send the list of parameters to a python program so it can call PUQ
+# and get a CSV file containing the parameter values to use for the runs.
+itcl::body Rappture::Task::_get_params {varlist uq_type uq_args} {
+    set pid [pid]
+    # puts "get_params.py $pid $varlist $uq_type $uq_args"
+    if {[catch {exec get_params.py $pid $varlist $uq_type $uq_args}]} {
+        set fp [open "uq_debug.err" r]
+        set rdata [read $fp]
+        close $fp
+        puts "get_params.py failed: $rdata"
+        error "get_params.py: $rdata"
+    }
+    return params[pid].csv
+}