[3501] | 1 | #!/bin/sh |
---|
| 2 | # ---------------------------------------------------------------------- |
---|
| 3 | # RPTIMES |
---|
| 4 | # |
---|
| 5 | # Scans through a series of run.xml files and puts their runtime info |
---|
| 6 | # into a SQLite database. This database can be used to predict |
---|
| 7 | # future runtimes for runs with similar parameters. |
---|
| 8 | # |
---|
| 9 | # USAGE: |
---|
| 10 | # % rptimes <run.xml> ?<run.xml>...? |
---|
[3625] | 11 | # % rptimes -v <sqliteDB> |
---|
[3501] | 12 | # |
---|
| 13 | # Exits with status 0 if successful, and non-zero if any run.xml |
---|
| 14 | # files cannot be processed. |
---|
| 15 | # |
---|
[3625] | 16 | # Creates an SQLite DB for each unique tool referenced by a run.xml |
---|
| 17 | # file. Each DB file contains a table of parameters and a table of |
---|
| 18 | # "jobs" info. The jobs table includes CPU time and Wall time |
---|
| 19 | # measurements. |
---|
[3501] | 20 | # |
---|
| 21 | # ====================================================================== |
---|
| 22 | # AUTHOR: Michael McLennan, Purdue University |
---|
| 23 | # Copyright (c) 2004-2013 HUBzero Foundation, LLC |
---|
| 24 | # |
---|
| 25 | # See the file "license.terms" for information on usage and |
---|
| 26 | # redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES. |
---|
| 27 | # ====================================================================== |
---|
| 28 | # \ |
---|
| 29 | exec tclsh "$0" ${1+"$@"} |
---|
| 30 | # tclsh executes the rest... |
---|
| 31 | |
---|
| 32 | package require Rappture |
---|
| 33 | package require sqlite3 |
---|
| 34 | package require md5 |
---|
| 35 | package require base64 |
---|
| 36 | |
---|
| 37 | if {[llength $argv] < 1} { |
---|
| 38 | puts stderr "USAGE: rptimes run.xml ?run.xml...?" |
---|
[3625] | 39 | puts stderr "USAGE: rptimes -v tool.sql3" |
---|
[3501] | 40 | exit 1 |
---|
| 41 | } |
---|
| 42 | |
---|
[3625] | 43 | set op "ingest" |
---|
| 44 | while {[llength $argv] > 0} { |
---|
| 45 | set flag [lindex $argv 0] |
---|
| 46 | if {![string match -* $flag]} { |
---|
| 47 | break |
---|
| 48 | } |
---|
| 49 | switch -- $flag { |
---|
| 50 | -v { |
---|
| 51 | set op "view" |
---|
| 52 | if {[llength $argv] < 2} { |
---|
| 53 | puts stderr "USAGE: rptimes -v tool.sql3" |
---|
| 54 | exit 1 |
---|
| 55 | } |
---|
| 56 | set dbfile [lindex $argv 1] |
---|
| 57 | set argv [lrange $argv 2 end] |
---|
| 58 | } |
---|
| 59 | default { |
---|
| 60 | puts stderr "bad flag \"$flag\": should be -v" |
---|
| 61 | exit 1 |
---|
| 62 | } |
---|
| 63 | } |
---|
| 64 | } |
---|
| 65 | |
---|
[3501] | 66 | # ---------------------------------------------------------------------- |
---|
[3507] | 67 | # USAGE: normval <libObj> <rappturePath> ?-default? |
---|
| 68 | # |
---|
| 69 | # Returns information about the normalized value of a particular input |
---|
| 70 | # in the Rappture run <libObj> at the specified <rappturePath> for the |
---|
| 71 | # input. The optional -default switch causes the processing to apply |
---|
| 72 | # to the default value instead of the current value. |
---|
| 73 | # |
---|
| 74 | # Returns info as a list of values in the form: |
---|
| 75 | # |
---|
| 76 | # <rappturePath> <type> <normValue> |
---|
| 77 | # <rappturePath2> <type2> <normValue2> |
---|
| 78 | # ... |
---|
| 79 | # |
---|
| 80 | # Most inputs return just 3 values: path/type/value. But some inputs |
---|
| 81 | # split into multiple parts and return several triples. Images, for |
---|
| 82 | # example, return a hash for the image itself, but also width/height |
---|
| 83 | # parameters. The width/height may actually be the better predictors |
---|
| 84 | # of CPU time required. |
---|
| 85 | # ---------------------------------------------------------------------- |
---|
| 86 | proc normval {libobj path {what "-current"}} { |
---|
| 87 | set rlist "" |
---|
[3501] | 88 | |
---|
[3507] | 89 | # get the default or the current raw value |
---|
| 90 | set raw "" |
---|
| 91 | switch -- $what { |
---|
| 92 | -default { |
---|
| 93 | set raw [$libobj get $path.default] |
---|
| 94 | } |
---|
| 95 | -current { |
---|
| 96 | if {[$libobj element $path.current] ne ""} { |
---|
| 97 | set raw [$libobj get $path.current] |
---|
| 98 | } else { |
---|
| 99 | set raw [$libobj get $path.default] |
---|
| 100 | } |
---|
| 101 | } |
---|
| 102 | default { |
---|
| 103 | error "bad option \"$what\": should be -current or -default" |
---|
| 104 | } |
---|
| 105 | } |
---|
| 106 | |
---|
| 107 | # normalize the value depending on the type |
---|
| 108 | switch -- [$libobj element -as type $path] { |
---|
| 109 | integer { |
---|
| 110 | lappend rlist $path "INTEGER" $raw |
---|
| 111 | } |
---|
| 112 | number { |
---|
| 113 | set norm "" |
---|
| 114 | if {$raw ne ""} { |
---|
| 115 | # then normalize to default units |
---|
| 116 | set units [$libobj get $path.units] |
---|
| 117 | if {$units ne ""} { |
---|
| 118 | set norm [Rappture::Units::convert $raw \ |
---|
| 119 | -context $units -to $units -units off] |
---|
| 120 | } |
---|
| 121 | } |
---|
| 122 | lappend rlist $path "REAL" $norm |
---|
| 123 | } |
---|
| 124 | boolean - choice - loader - periodicelement { |
---|
| 125 | lappend rlist $path "TEXT" $raw |
---|
| 126 | } |
---|
| 127 | image { |
---|
| 128 | # convert long string inputs into a unique (short) hash |
---|
| 129 | set norm [base64::encode [md5::md5 $raw]] |
---|
| 130 | if {[catch {package require Img; wm withdraw .} result]} { |
---|
| 131 | error "can't analyze <image> types: $result" |
---|
| 132 | } |
---|
| 133 | if {[catch {image create photo -data $raw} result]} { |
---|
| 134 | error "can't analyze <image> data: $result" |
---|
| 135 | } |
---|
| 136 | set width [image width $result] |
---|
| 137 | set height [image height $result] |
---|
| 138 | |
---|
| 139 | lappend rlist $path "TEXT" $norm |
---|
| 140 | lappend rlist $path-WIDTH "INTEGER" $width |
---|
| 141 | lappend rlist $path-HEIGHT "INTEGER" $height |
---|
| 142 | } |
---|
| 143 | string { |
---|
| 144 | # convert long string inputs into a unique (short) hash |
---|
| 145 | set norm [base64::encode [md5::md5 $raw]] |
---|
| 146 | lappend rlist $path "TEXT" $norm |
---|
| 147 | } |
---|
| 148 | structure { |
---|
| 149 | # oops! structure doesn't have a clear .current value |
---|
| 150 | # use the XML dump of the data as its "value" |
---|
| 151 | if {$what eq "-current"} { |
---|
| 152 | if {[catch {$libobj xml $path.current} raw]} { |
---|
| 153 | if {[catch {$libobj xml $path.default} raw]} { |
---|
| 154 | set raw "" |
---|
| 155 | } |
---|
| 156 | } |
---|
| 157 | } elseif {[catch {$libobj xml $path.default} raw]} { |
---|
| 158 | set raw "" |
---|
| 159 | } |
---|
| 160 | set norm [base64::encode [md5::md5 $raw]] |
---|
| 161 | lappend rlist $path "TEXT" $norm |
---|
| 162 | } |
---|
| 163 | default { |
---|
| 164 | # for anything else, punt and use the raw value |
---|
| 165 | lappend rlist $path "TEXT" $raw |
---|
| 166 | } |
---|
| 167 | } |
---|
| 168 | return $rlist |
---|
| 169 | } |
---|
| 170 | |
---|
[3624] | 171 | # ---------------------------------------------------------------------- |
---|
| 172 | # USAGE: escapeQuotes <string> |
---|
[3501] | 173 | # |
---|
[3624] | 174 | # Escapes single quotes in the given <string> by escaping the quote. |
---|
| 175 | # In SQLite, this is done by doubling the quote. This makes it |
---|
| 176 | # possible to embed the string in another quoted string like |
---|
| 177 | # 'hello ''world'''. Returns a string with escaped quotes. |
---|
| 178 | # ---------------------------------------------------------------------- |
---|
| 179 | proc escapeQuotes {str} { |
---|
| 180 | regsub -all {([^\'])\'} $str {\1''} str |
---|
| 181 | return $str |
---|
| 182 | } |
---|
| 183 | |
---|
| 184 | # |
---|
[3625] | 185 | # Handle "view" operation |
---|
| 186 | # |
---|
| 187 | if {$op eq "view"} { |
---|
| 188 | sqlite3 db $dbfile |
---|
| 189 | |
---|
| 190 | puts "PARAMETERS:" |
---|
| 191 | db eval {SELECT nickName,rappturePath,type FROM parameters;} data { |
---|
| 192 | puts " - $data(nickName) ($data(type)) = $data(rappturePath)" |
---|
| 193 | } |
---|
| 194 | |
---|
| 195 | puts "\nJOBS:" |
---|
| 196 | set n 0 |
---|
| 197 | db eval {SELECT runToken,date,cpuTime,wallTime FROM jobs;} data { |
---|
| 198 | puts [format " [incr n]) [clock format $data(date)]: %6.2f CPU, %6.2f Wall" $data(cpuTime) $data(wallTime)] |
---|
| 199 | } |
---|
| 200 | |
---|
| 201 | catch {db close} |
---|
| 202 | exit 0 |
---|
| 203 | } |
---|
| 204 | |
---|
| 205 | # |
---|
[3501] | 206 | # Run through each run.xml file and load params and execution time |
---|
| 207 | # |
---|
| 208 | set status 0 |
---|
| 209 | foreach fname $argv { |
---|
| 210 | set db "" |
---|
| 211 | set err "" |
---|
| 212 | if {[catch {Rappture::library $fname} libobj]} { |
---|
| 213 | set err "failed: $fname ($libobj)" |
---|
| 214 | } |
---|
| 215 | |
---|
| 216 | set app [$libobj get tool.id] |
---|
| 217 | set rev [$libobj get tool.version.application.revision] |
---|
| 218 | if {$err eq ""} { |
---|
| 219 | if {$app eq "" || $rev eq ""} { |
---|
| 220 | set err "failed: $fname (missing app info -- is tool deployed?)" |
---|
| 221 | } else { |
---|
| 222 | set dbfile "${app}_r$rev.sql3" |
---|
| 223 | if {![file exists $dbfile]} { |
---|
| 224 | sqlite3 db $dbfile |
---|
| 225 | db eval {CREATE TABLE parameters(nickName TEXT PRIMARY KEY, rappturePath TEXT, defValue TEXT, type TEXT);} |
---|
| 226 | db eval {CREATE TABLE jobs(runToken TEXT, date TEXT, cpuTime REAL, wallTime REAL, nCpus INTEGER, venue TEXT);} |
---|
| 227 | } else { |
---|
| 228 | sqlite3 db $dbfile |
---|
| 229 | } |
---|
| 230 | } |
---|
| 231 | } |
---|
| 232 | |
---|
| 233 | if {$err eq "" && [set cput [$libobj get output.cputime]] eq ""} { |
---|
| 234 | set err "failed: $fname (missing cpu time)" |
---|
| 235 | } |
---|
| 236 | if {$err eq "" && [set wallt [$libobj get output.walltime]] eq ""} { |
---|
| 237 | set err "failed: $fname (missing wall time)" |
---|
| 238 | } |
---|
| 239 | |
---|
| 240 | set date "?" |
---|
| 241 | if {$err eq "" && [$libobj get output.time] ne "" && [catch {clock scan [$libobj get output.time]} d] == 0} { |
---|
| 242 | set date $d |
---|
| 243 | } |
---|
| 244 | |
---|
| 245 | if {$err eq "" && [$libobj get output.venue.name] ne ""} { |
---|
| 246 | set venue [$libobj get output.venue.name] |
---|
| 247 | } else { |
---|
| 248 | set venue "" |
---|
| 249 | } |
---|
| 250 | |
---|
| 251 | if {$err eq "" && [$libobj get output.venue.ncpus] ne ""} { |
---|
| 252 | set ncpus [$libobj get output.venue.ncpus] |
---|
| 253 | } else { |
---|
| 254 | set ncpus 1 |
---|
| 255 | } |
---|
| 256 | |
---|
| 257 | if {$err eq ""} { |
---|
| 258 | set runtoken [base64::encode [md5::md5 [$libobj xml]]] |
---|
| 259 | |
---|
| 260 | set cols "" |
---|
| 261 | set vals "" |
---|
| 262 | foreach path [Rappture::entities $libobj input] { |
---|
[3507] | 263 | # get the nickname (column name) for this paraemter |
---|
| 264 | set id [db eval "SELECT nickName from parameters where rappturePath='$path'"] |
---|
| 265 | if {$id eq ""} { |
---|
| 266 | # haven't seen this parameter before -- add it |
---|
| 267 | foreach {rp type def} [normval $libobj $path -default] { |
---|
| 268 | set num [db eval "SELECT COUNT(nickName) from parameters;"] |
---|
| 269 | set id [format "x%03d" [incr num]] |
---|
[3501] | 270 | |
---|
[3624] | 271 | db eval "INSERT INTO parameters values('$id','$rp','[escapeQuotes $def]','$type')" |
---|
[3507] | 272 | db eval "ALTER TABLE jobs ADD COLUMN $id $type;" |
---|
[3501] | 273 | } |
---|
| 274 | } |
---|
| 275 | |
---|
[3507] | 276 | # add the current value onto the values we're building up for ALTER |
---|
| 277 | foreach {raw norm} [Rappture::LibraryObj::value $libobj $path] break |
---|
| 278 | |
---|
| 279 | foreach {rp type val} [normval $libobj $path] { |
---|
[3501] | 280 | set num [db eval "SELECT COUNT(nickName) from parameters;"] |
---|
[3507] | 281 | set id [db eval "SELECT nickName FROM parameters WHERE rappturePath='$rp'"] |
---|
| 282 | if {$id eq ""} { |
---|
| 283 | set err "INTERNAL ERROR: couldn't find nickName for existing parameter $rp" |
---|
| 284 | break |
---|
| 285 | } |
---|
| 286 | lappend cols $id |
---|
| 287 | if {$type eq "TEXT"} { |
---|
[3624] | 288 | lappend vals '[escapeQuotes $val]' |
---|
| 289 | } elseif {$val ne ""} { |
---|
| 290 | lappend vals $val |
---|
[3507] | 291 | } else { |
---|
[3624] | 292 | lappend vals '' |
---|
[3507] | 293 | } |
---|
[3501] | 294 | } |
---|
[3507] | 295 | } |
---|
[3501] | 296 | |
---|
[3507] | 297 | if {$err eq ""} { |
---|
| 298 | # add the info for this job |
---|
| 299 | db eval "DELETE from jobs WHERE runToken='$runtoken';" |
---|
| 300 | db eval "INSERT INTO jobs (runToken,date,cpuTime,wallTime,nCpus,venue,[join $cols ,]) values ('$runtoken','$date',$cput,$wallt,$ncpus,'$venue',[join $vals ,]);" |
---|
[3501] | 301 | } |
---|
| 302 | } |
---|
| 303 | |
---|
| 304 | if {$err ne ""} { |
---|
| 305 | puts stderr $err |
---|
| 306 | set status 1 |
---|
| 307 | } |
---|
[3507] | 308 | catch {db close} |
---|
[3501] | 309 | } |
---|
| 310 | |
---|
| 311 | exit $status |
---|