1 | #!/bin/sh |
---|
2 | # ---------------------------------------------------------------------- |
---|
3 | # RPTIMES |
---|
4 | # |
---|
5 | # Scans through a series of run.xml files and puts their runtime info |
---|
6 | # into a SQLite database. This database can be used to predict |
---|
7 | # future runtimes for runs with similar parameters. |
---|
8 | # |
---|
9 | # USAGE: |
---|
10 | # % rptimes <run.xml> ?<run.xml>...? |
---|
11 | # % rptimes -v <sqliteDB> |
---|
12 | # |
---|
13 | # Exits with status 0 if successful, and non-zero if any run.xml |
---|
14 | # files cannot be processed. |
---|
15 | # |
---|
16 | # Creates an SQLite DB for each unique tool referenced by a run.xml |
---|
17 | # file. Each DB file contains a table of parameters and a table of |
---|
18 | # "jobs" info. The jobs table includes CPU time and Wall time |
---|
19 | # measurements. |
---|
20 | # |
---|
21 | # ====================================================================== |
---|
22 | # AUTHOR: Michael McLennan, Purdue University |
---|
23 | # Copyright (c) 2004-2013 HUBzero Foundation, LLC |
---|
24 | # |
---|
25 | # See the file "license.terms" for information on usage and |
---|
26 | # redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES. |
---|
27 | # ====================================================================== |
---|
28 | # \ |
---|
29 | exec tclsh "$0" ${1+"$@"} |
---|
30 | # tclsh executes the rest... |
---|
31 | |
---|
32 | package require Rappture |
---|
33 | package require sqlite3 |
---|
34 | package require md5 |
---|
35 | package require base64 |
---|
36 | |
---|
37 | if {[llength $argv] < 1} { |
---|
38 | puts stderr "USAGE: rptimes run.xml ?run.xml...?" |
---|
39 | puts stderr "USAGE: rptimes -v tool.sql3" |
---|
40 | exit 1 |
---|
41 | } |
---|
42 | |
---|
43 | set op "ingest" |
---|
44 | while {[llength $argv] > 0} { |
---|
45 | set flag [lindex $argv 0] |
---|
46 | if {![string match -* $flag]} { |
---|
47 | break |
---|
48 | } |
---|
49 | switch -- $flag { |
---|
50 | -v { |
---|
51 | set op "view" |
---|
52 | if {[llength $argv] < 2} { |
---|
53 | puts stderr "USAGE: rptimes -v tool.sql3" |
---|
54 | exit 1 |
---|
55 | } |
---|
56 | set dbfile [lindex $argv 1] |
---|
57 | set argv [lrange $argv 2 end] |
---|
58 | } |
---|
59 | default { |
---|
60 | puts stderr "bad flag \"$flag\": should be -v" |
---|
61 | exit 1 |
---|
62 | } |
---|
63 | } |
---|
64 | } |
---|
65 | |
---|
66 | # ---------------------------------------------------------------------- |
---|
67 | # USAGE: normval <libObj> <rappturePath> ?-default? |
---|
68 | # |
---|
69 | # Returns information about the normalized value of a particular input |
---|
70 | # in the Rappture run <libObj> at the specified <rappturePath> for the |
---|
71 | # input. The optional -default switch causes the processing to apply |
---|
72 | # to the default value instead of the current value. |
---|
73 | # |
---|
74 | # Returns info as a list of values in the form: |
---|
75 | # |
---|
76 | # <rappturePath> <type> <normValue> |
---|
77 | # <rappturePath2> <type2> <normValue2> |
---|
78 | # ... |
---|
79 | # |
---|
80 | # Most inputs return just 3 values: path/type/value. But some inputs |
---|
81 | # split into multiple parts and return several triples. Images, for |
---|
82 | # example, return a hash for the image itself, but also width/height |
---|
83 | # parameters. The width/height may actually be the better predictors |
---|
84 | # of CPU time required. |
---|
85 | # ---------------------------------------------------------------------- |
---|
86 | proc normval {libobj path {what "-current"}} { |
---|
87 | set rlist "" |
---|
88 | |
---|
89 | # get the default or the current raw value |
---|
90 | set raw "" |
---|
91 | switch -- $what { |
---|
92 | -default { |
---|
93 | set raw [$libobj get $path.default] |
---|
94 | } |
---|
95 | -current { |
---|
96 | if {[$libobj element $path.current] ne ""} { |
---|
97 | set raw [$libobj get $path.current] |
---|
98 | } else { |
---|
99 | set raw [$libobj get $path.default] |
---|
100 | } |
---|
101 | } |
---|
102 | default { |
---|
103 | error "bad option \"$what\": should be -current or -default" |
---|
104 | } |
---|
105 | } |
---|
106 | |
---|
107 | # normalize the value depending on the type |
---|
108 | switch -- [$libobj element -as type $path] { |
---|
109 | integer { |
---|
110 | lappend rlist $path "INTEGER" $raw |
---|
111 | } |
---|
112 | number { |
---|
113 | set norm "" |
---|
114 | if {$raw ne ""} { |
---|
115 | # then normalize to default units |
---|
116 | set units [$libobj get $path.units] |
---|
117 | if {$units ne ""} { |
---|
118 | set norm [Rappture::Units::convert $raw \ |
---|
119 | -context $units -to $units -units off] |
---|
120 | } |
---|
121 | } |
---|
122 | lappend rlist $path "REAL" $norm |
---|
123 | } |
---|
124 | boolean - choice - loader - periodicelement { |
---|
125 | lappend rlist $path "TEXT" $raw |
---|
126 | } |
---|
127 | image { |
---|
128 | # convert long string inputs into a unique (short) hash |
---|
129 | set norm [base64::encode [md5::md5 $raw]] |
---|
130 | if {[catch {package require Img; wm withdraw .} result]} { |
---|
131 | error "can't analyze <image> types: $result" |
---|
132 | } |
---|
133 | if {[catch {image create photo -data $raw} result]} { |
---|
134 | error "can't analyze <image> data: $result" |
---|
135 | } |
---|
136 | set width [image width $result] |
---|
137 | set height [image height $result] |
---|
138 | |
---|
139 | lappend rlist $path "TEXT" $norm |
---|
140 | lappend rlist $path-WIDTH "INTEGER" $width |
---|
141 | lappend rlist $path-HEIGHT "INTEGER" $height |
---|
142 | } |
---|
143 | string { |
---|
144 | # convert long string inputs into a unique (short) hash |
---|
145 | set norm [base64::encode [md5::md5 $raw]] |
---|
146 | lappend rlist $path "TEXT" $norm |
---|
147 | } |
---|
148 | structure { |
---|
149 | # oops! structure doesn't have a clear .current value |
---|
150 | # use the XML dump of the data as its "value" |
---|
151 | if {$what eq "-current"} { |
---|
152 | if {[catch {$libobj xml $path.current} raw]} { |
---|
153 | if {[catch {$libobj xml $path.default} raw]} { |
---|
154 | set raw "" |
---|
155 | } |
---|
156 | } |
---|
157 | } elseif {[catch {$libobj xml $path.default} raw]} { |
---|
158 | set raw "" |
---|
159 | } |
---|
160 | set norm [base64::encode [md5::md5 $raw]] |
---|
161 | lappend rlist $path "TEXT" $norm |
---|
162 | } |
---|
163 | default { |
---|
164 | # for anything else, punt and use the raw value |
---|
165 | lappend rlist $path "TEXT" $raw |
---|
166 | } |
---|
167 | } |
---|
168 | return $rlist |
---|
169 | } |
---|
170 | |
---|
171 | # ---------------------------------------------------------------------- |
---|
172 | # USAGE: escapeQuotes <string> |
---|
173 | # |
---|
174 | # Escapes single quotes in the given <string> by escaping the quote. |
---|
175 | # In SQLite, this is done by doubling the quote. This makes it |
---|
176 | # possible to embed the string in another quoted string like |
---|
177 | # 'hello ''world'''. Returns a string with escaped quotes. |
---|
178 | # ---------------------------------------------------------------------- |
---|
179 | proc escapeQuotes {str} { |
---|
180 | regsub -all {([^\'])\'} $str {\1''} str |
---|
181 | return $str |
---|
182 | } |
---|
183 | |
---|
184 | # |
---|
185 | # Handle "view" operation |
---|
186 | # |
---|
187 | if {$op eq "view"} { |
---|
188 | sqlite3 db $dbfile |
---|
189 | |
---|
190 | puts "PARAMETERS:" |
---|
191 | db eval {SELECT nickName,rappturePath,type FROM parameters;} data { |
---|
192 | puts " - $data(nickName) ($data(type)) = $data(rappturePath)" |
---|
193 | } |
---|
194 | |
---|
195 | puts "\nJOBS:" |
---|
196 | set n 0 |
---|
197 | db eval {SELECT runToken,date,cpuTime,wallTime FROM jobs;} data { |
---|
198 | puts [format " [incr n]) [clock format $data(date)]: %6.2f CPU, %6.2f Wall" $data(cpuTime) $data(wallTime)] |
---|
199 | } |
---|
200 | |
---|
201 | catch {db close} |
---|
202 | exit 0 |
---|
203 | } |
---|
204 | |
---|
205 | # |
---|
206 | # Run through each run.xml file and load params and execution time |
---|
207 | # |
---|
208 | set status 0 |
---|
209 | foreach fname $argv { |
---|
210 | set db "" |
---|
211 | set err "" |
---|
212 | if {[catch {Rappture::library $fname} libobj]} { |
---|
213 | set err "failed: $fname ($libobj)" |
---|
214 | } |
---|
215 | |
---|
216 | set app [$libobj get tool.id] |
---|
217 | set rev [$libobj get tool.version.application.revision] |
---|
218 | if {$err eq ""} { |
---|
219 | if {$app eq "" || $rev eq ""} { |
---|
220 | set err "failed: $fname (missing app info -- is tool deployed?)" |
---|
221 | } else { |
---|
222 | set dbfile "${app}_r$rev.sql3" |
---|
223 | if {![file exists $dbfile]} { |
---|
224 | sqlite3 db $dbfile |
---|
225 | db eval {CREATE TABLE parameters(nickName TEXT PRIMARY KEY, rappturePath TEXT, defValue TEXT, type TEXT);} |
---|
226 | db eval {CREATE TABLE jobs(runToken TEXT, date TEXT, cpuTime REAL, wallTime REAL, nCpus INTEGER, venue TEXT);} |
---|
227 | } else { |
---|
228 | sqlite3 db $dbfile |
---|
229 | } |
---|
230 | } |
---|
231 | } |
---|
232 | |
---|
233 | if {$err eq "" && [set cput [$libobj get output.cputime]] eq ""} { |
---|
234 | set err "failed: $fname (missing cpu time)" |
---|
235 | } |
---|
236 | if {$err eq "" && [set wallt [$libobj get output.walltime]] eq ""} { |
---|
237 | set err "failed: $fname (missing wall time)" |
---|
238 | } |
---|
239 | |
---|
240 | set date "?" |
---|
241 | if {$err eq "" && [$libobj get output.time] ne "" && [catch {clock scan [$libobj get output.time]} d] == 0} { |
---|
242 | set date $d |
---|
243 | } |
---|
244 | |
---|
245 | if {$err eq "" && [$libobj get output.venue.name] ne ""} { |
---|
246 | set venue [$libobj get output.venue.name] |
---|
247 | } else { |
---|
248 | set venue "" |
---|
249 | } |
---|
250 | |
---|
251 | if {$err eq "" && [$libobj get output.venue.ncpus] ne ""} { |
---|
252 | set ncpus [$libobj get output.venue.ncpus] |
---|
253 | } else { |
---|
254 | set ncpus 1 |
---|
255 | } |
---|
256 | |
---|
257 | if {$err eq ""} { |
---|
258 | set runtoken [base64::encode [md5::md5 [$libobj xml]]] |
---|
259 | |
---|
260 | set cols "" |
---|
261 | set vals "" |
---|
262 | foreach path [Rappture::entities $libobj input] { |
---|
263 | # get the nickname (column name) for this paraemter |
---|
264 | set id [db eval "SELECT nickName from parameters where rappturePath='$path'"] |
---|
265 | if {$id eq ""} { |
---|
266 | # haven't seen this parameter before -- add it |
---|
267 | foreach {rp type def} [normval $libobj $path -default] { |
---|
268 | set num [db eval "SELECT COUNT(nickName) from parameters;"] |
---|
269 | set id [format "x%03d" [incr num]] |
---|
270 | |
---|
271 | db eval "INSERT INTO parameters values('$id','$rp','[escapeQuotes $def]','$type')" |
---|
272 | db eval "ALTER TABLE jobs ADD COLUMN $id $type;" |
---|
273 | } |
---|
274 | } |
---|
275 | |
---|
276 | # add the current value onto the values we're building up for ALTER |
---|
277 | foreach {raw norm} [Rappture::LibraryObj::value $libobj $path] break |
---|
278 | |
---|
279 | foreach {rp type val} [normval $libobj $path] { |
---|
280 | set num [db eval "SELECT COUNT(nickName) from parameters;"] |
---|
281 | set id [db eval "SELECT nickName FROM parameters WHERE rappturePath='$rp'"] |
---|
282 | if {$id eq ""} { |
---|
283 | set err "INTERNAL ERROR: couldn't find nickName for existing parameter $rp" |
---|
284 | break |
---|
285 | } |
---|
286 | lappend cols $id |
---|
287 | if {$type eq "TEXT"} { |
---|
288 | lappend vals '[escapeQuotes $val]' |
---|
289 | } elseif {$val ne ""} { |
---|
290 | lappend vals $val |
---|
291 | } else { |
---|
292 | lappend vals '' |
---|
293 | } |
---|
294 | } |
---|
295 | } |
---|
296 | |
---|
297 | if {$err eq ""} { |
---|
298 | # add the info for this job |
---|
299 | db eval "DELETE from jobs WHERE runToken='$runtoken';" |
---|
300 | db eval "INSERT INTO jobs (runToken,date,cpuTime,wallTime,nCpus,venue,[join $cols ,]) values ('$runtoken','$date',$cput,$wallt,$ncpus,'$venue',[join $vals ,]);" |
---|
301 | } |
---|
302 | } |
---|
303 | |
---|
304 | if {$err ne ""} { |
---|
305 | puts stderr $err |
---|
306 | set status 1 |
---|
307 | } |
---|
308 | catch {db close} |
---|
309 | } |
---|
310 | |
---|
311 | exit $status |
---|