source: trunk/tester/test.tcl @ 2061

Last change on this file since 2061 was 2055, checked in by braffert, 14 years ago

Regression tester: cleaning up and adding comments
File size: 15.7 KB
Line 
1# ----------------------------------------------------------------------
2#  COMPONENT: test - run a test and query the results
3#
4#  Encapsulates the testing logic, to keep it isolated from the rest of
5#  the tester GUI.  Constructor requires the location of the tool.xml
6#  for the new version, and the test xml file containing the golden set
7#  of results.
8# ======================================================================
9#  AUTHOR:  Ben Rafferty, Purdue University
10#  Copyright (c) 2010  Purdue Research Foundation
11#
12#  See the file "license.terms" for information on usage and
13#  redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
14# ======================================================================
15
16namespace eval Rappture::Tester::Test { #forward declaration }
17
18itcl::class Rappture::Tester::Test {
19
20    constructor {toolxml testxml} { #defined later }
21    destructor { #defined later }
22
23    private variable _added ""
24    private variable _diffs ""
25    private variable _missing ""
26    private variable _ran no
27    private variable _testxml
28    private variable _toolxml
29    private variable _result ""
30    private variable _runfile ""
31    private variable _testobj ""
32    private variable _toolobj ""
33    private variable _runobj ""
34
35    public method getAdded {}
36    public method getDiffs {}
37    public method getInputs {{path input}}
38    public method getMissing {}
39    public method getOutputs {{path output}}
40    public method getResult {}
41    public method getRunfile {}
42    public method getRunobj {}
43    public method getTestobj {}
44    public method getTestxml {}
45    public method hasRan {}
46    public method regoldenize {}
47    public method run {}
48
49    private method added {lib1 lib2 {path output}}
50    private method compareElements {lib1 lib2 path}
51    private method diffs {lib1 lib2 {path output}}
52    private method makeDriver {}
53    private method merge {toolobj golden driver {path input}}
54    private method missing {lib1 lib2 {path output}}
55
56}
57
58# ----------------------------------------------------------------------
59# CONSTRUCTOR
60# ----------------------------------------------------------------------
61itcl::body Rappture::Tester::Test::constructor {toolxml testxml} {
62    set _toolxml $toolxml
63    set _testxml $testxml
64    set _toolobj [Rappture::library $toolxml]
65    if {![Rappture::library isvalid $_toolobj]} {
66        error "$toolxml does not represent a valid library object"
67    }
68    set _testobj [Rappture::library $testxml]
69    if {![Rappture::library isvalid $_testobj]} {
70        error "$testxml does not represent a valid library object"
71    }
72    # HACK: Add a new input to differentiate between results
73    $_testobj put input.TestRun.current "Golden"
74}
75
76# ----------------------------------------------------------------------
77# DESTRUCTOR
78# ----------------------------------------------------------------------
79itcl::body Rappture::Tester::Test::destructor {} {
80    itcl::delete object $_toolobj
81    itcl::delete object $_testobj
82    if {$_ran} {
83        itcl::delete object $_runobj
84    }
85}
86
87# ----------------------------------------------------------------------
88# USAGE: getAdded
89#
90# Return a list of paths that have been added that do not exist in the
91# golden results.  Throws an error if the test has not been ran.
92# ----------------------------------------------------------------------
93itcl::body Rappture::Tester::Test::getAdded {} {
94    if {!$_ran} {
95        error "Test has not yet been ran."
96    }
97    return $_added
98}
99
100# ----------------------------------------------------------------------
101# USAGE: getDiffs
102#
103# Returns a list of paths that exist in both the golden and new results,
104# but contain data that does not match according to the compareElements
105# method.  Throws an error if the test has not been ran.
106# ----------------------------------------------------------------------
107itcl::body Rappture::Tester::Test::getDiffs {} {
108    if {!$_ran} {
109        error "Test has not yet been ran."
110    }
111    return $_diffs
112}
113
114# -----------------------------------------------------------------------
115# USAGE: getInputs
116#
117# Returns a list of key value pairs for all inputs given in the test xml.
118# Each key is the path to the input element, and each key is its current
119# value.
120# -----------------------------------------------------------------------
121itcl::body Rappture::Tester::Test::getInputs {{path input}} {
122    set retval [list]
123    foreach child [$_testobj children $path] {
124        set fullpath $path.$child
125        if {$fullpath != "input.TestRun"} {
126            set val [$_testobj get $fullpath.current]
127            if {$val != ""} {
128                lappend retval $fullpath $val
129            }
130        }
131        append retval [getInputs $fullpath]
132    }
133    return $retval
134}
135
136# ----------------------------------------------------------------------
137# USAGE: getMissing
138#
139# Return a list of paths that are present in the golden results, but are
140# missing in the new test results.  Throws an error if the test has not
141# been ran.
142# ----------------------------------------------------------------------
143itcl::body Rappture::Tester::Test::getMissing {} {
144    if {!$_ran} {
145        error "Test has not yet been ran."
146    }
147    return $_missing
148}
149
150# ----------------------------------------------------------------------
151# USAGE: getOutputs
152#
153# Returns a list of key value pairs for all outputs in the runfile
154# generated by the last run of the test.  Each key is the path to the
155# element, and each value is its status (ok, diff, added, or missing).
156# Throws an error if the test has not been ran.
157# ----------------------------------------------------------------------
158itcl::body Rappture::Tester::Test::getOutputs {{path output}} {
159    if {!$_ran} {
160        error "Test has not yet been ran."
161    }
162    set retval [list]
163    foreach child [$_runobj children $path] {
164        set fullpath $path.$child
165        if {$fullpath != "output.time" && $fullpath != "output.user" \
166            && $fullpath != "output.status"} {
167            if {[lsearch $fullpath [getDiffs]] != -1} {
168                set status diff
169            } elseif {[lsearch $fullpath [getAdded]] != -1} {
170                set status added
171            } else {
172                if {[$_runobj get $fullpath] != ""} {
173                    set status ok
174                } else {
175                    set status ""
176                }
177            }
178            lappend retval $fullpath $status
179        }
180        append retval " [getOutputs $fullpath]"
181    }
182    # We won't find missing elements by searching through runobj.  Instead,
183    # tack on all missing items at the end (only do this once)
184    if {$path == "output"} {
185        foreach item $_missing {
186            lappend retval $item missing
187        }
188    }
189    return $retval
190}
191
192# ----------------------------------------------------------------------
193# USAGE: getResult
194#
195# Returns the result of the test - either Pass, Fail, or Error.  Throws
196# an error if the test has not been ran.
197# ----------------------------------------------------------------------
198itcl::body Rappture::Tester::Test::getResult {} {
199    if {!$_ran} {
200        error "Test has not yet been ran."
201    }
202    return $_result
203}
204
205# ----------------------------------------------------------------------
206# USAGE: getRunfile
207#
208# Returns the location of the runfile generated by the previous run of
209# the test.  Throws an error if the test has not been ran.
210# ----------------------------------------------------------------------
211itcl::body Rappture::Tester::Test::getRunfile {} {
212    if {!$_ran} {
213        error "Test has not yet been ran."
214    }
215    return $_runfile
216}
217
218# -----------------------------------------------------------------------
219# USAGE: getRunobj
220#
221# Returns the library object generated by the previous run of the test.
222# Throws an error if the test has not been ran.
223# -----------------------------------------------------------------------
224itcl::body Rappture::Tester::Test::getRunobj {} {
225    if {!$_ran} {
226        error "Test has not yet been ran."
227    }
228    return $_runobj
229}
230
231# ----------------------------------------------------------------------
232# USAGE: getTestxml
233#
234# Returns the location of the test xml file containing the set of golden
235# results.
236# ----------------------------------------------------------------------
237itcl::body Rappture::Tester::Test::getTestxml {} {
238    return $_testxml
239}
240
241# ----------------------------------------------------------------------
242# USAGE: getTestobj
243#
244# Returns the test library object containing the set of golden results.
245# ----------------------------------------------------------------------
246itcl::body Rappture::Tester::Test::getTestobj {} {
247    return $_testobj
248}
249
250# ----------------------------------------------------------------------
251# USAGE: hasRan
252#
253# Returns yes if the test has been ran (with the run method), returns
254# no otherwise.
255# ----------------------------------------------------------------------
256itcl::body Rappture::Tester::Test::hasRan {} {
257    return $_ran
258}
259
260# ----------------------------------------------------------------------
261# USAGE: regoldenize
262#
263# Regoldenize the test by overwriting the test xml containin the golden
264# results with the data in the runfile generated by the last run.  Copy
265# test label and description into the new file.  Update the test's
266# result attributes to reflect the changes. Throws an error if the test
267# has not been ran.
268# ----------------------------------------------------------------------
269itcl::body Rappture::Tester::Test::regoldenize {} {
270    if {!$_ran} {
271        error "Test has not yet been ran."
272    }
273    $_runobj put test.label [$_testobj get test.label]
274    $_runobj put test.description [$_testobj get test.description]
275    set fid [open $_testxml w]
276    puts $fid "<?xml version=\"1.0\"?>"
277    puts $fid [$_runobj xml]
278    close $fid
279    set _testobj $_runobj
280    set _result Pass
281    set _diffs ""
282    set _added ""
283    set _missing ""
284}
285
286
287# ----------------------------------------------------------------------
288# USAGE: run
289#
290# Kicks off a new simulation and checks the results against the golden
291# set of results.  Set private attributes accordingly so that they can
292# later be retrieved via the public accessors.
293# ----------------------------------------------------------------------
294itcl::body Rappture::Tester::Test::run {} {
295    # Delete existing library if rerun
296    if {$_ran} {
297        itcl::delete object $_runobj
298    }
299    set driver [makeDriver]
300    set tool [Rappture::Tool ::#auto $driver [file dirname $_toolxml]]
301    foreach {status _runobj} [eval $tool run] break
302    set _ran yes
303    if {$status == 0 && [Rappture::library isvalid $_runobj]} {
304        # HACK: Add a new input to differentiate between results
305        $_runobj put input.TestRun.current "Test result"
306        set _diffs [diffs $_testobj $_runobj]
307        set _missing [missing $_testobj $_runobj]
308        set _added [added $_testobj $_runobj]
309        set _runfile [$tool getRunFile]
310        if {$_diffs == "" && $_missing == "" && $_added == ""} {
311            set _result Pass
312        } else {
313            set _result Fail
314        }
315    } else {
316        set _runobj ""
317        set _result Error
318    }
319}
320
321# ----------------------------------------------------------------------
322# USAGE: added lib1 lib2 ?path?
323#
324# Compares two library objects and returns a list of paths that have
325# been added in the second library and do not exist in the first.
326# Return value will contain all differences that occur as descendants of
327# an optional starting path.  If the path argument is not given, then
328# only the output sections will be compared.
329# ----------------------------------------------------------------------
330itcl::body Rappture::Tester::Test::added {lib1 lib2 {path output}} {
331    set paths [list]
332    foreach child [$lib2 children $path] {
333        foreach p [added $lib1 $lib2 $path.$child] {
334            lappend paths $p
335        }
336    }
337    if {[$lib1 get $path] == "" && [$lib2 get $path] != ""} {
338        lappend paths $path
339    }
340    return $paths
341}
342
343# ----------------------------------------------------------------------
344# USAGE: compareElements <lib1> <lib2> <path>
345#
346# Compare data found in two library objects at the given path.  Returns
347# 1 if match, 0 if no match.  For now, just check if ascii identical.
348# Later, we can do something more sophisticated for different types of
349# elements.
350# ----------------------------------------------------------------------
351itcl::body Rappture::Tester::Test::compareElements {lib1 lib2 path} {
352    set val1 [$lib1 get $path]
353    set val2 [$lib2 get $path]
354    return [expr {$val1} != {$val2}]
355}
356
357# ----------------------------------------------------------------------
358# USAGE: diffs <lib1> <lib2> ?path?
359#
360# Compares two library objects and returns a list of paths that do not
361# match.  Only paths which exist in both libraries are considered.
362# Return value will contain all differences that occur as descendants of
363# an optional starting path.  If the path argument is not given, then
364# only the output sections will be compared.
365# ----------------------------------------------------------------------
366itcl::body Rappture::Tester::Test::diffs {lib1 lib2 {path output}} {
367    set paths [list]
368    set clist1 [$lib1 children $path]
369    set clist2 [$lib2 children $path]
370    foreach child $clist1 {
371        # Ignore if not present in both libraries
372        if {[lsearch -exact $clist2 $child] != -1} {
373            foreach p [diffs $lib1 $lib2 $path.$child] {
374                lappend paths $p
375            }
376        }
377    }
378    if {[compareElements $lib1 $lib2 $path]} {
379        # Ignore output.time and output.user
380        if {$path != "output.time" && $path != "output.user"} {
381            lappend paths $path
382        }
383    }
384    return $paths
385}
386
387# ----------------------------------------------------------------------
388# USAGE: makeDriver
389#
390# Builds and returns a driver library object to be used for running the
391# test specified by testxml.  Copy current values from test xml into the
392# newly created driver.  If any inputs are present in the new tool.xml
393# which do not exist in the test xml, use the default value.
394# ----------------------------------------------------------------------
395itcl::body Rappture::Tester::Test::makeDriver {} {
396    set driver [Rappture::library $_toolxml]
397    return [merge $_toolobj $_testobj $driver]
398}
399
400# ----------------------------------------------------------------------
401# USAGE: merge <toolobj> <golden> <driver> ?path?
402#
403# Used to recursively build up a driver library object for running a
404# test.  Should not be called directly - see makeDriver.
405# ----------------------------------------------------------------------
406itcl::body Rappture::Tester::Test::merge {toolobj golden driver {path input}} {
407    foreach child [$toolobj children $path] {
408        set val [$golden get $path.$child.current]
409        if {$val != ""} {
410            $driver put $path.$child.current $val
411        } else {
412            set def [$toolobj get $path.$child.default]
413            if {$def != ""} {
414                $driver put $path.$child.current $def
415            }
416        }
417        merge $toolobj $golden $driver $path.$child
418    }
419    return $driver
420}
421
422# ----------------------------------------------------------------------
423# USAGE: added lib1 lib2 ?path?
424#
425# Compares two library objects and returns a list of paths that do not
426# exist in the first library and have been added in the second.
427# Return value will contain all differences that occur as descendants of
428# an optional starting path.  If the path argument is not given, then
429# only the output sections will be compared.
430# ----------------------------------------------------------------------
431itcl::body Rappture::Tester::Test::missing {lib1 lib2 {path output}} {
432    set paths [list]
433    foreach child [$lib1 children $path] {
434        foreach p [missing $lib1 $lib2 $path.$child] {
435            lappend paths $p
436        }
437    }
438    if {[$lib1 get $path] != "" && [$lib2 get $path] == ""} {
439        lappend paths $path
440    }
441    return $paths
442}
Note: See TracBrowser for help on using the repository browser.