source: trunk/tester/test.tcl @ 2053

Last change on this file since 2053 was 2053, checked in by braffert, 14 years ago

Regression tester: adding tabs to view heirarchy of inputs and outputs for each test
File size: 15.4 KB
Line 
1# ----------------------------------------------------------------------
2#  COMPONENT: test - run a test and query the results
3#
4#  Encapsulates the testing logic, to keep it isolated from the rest of
5#  the tester GUI.  Constructor requires the location of the tool.xml
6#  for the new version, and the test xml file containing the golden set
7#  of results.
8# ======================================================================
9#  AUTHOR:  Ben Rafferty, Purdue University
10#  Copyright (c) 2010  Purdue Research Foundation
11#
12#  See the file "license.terms" for information on usage and
13#  redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
14# ======================================================================
15
16namespace eval Rappture::Tester::Test { #forward declaration }
17
18itcl::class Rappture::Tester::Test {
19
20    constructor {toolxml testxml} { #defined later }
21    destructor { #defined later }
22
23    private variable _added ""
24    private variable _diffs ""
25    private variable _missing ""
26    private variable _ran no
27    private variable _testxml
28    private variable _toolxml
29    private variable _result ""
30    private variable _runfile ""
31    private variable _testobj ""
32    private variable _toolobj ""
33    private variable _runobj ""
34
35    public method getAdded {}
36    public method getDiffs {}
37    public method getInputs {{path input}}
38    public method getMissing {}
39    public method getOutputs {{path output}}
40    public method getResult {}
41    public method getRunfile {}
42    public method getRunobj {}
43    public method getTestobj {}
44    public method getTestxml {}
45    public method hasRan {}
46    public method regoldenize {}
47    public method run {}
48
49    private method added {lib1 lib2 {path output}}
50    private method compareElements {lib1 lib2 path}
51    private method diffs {lib1 lib2 {path output}}
52    private method makeDriver {}
53    private method merge {toolobj golden driver {path input}}
54    private method missing {lib1 lib2 {path output}}
55
56}
57
58# ----------------------------------------------------------------------
59# CONSTRUCTOR
60# ----------------------------------------------------------------------
61itcl::body Rappture::Tester::Test::constructor {toolxml testxml} {
62    set _toolxml $toolxml
63    set _testxml $testxml
64    set _toolobj [Rappture::library $toolxml]
65    if {![Rappture::library isvalid $_toolobj]} {
66        error "$toolxml does not represent a valid library object"
67    }
68    set _testobj [Rappture::library $testxml]
69    if {![Rappture::library isvalid $_testobj]} {
70        error "$testxml does not represent a valid library object"
71    }
72    # HACK: Add a new input to differentiate between results
73    $_testobj put input.TestRun.current "Golden"
74}
75
76# ----------------------------------------------------------------------
77# DESTRUCTOR
78# ----------------------------------------------------------------------
79itcl::body Rappture::Tester::Test::destructor {} {
80    itcl::delete object $_toolobj
81    itcl::delete object $_testobj
82    if {$_ran} {
83        itcl::delete object $_runobj
84    }
85}
86
87# ----------------------------------------------------------------------
88# USAGE: getAdded
89#
90# Return a list of paths that have been added that do not exist in the
91# golden results.  Throws an error if the test has not been ran.
92# ----------------------------------------------------------------------
93itcl::body Rappture::Tester::Test::getAdded {} {
94    if {!$_ran} {
95        error "Test has not yet been ran."
96    }
97    return $_added
98}
99
100# ----------------------------------------------------------------------
101# USAGE: getDiffs
102#
103# Returns a list of paths that exist in both the golden and new results,
104# but contain data that does not match according to the compareElements
105# method.  Throws an error if the test has not been ran.
106# ----------------------------------------------------------------------
107itcl::body Rappture::Tester::Test::getDiffs {} {
108    if {!$_ran} {
109        error "Test has not yet been ran."
110    }
111    return $_diffs
112}
113
114# -----------------------------------------------------------------------
115# USAGE: getInputs
116#
117# TODO
118# -----------------------------------------------------------------------
119itcl::body Rappture::Tester::Test::getInputs {{path input}} {
120    set retval [list]
121    foreach child [$_testobj children $path] {
122        set fullpath $path.$child
123        if {$fullpath != "input.TestRun"} {
124            set val [$_testobj get $fullpath.current]
125            if {$val != ""} {
126                lappend retval $fullpath $val
127            }
128        }
129        append retval [getInputs $fullpath]
130    }
131    return $retval
132}
133
134# ----------------------------------------------------------------------
135# USAGE: getMissing
136#
137# Return a list of paths that are present in the golden results, but are
138# missing in the new test results.  Throws an error if the test has not
139# been ran.
140# ----------------------------------------------------------------------
141itcl::body Rappture::Tester::Test::getMissing {} {
142    if {!$_ran} {
143        error "Test has not yet been ran."
144    }
145    return $_missing
146}
147
148# ----------------------------------------------------------------------
149# USAGE: getOutputs
150#
151# TODO
152# ----------------------------------------------------------------------
153itcl::body Rappture::Tester::Test::getOutputs {{path output}} {
154    if {!$_ran} {
155        error "Test has not yet been ran."
156    }
157    set retval [list]
158    foreach child [$_runobj children $path] {
159        set fullpath $path.$child
160        if {$fullpath != "output.time" && $fullpath != "output.user" && $fullpath != "output.status"} {
161            if {[lsearch $fullpath [getDiffs]] != -1} {
162                set status diff
163            } elseif {[lsearch $fullpath [getAdded]] != -1} {
164                set status added
165            #} elseif {[lsearch $fullpath [getMissing]] != -1} {
166            #    set status missing
167            } else {
168                if {[$_runobj get $fullpath] != ""} {
169                    set status ok
170                } else {
171                    set status ""
172                }
173            }
174            lappend retval $fullpath $status
175        }
176        append retval " [getOutputs $fullpath]"
177    }
178    # We won't find missing elements by searching through runobj.  Instead,
179    # tack on all missing items at the end (only do this once)
180    if {$path == "output"} {
181        foreach item $_missing {
182            lappend retval $item missing
183        }
184    }
185    return $retval
186}
187
188# ----------------------------------------------------------------------
189# USAGE: getResult
190#
191# Returns the result of the test - either Pass, Fail, or Error.  Throws
192# an error if the test has not been ran.
193# ----------------------------------------------------------------------
194itcl::body Rappture::Tester::Test::getResult {} {
195    if {!$_ran} {
196        error "Test has not yet been ran."
197    }
198    return $_result
199}
200
201# ----------------------------------------------------------------------
202# USAGE: getRunfile
203#
204# Returns the location of the runfile generated by the previous run of
205# the test.  Throws an error if the test has not been ran.
206# ----------------------------------------------------------------------
207itcl::body Rappture::Tester::Test::getRunfile {} {
208    if {!$_ran} {
209        error "Test has not yet been ran."
210    }
211    return $_runfile
212}
213
214# -----------------------------------------------------------------------
215# USAGE: getRunobj
216#
217# Returns the library object generated by the previous run of the test.
218# Throws an error if the test has not been ran.
219# -----------------------------------------------------------------------
220itcl::body Rappture::Tester::Test::getRunobj {} {
221    if {!$_ran} {
222        error "Test has not yet been ran."
223    }
224    return $_runobj
225}
226
227# ----------------------------------------------------------------------
228# USAGE: getTestxml
229#
230# Returns the location of the test xml file containing the set of golden
231# results.
232# ----------------------------------------------------------------------
233itcl::body Rappture::Tester::Test::getTestxml {} {
234    return $_testxml
235}
236
237# ----------------------------------------------------------------------
238# USAGE: getTestobj
239#
240# Returns the test library object containing the set of golden results.
241# ----------------------------------------------------------------------
242itcl::body Rappture::Tester::Test::getTestobj {} {
243    return $_testobj
244}
245
246# ----------------------------------------------------------------------
247# USAGE: hasRan
248#
249# Returns yes if the test has been ran (with the run method), returns
250# no otherwise.
251# ----------------------------------------------------------------------
252itcl::body Rappture::Tester::Test::hasRan {} {
253    return $_ran
254}
255
256# ----------------------------------------------------------------------
257# USAGE: regoldenize
258#
259# Regoldenize the test by overwriting the test xml containin the golden
260# results with the data in the runfile generated by the last run.  Copy
261# test label and description into the new file.  Update the test's
262# result attributes to reflect the changes. Throws an error if the test
263# has not been ran.
264# ----------------------------------------------------------------------
265itcl::body Rappture::Tester::Test::regoldenize {} {
266    if {!$_ran} {
267        error "Test has not yet been ran."
268    }
269    $_runobj put test.label [$_testobj get test.label]
270    $_runobj put test.description [$_testobj get test.description]
271    set fid [open $_testxml w]
272    puts $fid "<?xml version=\"1.0\"?>"
273    puts $fid [$_runobj xml]
274    close $fid
275    set _testobj $_runobj
276    set _result Pass
277    set _diffs ""
278    set _added ""
279    set _missing ""
280}
281
282
283# ----------------------------------------------------------------------
284# USAGE: run
285#
286# Kicks off a new simulation and checks the results against the golden
287# set of results.  Set private attributes accordingly so that they can
288# later be retrieved via the public accessors.
289# ----------------------------------------------------------------------
290itcl::body Rappture::Tester::Test::run {} {
291    # Delete existing library if rerun
292    if {$_ran} {
293        itcl::delete object $_runobj
294    }
295    set driver [makeDriver]
296    set tool [Rappture::Tool ::#auto $driver [file dirname $_toolxml]]
297    foreach {status _runobj} [eval $tool run] break
298    set _ran yes
299    if {$status == 0 && [Rappture::library isvalid $_runobj]} {
300        # HACK: Add a new input to differentiate between results
301        $_runobj put input.TestRun.current "Test result"
302        set _diffs [diffs $_testobj $_runobj]
303        set _missing [missing $_testobj $_runobj]
304        set _added [added $_testobj $_runobj]
305        set _runfile [$tool getRunFile]
306        if {$_diffs == "" && $_missing == "" && $_added == ""} {
307            set _result Pass
308        } else {
309            set _result Fail
310        }
311    } else {
312        set _runobj ""
313        set _result Error
314    }
315}
316
317# ----------------------------------------------------------------------
318# USAGE: added lib1 lib2 ?path?
319#
320# Compares two library objects and returns a list of paths that have
321# been added in the second library and do not exist in the first.
322# Return value will contain all differences that occur as descendants of
323# an optional starting path.  If the path argument is not given, then
324# only the output sections will be compared.
325# ----------------------------------------------------------------------
326itcl::body Rappture::Tester::Test::added {lib1 lib2 {path output}} {
327    set paths [list]
328    foreach child [$lib2 children $path] {
329        foreach p [added $lib1 $lib2 $path.$child] {
330            lappend paths $p
331        }
332    }
333    if {[$lib1 get $path] == "" && [$lib2 get $path] != ""} {
334        lappend paths $path
335    }
336    return $paths
337}
338
339# ----------------------------------------------------------------------
340# USAGE: compareElements <lib1> <lib2> <path>
341#
342# Compare data found in two library objects at the given path.  Returns
343# 1 if match, 0 if no match.  For now, just check if ascii identical.
344# Later, we can do something more sophisticated for different types of
345# elements.
346# ----------------------------------------------------------------------
347itcl::body Rappture::Tester::Test::compareElements {lib1 lib2 path} {
348    set val1 [$lib1 get $path]
349    set val2 [$lib2 get $path]
350    return [expr {$val1} != {$val2}]
351}
352
353# ----------------------------------------------------------------------
354# USAGE: diffs <lib1> <lib2> ?path?
355#
356# Compares two library objects and returns a list of paths that do not
357# match.  Only paths which exist in both libraries are considered.
358# Return value will contain all differences that occur as descendants of
359# an optional starting path.  If the path argument is not given, then
360# only the output sections will be compared.
361# ----------------------------------------------------------------------
362itcl::body Rappture::Tester::Test::diffs {lib1 lib2 {path output}} {
363    set paths [list]
364    set clist1 [$lib1 children $path]
365    set clist2 [$lib2 children $path]
366    foreach child $clist1 {
367        # Ignore if not present in both libraries
368        if {[lsearch -exact $clist2 $child] != -1} {
369            foreach p [diffs $lib1 $lib2 $path.$child] {
370                lappend paths $p
371            }
372        }
373    }
374    if {[compareElements $lib1 $lib2 $path]} {
375        # Ignore output.time and output.user
376        if {$path != "output.time" && $path != "output.user"} {
377            lappend paths $path
378        }
379    }
380    return $paths
381}
382
383# ----------------------------------------------------------------------
384# USAGE: makeDriver
385#
386# Builds and returns a driver library object to be used for running the
387# test specified by testxml.  Copy current values from test xml into the
388# newly created driver.  If any inputs are present in the new tool.xml
389# which do not exist in the test xml, use the default value.
390# ----------------------------------------------------------------------
391itcl::body Rappture::Tester::Test::makeDriver {} {
392    set driver [Rappture::library $_toolxml]
393    return [merge $_toolobj $_testobj $driver]
394}
395
396# ----------------------------------------------------------------------
397# USAGE: merge <toolobj> <golden> <driver> ?path?
398#
399# Used to recursively build up a driver library object for running a
400# test.  Should not be called directly - see makeDriver.
401# ----------------------------------------------------------------------
402itcl::body Rappture::Tester::Test::merge {toolobj golden driver {path input}} {
403    foreach child [$toolobj children $path] {
404        set val [$golden get $path.$child.current]
405        if {$val != ""} {
406            $driver put $path.$child.current $val
407        } else {
408            set def [$toolobj get $path.$child.default]
409            if {$def != ""} {
410                $driver put $path.$child.current $def
411            }
412        }
413        merge $toolobj $golden $driver $path.$child
414    }
415    return $driver
416}
417
418# ----------------------------------------------------------------------
419# USAGE: added lib1 lib2 ?path?
420#
421# Compares two library objects and returns a list of paths that do not
422# exist in the first library and have been added in the second.
423# Return value will contain all differences that occur as descendants of
424# an optional starting path.  If the path argument is not given, then
425# only the output sections will be compared.
426# ----------------------------------------------------------------------
427itcl::body Rappture::Tester::Test::missing {lib1 lib2 {path output}} {
428    set paths [list]
429    foreach child [$lib1 children $path] {
430        foreach p [missing $lib1 $lib2 $path.$child] {
431            lappend paths $p
432        }
433    }
434    if {[$lib1 get $path] != "" && [$lib2 get $path] == ""} {
435        lappend paths $path
436    }
437    return $paths
438}
Note: See TracBrowser for help on using the repository browser.