source: branches/blt4/tester/test.tcl @ 2047

Last change on this file since 2047 was 2047, checked in by gah, 11 years ago

update from r6 branch

File size: 13.4 KB
Line 
1# ----------------------------------------------------------------------
2#  COMPONENT: test - run a test and query the results
3#
4#  Encapsulates the testing logic, to keep it isolated from the rest of
5#  the tester GUI.  Constructor requires the location of the tool.xml
6#  for the new version, and the test xml file containing the golden set
7#  of results.
8# ======================================================================
9#  AUTHOR:  Ben Rafferty, Purdue University
10#  Copyright (c) 2010  Purdue Research Foundation
11#
12#  See the file "license.terms" for information on usage and
13#  redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
14# ======================================================================
15
16namespace eval Rappture::Tester::Test { #forward declaration }
17
18itcl::class Rappture::Tester::Test {
19
20    constructor {toolxml testxml} { #defined later }
21    destructor { #defined later }
22
23    private variable _added ""
24    private variable _diffs ""
25    private variable _missing ""
26    private variable _ran no
27    private variable _testxml
28    private variable _toolxml
29    private variable _result ""
30    private variable _runfile ""
31    private variable _testobj ""
32    private variable _toolobj ""
33    private variable _runobj ""
34
35    public method getAdded {}
36    public method getDiffs {}
37    public method getMissing {}
38    public method getResult {}
39    public method getRunfile {}
40    public method getRunobj {}
41    public method getTestobj {}
42    public method getTestxml {}
43    public method hasRan {}
44    public method regoldenize {}
45    public method run {}
46
47    private method added {lib1 lib2 {path output}}
48    private method compareElements {lib1 lib2 path}
49    private method diffs {lib1 lib2 {path output}}
50    private method makeDriver {}
51    private method merge {toolobj golden driver {path input}}
52    private method missing {lib1 lib2 {path output}}
53
54}
55
56# ----------------------------------------------------------------------
57# CONSTRUCTOR
58# ----------------------------------------------------------------------
59itcl::body Rappture::Tester::Test::constructor {toolxml testxml} {
60    set _toolxml $toolxml
61    set _testxml $testxml
62    set _toolobj [Rappture::library $toolxml]
63    if {![Rappture::library isvalid $_toolobj]} {
64        error "$toolxml does not represent a valid library object"
65    }
66    set _testobj [Rappture::library $testxml]
67    if {![Rappture::library isvalid $_testobj]} {
68        error "$testxml does not represent a valid library object"
69    }
70    # HACK: Add a new input to differentiate between results
71    $_testobj put input.run.current "Golden"
72}
73
74# ----------------------------------------------------------------------
75# DESTRUCTOR
76# ----------------------------------------------------------------------
77itcl::body Rappture::Tester::Test::destructor {} {
78    itcl::delete object $_toolobj
79    itcl::delete object $_testobj
80    if {$_ran} {
81        itcl::delete object $_runobj
82    }
83}
84
85# ----------------------------------------------------------------------
86# USAGE: getAdded
87#
88# Return a list of paths that have been added that do not exist in the
89# golden results.  Throws an error if the test has not been ran.
90# ----------------------------------------------------------------------
91itcl::body Rappture::Tester::Test::getAdded {} {
92    if {!$_ran} {
93        error "Test has not yet been ran."
94    }
95    return $_added
96}
97
98# ----------------------------------------------------------------------
99# USAGE: getDiffs
100#
101# Returns a list of paths that exist in both the golden and new results,
102# but contain data that does not match according to the compareElements
103# method.  Throws an error if the test has not been ran.
104# ----------------------------------------------------------------------
105itcl::body Rappture::Tester::Test::getDiffs {} {
106    if {!$_ran} {
107        error "Test has not yet been ran."
108    }
109    return $_diffs
110}
111
112# ----------------------------------------------------------------------
113# USAGE: getMissing
114#
115# Return a list of paths that are present in the golden results, but are
116# missing in the new test results.  Throws an error if the test has not
117# been ran.
118# ----------------------------------------------------------------------
119itcl::body Rappture::Tester::Test::getMissing {} {
120    if {!$_ran} {
121        error "Test has not yet been ran."
122    }
123    return $_missing
124}
125
126# ----------------------------------------------------------------------
127# USAGE: getResult
128#
129# Returns the result of the test - either Pass, Fail, or Error.  Throws
130# an error if the test has not been ran.
131# ----------------------------------------------------------------------
132itcl::body Rappture::Tester::Test::getResult {} {
133    if {!$_ran} {
134        error "Test has not yet been ran."
135    }
136    return $_result
137}
138
139# ----------------------------------------------------------------------
140# USAGE: getRunfile
141#
142# Returns the location of the runfile generated by the previous run of
143# the test.  Throws an error if the test has not been ran.
144# ----------------------------------------------------------------------
145itcl::body Rappture::Tester::Test::getRunfile {} {
146    if {!$_ran} {
147        error "Test has not yet been ran."
148    }
149    return $_runfile
150}
151
152# -----------------------------------------------------------------------
153# USAGE: getRunobj
154#
155# Returns the library object generated by the previous run of the test.
156# Throws an error if the test has not been ran.
157# -----------------------------------------------------------------------
158itcl::body Rappture::Tester::Test::getRunobj {} {
159    if {!$_ran} {
160        error "Test has not yet been ran."
161    }
162    return $_runobj
163}
164
165# ----------------------------------------------------------------------
166# USAGE: getTestxml
167#
168# Returns the location of the test xml file containing the set of golden
169# results.
170# ----------------------------------------------------------------------
171itcl::body Rappture::Tester::Test::getTestxml {} {
172    return $_testxml
173}
174
175# ----------------------------------------------------------------------
176# USAGE: getTestobj
177#
178# Returns the test library object containing the set of golden results.
179# ----------------------------------------------------------------------
180itcl::body Rappture::Tester::Test::getTestobj {} {
181    return $_testobj
182}
183
184# ----------------------------------------------------------------------
185# USAGE: hasRan
186#
187# Returns yes if the test has been ran (with the run method), returns
188# no otherwise.
189# ----------------------------------------------------------------------
190itcl::body Rappture::Tester::Test::hasRan {} {
191    return $_ran
192}
193
194# ----------------------------------------------------------------------
195# USAGE: regoldenize
196#
197# Regoldenize the test by overwriting the test xml containin the golden
198# results with the data in the runfile generated by the last run.  Copy
199# test label and description into the new file.  Update the test's
200# result attributes to reflect the changes. Throws an error if the test
201# has not been ran.
202# ----------------------------------------------------------------------
203itcl::body Rappture::Tester::Test::regoldenize {} {
204    if {!$_ran} {
205        error "Test has not yet been ran."
206    }
207    $_runobj put test.label [$_testobj get test.label]
208    $_runobj put test.description [$_testobj get test.description]
209    set fid [open $_testxml w]
210    puts $fid "<?xml version=\"1.0\"?>"
211    puts $fid [$_runobj xml]
212    close $fid
213    set _testobj $_runobj
214    set _result Pass
215    set _diffs ""
216    set _added ""
217    set _missing ""
218}
219
220
221# ----------------------------------------------------------------------
222# USAGE: run
223#
224# Kicks off a new simulation and checks the results against the golden
225# set of results.  Set private attributes accordingly so that they can
226# later be retrieved via the public accessors.
227# ----------------------------------------------------------------------
228itcl::body Rappture::Tester::Test::run {} {
229    # Delete existing library if rerun
230    if {$_ran} {
231        itcl::delete object $_runobj
232    }
233    set driver [makeDriver]
234    set tool [Rappture::Tool ::#auto $driver [file dirname $_toolxml]]
235    foreach {status _runobj} [eval $tool run] break
236    set _ran yes
237    if {$status == 0 && [Rappture::library isvalid $_runobj]} {
238        # HACK: Add a new input to differentiate between results
239        $_runobj put input.run.current "Test result"
240        set _diffs [diffs $_testobj $_runobj]
241        set _missing [missing $_testobj $_runobj]
242        set _added [added $_testobj $_runobj]
243        set _runfile [$tool getRunFile]
244        if {$_diffs == "" && $_missing == "" && $_added == ""} {
245            set _result Pass
246        } else {
247            set _result Fail
248        }
249    } else {
250        set _result Error
251    }
252}
253
254# ----------------------------------------------------------------------
255# USAGE: added lib1 lib2 ?path?
256#
257# Compares two library objects and returns a list of paths that have
258# been added in the second library and do not exist in the first.
259# Return value will contain all differences that occur as descendants of
260# an optional starting path.  If the path argument is not given, then
261# only the output sections will be compared.
262# ----------------------------------------------------------------------
263itcl::body Rappture::Tester::Test::added {lib1 lib2 {path output}} {
264    set paths [list]
265    foreach child [$lib2 children $path] {
266        foreach p [added $lib1 $lib2 $path.$child] {
267            lappend paths $p
268        }
269    }
270    if {[$lib1 get $path] == "" && [$lib2 get $path] != ""} {
271        lappend paths $path
272    }
273    return $paths
274}
275
276# ----------------------------------------------------------------------
277# USAGE: compareElements <lib1> <lib2> <path>
278#
279# Compare data found in two library objects at the given path.  Returns
280# 1 if match, 0 if no match.  For now, just check if ascii identical.
281# Later, we can do something more sophisticated for different types of
282# elements.
283# ----------------------------------------------------------------------
284itcl::body Rappture::Tester::Test::compareElements {lib1 lib2 path} {
285    set val1 [$lib1 get $path]
286    set val2 [$lib2 get $path]
287    return [expr {$val1} != {$val2}]
288}
289
290# ----------------------------------------------------------------------
291# USAGE: diffs <lib1> <lib2> ?path?
292#
293# Compares two library objects and returns a list of paths that do not
294# match.  Only paths which exist in both libraries are considered.
295# Return value will contain all differences that occur as descendants of
296# an optional starting path.  If the path argument is not given, then
297# only the output sections will be compared.
298# ----------------------------------------------------------------------
299itcl::body Rappture::Tester::Test::diffs {lib1 lib2 {path output}} {
300    set paths [list]
301    set clist1 [$lib1 children $path]
302    set clist2 [$lib2 children $path]
303    foreach child $clist1 {
304        # Ignore if not present in both libraries
305        if {[lsearch -exact $clist2 $child] != -1} {
306            foreach p [diffs $lib1 $lib2 $path.$child] {
307                lappend paths $p
308            }
309        }
310    }
311    if {[compareElements $lib1 $lib2 $path]} {
312        # Ignore output.time and output.user
313        if {$path != "output.time" && $path != "output.user"} {
314            lappend paths $path
315        }
316    }
317    return $paths
318}
319
320# ----------------------------------------------------------------------
321# USAGE: makeDriver
322#
323# Builds and returns a driver library object to be used for running the
324# test specified by testxml.  Copy current values from test xml into the
325# newly created driver.  If any inputs are present in the new tool.xml
326# which do not exist in the test xml, use the default value.
327# ----------------------------------------------------------------------
328itcl::body Rappture::Tester::Test::makeDriver {} {
329    set driver [Rappture::library $_toolxml]
330    return [merge $_toolobj $_testobj $driver]
331}
332
333# ----------------------------------------------------------------------
334# USAGE: merge <toolobj> <golden> <driver> ?path?
335#
336# Used to recursively build up a driver library object for running a
337# test.  Should not be called directly - see makeDriver.
338# ----------------------------------------------------------------------
339itcl::body Rappture::Tester::Test::merge {toolobj golden driver {path input}} {
340    foreach child [$toolobj children $path] {
341        set val [$golden get $path.$child.current]
342        if {$val != ""} {
343            $driver put $path.$child.current $val
344        } else {
345            set def [$toolobj get $path.$child.default]
346            if {$def != ""} {
347                $driver put $path.$child.current $def
348            }
349        }
350        merge $toolobj $golden $driver $path.$child
351    }
352    return $driver
353}
354
355# ----------------------------------------------------------------------
356# USAGE: added lib1 lib2 ?path?
357#
358# Compares two library objects and returns a list of paths that do not
359# exist in the first library and have been added in the second.
360# Return value will contain all differences that occur as descendants of
361# an optional starting path.  If the path argument is not given, then
362# only the output sections will be compared.
363# ----------------------------------------------------------------------
364itcl::body Rappture::Tester::Test::missing {lib1 lib2 {path output}} {
365    set paths [list]
366    foreach child [$lib1 children $path] {
367        foreach p [missing $lib1 $lib2 $path.$child] {
368            lappend paths $p
369        }
370    }
371    if {[$lib1 get $path] != "" && [$lib2 get $path] == ""} {
372        lappend paths $path
373    }
374    return $paths
375}
Note: See TracBrowser for help on using the repository browser.