Changeset 2013 for trunk/tester


Ignore:
Timestamp:
Dec 17, 2010 8:26:01 PM (9 years ago)
Author:
braffert
Message:

Developing regression tester

Location:
trunk/tester
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • trunk/tester/compare.tcl

    r1977 r2013  
    5252}
    5353
    54 # ----------------------------------------------------------------------
    55 # USAGE: makeDriver <tool.xml> <test.xml>
    56 #
    57 # Builds and returns a driver library object to be used for running the
    58 # test specified by testxml.  Copy current values from test xml into the
    59 # newly created driver.  If any inputs are present in the new tool.xml
    60 # which do not exist in the test xml, use the default value.
    61 # ----------------------------------------------------------------------
    62 proc Rappture::Tester::makeDriver {toolxml testxml} {
    63     # TODO: Test with various cases, especially with missing input elements
    64     # TODO: Any way to copy an object rather than creating a duplicate?
    65     # TODO: Sensible way of combining this proc with "merge" below?
    66     set toolobj [Rappture::library $toolxml]
    67     set golden [Rappture::library $testxml]
    68     set driver [Rappture::library $toolxml]
    69     return [Rappture::Tester::merge $toolobj $golden $driver]
    70 }
    71 
    72 # ----------------------------------------------------------------------
    73 # USAGE: merge <toolobj> <golden> <driver> ?path?
    74 #
    75 # Used to recursively build up a driver library object for running a
    76 # test.  Should not be called directly - see makeDriver.
    77 # ----------------------------------------------------------------------
    78 proc Rappture::Tester::merge {toolobj golden driver {path input}} {
    79     foreach child [$toolobj children $path] {
    80         set val [$golden get $path.$child.current]
    81         if {$val != ""} {
    82             $driver put $path.$child.current $val
    83         } else {
    84             set def [$toolobj get $path.$child.default]
    85             if {$def != ""} {
    86                 $driver put $path.$child.current $def
    87             }
    88         }
    89         Rappture::Tester::merge $toolobj $golden $driver $path.$child
    90     }
    91     return $driver
    92 }
    93 
    94 
  • trunk/tester/tester.tcl

    r2012 r2013  
    7171    -command Rappture::Tester::runSelected \
    7272    -selectcommand Rappture::Tester::selectionHandler]
    73 .pw add [Rappture::Tester::TestView .view $params(-tool)]
     73.pw add [frame .right]
     74Rappture::Tester::TestView .right.view $params(-tool)
     75button .right.regoldenize -text "Regoldenize" -state disabled \
     76    -command Rappture::Tester::regoldenize
     77pack .right.regoldenize -side bottom -anchor e
     78pack .right.view -side bottom -expand yes -fill both
    7479pack .pw -expand yes -fill both
     80set curselection ""
     81
    7582# TODO: Handle resizing better
    7683# TODO: Fix error that occurs only when you click and hold on a test
    7784#       while the right hand side is empty
    7885
    79 set curselection ""
    80 
    81 # ----------------------------------------------------------------------
    82 # USAGE: runAll ?-force?
    83 #
    84 # When this method is invoked, all tests contained in the TestTree will
    85 # be ran sequentially.
    86 # ----------------------------------------------------------------------
    87 proc Rappture::Tester::runAll {args} {
    88     foreach id [.tree getTests] {
    89         runTest $id $args
    90     }
    91     .view update [.tree getData focus]
    92 }
    93 
    94 # ----------------------------------------------------------------------
    95 # USAGE: runSelected ?-force?
     86# ----------------------------------------------------------------------
     87# USAGE: runSelected
    9688#
    9789# When this method is invoked, all tests that are currently selected
     
    10092# TODO: Parallel exection
    10193# ----------------------------------------------------------------------
    102 proc Rappture::Tester::runSelected {args} {
     94proc Rappture::Tester::runSelected {} {
    10395    foreach id [.tree getSelected] {
    104         runTest $id $args
    105     }
    106     .view update [.tree getData focus]
    107 }
    108 
    109 # ----------------------------------------------------------------------
    110 # USAGE: runTest <id> ?-ifneeded?
     96        runTest $id
     97    }
     98    selectionHandler -forcerefresh
     99}
     100
     101# ----------------------------------------------------------------------
     102# USAGE: runTest <id>
    111103#
    112104# Called by runAll and runSelected to run a single test at the tree node
    113105# specified by the given id.  In most cases, this method should not be
    114 # called directly.  A driver object is generated by the makeDriver
    115 # procedure in compare.tcl, and the results given by the new version are
    116 # compared to the golden result by the compare procedure in compare.tcl
    117 # ----------------------------------------------------------------------
    118 proc Rappture::Tester::runTest {id args} {
    119     array set data [.tree getData $id]
     106# called directly.  A driver object appropriate for running the test is
     107# generated by the makeDriver procedure, and the results given by the
     108# new version are compared to the golden result by the compare
     109# procedure.
     110# ----------------------------------------------------------------------
     111proc Rappture::Tester::runTest {id} {
     112    array set darray [.tree getData $id]
    120113    global params
    121     if {$data(ran) && [lsearch -exact $args "-ifneeded"] != -1} {
    122         # Already ran. Skip.
    123         return
    124     }
    125     set data(result) "Running"
    126     .tree setData $id [array get data]
    127 
    128     set driver [Rappture::Tester::makeDriver $params(-tool) $data(testxml)]
     114    set darray(result) "Running"
     115    .tree setData $id [array get darray]
     116
     117    set driver [Rappture::Tester::makeDriver $params(-tool) $darray(testxml)]
    129118    set tool [Rappture::Tool ::#auto $driver [file dirname $params(-tool)]]
    130119    foreach {status result} [eval $tool run] break
    131     set data(ran) yes
     120    set darray(ran) yes
    132121    if {$status == 0 && [Rappture::library isvalid $result]} {
    133         set golden [Rappture::library $data(testxml)]
     122        set golden [Rappture::library $darray(testxml)]
    134123        set diffs [Rappture::Tester::compare $golden $result output]
    135124        if {$diffs == ""} {
    136             set data(result) Pass
     125            set darray(result) Pass
    137126        } else {
    138             set data(result) Fail
     127            set darray(result) Fail
    139128        }
    140         set data(diffs) $diffs
    141         set data(runfile) [$tool getRunFile]
     129        set darray(diffs) $diffs
     130        set darray(runfile) [$tool getRunFile]
    142131    } else {
    143         set data(result) Error
    144         set data(runfile) ""
    145     }
    146     .tree setData $id [array get data]
    147 }
    148 
    149 # ----------------------------------------------------------------------
    150 # USAGE: selectionHandler
     132        set darray(result) Error
     133        set darray(runfile) ""
     134    }
     135    .tree setData $id [array get darray]
     136}
     137
     138# ----------------------------------------------------------------------
     139# USAGE: selectionHandler ?-forcerefresh?
    151140#
    152141# Used internally to communicate between the test tree and the right
     
    154143# node's data to the right hand side.
    155144# ----------------------------------------------------------------------
    156 proc Rappture::Tester::selectionHandler {} {
     145proc Rappture::Tester::selectionHandler {args} {
     146    if {[llength $args] > 1} {
     147        error "wrong # args: should be \"selectionHandler ?-forcerefresh?\""
     148    }
    157149    global curselection
    158     set sel [.tree getSelected]
    159     if {$sel != $curselection} {
    160         .view update [.tree getData focus]
     150    set sel [.tree getFocus]
     151    if {$sel != $curselection || [lindex $args 0] == "-forcerefresh"} {
     152        set data [.tree getData focus]
     153        .right.view configure -data $data
     154        array set darray $data
     155        if {[array names darray] != "" && $darray(ran)} {
     156            .right.regoldenize configure -state normal
     157        } else {
     158            .right.regoldenize configure -state disabled
     159        }
    161160        set curselection $sel
    162161    }
    163162}
    164163
     164# ----------------------------------------------------------------------
     165# USAGE: makeDriver <tool.xml> <test.xml>
     166#
     167# Builds and returns a driver library object to be used for running the
     168# test specified by testxml.  Copy current values from test xml into the
     169# newly created driver.  If any inputs are present in the new tool.xml
     170# which do not exist in the test xml, use the default value.
     171# ----------------------------------------------------------------------
     172proc Rappture::Tester::makeDriver {toolxml testxml} {
     173    # TODO: Test with various cases, especially with missing input elements
     174    # TODO: Any way to copy an object rather than creating a duplicate?
     175    # TODO: Sensible way of combining this proc with "merge" below?
     176    set toolobj [Rappture::library $toolxml]
     177    set golden [Rappture::library $testxml]
     178    set driver [Rappture::library $toolxml]
     179    return [Rappture::Tester::merge $toolobj $golden $driver]
     180}
     181
     182# ----------------------------------------------------------------------
     183# USAGE: merge <toolobj> <golden> <driver> ?path?
     184#
     185# Used to recursively build up a driver library object for running a
     186# test.  Should not be called directly - see makeDriver.
     187# ----------------------------------------------------------------------
     188proc Rappture::Tester::merge {toolobj golden driver {path input}} {
     189    foreach child [$toolobj children $path] {
     190        set val [$golden get $path.$child.current]
     191        if {$val != ""} {
     192            $driver put $path.$child.current $val
     193        } else {
     194            set def [$toolobj get $path.$child.default]
     195            if {$def != ""} {
     196                $driver put $path.$child.current $def
     197            }
     198        }
     199        Rappture::Tester::merge $toolobj $golden $driver $path.$child
     200    }
     201    return $driver
     202}
     203
     204# ----------------------------------------------------------------------
     205# TODO
     206# ----------------------------------------------------------------------
     207proc Rappture::Tester::regoldenize {} {
     208    set data [.tree getData focus]
     209    array set darray $data
     210    if {[tk_messageBox -type yesno -icon warning -message "Are you sure you want to regoldenize?\n$darray(testxml) will be overwritten."]} {
     211        # Copy test info from existing test xml
     212        set newlib [Rappture::library $darray(runfile)]
     213        set oldlib [Rappture::library $darray(testxml)]
     214        $newlib put test.label [$oldlib get test.label]
     215        $newlib put test.description [$oldlib get test.description]
     216        # Overwrite test xml
     217        set fid [open $darray(testxml) w]
     218        puts $fid "<?xml version=\"1.0\"?>"
     219        puts $fid [$newlib xml]
     220        close $fid
     221        # Update tree data accordingly
     222        set darray(result) "Pass"
     223        .tree setData focus [array get darray]
     224        # Refresh right hand side view
     225        selectionHandler -forcerefresh
     226    }
     227}
     228
  • trunk/tester/testtree.tcl

    r2012 r2013  
    4141    public method getTests {{id 0}}
    4242    public method getSelected {}
     43    public method getFocus {}
    4344    public method getData {id}
    4445    public method setData {id data}
     
    195196
    196197# ----------------------------------------------------------------------
     198# USAGE: getFocus
     199#
     200# Returns index of the focused node in the treeview.
     201# ----------------------------------------------------------------------
     202itcl::body Rappture::Tester::TestTree::getFocus {} {
     203    return [$itk_component(treeview) index focus]
     204}
     205
     206# ----------------------------------------------------------------------
    197207# USAGE: getData <id>
    198208#
  • trunk/tester/testview.tcl

    r2012 r2013  
    3535    protected method showDescription {text}
    3636    public method update {datapairs}
    37     public method clear
    38 
     37    public method reset
     38
     39    public variable data
    3940    protected variable _toolobj
    4041
     
    7677    $itk_component(tabs) insert end "Result" -ipady 25 -fill both \
    7778        -state disabled
    78     pack $itk_component(tabs) -expand yes -fill both -side top
    7979
    8080    itk_component add analyzer {
     
    8888    }
    8989    $itk_component(tabs) tab configure "Result" -window $itk_component(result)
     90    pack $itk_component(tabs) -expand yes -fill both -side top
    9091
    9192    eval itk_initialize $args
     93}
     94
     95# ----------------------------------------------------------------------
     96# When the -data configuration option is modified, update the display
     97# accordingly.  The data passed in should be a list of key value pairs
     98# from the TestTree widget.
     99# ----------------------------------------------------------------------
     100itcl::configbody Rappture::Tester::TestView::data {
     101    array set darray $data
     102    # Data array is empty for branch nodes.
     103    if {[array names darray] != ""} {
     104        if {$darray(ran)} {
     105            switch $darray(result) {
     106                Pass {showStatus "Test passed."}
     107                Fail {showStatus "Test failed."}
     108                Error {showStatus "Error while running test."}
     109            }
     110            if {$darray(runfile) != ""} {
     111                # HACK: Add a new input to differentiate between golden and
     112                # test result.  Otherwise, the slider at the bottom won't
     113                # be enabled.
     114                set golden [Rappture::library $darray(testxml)]
     115                $golden put input.run.current "Golden"
     116                set result [Rappture::library $darray(runfile)]
     117                $result put input.run.current "Test"
     118                updateAnalyzer $golden $result
     119                updateResult $data
     120            } else {
     121                updateResult
     122            }
     123        } else {
     124            showStatus "Test has not yet ran."
     125            updateResult
     126            if {$darray(testxml) != ""} {
     127                updateAnalyzer [Rappture::library $darray(testxml)]
     128            }
     129        }
     130        set descr [[Rappture::library $darray(testxml)] get test.description]
     131        if {$descr == ""} {
     132            set descr "No description."
     133        }
     134        showDescription $descr
     135    } else {
     136       # Clear everything if branch node selected
     137       reset
     138    }
    92139}
    93140
     
    97144
    98145# ----------------------------------------------------------------------
    99 # USAGE: clear
     146# USAGE: reset
    100147#
    101148# Resets the entire TestView widget back to the default state.
    102149# ----------------------------------------------------------------------
    103 itcl::body Rappture::Tester::TestView::clear {} {
     150itcl::body Rappture::Tester::TestView::reset {} {
    104151    updateAnalyzer
    105152    updateResult
    106     # TODO: Switch back to analyzer tab when disabling result tab
    107     $itk_component(tabs) focus [$itk_component(tabs) index -name "Analyzer"]
    108     $itk_component(tabs) tab configure "Result" -state disabled
    109153    showStatus ""
    110154    showDescription ""
     
    117161# both the golden result as well as the test result.  Clears the
    118162# analyzer space if no arguments are given.
    119 # Destroys the existing analyzer widget and creates a new one. 
     163# HACK: Destroys the existing analyzer widget and creates a new one. 
    120164# Eventually this should be able to keep the same widget and swap in
    121165# and out different result sets.
     
    138182
    139183# ----------------------------------------------------------------------
    140 # USAGE: updateResult ?datapairs?
     184# USAGE: updateResult ?data?
    141185#
    142186# Given a set of key value pairs from the test tree, update the result
     
    146190itcl::body Rappture::Tester::TestView::updateResult {args} {
    147191    if {[llength $args] == 0} {
    148         $itk_component(result) delete 0.0 end
    149         $itk_component(tabs) focus [$itk_component(tabs) index -name "Result"]
     192        $itk_component(result) delete 0.0 end
     193        # TODO: Switch back to analyzer tab.  Why doesn't this work?
     194        $itk_component(tabs) invoke \
     195            [$itk_component(tabs) index -name "Analyzer"]
    150196        $itk_component(tabs) tab configure "Result" -state disabled
    151197        return
    152198    } elseif {[llength $args] == 1} {
    153         array set data [lindex $args 0]
     199        array set darray [lindex $args 0]
    154200        $itk_component(tabs) tab configure "Result" -state normal
    155201        $itk_component(result) delete 0.0 end
    156         $itk_component(result) insert end "Test xml: $data(testxml)\n"
    157         $itk_component(result) insert end "Runfile: $data(runfile)\n"
    158         if {$data(result) == "Fail"} {
    159             $itk_component(result) insert end "Diffs: $data(diffs)\n"
     202        $itk_component(result) insert end "Test xml: $darray(testxml)\n"
     203        $itk_component(result) insert end "Runfile: $darray(runfile)\n"
     204        if {$darray(result) == "Fail"} {
     205            $itk_component(result) insert end "Diffs: $darray(diffs)\n"
    160206        }
    161207    } else {
    162         error "wrong # args: should be \"updateResult ?datapairs?\""
     208        error "wrong # args: should be \"updateResult ?data?\""
    163209    }
    164210}
     
    192238}
    193239
    194 # ----------------------------------------------------------------------
    195 # USAGE: update <datapairs>
    196 #
    197 # Given a list of key value pairs from the test tree, update both the
    198 # analyzer and the result tab.
    199 # ----------------------------------------------------------------------
    200 itcl::body Rappture::Tester::TestView::update {datapairs} {
    201     array set data $datapairs
    202     # Data array is empty for branch nodes.
    203     if {[array names data] != ""} {
    204         if {$data(ran)} {
    205             switch $data(result) {
    206                 Pass {showStatus "Test passed."}
    207                 Fail {showStatus "Test failed."}
    208                 Error {showStatus "Error while running test."}
    209             }
    210             if {$data(runfile) != ""} {
    211                 # HACK: Add a new input to differentiate between golden and
    212                 # test result.  Otherwise, the slider at the bottom won't
    213                 # be enabled.
    214                 set golden [Rappture::library $data(testxml)]
    215                 $golden put input.run.current "Golden"
    216                 set result [Rappture::library $data(runfile)]
    217                 $result put input.run.current "Test"
    218                 updateAnalyzer $golden $result
    219                 updateResult $datapairs
    220             } else {
    221                 updateResult
    222             }
    223         } else {
    224             showStatus "Test has not yet ran."
    225             updateResult
    226             if {$data(testxml) != ""} {
    227                 updateAnalyzer [Rappture::library $data(testxml)]
    228             }
    229         }
    230         set descr [[Rappture::library $data(testxml)] get test.description]
    231         if {$descr == ""} {
    232             set descr "No description."
    233         }
    234         showDescription $descr
    235     } else {
    236        # clear everything if branch node selected
    237        clear
    238     }
    239 }
    240 
Note: See TracChangeset for help on using the changeset viewer.