Index: trunk/p2p/Makefile
===================================================================
--- trunk/p2p/Makefile	(revision 1257)
+++ trunk/p2p/Makefile	(revision 1273)
@@ -4,3 +4,5 @@
 all:
 	$(CC) $(CFLAGS) perftest.c -o perftest
-	./mkindex handler.tcl server.tcl client.tcl log.tcl random.tcl
+	./mkindex handler.tcl server.tcl client.tcl log.tcl random.tcl \
+		options.tcl protocols.tcl statemachine.tcl wonks.tcl \
+		foreman.tcl
Index: trunk/p2p/authority.tcl
===================================================================
--- trunk/p2p/authority.tcl	(revision 1257)
+++ trunk/p2p/authority.tcl	(revision 1273)
@@ -31,5 +31,5 @@
 
 # register with the central authority at this frequency
-set worker_options(time_between_registers) 60000
+set worker_options(time_between_authority_checks) 60000
 
 # workers should try to connect with this many other peers
@@ -37,6 +37,66 @@
 
 # ======================================================================
+#  PROTOCOL: hubzero:authority<-worker/1
+#
+#  The worker initiates communication with the authority by sending
+#  these messages to the authority to request information about
+#  other peers.
+# ======================================================================
+p2p::protocol::register hubzero:authority<-worker/1 {
+
+    # ------------------------------------------------------------------
+    #  INCOMING: listening <port>
+    #  Workers use this to tell the authority what port they're
+    #  listening on.  Other workers should connect to this port.
+    # ------------------------------------------------------------------
+    define listening {port} {
+        global client2address workers
+        variable handler
+        variable cid
+
+        # register this address on the list of workers
+        set addr "$client2address($cid):$port"
+        set workers($addr) "--"
+
+        # name this connection for easier debugging
+        $handler connectionName $cid $addr
+
+        return ""
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: peers
+    # ------------------------------------------------------------------
+    define peers {} {
+        global workers
+        return [list peers [array names workers]]
+    }
+}
+
+# ======================================================================
+#  PROTOCOL: hubzero:authority<-foreman/1
+#
+#  The foreman initiates communication with the authority by sending
+#  these messages to request information about workers, to estimate
+#  requirements for a job, to put funds in escrow, and to finalize
+#  the transaction.
+# ======================================================================
+p2p::protocol::register hubzero:authority<-foreman/1 {
+    # ------------------------------------------------------------------
+    #  INCOMING: workers
+    #  Foremen use this to request the list of known workers, and then
+    #  pick one worker to act as their connection to the P2P network.
+    # ------------------------------------------------------------------
+    define workers {} {
+        global workers
+        return [list workers [array names workers]]
+    }
+}
+
+# ======================================================================
 # set up a server at the first open port above 9001
-set server [Server ::#auto 9001? \
+set server [p2p::server \
+    -port 9001? \
+    -protocols {hubzero:authority<-worker hubzero:authority<-foreman} \
     -servername authority \
     -onconnect authority_client_address \
@@ -66,49 +126,20 @@
 proc authority_client_protocol {cid protocol} {
     global worker_options client2address
-    if {"DEFAULT" != $protocol} {
-        puts $cid [list protocol $protocol]
-        puts $cid "options [array get worker_options] ip $client2address($cid)"
+    switch -glob -- $protocol {
+        *<-worker* {
+            puts $cid [list protocol hubzero:worker<-authority/1]
+            puts $cid "options [array get worker_options] ip $client2address($cid)"
+        }
+        *<-foreman* {
+            puts $cid [list protocol hubzero:foreman<-authority/1]
+        }
+        DEF* {
+            # do nothing
+        }
+        default {
+            error "don't recognize protocol \"$protocol\""
+        }
     }
 }
 
-# ======================================================================
-#  PROTOCOL: hub:worker/1
-# ======================================================================
-$server protocol hubzero:worker/1
-
-# ----------------------------------------------------------------------
-#  DIRECTIVE: exception <message>
-# ----------------------------------------------------------------------
-$server define hubzero:worker/1 exception {args} {
-    log system "error from worker: $args"
-}
-
-# ----------------------------------------------------------------------
-#  DIRECTIVE: listening <port>
-#  Workers use this to tell the authority what port they're listening
-#  on.  Other workers should connect to this port.
-# ----------------------------------------------------------------------
-$server define hubzero:worker/1 listening {port} {
-    global client2address workers
-    variable handler
-    variable cid
-
-    # register this address on the list of workers
-    set addr "$client2address($cid):$port"
-    set workers($addr) "--"
-
-    # name this connection for easier debugging
-    $handler connectionName $cid $addr
-
-    return ""
-}
-
-# ----------------------------------------------------------------------
-#  DIRECTIVE: peers
-# ----------------------------------------------------------------------
-$server define hubzero:worker/1 peers {} {
-    global workers
-    return [list peers [array names workers]]
-}
-
 vwait main-loop
Index: trunk/p2p/client.tcl
===================================================================
--- trunk/p2p/client.tcl	(revision 1257)
+++ trunk/p2p/client.tcl	(revision 1273)
@@ -11,4 +11,44 @@
 # ======================================================================
 package require Itcl
+
+namespace eval p2p { # forward declaration }
+
+# ======================================================================
+#  USAGE: p2p::client ?-option value -option value ...?
+#
+#  Used to create a client connection to a peer-to-peer server.
+#  Recognizes the following options:
+#    -address ........... connect to server at this host:port
+#    -sendprotocol ...... tell server we're speaking this protocol
+#    -receiveprotocol ... server replies back with these commands
+# ======================================================================
+proc p2p::client {args} {
+    array set options {
+        -address ?
+        -sendprotocol ""
+        -receiveprotocol ""
+    }
+    foreach {key val} $args {
+        if {![info exists options($key)]} {
+            error "bad option \"$key\": should be [join [lsort [array names options]] {, }]"
+        }
+        set options($key) $val
+    }
+
+    # create the client
+    set client [eval Client ::#auto $options(-address)]
+
+    # install the protocol for incoming commands
+    p2p::protocol::init $client $options(-receiveprotocol)
+
+    # tell the server what protocol we'll be speaking
+    $client send [list protocol $options(-sendprotocol)]
+
+    return $client
+}
+
+# ======================================================================
+#  CLASS: Client
+# ======================================================================
 
 itcl::class Client {
Index: trunk/p2p/foreman.tcl
===================================================================
--- trunk/p2p/foreman.tcl	(revision 1273)
+++ trunk/p2p/foreman.tcl	(revision 1273)
@@ -0,0 +1,199 @@
+# ----------------------------------------------------------------------
+#  P2P: foreman node in P2P mesh, sending jobs out for execution
+#
+#  This file provides an API for clients to connect to the P2P network,
+#  solicit bids on jobs, and send them off for execution.
+# ----------------------------------------------------------------------
+#  Michael McLennan (mmclennan@purdue.edu)
+# ======================================================================
+#  Copyright (c) 2008  Purdue Research Foundation
+#
+#  See the file "license.terms" for information on usage and
+#  redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
+# ======================================================================
+package require Rappture
+
+# recognize other library files in this same directory
+set dir [file dirname [info script]]
+lappend auto_path $dir
+
+# handle log file for this foreman
+log channel error on
+log channel debug on
+log channel system on
+
+proc ::bgerror {err} { log error "ERROR: $err $::errorInfo" }
+
+# set of connections for authority servers
+p2p::options register authority_hosts 127.0.0.1:9001
+
+# ======================================================================
+#  PROTOCOL: hubzero:foreman<-authority/1
+#
+#  The foreman initiates communication with the authority, and the
+#  authority responds by sending the following messages back.
+# ======================================================================
+p2p::protocol::register hubzero:foreman<-authority/1 {
+    # ------------------------------------------------------------------
+    #  INCOMING: options <key1> <value1> <key2> <value2> ...
+    #  These option settings coming from the authority override the
+    #  option settings built into the client.  The authority probably
+    #  has a more up-to-date list of other authorities and better
+    #  policies for living within the network.
+    # ------------------------------------------------------------------
+    define options {args} {
+        foreach {key val} $args {
+            catch {p2p::options set $key $val}
+        }
+        return ""
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: workers <listOfAddresses>
+    #  This message comes in after this foreman has sent the "workers"
+    #  message to request the current list of workers.  The
+    #  <listOfAddresses> is a list of host:port addresses that this
+    #  foreman should contact to enter the p2p network.
+    # ------------------------------------------------------------------
+    define workers {wlist} {
+        global workers
+        set workers $wlist
+        foreman-solicit goto informed
+        return ""
+    }
+}
+
+# ======================================================================
+#  PROTOCOL: hubzero:foreman<-worker/1
+#
+#  The foreman initiates communication with a worker, and the
+#  worker responds by sending the following messages back.
+# ======================================================================
+p2p::protocol::register hubzero:foreman<-worker/1 {
+    # ------------------------------------------------------------------
+    #  INCOMING: proffer <token> <details>
+    # ------------------------------------------------------------------
+    define proffer {token details} {
+puts "PROFFER:\n$details"
+        after idle {foreman-solicit goto informed}
+        return ""
+    }
+}
+
+# ----------------------------------------------------------------------
+#  PEER-TO-PEER CONNECTION
+#
+#  This is the state machine representing the connection of the foreman
+#  to the peer-to-peer network.  We start in the "init" state, then
+#  move to "fetching" while waiting for an authority to return a list
+#  of workers, then move to "informed".  At any point, we can move
+#  from "informed" to "connected" and connect to one of the workers
+#  in the peer-to-peer network, then move back to "informed" when
+#  the information has been gathered.
+# ----------------------------------------------------------------------
+StateMachine foreman-solicit
+foreman-solicit statedata authoritycnx ""
+foreman-solicit statedata workercnx ""
+
+# sit here whenever we don't have a connection
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+foreman-solicit state init
+
+# sit here while we're connected to the authority and fetching data
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+foreman-solicit state fetching
+
+# when moving to the fetching state, connect to the authority
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+foreman-solicit transition init->fetching -onchange {
+    # connect to the authority and request a list of peers
+    statedata authoritycnx ""
+    foreach addr [randomize [p2p::options get authority_hosts]] {
+        if {[catch {p2p::client -address $addr \
+            -sendprotocol hubzero:authority<-foreman/1 \
+            -receiveprotocol hubzero:foreman<-authority/1} result] == 0} {
+            statedata authoritycnx $result
+            break
+        }
+    }
+
+    if {"" != [statedata authoritycnx]} {
+        [statedata authoritycnx] send "workers"
+    } else {
+        error "can't connect to any authority\nAUTHORITY LIST: [p2p::options get authority_hosts]"
+    }
+}
+
+# sit here after we've gotten a list of workers
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+foreman-solicit state informed
+
+# when moving to the fetching state, connect to the authority
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+foreman-solicit transition fetching->informed -onchange {
+    # connect to the authority and request a list of peers
+    if {"" != [statedata authoritycnx]} {
+        catch {itcl::delete object [statedata authoritycnx]}
+        statedata authoritycnx ""
+    }
+    after idle {foreman-solicit goto connected}
+}
+
+# sit here when we're connected to the p2p network
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+foreman-solicit state connected
+
+# when moving to the connected state, connect to the p2p worker
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+foreman-solicit transition informed->connected -onchange {
+    # connect to the p2p network and send a solicitation
+    global workers
+    statedata workercnx ""
+    foreach addr [randomize $workers 100] {
+        if {[catch {p2p::client -address $addr \
+            -sendprotocol hubzero:worker<-foreman/1 \
+            -receiveprotocol hubzero:foreman<-worker/1} result] == 0} {
+            statedata workercnx $result
+            break
+        }
+    }
+
+    if {"" != [statedata workercnx]} {
+        [statedata workercnx] send "solicit -job [expr {rand()}]"
+    } else {
+        after 2000 {foreman-solicit goto connected}
+        error "can't connect to any worker in the p2p network"
+    }
+}
+
+# when moving back to the informed state, disconnect from p2p network
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+foreman-solicit transition connected->informed -onchange {
+    # have an open connection? then close it
+    if {"" != [statedata workercnx]} {
+        catch {itcl::delete object [statedata workercnx]}
+        statedata workercnx ""
+    }
+}
+
+# ----------------------------------------------------------------------
+#  API
+# ----------------------------------------------------------------------
+namespace eval Rappture::foreman { # forward declaration }
+
+# ----------------------------------------------------------------------
+#  COMMAND:  Rappture::foreman::bids -callback <command>
+#
+#  Clients use this to solicit bids for jobs.  Connects to an
+#  authority, if necessary, to determine a list of workers, and then
+#  connects to one of the workers to kick off the bid process.
+# ----------------------------------------------------------------------
+proc Rappture::foreman::bids {args} {
+    global workers
+
+    if {![info exists workers] || [llength $workers] == 0} {
+        foreman-solicit goto fetching
+    } else {
+        foreman-solicit goto connected
+    }
+}
Index: trunk/p2p/handler.tcl
===================================================================
--- trunk/p2p/handler.tcl	(revision 1257)
+++ trunk/p2p/handler.tcl	(revision 1273)
@@ -34,4 +34,7 @@
             return ""
         }
+        define DEFAULT exception {message} {
+            log error "ERROR: $message"
+        }
 
         eval configure $args
@@ -49,4 +52,5 @@
     public method protocol {name}
     public method define {protocol name arglist body}
+    public method connections {{protocol *}}
     public method connectionName {cid {name ""}}
     public method connectionSpeaks {cid protocol}
@@ -54,4 +58,5 @@
     protected method handle {cid}
     protected method finalize {protocol}
+    protected method dropped {cid}
     protected method handlerType {}
 }
@@ -101,4 +106,22 @@
 
 # ----------------------------------------------------------------------
+#  USAGE: connections ?<protocol>?
+#
+#  Returns a list of file handles for current connections that match
+#  the glob-style <protocol> pattern.  If no pattern is given, then
+#  it returns all connections.  Each handle can be passed to
+#  connectionName and connectionSpeaks to get more information.
+# ----------------------------------------------------------------------
+itcl::body Handler::connections {{pattern *}} {
+    set rlist ""
+    foreach cid [array names _protocol] {
+        if {[string match $pattern $_protocol($cid)]} {
+            lappend rlist $cid
+        }
+    }
+    return $rlist
+}
+
+# ----------------------------------------------------------------------
 #  USAGE: connectionName <sockId> ?<name>?
 #
@@ -144,10 +167,5 @@
 itcl::body Handler::handle {cid} {
     if {[gets $cid request] < 0} {
-        log system "dropped: [connectionName $cid]"
-        # connection has connection -- forget this entity
-        catch {close $cid}
-        catch {unset _buffer($cid)}
-        catch {unset _protocol($cid)}
-        catch {unset _cname($cid)}
+        dropped $cid
     } elseif {[info exists _protocol($cid)]} {
         # complete command? then process it below...
@@ -171,9 +189,9 @@
             if {[catch {$_parser($protocol) eval $request} result] == 0} {
                 if {[string length $result] > 0} {
-                    puts $cid $result
+                    catch {puts $cid $result}
                     append mesg "ok: $result"
                 }
             } else {
-                puts $cid [list exception $result]
+                catch {puts $cid [list exception $result]}
                 append mesg "exception: $result"
             }
@@ -204,4 +222,21 @@
 
 # ----------------------------------------------------------------------
+#  USAGE: dropped <cid>
+#
+#  Invoked automatically whenever a client connection drops, to
+#  log the event and remove all trace of the client.  Derived classes
+#  can override this method to perform other functions too.
+# ----------------------------------------------------------------------
+itcl::body Handler::dropped {cid} {
+    log system "dropped: [connectionName $cid]"
+
+    # connection has connection -- forget this entity
+    catch {close $cid}
+    catch {unset _buffer($cid)}
+    catch {unset _protocol($cid)}
+    catch {unset _cname($cid)}
+}
+
+# ----------------------------------------------------------------------
 #  USAGE: handlerType
 #
Index: trunk/p2p/options.tcl
===================================================================
--- trunk/p2p/options.tcl	(revision 1273)
+++ trunk/p2p/options.tcl	(revision 1273)
@@ -0,0 +1,73 @@
+# ----------------------------------------------------------------------
+#  P2P: options
+#
+#  This code manages option settings within all of the programs in the
+#  peer to peer job management system.  Options are soft-coded
+#  parameters controlling things like how many peers each node should
+#  be connected to.
+# ----------------------------------------------------------------------
+#  Michael McLennan (mmclennan@purdue.edu)
+# ======================================================================
+#  Copyright (c) 2008  Purdue Research Foundation
+#
+#  See the file "license.terms" for information on usage and
+#  redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
+# ======================================================================
+package require Rappture
+
+namespace eval p2p {
+    variable options  ;# array of option settings for this program
+}
+
+# ======================================================================
+#  OPTIONS
+#  These options get the worker started, but new option settings
+#  some from the authority after this worker first connects.
+# ----------------------------------------------------------------------
+#  USAGE:  p2p::options register <name> <defaultValue>
+#  USAGE:  p2p::options set <name> <newValue>
+#  USAGE:  p2p::options get <name>
+#
+#  Use these commands to register and get/set various option settings
+#  used within these programs.  The option settings start with default
+#  values but can be overridden at any point, for example by the
+#  "authority" program.
+# ======================================================================
+proc p2p::options {op args} {
+    global options
+
+    switch -- $op {
+        register {
+            if {[llength $args] != 2} {
+                error "wrong # args: should be \"p2p::options register name default\""
+            }
+            set opt [lindex $args 0]
+            set def [lindex $args 1]
+            if {[info exists options($opt)]} {
+                error "option \"$opt\" is already registered"
+            }
+            set options($opt) $def
+        }
+        get {
+            if {[llength $args] != 1} {
+                error "wrong # args: should be \"p2p::options get name\""
+            }
+            set opt [lindex $args 0]
+            if {![info exists options($opt)]} {
+                error "option \"$opt\" is not defined"
+            }
+            return $options($opt)
+        }
+        set {
+            if {[llength $args] != 2} {
+                error "wrong # args: should be \"p2p::options set name default\""
+            }
+            set opt [lindex $args 0]
+            set val [lindex $args 1]
+            if {![info exists options($opt)]} {
+                error "option \"$opt\" is not defined"
+            }
+            set options($opt) $val
+        }
+    }
+}
Index: trunk/p2p/perftest.c
===================================================================
--- trunk/p2p/perftest.c	(revision 1257)
+++ trunk/p2p/perftest.c	(revision 1273)
@@ -91,5 +91,5 @@
              + tend.tv_usec - tstart.tv_usec;
 
-    printf("%ld\n", tval);
+    printf("rappture-wonks/v1 %ld\n", tval);
     exit(0);
 }
Index: trunk/p2p/protocols.tcl
===================================================================
--- trunk/p2p/protocols.tcl	(revision 1273)
+++ trunk/p2p/protocols.tcl	(revision 1273)
@@ -0,0 +1,114 @@
+# ----------------------------------------------------------------------
+#  P2P: protocol initialization and registration
+#
+#  This file contains p2p::init and p2p::connect routines used to
+#  add protocols to Client and Server objects in the p2p system.
+#  New protocols are stored in the various p-*.tcl files.  This
+#  code loads those files, obtains a list of known protocols, and
+#  then registers the latest version of a protocol in a given
+#  Client or Server connection.
+# ----------------------------------------------------------------------
+#  Michael McLennan (mmclennan@purdue.edu)
+# ======================================================================
+#  Copyright (c) 2008  Purdue Research Foundation
+#
+#  See the file "license.terms" for information on usage and
+#  redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
+# ======================================================================
+
+namespace eval p2p { # forward declaration }
+
+namespace eval p2p::protocol {
+    variable name2defn  ;# maps protocol name => body of code defining it
+    variable version    ;# maps protocol name => highest /version number
+    variable _tmpobj    ;# used during init/define calls
+    variable _tmpproto  ;# used during init/define calls
+}
+
+# ======================================================================
+#  USAGE: p2p::protocol::register <name> <code>
+#
+#  Various p-*.tcl files use this to define a particular protocol
+#  called <name>.  Calling p2p::protocol::init adds that protocol to
+#  a particular Handler object.  This is done by executing the <code>,
+#  which invokes a series of "define" operations to define the
+#  protocol handlers.
+# ======================================================================
+proc p2p::protocol::register {name code} {
+    variable name2defn
+    variable version
+
+    # store the code needed for this version for later call to init
+    if {[info exists name2defn($name)]} {
+        error "protocol \"$name\" already exists"
+    }
+    set name2defn($name) $code
+
+    # extract the version number from the protocol and save the highest num
+    if {![regexp {(.+)/([0-9]+)$} $name match root vnum]} {
+        set root $name
+        set vnum 1
+    }
+    if {![info exists version($root)] || $vnum > $version($root)} {
+        set version($root) $vnum
+    }
+}
+
+# ======================================================================
+#  USAGE: p2p::protocol::init <handlerObj> <protocol>
+#
+#  Adds all protocol versions matching the <protocol> root name to
+#  the given <handlerObj> object.  For example, the <protocol> might
+#  be "a->b", and this routine will add "a->b/1", "a->b/2", etc.
+#  Returns the highest version of the protocol added to the object.
+# ======================================================================
+proc p2p::protocol::init {obj protocol} {
+    variable name2defn
+    variable version
+    variable _tmpobj
+    variable _tmpproto
+
+    # trim off any version number included in the protocol name
+    regexp {(.+)/[0-9]+$} $protocol match protocol
+
+    set plist $protocol
+    eval lappend plist [array names name2defn $protocol/*]
+    set _tmpobj $obj  ;# used via "define" statements in "catch" below
+
+    set added 0
+    foreach name $plist {
+        if {[info exists name2defn($name)]} {
+            $obj protocol $name
+            $obj define $name exception {message} {
+                log error "ERROR from client:  $message"
+            }
+
+            # define the rest of the protocol as registered earlier
+            set _tmpproto $name  ;# used via "define" statements below
+            if {[catch $name2defn($name) err]} {
+                error $err "$err\n    (while adding protocol $name to $obj)"
+            }
+            incr added
+        }
+    }
+
+    if {$added == 0} {
+        error "can't find protocol definition matching \"$protocol\""
+    }
+    return $version($protocol)
+}
+
+# ======================================================================
+#  USAGE: p2p::protocol::define <name> <args> <body>
+#
+#  This gets called implicitly by the bodies of code invoked in the
+#  p2p::protocol::init statements above.  Code bodies should contain
+#  a series of "define" statements, and this is the "define" that
+#  gets invoked, which passes the call onto the _tmpobj where the
+#  protocol is being installed.
+# ======================================================================
+proc p2p::protocol::define {args} {
+    variable _tmpobj
+    variable _tmpproto
+    eval $_tmpobj define $_tmpproto $args
+}
Index: trunk/p2p/random.tcl
===================================================================
--- trunk/p2p/random.tcl	(revision 1257)
+++ trunk/p2p/random.tcl	(revision 1273)
@@ -38,2 +38,12 @@
     return $rlist
 }
+
+# ----------------------------------------------------------------------
+#  USAGE: random <list>
+#
+#  Picks one element at random from the given <list>.
+# ----------------------------------------------------------------------
+proc random {entries} {
+    set nrand [expr {int(rand()*[llength $entries])}]
+    return [lindex $entries $nrand]
+}
Index: trunk/p2p/server.tcl
===================================================================
--- trunk/p2p/server.tcl	(revision 1257)
+++ trunk/p2p/server.tcl	(revision 1273)
@@ -11,4 +11,44 @@
 package require Itcl
 
+namespace eval p2p { # forward declaration }
+
+# ======================================================================
+#  USAGE: p2p::server ?-option value -option value ...?
+#
+#  Used to create a new peer-to-peer server object for this program.
+#  Recognizes the following options:
+#    -port ........ port number that the server listens on
+#    -protocols ... list of protocol root names that the server handles
+#               ... any other option supported by Server class
+# ======================================================================
+proc p2p::server {args} {
+    set port "?"
+    set protocols ""
+    set options ""
+    foreach {key val} $args {
+        switch -- $key {
+            -port { set port $val }
+            -protocols { set protocols $val }
+            default { lappend options $key $val }
+        }
+    }
+
+    if {[llength $protocols] == 0} {
+        error "server needs at least one value for -protocols"
+    }
+
+    # create the server
+    set server [eval Server ::#auto $port $options]
+
+    # install the protocols that this server recognizes
+    foreach name $protocols {
+        p2p::protocol::init $server $name
+    }
+    return $server
+}
+
+# ======================================================================
+#  CLASS: Server
+# ======================================================================
 itcl::class Server {
     inherit Handler
@@ -24,4 +64,7 @@
     # this code fragment gets invoked when client declares the protocol
     public variable onprotocol ""
+
+    # this code fragment gets invoked when client drops
+    public variable ondisconnect ""
 
     constructor {port args} {
@@ -59,6 +102,8 @@
 
     public method port {}
+    public method broadcast {args}
     public method connectionSpeaks {cid protocol}
 
+    protected method dropped {cid}
     protected method handlerType {}
 
@@ -80,4 +125,62 @@
 
 # ----------------------------------------------------------------------
+#  USAGE: broadcast ?-protocol <name>? ?-avoid <avoidList>? <message>
+#
+#  Sends a <message> to all clients connected to this server.  If a
+#  client address appears on the -avoid list, then that client is
+#  avoided.  If the -protocol is specified, then the message is sent
+#  only to clients who match the glob-style pattern for the protocol
+#  name.
+# ----------------------------------------------------------------------
+itcl::body Server::broadcast {args} {
+    set pattern "*"
+    set avoidList ""
+    set i 0
+    while {$i < [llength $args]} {
+        set option [lindex $args $i]
+        if {[string index $option 0] == "-"} {
+            switch -- $option {
+                -protocol {
+                    set pattern [lindex $args [expr {$i+1}]]
+                    incr i 2
+                }
+                -avoid {
+                    set avoidList [lindex $args [expr {$i+1}]]
+                    incr i 2
+                }
+                -- {
+                    incr i
+                    break
+                }
+                default {
+                    error "bad option \"$option\": should be -avoid, -protocol, or --"
+                }
+            }
+        } else {
+            break
+        }
+    }
+    if {$i != [llength $args]-1} {
+        error "wrong # args: should be \"broadcast ?-protocol pattern? ?-avoid clients? message\""
+    }
+    set message [lindex $args end]
+
+    set nmesgs 0
+    foreach cid [connections $pattern] {
+        set addr [lindex [connectionName $cid] 0]  ;# x.x.x.x (sockN)
+        if {[llength $avoidList] == 0 || [lsearch $avoidList $addr] < 0} {
+puts "  inbound => [connectionName $cid]"
+            if {[catch {puts $cid $message} result] == 0} {
+                incr nmesgs
+            } else {
+                log error "ERROR: broadcast failed for $cid: $result"
+                log error "  (message was $message)"
+            }
+        }
+    }
+    return $nmesgs
+}
+
+# ----------------------------------------------------------------------
 #  USAGE: accept <cid> <addr> <port>
 #
@@ -116,4 +219,22 @@
 
 # ----------------------------------------------------------------------
+#  USAGE: dropped <cid>
+#
+#  Invoked automatically whenever a client connection drops, to
+#  log the event and remove all trace of the client.  Invokes any
+#  command hook for this server to note the fact that the client
+#  has dropped.
+# ----------------------------------------------------------------------
+itcl::body Server::dropped {cid} {
+    # if there's a callback to handle the drop, execute it here
+    if {[string length $ondisconnect] > 0} {
+        uplevel #0 [list $ondisconnect [connectionName $cid]]
+    }
+
+    # call the base class method to clean up after the client
+    chain $cid
+}
+
+# ----------------------------------------------------------------------
 #  USAGE: handlerType
 #
Index: trunk/p2p/statemachine.tcl
===================================================================
--- trunk/p2p/statemachine.tcl	(revision 1273)
+++ trunk/p2p/statemachine.tcl	(revision 1273)
@@ -0,0 +1,210 @@
+# ----------------------------------------------------------------------
+#  LIBRARY: statemachine
+#
+#  This object comes in handy when building the state machines that
+#  drive the behavior of clients and servers in the peer-to-peer system.
+#  Each state machine has a number of recognized states and trasitions
+#  between the states that kick off actions within the system.
+# ----------------------------------------------------------------------
+#  Michael McLennan (mmclennan@purdue.edu)
+# ======================================================================
+#  Copyright (c) 2008  Purdue Research Foundation
+#
+#  See the file "license.terms" for information on usage and
+#  redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
+# ======================================================================
+package require Itcl
+
+# ======================================================================
+#  CLASS: StateMachine
+# ======================================================================
+itcl::class StateMachine {
+    private variable _states  ;# maps states to -enter/-leave commands
+    private variable _trans   ;# maps s1->s2 transitions to commands
+    private variable _current ""
+    private variable _slots   ;# maps statedata name => value
+    private variable _intrans 0
+
+    constructor {args} {
+        eval configure $args
+        set _states(all) ""
+    }
+
+    public method state {name options}
+    public method transition {name options}
+    public method current {} { return $_current }
+    public method all {} { return $_states(all) }
+    public method goto {state}
+    public method statedata {args}
+}
+
+# ----------------------------------------------------------------------
+#  USAGE: state <name> ?-option value -option value ...?
+#
+#  Defines a new state in the state machine.  Recognizes the following
+#  options:
+#    -onenter ... command invoked when state is entered
+#    -onleave ... command invoked when state is left
+# ----------------------------------------------------------------------
+itcl::body StateMachine::state {name args} {
+    array set options {
+        -onenter ""
+        -onleave ""
+    }
+    foreach {key val} $args {
+        if {![info exists options($key)]} {
+            error "bad option \"$key\": should be [join [lsort [array names options]] {, }]"
+        }
+        set options($key) $val
+    }
+
+    if {![regexp {^[-a-zA-Z0-9_]+$} $name]} {
+        error "bad state name \"$name\": should be alphanumeric, including - or _"
+    }
+
+    if {[lsearch $_states(all) $name] >= 0} {
+        error "state \"$name\" already defined"
+    }
+    lappend _states(all) $name
+    set _states($name-onenter) $options(-onenter)
+    set _states($name-onleave) $options(-onleave)
+
+    # start in the first state by default
+    if {$_current == ""} {
+        goto $name
+    }
+    return $name
+}
+
+# ----------------------------------------------------------------------
+#  USAGE: transition <name> ?-option value -option value ...?
+#
+#  Defines the transition from one state to another.  The transition
+#  <name> should be of the form "s1->s2", where "s1" and "s2" are
+#  recognized state names.  Recognizes the following options:
+#    -onchange ... command invoked when transition is followed
+# ----------------------------------------------------------------------
+itcl::body StateMachine::transition {name args} {
+    array set options {
+        -onchange ""
+    }
+    foreach {key val} $args {
+        if {![info exists options($key)]} {
+            error "bad option \"$key\": should be [join [lsort [array names options]] {, }]"
+        }
+        set options($key) $val
+    }
+
+    if {![regexp {^([-a-zA-Z0-9_]+)->([-a-zA-Z0-9_]+)$} $name match state1 state2]} {
+        error "bad transition name \"$name\": should have the form s1->s2"
+    }
+    if {[lsearch $_states(all) $state1] < 0} {
+        error "unrecognized starting state \"$state1\" for transition"
+    }
+    if {[lsearch $_states(all) $state2] < 0} {
+        error "unrecognized ending state \"$state2\" for transition"
+    }
+
+    set _trans($state1->$state2-onchange) $options(-onchange)
+    return $name
+}
+
+# ----------------------------------------------------------------------
+#  USAGE: goto <name>
+#
+#  Forces the state machine to undergo a transition from the current
+#  state to the new state <name>.  If the current state has a -onleave
+#  hook, it is executed first.  If there is a transition between the
+#  states with a -onchange hook, it is executed next.  If the new
+#  state has a -onenter hook, it is executed last.  If any of the
+#  command hooks along the way fail, then the state machine will
+#  drop back to its current state and return the error.
+# ----------------------------------------------------------------------
+itcl::body StateMachine::goto {state} {
+    if {$_intrans} {
+        error "can't change states while making a transition"
+    }
+
+    #
+    # NOTE: Use _'s for all local variables in this routine.  We eval
+    #   the code fragments or -onenter, -onleave, -onchange directly
+    #   in the context of this method.  This keeps them from polluting
+    #   the global scope and gives them access to the "statedata"
+    #   method.  But it could have side effects if the code fragments
+    #   accidentally redefined local variables.  So start all local
+    #   variables with _ to avoid any collisions.
+    #
+    set _sname $state
+
+    if {[lsearch $_states(all) $_sname] < 0} {
+        error "bad state name \"$_sname\": should be one of [join [lsort $_states(all)] {, }]"
+    }
+    set _intrans 1
+
+    # execute any -onleave hook for the current state
+    set _goback 0
+    if {"" != $_current && "" != $_states($_current-onleave)} {
+        if {[catch $_states($_current-onleave) _result]} {
+            set _goback 1
+        }
+    }
+
+    # execute any -onchange hook for the transition
+    if {!$_goback && [info exists _trans($_current->$_sname-onchange)]} {
+        if {[catch $_trans($_current->$_sname-onchange) _result]} {
+            set _goback 1
+        }
+    }
+
+    # execute any -onenter hook for the new state
+    if {!$_goback && "" != $_states($_sname-onenter)} {
+        if {[catch $_states($_sname-onenter) _result]} {
+            set _goback 1
+        }
+    }
+
+    if {$_goback} {
+        if {"" != $_current && "" != $_states($_current-onenter)} {
+            catch $_states($_current-onenter)
+        }
+        set _intrans 0
+        error $_result "$_result\n    (while transitioning from $_current to $_sname)"
+    }
+
+    set _intrans 0
+    set _current $_sname
+    return $_current
+}
+
+# ----------------------------------------------------------------------
+#  USAGE: statedata ?<name>? ?<value>?
+#
+#  Used to get/set special data values stored within the state
+#  machine.  These are like local variables, but can be shared
+#  across the code fragments associated with states and transitions.
+#  We could also use global variables for this, but this allows
+#  each state machine to store its own values.
+# ----------------------------------------------------------------------
+itcl::body StateMachine::statedata {args} {
+    switch [llength $args] {
+        0 {
+            return [array get _slots]
+        }
+        1 {
+            set name [lindex $args 0]
+            if {[info exists _slots($name)]} {
+                return $_slots($name)
+            }
+            return ""
+        }
+        2 {
+            set name [lindex $args 0]
+            set value [lindex $args 1]
+            set _slots($name) $value
+            return $value
+        }
+        default {
+            error "wrong # args: should be \"statedata ?name? ?value?\""
+        }
+    }
+}
Index: trunk/p2p/tclIndex
===================================================================
--- trunk/p2p/tclIndex	(revision 1257)
+++ trunk/p2p/tclIndex	(revision 1273)
@@ -10,14 +10,20 @@
 set auto_index(::Handler::protocol) [list source [file join $dir handler.tcl]]
 set auto_index(::Handler::define) [list source [file join $dir handler.tcl]]
+set auto_index(::Handler::connections) [list source [file join $dir handler.tcl]]
 set auto_index(::Handler::connectionName) [list source [file join $dir handler.tcl]]
 set auto_index(::Handler::connectionSpeaks) [list source [file join $dir handler.tcl]]
 set auto_index(::Handler::handle) [list source [file join $dir handler.tcl]]
 set auto_index(::Handler::finalize) [list source [file join $dir handler.tcl]]
+set auto_index(::Handler::dropped) [list source [file join $dir handler.tcl]]
 set auto_index(::Handler::handlerType) [list source [file join $dir handler.tcl]]
+set auto_index(::p2p::server) [list source [file join $dir server.tcl]]
 set auto_index(Server) [list source [file join $dir server.tcl]]
 set auto_index(::Server::port) [list source [file join $dir server.tcl]]
+set auto_index(::Server::broadcast) [list source [file join $dir server.tcl]]
 set auto_index(::Server::accept) [list source [file join $dir server.tcl]]
 set auto_index(::Server::connectionSpeaks) [list source [file join $dir server.tcl]]
+set auto_index(::Server::dropped) [list source [file join $dir server.tcl]]
 set auto_index(::Server::handlerType) [list source [file join $dir server.tcl]]
+set auto_index(::p2p::client) [list source [file join $dir client.tcl]]
 set auto_index(Client) [list source [file join $dir client.tcl]]
 set auto_index(::Client::send) [list source [file join $dir client.tcl]]
@@ -26,2 +32,16 @@
 set auto_index(log) [list source [file join $dir log.tcl]]
 set auto_index(randomize) [list source [file join $dir random.tcl]]
+set auto_index(random) [list source [file join $dir random.tcl]]
+set auto_index(::p2p::options) [list source [file join $dir options.tcl]]
+set auto_index(::p2p::protocol::register) [list source [file join $dir protocols.tcl]]
+set auto_index(::p2p::protocol::init) [list source [file join $dir protocols.tcl]]
+set auto_index(::p2p::protocol::define) [list source [file join $dir protocols.tcl]]
+set auto_index(StateMachine) [list source [file join $dir statemachine.tcl]]
+set auto_index(::StateMachine::state) [list source [file join $dir statemachine.tcl]]
+set auto_index(::StateMachine::transition) [list source [file join $dir statemachine.tcl]]
+set auto_index(::StateMachine::goto) [list source [file join $dir statemachine.tcl]]
+set auto_index(::StateMachine::statedata) [list source [file join $dir statemachine.tcl]]
+set auto_index(::p2p::wonks::init) [list source [file join $dir wonks.tcl]]
+set auto_index(::p2p::wonks::current) [list source [file join $dir wonks.tcl]]
+set auto_index(bgerror) [list source [file join $dir foreman.tcl]]
+set auto_index(::Rappture::foreman::bids) [list source [file join $dir foreman.tcl]]
Index: trunk/p2p/test.tcl
===================================================================
--- trunk/p2p/test.tcl	(revision 1257)
+++ trunk/p2p/test.tcl	(revision 1273)
@@ -23,4 +23,9 @@
 set nodes(all) ""
 set nodeRadius 15
+
+option add *highlightBackground [. cget -background]
+option add *client*background gray
+option add *client*highlightBackground gray
+option add *client*troughColor darkGray
 
 # ======================================================================
@@ -162,4 +167,24 @@
 #  Build the main interface
 # ======================================================================
+frame .client -borderwidth 8 -relief flat
+pack .client -side right -fill y
+button .client.getbids -text "Get Bids:" -command test_bids
+pack .client.getbids -side top -anchor w
+frame .client.cntls
+pack .client.cntls -side bottom -fill x
+button .client.cntls.run -text "Spend" -command test_spend
+pack .client.cntls.run -side left
+entry .client.cntls.points -width 8
+pack .client.cntls.points -side left
+label .client.cntls.pointsl -text "points"
+pack .client.cntls.pointsl -side left
+
+frame .client.bids
+pack .client.bids -side bottom -expand yes -fill both
+scrollbar .client.bids.ysbar -orient vertical -command {.client.bids.info yview}
+pack .client.bids.ysbar -side right -fill y
+listbox .client.bids.info -yscrollcommand {.client.bids.ysbar set}
+pack .client.bids.info -side left -expand yes -fill both
+
 frame .cntls
 pack .cntls -fill x
@@ -181,4 +206,10 @@
 }
 pack .cntls.layout -side left -padx 4 -pady 2
+
+entry .cntls.workers -width 5
+pack .cntls.workers -side right -padx {0 4} -pady 2
+.cntls.workers insert end "3"
+label .cntls.workersl -text "Workers:"
+pack .cntls.workersl -side right -pady 2
 
 frame .player
@@ -238,5 +269,5 @@
 
     # launch a series of workers
-    for {set i 0} {$i < 20} {incr i} {
+    for {set i 0} {$i < [.cntls.workers get]} {incr i} {
         lappend processes [exec tclsh worker.tcl &]
         after [expr {int(rand()*5000)}]
@@ -266,4 +297,9 @@
         set info [read $fid]
         close $fid
+
+        if {[regexp -- {foreman<-} $info]} {
+            # skip log file from foreman
+            continue
+        }
 
         # get the address for this host
@@ -322,4 +358,9 @@
         close $fid
 puts "\nscanning $fname"
+
+        if {[regexp -- {foreman<-} $info]} {
+            # skip log file from foreman
+            continue
+        }
 
         catch {unset started}
@@ -358,5 +399,5 @@
                     unset started(connect$cid-addr)
 
-                } elseif {[regexp {server message from ([a-zA-Z0-9\.]+:[0-9]+) \(([a-z0-9]+)\): +(.+) => (.*)} $mesg match addr cid cmd result]} {
+                } elseif {[regexp {(server|client) message from ([a-zA-Z0-9\.]+:[0-9]+) \(([a-z0-9]+)\): +(.+) => (.*)} $mesg match which addr cid cmd result]} {
                     if {![string match identity* $cmd]} {
                         append actions($tval) $mesg \n
@@ -434,4 +475,10 @@
             set from $nodes($fname)
             set addr $started($key)
+
+            if {![info exists nodes($addr-x)]} {
+                unset started(connect$cid-time)
+                unset started(connect$cid-addr)
+                continue
+            }
             set x0 $nodes($from-x)
             set y0 $nodes($from-y)
@@ -558,2 +605,8 @@
     pack .diagram.$view -expand yes -fill both
 }
+
+proc test_bids {} {
+    set info [Rappture::foreman::bids]
+    .client.bids.info delete 0 end
+    eval .client.bids.info insert end $info
+}
Index: trunk/p2p/wonks.tcl
===================================================================
--- trunk/p2p/wonks.tcl	(revision 1273)
+++ trunk/p2p/wonks.tcl	(revision 1273)
@@ -0,0 +1,202 @@
+# ----------------------------------------------------------------------
+#  P2P: wonks
+#
+#  This code manages the calculation of "wonks" within a worker node.
+#  Wonks are measured by executing the "perftest" program on the
+#  worker node to see how fast it can finish the calculation.
+# ----------------------------------------------------------------------
+#  Michael McLennan (mmclennan@purdue.edu)
+# ======================================================================
+#  Copyright (c) 2008  Purdue Research Foundation
+#
+#  See the file "license.terms" for information on usage and
+#  redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
+# ======================================================================
+package require Rappture
+
+namespace eval p2p { # forward declaration }
+
+namespace eval p2p::wonks {
+    # remember this directory -- perftest sits here too
+    variable dir [file dirname [info script]]
+
+    # version info from perftest
+    variable version "?"
+}
+
+# number of seconds between each check of system load
+p2p::options register wonks_time_between_checks 60000
+
+# fraction between 0-1 indicating a significant change in wonks
+p2p::options register wonk_tolerance 0.1
+
+# ----------------------------------------------------------------------
+#  WONKS CHECK
+#
+#  This is the state machine controlling the periodic testing for
+#  the available wonks on this machine.  The "perftest" program is
+#  executed from time to time to compute the available wonks, and
+#  the results are stored in an array of tuples.  At any point, we
+#  can estimate the number of wonks by getting the load and available
+#  memory and interpolating within the available tuple data.
+# ----------------------------------------------------------------------
+StateMachine wonks-check
+
+# sit here at times between sampling
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+wonks-check state idle -onenter {
+    set delay [p2p::options get wonks_time_between_checks]
+    after $delay {wonks-check goto checking}
+} -onleave {
+    after cancel {wonks-check goto checking}
+}
+
+# sit here while we're sampling
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+wonks-check state checking
+
+# when moving to the checking state, test available wonks
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+wonks-check transition idle->checking -onchange {
+    set data [wonks-check statedata perftable]
+
+    set getWonks 0  ;# probably won't have to use perftest
+
+    # measure system load and estimate the available wonks
+    foreach {load mem} [Rappture::sysinfo load1 freeswap] break
+    set load [format "%.1f" $load]
+    set mem [expr {$mem/1048576}]  ;# in megabytes
+    set wval 0
+    set nsamples 0
+
+    if {![$data inrange [list $load $mem -- --]]} {
+        # outside range of data -- must call perftest
+        set getWonks 1
+    } else {
+        set rec [$data find -nearest [list $load $mem -- --]]
+        set prevload [lindex $rec 0]
+        set prevmem [lindex $rec 1]
+        set tol [p2p::options get wonk_tolerance]
+
+        set closeEnough 1
+        if {"" == $prevload || ($load+$prevload > 0
+              && abs($load-$prevload)/(0.5*double($load+$prevload)) > $tol)} {
+            # load differs by more than the tolerance -- must test
+            set closeEnough 0
+        } elseif {"" == $prevmem || ($mem+$prevmem > 0
+              && abs($mem-$prevmem)/(0.5*double($mem+$prevmem)) > $tol)} {
+            set closeEnough 0
+        }
+
+        if {$closeEnough} {
+            # close enough to some other sample point
+            # update the existing data point in the table
+            set load $prevload
+            set mem $prevmem
+
+            set wval [lindex $rec 2]
+            set nsamples [lindex $rec 3]
+            if {$nsamples < 5} {
+                # not very many samples? then do another
+                set getWonks 1
+            } elseif {$nsamples < 50 && rand() > 0.9} {
+                # if we have a lot of samples do another from time to time
+                set getWonks 1
+            }
+        }
+    }
+
+    # if we need a new reading, exec the perftest program...
+    if {$getWonks} {
+        # Run the program, but don't use exec, since it blocks.
+        # Instead, run it in the background and harvest its output later.
+        set prog [open "| [file join $p2p::wonks::dir perftest]" r]
+        fileevent $prog readable {wonks-check goto update}
+        wonks-check statedata sysinfo [list $load $mem $wval $nsamples]
+        wonks-check statedata prog $prog
+    } else {
+        after idle {wonks-check goto idle}
+    }
+}
+
+# go here when perftest has finished
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+wonks-check state update
+
+# when moving to the update state, store the data from perftest
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+wonks-check transition checking->update -onchange {
+    set data [wonks-check statedata perftable]
+    set prog [wonks-check statedata prog]
+
+    if {[catch {read $prog} msg] == 0} {
+        if {[regexp {^([^ ]+) +([0-9]+)} $msg match vstr wval]} {
+            set p2p::wonks::version $vstr
+            foreach {load mem oldwval nsamples} [wonks-check statedata sysinfo] break
+            if {$nsamples >= 1} {
+                # average new value with old values
+                set wval [expr {$wval/double($nsamples+1)
+                                 + $nsamples/double($nsamples+1)*$oldwval}]
+            }
+            $data delete [list $load $mem -- --]
+            $data add [list $load $mem $wval [incr nsamples]]
+        } else {
+            log error "ERROR: performance test failed: $msg"
+        }
+    }
+    catch {close $prog}
+    wonks-check statedata prog ""
+    wonks-check statedata sysinfo ""
+
+    after idle {wonks-check goto idle}
+}
+
+# ----------------------------------------------------------------------
+#  COMMAND:  p2p::wonks::init
+#
+#  Invoked when a worker node starts up to initialize this part of
+#  the package.  This triggers autoloading of the file, and also
+#  initializes variables used by the functions in this file.
+# ----------------------------------------------------------------------
+proc p2p::wonks::init {} {
+    variable version
+    set version "?"
+
+    set data [Rappture::tuples ::#auto]
+    $data dimension add load1
+    $data dimension add freeswap
+    $data dimension add wonks
+    $data dimension add nsamples
+    wonks-check statedata perftable $data
+
+    wonks-check goto idle
+}
+
+# ----------------------------------------------------------------------
+#  COMMAND:  p2p::wonks::current
+#
+#  Used to estimate the current performance of this node as a number
+#  of wonks.  Takes
+# ----------------------------------------------------------------------
+proc p2p::wonks::current {} {
+    foreach {load mem} [Rappture::sysinfo load1 freeswap] break
+    set load [format "%.1f" $load]
+    set mem [expr {$mem/1048576}]  ;# in megabytes
+
+    set data [wonks-check statedata perftable]
+    if {[$data inrange [list $load $mem -- --]]} {
+        set wval [$data interpolate wonks [list $load $mem -- --]]
+        return [list $p2p::wonks::version [expr {round($wval)}]]
+    }
+
+    # don't have any data -- we're forced to exec perftest and block
+    if {[catch {exec [file join $p2p::wonks::dir perftest]} msg] == 0
+          && [regexp {^([^ ]+) +([0-9]+)} $msg match vstr wval]} {
+        $data add [list $load $mem $wval 1]
+        set p2p::wonks::version $vstr
+        return [list $vstr $wval]
+    }
+
+    # can't even exec perftest -- return some large bogus value
+    return [list bogus 1000000]
+}
Index: trunk/p2p/worker.tcl
===================================================================
--- trunk/p2p/worker.tcl	(revision 1257)
+++ trunk/p2p/worker.tcl	(revision 1273)
@@ -21,124 +21,50 @@
 
 # handle log file for this worker
+log channel error on
+log channel system on
 log channel debug on
-log channel system on
-
-# set up a server at the first open port above 9101
-set server [Server ::#auto 9101? -servername worker \
-    -onprotocol worker_peers_protocol]
-
-# ======================================================================
-#  OPTIONS
-#  These options get the worker started, but new option settings
-#  some from the authority after this worker first connects.
-# ======================================================================
-set authority ""     ;# file handle for authority connection
+
+proc ::bgerror {err} { log error "ERROR: $err $::errorInfo" }
+
+
 set myaddress "?:?"  ;# address/port for this worker
 
 # set of connections for authority servers
-set options(authority_hosts) 127.0.0.1:9001
+p2p::options register authority_hosts 127.0.0.1:9001
 
 # register with the central authority at this frequency
-set options(time_between_registers) 10000
+p2p::options register time_between_authority_checks 60000
+
+# rebuild the peer-to-peer network at this frequency
+p2p::options register time_between_network_rebuilds 600000
 
 # this worker should try to connect with this many other peers
-set options(max_peer_connections) 4
-
-# number of seconds between each check of system load
-set options(time_between_load_checks) 60000
-
-# ======================================================================
-#  PEERS
-# ======================================================================
-# eventually set to the list of all peers in the network
-set peers(all) ""
-
-# list of peers that we're testing connections with
-set peers(testing) ""
-
-# ======================================================================
-#  SYSTEM LOAD
-# ======================================================================
-# Check system load every so often.  When the load changes
-# significantly, execute a "perftest" run to compute the current
-# number of wonks available to this worker.
-
-set sysload(lastLoad) -1
-set sysload(wonks) -1
-set sysload(measured) -1
-
-after idle worker_load_check
-
-# ======================================================================
-#  PROTOCOL: hubzero:peer/1
-#  This protocol gets used when another worker connects to this one.
-# ======================================================================
-$server protocol hubzero:peer/1
-
-$server define hubzero:peer/1 exception {message} {
-    log system "ERROR: $message"
-}
-
-# ----------------------------------------------------------------------
-#  DIRECTIVE: identity
-#  Used for debugging, so that each client can identify itself by
-#  name to this worker.
-# ----------------------------------------------------------------------
-$server define hubzero:peer/1 identity {name} {
-    variable cid
-    variable handler
-    $handler connectionName $cid $name
-    return ""
-}
-
-# ----------------------------------------------------------------------
-#  DIRECTIVE: ping
-# ----------------------------------------------------------------------
-$server define hubzero:peer/1 ping {} {
-    return "pong"
-}
-
-# ======================================================================
-#  PROTOCOL: hubzero:worker/1
-#  The authority_connect procedure connects this worker to an authority
-#  and begins the authority/worker protocol.  The directives below
-#  handle the incoming (authority->worker) traffic.
-# ======================================================================
-proc authority_connect {} {
-    global authority options
-    log system "connecting to authorities..."
-
-    authority_disconnect
-
-    # scan through list of authorities and try to connect
-    foreach addr [randomize $options(authority_hosts)] {
-        if {[catch {Client ::#auto $addr} result] == 0} {
-            set authority $result
-            break
-        }
-    }
-
-    if {"" == $authority} {
-        error "can't connect to any known authorities"
-    }
-
-    # add handlers to process the incoming commands...
-    $authority protocol hubzero:worker/1
-    $authority define hubzero:worker/1 exception {args} {
-        log system "error from authority: $args"
-    }
-
-    # ------------------------------------------------------------------
-    #  DIRECTIVE: options <key1> <value1> <key2> <value2> ...
+p2p::options register max_peer_connections 4
+
+# workers propagate messages until time-to-live reaches 0
+p2p::options register peer_time_to_live 4
+
+# ======================================================================
+#  PROTOCOL: hubzero:worker<-authority/1
+#
+#  The worker initiates communication with the authority, and the
+#  authority responds by sending these messages.
+# ======================================================================
+p2p::protocol::register hubzero:worker<-authority/1 {
+
+    # ------------------------------------------------------------------
+    #  INCOMING: options <key1> <value1> <key2> <value2> ...
     #  These option settings coming from the authority override the
-    #  option settings built into the client at the top of this
-    #  script.  The authority probably has a more up-to-date list
-    #  of other authorities and better policies for living within
-    #  the network.
-    # ------------------------------------------------------------------
-    $authority define hubzero:worker/1 options {args} {
-        global options server myaddress
+    #  option settings built into the client.  The authority probably
+    #  has a more up-to-date list of other authorities and better
+    #  policies for living within the network.
+    # ------------------------------------------------------------------
+    define options {args} {
+        global server myaddress
 
         array set options $args
+        foreach key [array names options] {
+            catch {p2p::options set $key $options($key)}
+        }
 
         if {[info exists options(ip)]} {
@@ -150,5 +76,5 @@
 
     # ------------------------------------------------------------------
-    #  DIRECTIVE: peers <listOfAddresses>
+    #  INCOMING: peers <listOfAddresses>
     #  This message comes in after this worker has sent the "peers"
     #  message to request the current list of peers.  The
@@ -156,19 +82,20 @@
     #  worker should contact to enter the p2p network.
     # ------------------------------------------------------------------
-    $authority define hubzero:worker/1 peers {plist} {
+    define peers {plist} {
         global peers
         set peers(all) $plist
-        after idle worker_peers_update
+        after idle {peer-network goto measure}
 
         # now that we've gotten the peers, we're done with the authority
-        authority_disconnect
-    }
-
-    # ------------------------------------------------------------------
-    #  DIRECTIVE: identity
+        authority-connection goto idle
+        return ""
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: identity
     #  Used for debugging, so that each client can identify itself by
     #  name to this worker.
     # ------------------------------------------------------------------
-    $authority define hubzero:worker/1 identity {name} {
+    define identity {name} {
         variable cid
         variable handler
@@ -176,125 +103,381 @@
         return ""
     }
-
-    $authority send "protocol hubzero:worker/1"
-    return $authority
-}
-
-proc authority_disconnect {} {
-    global authority
-    if {"" != $authority} {
-        catch {itcl::delete object $authority}
-        set authority ""
-    }
-}
-
-# ======================================================================
-#  USEFUL ROUTINES
-# ======================================================================
-# ----------------------------------------------------------------------
-#  COMMAND:  worker_register
-#
-#  Invoked when this worker first starts up and periodically thereafter
-#  to register the worker with the central authority, and to request
-#  a list of peers that this peer can talk to.
-# ----------------------------------------------------------------------
-proc worker_register {} {
-    global options server
+}
+
+# ======================================================================
+#  PROTOCOL: hubzero:worker<-foreman/1
+#
+#  The foreman initiates communication with a worker, and sends the
+#  various messages supported below.
+# ======================================================================
+p2p::protocol::register hubzero:worker<-foreman/1 {
+
+    # ------------------------------------------------------------------
+    #  INCOMING: solicit
+    #  Foremen send this message to solicit bids for a simulation job.
+    # ------------------------------------------------------------------
+    define solicit {args} {
+log debug "SOLICIT from foreman: $args"
+        variable cid
+        log debug "solicitation request from foreman: $args"
+        eval Solicitation ::#auto $args -connection $cid
+        return ""
+    }
+}
+
+# ======================================================================
+#  PROTOCOL: hubzero:workers<-workerc/1
+#
+#  Workers initiate connections with other workers as peers.  The
+#  following messages are sent by worker clients to the worker server.
+# ======================================================================
+p2p::protocol::register hubzero:workers<-workerc/1 {
+    # ------------------------------------------------------------------
+    #  INCOMING: identity
+    #  Used for debugging, so that each client can identify itself by
+    #  name to this worker.
+    # ------------------------------------------------------------------
+    define identity {name} {
+        variable cid
+        variable handler
+        $handler connectionName $cid $name
+        return ""
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: ping
+    #  If another worker sends "ping", they are trying to measure the
+    #  speed of the connection.  Send back a "pong" message.  If this
+    #  worker is close by, the other worker will stay connected to it.
+    # ------------------------------------------------------------------
+    define ping {} {
+        return "pong"
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: solicit -job info -path hosts -token xxx
+    #  Workers send this message on to their peers to solicit bids
+    #  for a simulation job.
+    # ------------------------------------------------------------------
+    define solicit {args} {
+log debug "SOLICIT from peer: $args"
+        variable cid
+        log debug "solicitation request from peer: $args"
+        eval Solicitation ::#auto $args -connection $cid
+        return ""
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: proffer <token> <details>
+    #  Workers send this message back after a "solicit" request with
+    #  details about what they can offer in terms of CPU power.  When
+    #  a worker has received all replies from its peers, it sends back
+    #  its own proffer message to the client that started the
+    #  solicitation.
+    # ------------------------------------------------------------------
+    define proffer {token details} {
+log debug "PROFFER from peer: $token $details"
+        Solicitation::proffer $token $details
+        return ""
+    }
+}
+
+# ======================================================================
+#  PROTOCOL: hubzero:workerc<-workers/1
+#
+#  The following messages are received by a worker client in response
+#  to the requests that they send to a worker server.
+# ======================================================================
+p2p::protocol::register hubzero:workerc<-workers/1 {
+    # ------------------------------------------------------------------
+    #  INCOMING: identity
+    #  Used for debugging, so that each client can identify itself by
+    #  name to this worker.
+    # ------------------------------------------------------------------
+    define identity {name} {
+        variable cid
+        variable handler
+        $handler connectionName $cid $name
+        return ""
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: pong
+    #  When forming the peer-to-peer network, workers send ping/pong
+    #  messages to one another to measure the latency.
+    # ------------------------------------------------------------------
+    define pong {} {
+        global peers
+        variable handler
+        set now [clock clicks -milliseconds]
+        set delay [expr {$now - $peers(ping-$handler-start)}]
+        set peers(ping-$handler-latency) $delay
+        if {[incr peers(responses)] >= [llength $peers(testing)]} {
+            after cancel {peer-network goto finalize}
+            after idle {peer-network goto finalize}
+        }
+        return ""
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: solicit -job info -path hosts -token xxx
+    #  Workers send this message on to their peers to solicit bids
+    #  for a simulation job.
+    # ------------------------------------------------------------------
+    define solicit {args} {
+log debug "SOLICIT from peer: $args"
+        variable cid
+        log debug "solicitation request from peer: $args"
+        eval Solicitation ::#auto $args -connection $cid
+        return ""
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: proffer <token> <details>
+    #  Workers send this message back after a "solicit" request with
+    #  details about what they can offer in terms of CPU power.  When
+    #  a worker has received all replies from its peers, it sends back
+    #  its own proffer message to the client that started the
+    #  solicitation.
+    # ------------------------------------------------------------------
+    define proffer {token details} {
+log debug "PROFFER from peer: $token $details"
+        Solicitation::proffer $token $details
+        return ""
+    }
+}
+
+# ----------------------------------------------------------------------
+#  COMMAND:  broadcast_to_peers <message> ?<avoidList>?
+#
+#  Used to broadcast a message out to all peers.  The <message> must
+#  be one of the commands defined in the worker protocol.  If there
+#  are no peer connections yet, this command does nothing.  If any
+#  peer appears on the <avoidList>, it is skipped.  Returns the
+#  number of messages sent out, so this peer knows how many replies
+#  to wait for.
+# ----------------------------------------------------------------------
+proc broadcast_to_peers {message {avoidList ""}} {
+    global server peers
+
+    #
+    # Build a list of all peer addresses, so we know who we're
+    # going to send this message out to.
+    #
+    set recipients ""
+    foreach key [array names peers current-*] {
+        set addr [$peers($key) address]
+        if {[lsearch $avoidList $addr] < 0} {
+            lappend recipients $addr
+        }
+    }
+    foreach cid [$server connections hubzero:workers<-workerc/1] {
+        set addr [lindex [$server connectionName $cid] 0]  ;# x.x.x.x (sockN)
+        if {[llength $avoidList] == 0 || [lsearch $avoidList $addr] < 0} {
+            lappend recipients $addr
+        }
+    }
+
+    #
+    # If the message has any @RECIPIENTS fields, replace them
+    # with the list of recipients
+    #
+    regsub -all @RECIPIENTS $message $recipients message
+
+    #
+    # Send the message out to all peers.  Keep a count and double-check
+    # it against the list of recipients generated above.
+    #
+    set nmesgs 0
+
+    # send off to other workers that this one has connected to
+    foreach key [array names peers current-*] {
+        set addr [$peers($key) address]
+        if {[lsearch $avoidList $addr] < 0} {
+            if {[catch {$peers($key) send $message} err] == 0} {
+                incr nmesgs
+            } else {
+                log error "ERROR: broadcast failed to [$peers($key) address]: $result\n  (message was \"$message\")"
+            }
+        }
+    }
+
+    # send off to other workers that connected to this one
+    foreach cid [$server connections hubzero:workers<-workerc/1] {
+        set addr [lindex [$server connectionName $cid] 0]  ;# x.x.x.x (sockN)
+        if {[llength $avoidList] == 0 || [lsearch $avoidList $addr] < 0} {
+            if {[catch {puts $cid $message} result] == 0} {
+                incr nmesgs
+            } else {
+                log error "ERROR: broadcast failed for $cid: $result"
+                log error "  (message was $message)"
+            }
+        }
+    }
+
+    # did we send the right number of messages?
+    if {[llength $recipients] != $nmesgs} {
+        log error "ERROR: sent only $nmesgs messages to peers {$recipients}"
+    }
+
+    return $nmesgs
+}
+
+# ----------------------------------------------------------------------
+#  COMMAND:  worker_got_protocol
+#
+#  Invoked whenever a peer sends their protocol message to this
+#  worker.  Sends the same protocol name back, so the other worker
+#  understands what protocol we're speaking.
+# ----------------------------------------------------------------------
+proc worker_got_protocol {cid protocol} {
+    switch -glob -- $protocol {
+        *<-worker* {
+            puts $cid [list protocol hubzero:workerc<-workers/1]
+        }
+        *<-foreman* {
+            puts $cid [list protocol hubzero:foreman<-worker/1]
+        }
+        DEF* {
+            # do nothing
+        }
+        default {
+            error "don't recognize protocol \"$protocol\""
+        }
+    }
+}
+
+# ----------------------------------------------------------------------
+#  COMMAND:  worker_got_dropped <address>
+#
+#  Invoked whenever an inbound connection to this worker drops.
+#  At some point we should try updating our connections to other
+#  peers, to replace this missing connection.
+# ----------------------------------------------------------------------
+proc worker_got_dropped {addr} {
+    global peers
+    if {[info exists peers(current-$addr)]} {
+        unset peers(current-$addr)
+        after cancel {peer-network goto measure}
+        after 5000 {peer-network goto measure}
+log debug "peer dropped: $addr"
+    }
+}
+
+# ======================================================================
+#  Connect to one of the authorities to get a list of peers.  Then,
+#  sit in the event loop and process events.
+# ======================================================================
+p2p::wonks::init
+
+set server [p2p::server -port 9101? \
+        -protocols {hubzero:workers<-workerc hubzero:worker<-foreman} \
+        -servername worker \
+        -onprotocol worker_got_protocol \
+        -ondisconnect worker_got_dropped]
+
+# ----------------------------------------------------------------------
+#  AUTHORITY CONNECTION
+#
+#  This is the state machine representing the connection to the
+#  authority server.  We start in the "idle" state.  Whenever we
+#  try to move to "connected", we'll open a connection to an authority
+#  and request updated information on workers.  If that connection
+#  fails, we'll get kicked back to "idle" and we'll try again later.
+# ----------------------------------------------------------------------
+StateMachine authority-connection
+authority-connection statedata cnx ""
+
+# sit here whenever we don't have a connection
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+authority-connection state idle -onenter {
+    # have an open connection? then close it
+    if {"" != [statedata cnx]} {
+        catch {itcl::delete object [statedata cnx]}
+        statedata cnx ""
+    }
+    # try to connect again later
+    set delay [p2p::options get time_between_authority_checks]
+    after $delay {authority-connection goto connected}
+} -onleave {
+    after cancel {authority-connection goto connected}
+}
+
+# sit here after we're connected and waiting for a response
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+authority-connection state connected
+
+# when moving to the connected state, make a connection
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+authority-connection transition idle->connected -onchange {
+    global server
 
     # connect to the authority and request a list of peers
-    if {[catch {
-        set client [authority_connect]
-        $client send "listening [$server port]"
-        $client send "peers"
-    } result]} {
-        log system "ERROR: $result"
-    }
-
-    # register again at regular intervals in case the authority
-    # gets restarted in between.
-    after $options(time_between_registers) worker_register
-}
-
-# ----------------------------------------------------------------------
-#  COMMAND:  worker_load_check
-#
-#  Invoked when this worker first starts up and periodically thereafter
-#  to compute the system load available to the worker.  If the system
-#  load has changed significantly, then the "perftest" program is
-#  executed to get a measure of performance available.  This program
-#  returns a number of "wonks" which we report to peers as our available
-#  performance.
-# ----------------------------------------------------------------------
-proc worker_load_check {} {
-    global options sysload dir
-
-    # see if the load has changed significantly
-    set changed 0
-    if {$sysload(lastLoad) < 0} {
-        set changed 1
+    statedata cnx ""
+    foreach addr [randomize [p2p::options get authority_hosts]] {
+        if {[catch {p2p::client -address $addr \
+            -sendprotocol hubzero:authority<-worker/1 \
+            -receiveprotocol hubzero:worker<-authority/1} result] == 0} {
+            statedata cnx $result
+            break
+        }
+    }
+
+    if {"" != [statedata cnx]} {
+        [statedata cnx] send "listening [$server port]"
+        [statedata cnx] send "peers"
     } else {
-        set load [Rappture::sysinfo load5]
-        if {$load < 0.9*$sysload(lastLoad) || $load > 1.1*$sysload(lastLoad)} {
-            set changed 1
-        }
-    }
-
-    if {$changed} {
-        set sysload(lastLoad) [Rappture::sysinfo load5]
-puts "LOAD CHANGED: $sysload(lastLoad) [Rappture::sysinfo freeram freeswap]"
-        set sysload(measured) [clock seconds]
-
-        # Run the program, but don't use exec, since it blocks.
-        # Instead, run it in the background and harvest its output later.
-        set sysload(test) [open "| [file join $dir perftest]" r]
-        fileevent $sysload(test) readable worker_load_results
-    }
-
-    # monitor the system load at regular intervals
-    after $options(time_between_load_checks) worker_load_check
-}
-
-# ----------------------------------------------------------------------
-#  COMMAND:  worker_load_results
-#
-#  Invoked automatically when the "perftest" run finishes.  Reads the
-#  number of wonks reported on standard output and reports them to
-#  other peers.
-# ----------------------------------------------------------------------
-proc worker_load_results {} {
-    global sysload
-
-    if {[catch {read $sysload(test)} msg] == 0} {
-        if {[regexp {[0-9]+} $msg match]} {
-puts "WONKS: $match"
-            set sysload(wonks) $match
-        } else {
-            log system "ERROR: performance test failed: $msg"
-        }
-    }
-    catch {close $sysload(test)}
-    set sysload(test) ""
-}
-
-# ----------------------------------------------------------------------
-#  COMMAND:  worker_peers_update
-#
-#  Invoked when this worker has received a list of peers, and from
-#  time to time thereafter, to establish connections with other peers
-#  in the network.  This worker picks a number of peers randomly from
-#  the list of all available peers, and then pings each of them.
-#  Timing results from the pings are stored away, and when all pings
-#  have completed, this worker picks the best few connections and
-#  keeps those.
-# ----------------------------------------------------------------------
-proc worker_peers_update {} {
-    global options peers myaddress
-    worker_peers_cleanup
+        error "can't connect to any authority\nAUTHORITY LIST: [p2p::options get authority_hosts]"
+    }
+}
+
+# ----------------------------------------------------------------------
+#  PEER-TO-PEER CONNECTIONS
+#
+#  This is the state machine representing the network of worker peers.
+#  We start in the "idle" state.  From time to time we move to the
+#  "measure" state and attempt to establish connections with a set
+#  peers.  We then wait for ping/pong responses and move to "finalize".
+#  When we have all responses, we move to "finalize" and finalize all
+#  connections, and then move back to "idle".
+# ----------------------------------------------------------------------
+StateMachine peer-network
+peer-network statedata cnx ""
+
+# sit here when the peer network is okay
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+peer-network state idle -onenter {
+    # try to connect again later
+    set delay [p2p::options get time_between_network_rebuilds]
+    after $delay {peer-network goto measure}
+} -onleave {
+    after cancel {peer-network goto measure}
+}
+
+# sit here when we need to rebuild the peer-to-peer network
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+peer-network state measure
+
+# when moving to the start state, make connections to a bunch of peers
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+peer-network transition idle->measure -onchange {
+    global server myaddress peers
+
+    #
+    # Get a list of workers already connected to this one
+    # with an inbound connection to the server.
+    #
+    foreach cid [$server connections hubzero:workers<-workerc/1] {
+        set addr [lindex [$server connectionName $cid] 0]
+        set inbound($addr) $cid
+    }
 
     #
     # Pick a random group of peers and try to connect to them.
-    # Start with the existing peers, and then add a random
-    # bunch of others.
-    #
+    # Start with the existing peers to see if their connection
+    # is still favorable, and then add a random bunch of others.
+    #
+    set peers(testing) ""
+    set peers(responses) 0
     foreach key [array names peers current-*] {
         set peer $peers($key)
@@ -307,46 +490,28 @@
     # final number we want to talk to.
     #
-    set ntest [expr {10 * $options(max_peer_connections)}]
-    foreach addr [randomize $peers(all) $ntest] {
-        if {$addr == $myaddress} {
+    set maxpeers [p2p::options get max_peer_connections]
+    foreach addr [randomize $peers(all) [expr {10*$maxpeers}]] {
+        #
+        # Avoid connecting to ourself, or to peers that we're
+        # already connected to (either as inbound connections
+        # to the server or as outbound connections to others).
+        #
+        if {$addr == $myaddress
+              || [info exists peers(current-$addr)]
+              || [info exists inbound($addr)]} {
             continue
         }
-        if {![info exists peers(current-$addr)]
-              && [catch {Client ::#auto $addr} peer] == 0} {
-            # open a new connection to this address
-            lappend peers(testing) $peer
-            $peer protocol hubzero:peer/1
-
-            $peer define hubzero:peer/1 exception {message} {
-                log system "ERROR: $message"
-            }
-
-            $peer define hubzero:peer/1 identity {name} {
-                variable cid
-                variable handler
-                $handler connectionName $cid $name
-                return ""
-            }
-
-            # when we get a "pong" back, store the latency
-            $peer define hubzero:peer/1 pong {} {
-                global peers
-                variable handler
-puts "  pong from $handler"
-                set now [clock clicks -milliseconds]
-                set delay [expr {$now - $peers(ping-$handler-start)}]
-                set peers(ping-$handler-latency) $delay
-                incr peers(responses)
-                worker_peers_finalize
-                return ""
-            }
-
-            # start the ping/pong session with the peer
-            $peer send "protocol hubzero:peer/1"
-
-            # send tell this peer our name (for debugging)
-            if {[log channel debug]} {
-                $peer send "identity $myaddress"
-            }
+
+        if {[catch {p2p::client -address $addr \
+            -sendprotocol hubzero:workers<-workerc/1 \
+            -receiveprotocol hubzero:workerc<-workers/1} cnx]} {
+            continue
+        }
+        $cnx send "identity $myaddress"
+        lappend peers(testing) $cnx
+
+        # have enough connections to test?
+        if {[llength $peers(testing)] >= 2*$maxpeers} {
+            break
         }
     }
@@ -355,88 +520,73 @@
     # Now, loop through all peers and send a "ping" message.
     #
-    foreach peer $peers(testing) {
-puts "pinging $peer = [$peer address]..."
+    foreach cnx $peers(testing) {
         # mark the time and send a ping to this peer
-        set peers(ping-$peer-start) [clock clicks -milliseconds]
-        $peer send "ping"
+        set peers(ping-$cnx-start) [clock clicks -milliseconds]
+        $cnx send "ping"
     }
 
     # if this test takes too long, just give up
-    after 10000 worker_peers_finalize -force
-}
-
-# ----------------------------------------------------------------------
-#  COMMAND:  worker_peers_finalize ?-force?
-#
-#  Called after worker_peers_update has finished its business to
-#  clean up all of the connections that were open for testing.
-# ----------------------------------------------------------------------
-proc worker_peers_finalize {{option -check}} {
-    global peers options
-    if {$option == "-force" || $peers(responses) == [llength $peers(testing)]} {
-        # build a list:  {peer latency} {peer latency} ...
-        set plist ""
-        foreach obj $peers(testing) {
-            if {[info exists peers(ping-$obj-latency)]} {
-                lappend plist [list $obj $peers(ping-$obj-latency)]
-            }
-        }
-puts "-------------------\nFINALIZING $::myaddress"
-puts "PINGS: $plist"
-
-        # sort the list and extract the top peers
-        set plist [lsort -increasing -index 1 $plist]
-        set plist [lrange $plist 0 [expr {$options(max_peer_connections)-1}]]
-puts "TOP: $plist"
-
-        set current [array names peers current-*]
-puts "  current: $current"
-
-        # transfer the top peers to the "current" list
-        set all ""
-        foreach rec $plist {
-            set peer [lindex $rec 0]
-            set addr [$peer address]
-            set peers(current-$addr) $peer
-            lappend all $addr
-
-            # if it is already on the "current" list, then keep it
-            set i [lsearch $current current-$addr]
-            if {$i >= 0} {
-                set current [lreplace $current $i $i]
-            }
-            set i [lsearch $peers(testing) $peer]
-            if {$i >= 0} {
-                set peers(testing) [lreplace $peers(testing) $i $i]
-            }
-        }
-        log system "connected to peers: $all"
-puts "  new current: [array names peers current-*]"
-puts "  final: $all"
-puts "  leftover: $current $peers(testing)"
-
-        # get rid of old peers that we no longer want to talk to
-        foreach leftover $current {
-            itcl::delete object $peers($leftover)
-            unset peers($leftover)
-puts "  cleaned up $leftover (was on current list)"
-        }
-
-        # clean up after this test
-        worker_peers_cleanup
-    }
-}
-
-# ----------------------------------------------------------------------
-#  COMMAND:  worker_peers_cleanup
-#
-#  Called after worker_peers_update has finished its business to
-#  clean up all of the connections that were open for testing.
-# ----------------------------------------------------------------------
-proc worker_peers_cleanup {} {
+    after 10000 {peer-network goto finalize}
+}
+
+# sit here when we need to rebuild the peer-to-peer network
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+peer-network state finalize -onenter {
+    after cancel {peer-network goto finalize}
+
+    # everything is finalized now, so go to idle
+    after idle {peer-network goto idle}
+}
+
+# when moving to the finalize state, decide on the network of peers
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+peer-network transition measure->finalize -onchange {
     global peers
+
+    set maxpeers [p2p::options get max_peer_connections]
+
+    # build a list:  {peer latency} {peer latency} ...
+    set plist ""
+    foreach obj $peers(testing) {
+        if {[info exists peers(ping-$obj-latency)]} {
+            lappend plist [list $obj $peers(ping-$obj-latency)]
+        }
+    }
+
+    # sort the list and extract the top peers
+    set plist [lsort -increasing -index 1 $plist]
+    set plist [lrange $plist 0 [expr {$maxpeers-1}]]
+
+    set currentpeers [array names peers current-*]
+
+    # transfer the top peers to the "current" list
+    set all ""
+    foreach rec $plist {
+        set peer [lindex $rec 0]
+        set addr [$peer address]
+        set peers(current-$addr) $peer
+        lappend all $addr
+
+        # if it is already on the "current" list, then keep it
+        set i [lsearch $currentpeers current-$addr]
+        if {$i >= 0} {
+            set currentpeers [lreplace $currentpeers $i $i]
+        }
+        set i [lsearch $peers(testing) $peer]
+        if {$i >= 0} {
+            set peers(testing) [lreplace $peers(testing) $i $i]
+        }
+    }
+    log system "connected to peers: $all"
+
+    # get rid of old peers that we no longer want to talk to
+    foreach leftover $currentpeers {
+        itcl::delete object $peers($leftover)
+        unset peers($leftover)
+    }
+
+    # clean up after this test
     foreach obj $peers(testing) {
         catch {itcl::delete object $obj}
-puts "  cleaned up $obj (was on testing list)"
     }
     set peers(testing) ""
@@ -446,26 +596,110 @@
     }
     set peers(responses) 0
-
-    after cancel worker_peers_finalize -force
-}
-
-# ----------------------------------------------------------------------
-#  COMMAND:  worker_peers_protocol
-#
-#  Invoked whenever a peer sends their protocol message to this
-#  worker.  Sends the same protocol name back, so the other worker
-#  understands what protocol we're speaking.
-# ----------------------------------------------------------------------
-proc worker_peers_protocol {cid protocol} {
-    if {"DEFAULT" != $protocol} {
-        puts $cid [list protocol $protocol]
-    }
-}
-
-# ======================================================================
-#  Connect to one of the authorities to get a list of peers.  Then,
-#  sit in the event loop and process events.
-# ======================================================================
-after idle worker_register
+}
+
+# ----------------------------------------------------------------------
+#  JOB SOLICITATION
+#
+#  Each worker can receive a "solicit" request, asking for information
+#  about performance and price of peers available to work.  Each worker
+#  sends the request on to its peers, then gathers the information
+#  and sends it back to the client or to the peer requesting the
+#  information.  There can be multiple requests going on at once,
+#  and each may have different job types and return different info,
+#  so the class below helps to watch over each request until it has
+#  completed.
+# ----------------------------------------------------------------------
+itcl::class Solicitation {
+    public variable connection ""
+    public variable job ""
+    public variable path ""
+    public variable avoid ""
+    public variable token ""
+
+    private variable _serial ""
+    private variable _response ""
+    private variable _waitfor 0
+    private variable _timeout ""
+
+    constructor {args} {
+        eval configure $args
+
+        global myaddress
+        lappend path $myaddress
+        lappend avoid $myaddress
+        set _serial [incr counter]
+        set all($_serial) $this
+
+        set delay "idle"  ;# finalize after waiting for responses
+        set ttl [p2p::options get peer_time_to_live]
+        if {[llength $path] < $ttl} {
+            set mesg [list solicit -job $job -path $path -avoid "$avoid @RECIPIENTS" -token $_serial]
+            set _waitfor [broadcast_to_peers $mesg $avoid]
+log debug "WAIT FOR: $_waitfor"
+
+            if {$_waitfor > 0} {
+                # add a delay proportional to ttl + time for wonks measurement
+                set delay [expr {($ttl-[llength $path]-1)*1000 + 3000}]
+            }
+        }
+log debug "TIMEOUT: $delay"
+        set _timeout [after $delay [itcl::code $this finalize]]
+    }
+    destructor {
+        after cancel $_timeout
+        catch {unset all($_serial)}
+    }
+
+    # this adds the info from each proffer to this solicitation
+    method response {details} {
+        set addr [lindex $details 0]
+        append _response $details "\n"
+        if {[incr _waitfor -1] <= 0} {
+log debug "ALL RESPONSES"
+            finalize
+        }
+    }
+
+    # called to finalize when all peers have responded back
+    method finalize {} {
+        global myaddress
+
+        # filter out duplicate info from clients
+        set block ""
+        foreach line [split $_response \n] {
+            set addr [lindex $line 0]
+            if {"" != $addr && ![info exists response($addr)]} {
+                append block $line "\n"
+                set response($addr) 1
+            }
+        }
+        # add details about this client to the message
+        append block "$myaddress -job $job -cost 1 -wonks [p2p::wonks::current]"
+log debug "FINALIZE {$block}"
+
+        # send the composite results back to the caller
+        set cmd [list proffer $token $block]
+        if {[catch {puts $connection $cmd} err]} {
+            log error "ERROR while sending back proffer: $err"
+        }
+        itcl::delete object $this
+    }
+
+    proc proffer {serial details} {
+        if {[info exists all($serial)]} {
+            $all($serial) response $details
+        }
+    }
+
+    common counter 0 ;# generates serial nums for objects
+    common all       ;# maps serial num => solicitation object
+}
+
+# ----------------------------------------------------------------------
 log system "starting..."
+
+after idle {
+    authority-connection goto connected
+}
+
 vwait main-loop
