Changeset 1273

Timestamp:

Feb 5, 2009, 6:17:23 AM (16 years ago)

Author:

mmc

Message:

Major reorganization of p2p code, and support for solicit/proffer
messages.

Location:

trunk/p2p

Files:

• Makefile (modified) (1 diff)
• authority.tcl (modified) (3 diffs)
• client.tcl (modified) (1 diff)
• foreman.tcl (added)
• handler.tcl (modified) (7 diffs)
• options.tcl (added)
• perftest.c (modified) (1 diff)
• protocols.tcl (added)
• random.tcl (modified) (1 diff)
• server.tcl (modified) (5 diffs)
• statemachine.tcl (added)
• tclIndex (modified) (2 diffs)
• test.tcl (modified) (9 diffs)
• wonks.tcl (added)
• worker.tcl (modified) (7 diffs)

trunk/p2p/Makefile

@@ -4 +4 @@
 all:
         $(CC) $(CFLAGS) perftest.c -o perftest
-        ./mkindex handler.tcl server.tcl client.tcl log.tcl random.tcl
+        ./mkindex handler.tcl server.tcl client.tcl log.tcl random.tcl \
+                options.tcl protocols.tcl statemachine.tcl wonks.tcl \
+                foreman.tcl

trunk/p2p/authority.tcl

@@ -31 +31 @@
 
 # register with the central authority at this frequency
-set worker_options(time_between_registers) 60000
+set worker_options(time_between_authority_checks) 60000
 
 # workers should try to connect with this many other peers
@@ -37 +37 @@
 
 # ======================================================================
+#  PROTOCOL: hubzero:authority<-worker/1
+#
+#  The worker initiates communication with the authority by sending
+#  these messages to the authority to request information about
+#  other peers.
+# ======================================================================
+p2p::protocol::register hubzero:authority<-worker/1 {
+
+    # ------------------------------------------------------------------
+    #  INCOMING: listening <port>
+    #  Workers use this to tell the authority what port they're
+    #  listening on.  Other workers should connect to this port.
+    # ------------------------------------------------------------------
+    define listening {port} {
+        global client2address workers
+        variable handler
+        variable cid
+
+        # register this address on the list of workers
+        set addr "$client2address($cid):$port"
+        set workers($addr) "--"
+
+        # name this connection for easier debugging
+        $handler connectionName $cid $addr
+
+        return ""
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: peers
+    # ------------------------------------------------------------------
+    define peers {} {
+        global workers
+        return [list peers [array names workers]]
+    }
+}
+
+# ======================================================================
+#  PROTOCOL: hubzero:authority<-foreman/1
+#
+#  The foreman initiates communication with the authority by sending
+#  these messages to request information about workers, to estimate
+#  requirements for a job, to put funds in escrow, and to finalize
+#  the transaction.
+# ======================================================================
+p2p::protocol::register hubzero:authority<-foreman/1 {
+    # ------------------------------------------------------------------
+    #  INCOMING: workers
+    #  Foremen use this to request the list of known workers, and then
+    #  pick one worker to act as their connection to the P2P network.
+    # ------------------------------------------------------------------
+    define workers {} {
+        global workers
+        return [list workers [array names workers]]
+    }
+}
+
+# ======================================================================
 # set up a server at the first open port above 9001
-set server [Server ::#auto 9001? \
+set server [p2p::server \
+    -port 9001? \
+    -protocols {hubzero:authority<-worker hubzero:authority<-foreman} \
     -servername authority \
     -onconnect authority_client_address \
@@ -66 +126 @@
 proc authority_client_protocol {cid protocol} {
     global worker_options client2address
-    if {"DEFAULT" != $protocol} {
-        puts $cid [list protocol $protocol]
-        puts $cid "options [array get worker_options] ip $client2address($cid)"
+    switch -glob -- $protocol {
+        *<-worker* {
+            puts $cid [list protocol hubzero:worker<-authority/1]
+            puts $cid "options [array get worker_options] ip $client2address($cid)"
+        }
+        *<-foreman* {
+            puts $cid [list protocol hubzero:foreman<-authority/1]
+        }
+        DEF* {
+            # do nothing
+        }
+        default {
+            error "don't recognize protocol \"$protocol\""
+        }
     }
 }
 
-# ======================================================================
-#  PROTOCOL: hub:worker/1
-# ======================================================================
-$server protocol hubzero:worker/1
-
-# ----------------------------------------------------------------------
-#  DIRECTIVE: exception <message>
-# ----------------------------------------------------------------------
-$server define hubzero:worker/1 exception {args} {
-    log system "error from worker: $args"
-}
-
-# ----------------------------------------------------------------------
-#  DIRECTIVE: listening <port>
-#  Workers use this to tell the authority what port they're listening
-#  on.  Other workers should connect to this port.
-# ----------------------------------------------------------------------
-$server define hubzero:worker/1 listening {port} {
-    global client2address workers
-    variable handler
-    variable cid
-
-    # register this address on the list of workers
-    set addr "$client2address($cid):$port"
-    set workers($addr) "--"
-
-    # name this connection for easier debugging
-    $handler connectionName $cid $addr
-
-    return ""
-}
-
-# ----------------------------------------------------------------------
-#  DIRECTIVE: peers
-# ----------------------------------------------------------------------
-$server define hubzero:worker/1 peers {} {
-    global workers
-    return [list peers [array names workers]]
-}
-
 vwait main-loop

trunk/p2p/client.tcl

@@ -11 +11 @@
 # ======================================================================
 package require Itcl
+
+namespace eval p2p { # forward declaration }
+
+# ======================================================================
+#  USAGE: p2p::client ?-option value -option value ...?
+#
+#  Used to create a client connection to a peer-to-peer server.
+#  Recognizes the following options:
+#    -address ........... connect to server at this host:port
+#    -sendprotocol ...... tell server we're speaking this protocol
+#    -receiveprotocol ... server replies back with these commands
+# ======================================================================
+proc p2p::client {args} {
+    array set options {
+        -address ?
+        -sendprotocol ""
+        -receiveprotocol ""
+    }
+    foreach {key val} $args {
+        if {![info exists options($key)]} {
+            error "bad option \"$key\": should be [join [lsort [array names options]] {, }]"
+        }
+        set options($key) $val
+    }
+
+    # create the client
+    set client [eval Client ::#auto $options(-address)]
+
+    # install the protocol for incoming commands
+    p2p::protocol::init $client $options(-receiveprotocol)
+
+    # tell the server what protocol we'll be speaking
+    $client send [list protocol $options(-sendprotocol)]
+
+    return $client
+}
+
+# ======================================================================
+#  CLASS: Client
+# ======================================================================
 
 itcl::class Client {

trunk/p2p/handler.tcl

@@ -34 +34 @@
             return ""
         }
+        define DEFAULT exception {message} {
+            log error "ERROR: $message"
+        }
 
         eval configure $args
@@ -49 +52 @@
     public method protocol {name}
     public method define {protocol name arglist body}
+    public method connections {{protocol *}}
     public method connectionName {cid {name ""}}
     public method connectionSpeaks {cid protocol}
@@ -54 +58 @@
     protected method handle {cid}
     protected method finalize {protocol}
+    protected method dropped {cid}
     protected method handlerType {}
 }
@@ -101 +106 @@
 
 # ----------------------------------------------------------------------
+#  USAGE: connections ?<protocol>?
+#
+#  Returns a list of file handles for current connections that match
+#  the glob-style <protocol> pattern.  If no pattern is given, then
+#  it returns all connections.  Each handle can be passed to
+#  connectionName and connectionSpeaks to get more information.
+# ----------------------------------------------------------------------
+itcl::body Handler::connections {{pattern *}} {
+    set rlist ""
+    foreach cid [array names _protocol] {
+        if {[string match $pattern $_protocol($cid)]} {
+            lappend rlist $cid
+        }
+    }
+    return $rlist
+}
+
+# ----------------------------------------------------------------------
 #  USAGE: connectionName <sockId> ?<name>?
 #
@@ -144 +167 @@
 itcl::body Handler::handle {cid} {
     if {[gets $cid request] < 0} {
-        log system "dropped: [connectionName $cid]"
-        # connection has connection -- forget this entity
-        catch {close $cid}
-        catch {unset _buffer($cid)}
-        catch {unset _protocol($cid)}
-        catch {unset _cname($cid)}
+        dropped $cid
     } elseif {[info exists _protocol($cid)]} {
         # complete command? then process it below...
@@ -171 +189 @@
             if {[catch {$_parser($protocol) eval $request} result] == 0} {
                 if {[string length $result] > 0} {
-                    puts $cid $result
+                    catch {puts $cid $result}
                     append mesg "ok: $result"
                 }
             } else {
-                puts $cid [list exception $result]
+                catch {puts $cid [list exception $result]}
                 append mesg "exception: $result"
             }
@@ -204 +222 @@
 
 # ----------------------------------------------------------------------
+#  USAGE: dropped <cid>
+#
+#  Invoked automatically whenever a client connection drops, to
+#  log the event and remove all trace of the client.  Derived classes
+#  can override this method to perform other functions too.
+# ----------------------------------------------------------------------
+itcl::body Handler::dropped {cid} {
+    log system "dropped: [connectionName $cid]"
+
+    # connection has connection -- forget this entity
+    catch {close $cid}
+    catch {unset _buffer($cid)}
+    catch {unset _protocol($cid)}
+    catch {unset _cname($cid)}
+}
+
+# ----------------------------------------------------------------------
 #  USAGE: handlerType
 #

trunk/p2p/perftest.c

@@ -91 +91 @@
              + tend.tv_usec - tstart.tv_usec;
 
-    printf("%ld\n", tval);
+    printf("rappture-wonks/v1 %ld\n", tval);
     exit(0);
 }

trunk/p2p/random.tcl

@@ -38 +38 @@
     return $rlist
 }
+
+# ----------------------------------------------------------------------
+#  USAGE: random <list>
+#
+#  Picks one element at random from the given <list>.
+# ----------------------------------------------------------------------
+proc random {entries} {
+    set nrand [expr {int(rand()*[llength $entries])}]
+    return [lindex $entries $nrand]
+}

trunk/p2p/server.tcl

@@ -11 +11 @@
 package require Itcl
 
+namespace eval p2p { # forward declaration }
+
+# ======================================================================
+#  USAGE: p2p::server ?-option value -option value ...?
+#
+#  Used to create a new peer-to-peer server object for this program.
+#  Recognizes the following options:
+#    -port ........ port number that the server listens on
+#    -protocols ... list of protocol root names that the server handles
+#               ... any other option supported by Server class
+# ======================================================================
+proc p2p::server {args} {
+    set port "?"
+    set protocols ""
+    set options ""
+    foreach {key val} $args {
+        switch -- $key {
+            -port { set port $val }
+            -protocols { set protocols $val }
+            default { lappend options $key $val }
+        }
+    }
+
+    if {[llength $protocols] == 0} {
+        error "server needs at least one value for -protocols"
+    }
+
+    # create the server
+    set server [eval Server ::#auto $port $options]
+
+    # install the protocols that this server recognizes
+    foreach name $protocols {
+        p2p::protocol::init $server $name
+    }
+    return $server
+}
+
+# ======================================================================
+#  CLASS: Server
+# ======================================================================
 itcl::class Server {
     inherit Handler
@@ -24 +64 @@
     # this code fragment gets invoked when client declares the protocol
     public variable onprotocol ""
+
+    # this code fragment gets invoked when client drops
+    public variable ondisconnect ""
 
     constructor {port args} {
@@ -59 +102 @@
 
     public method port {}
+    public method broadcast {args}
     public method connectionSpeaks {cid protocol}
 
+    protected method dropped {cid}
     protected method handlerType {}
 
@@ -80 +125 @@
 
 # ----------------------------------------------------------------------
+#  USAGE: broadcast ?-protocol <name>? ?-avoid <avoidList>? <message>
+#
+#  Sends a <message> to all clients connected to this server.  If a
+#  client address appears on the -avoid list, then that client is
+#  avoided.  If the -protocol is specified, then the message is sent
+#  only to clients who match the glob-style pattern for the protocol
+#  name.
+# ----------------------------------------------------------------------
+itcl::body Server::broadcast {args} {
+    set pattern "*"
+    set avoidList ""
+    set i 0
+    while {$i < [llength $args]} {
+        set option [lindex $args $i]
+        if {[string index $option 0] == "-"} {
+            switch -- $option {
+                -protocol {
+                    set pattern [lindex $args [expr {$i+1}]]
+                    incr i 2
+                }
+                -avoid {
+                    set avoidList [lindex $args [expr {$i+1}]]
+                    incr i 2
+                }
+                -- {
+                    incr i
+                    break
+                }
+                default {
+                    error "bad option \"$option\": should be -avoid, -protocol, or --"
+                }
+            }
+        } else {
+            break
+        }
+    }
+    if {$i != [llength $args]-1} {
+        error "wrong # args: should be \"broadcast ?-protocol pattern? ?-avoid clients? message\""
+    }
+    set message [lindex $args end]
+
+    set nmesgs 0
+    foreach cid [connections $pattern] {
+        set addr [lindex [connectionName $cid] 0]  ;# x.x.x.x (sockN)
+        if {[llength $avoidList] == 0 || [lsearch $avoidList $addr] < 0} {
+puts "  inbound => [connectionName $cid]"
+            if {[catch {puts $cid $message} result] == 0} {
+                incr nmesgs
+            } else {
+                log error "ERROR: broadcast failed for $cid: $result"
+                log error "  (message was $message)"
+            }
+        }
+    }
+    return $nmesgs
+}
+
+# ----------------------------------------------------------------------
 #  USAGE: accept <cid> <addr> <port>
 #
@@ -116 +219 @@
 
 # ----------------------------------------------------------------------
+#  USAGE: dropped <cid>
+#
+#  Invoked automatically whenever a client connection drops, to
+#  log the event and remove all trace of the client.  Invokes any
+#  command hook for this server to note the fact that the client
+#  has dropped.
+# ----------------------------------------------------------------------
+itcl::body Server::dropped {cid} {
+    # if there's a callback to handle the drop, execute it here
+    if {[string length $ondisconnect] > 0} {
+        uplevel #0 [list $ondisconnect [connectionName $cid]]
+    }
+
+    # call the base class method to clean up after the client
+    chain $cid
+}
+
+# ----------------------------------------------------------------------
 #  USAGE: handlerType
 #

trunk/p2p/tclIndex

@@ -10 +10 @@
 set auto_index(::Handler::protocol) [list source [file join $dir handler.tcl]]
 set auto_index(::Handler::define) [list source [file join $dir handler.tcl]]
+set auto_index(::Handler::connections) [list source [file join $dir handler.tcl]]
 set auto_index(::Handler::connectionName) [list source [file join $dir handler.tcl]]
 set auto_index(::Handler::connectionSpeaks) [list source [file join $dir handler.tcl]]
 set auto_index(::Handler::handle) [list source [file join $dir handler.tcl]]
 set auto_index(::Handler::finalize) [list source [file join $dir handler.tcl]]
+set auto_index(::Handler::dropped) [list source [file join $dir handler.tcl]]
 set auto_index(::Handler::handlerType) [list source [file join $dir handler.tcl]]
+set auto_index(::p2p::server) [list source [file join $dir server.tcl]]
 set auto_index(Server) [list source [file join $dir server.tcl]]
 set auto_index(::Server::port) [list source [file join $dir server.tcl]]
+set auto_index(::Server::broadcast) [list source [file join $dir server.tcl]]
 set auto_index(::Server::accept) [list source [file join $dir server.tcl]]
 set auto_index(::Server::connectionSpeaks) [list source [file join $dir server.tcl]]
+set auto_index(::Server::dropped) [list source [file join $dir server.tcl]]
 set auto_index(::Server::handlerType) [list source [file join $dir server.tcl]]
+set auto_index(::p2p::client) [list source [file join $dir client.tcl]]
 set auto_index(Client) [list source [file join $dir client.tcl]]
 set auto_index(::Client::send) [list source [file join $dir client.tcl]]
@@ -26 +32 @@
 set auto_index(log) [list source [file join $dir log.tcl]]
 set auto_index(randomize) [list source [file join $dir random.tcl]]
+set auto_index(random) [list source [file join $dir random.tcl]]
+set auto_index(::p2p::options) [list source [file join $dir options.tcl]]
+set auto_index(::p2p::protocol::register) [list source [file join $dir protocols.tcl]]
+set auto_index(::p2p::protocol::init) [list source [file join $dir protocols.tcl]]
+set auto_index(::p2p::protocol::define) [list source [file join $dir protocols.tcl]]
+set auto_index(StateMachine) [list source [file join $dir statemachine.tcl]]
+set auto_index(::StateMachine::state) [list source [file join $dir statemachine.tcl]]
+set auto_index(::StateMachine::transition) [list source [file join $dir statemachine.tcl]]
+set auto_index(::StateMachine::goto) [list source [file join $dir statemachine.tcl]]
+set auto_index(::StateMachine::statedata) [list source [file join $dir statemachine.tcl]]
+set auto_index(::p2p::wonks::init) [list source [file join $dir wonks.tcl]]
+set auto_index(::p2p::wonks::current) [list source [file join $dir wonks.tcl]]
+set auto_index(bgerror) [list source [file join $dir foreman.tcl]]
+set auto_index(::Rappture::foreman::bids) [list source [file join $dir foreman.tcl]]

trunk/p2p/test.tcl

@@ -23 +23 @@
 set nodes(all) ""
 set nodeRadius 15
+
+option add *highlightBackground [. cget -background]
+option add *client*background gray
+option add *client*highlightBackground gray
+option add *client*troughColor darkGray
 
 # ======================================================================
@@ -162 +167 @@
 #  Build the main interface
 # ======================================================================
+frame .client -borderwidth 8 -relief flat
+pack .client -side right -fill y
+button .client.getbids -text "Get Bids:" -command test_bids
+pack .client.getbids -side top -anchor w
+frame .client.cntls
+pack .client.cntls -side bottom -fill x
+button .client.cntls.run -text "Spend" -command test_spend
+pack .client.cntls.run -side left
+entry .client.cntls.points -width 8
+pack .client.cntls.points -side left
+label .client.cntls.pointsl -text "points"
+pack .client.cntls.pointsl -side left
+
+frame .client.bids
+pack .client.bids -side bottom -expand yes -fill both
+scrollbar .client.bids.ysbar -orient vertical -command {.client.bids.info yview}
+pack .client.bids.ysbar -side right -fill y
+listbox .client.bids.info -yscrollcommand {.client.bids.ysbar set}
+pack .client.bids.info -side left -expand yes -fill both
+
 frame .cntls
 pack .cntls -fill x
@@ -181 +206 @@
 }
 pack .cntls.layout -side left -padx 4 -pady 2
+
+entry .cntls.workers -width 5
+pack .cntls.workers -side right -padx {0 4} -pady 2
+.cntls.workers insert end "3"
+label .cntls.workersl -text "Workers:"
+pack .cntls.workersl -side right -pady 2
 
 frame .player
@@ -238 +269 @@
 
     # launch a series of workers
-    for {set i 0} {$i < 20} {incr i} {
+    for {set i 0} {$i < [.cntls.workers get]} {incr i} {
         lappend processes [exec tclsh worker.tcl &]
         after [expr {int(rand()*5000)}]
@@ -266 +297 @@
         set info [read $fid]
         close $fid
+
+        if {[regexp -- {foreman<-} $info]} {
+            # skip log file from foreman
+            continue
+        }
 
         # get the address for this host
@@ -322 +358 @@
         close $fid
 puts "\nscanning $fname"
+
+        if {[regexp -- {foreman<-} $info]} {
+            # skip log file from foreman
+            continue
+        }
 
         catch {unset started}
@@ -358 +399 @@
                     unset started(connect$cid-addr)
 
-                } elseif {[regexp {server message from ([a-zA-Z0-9\.]+:[0-9]+) \(([a-z0-9]+)\): +(.+) => (.*)} $mesg match addr cid cmd result]} {
+                } elseif {[regexp {(server|client) message from ([a-zA-Z0-9\.]+:[0-9]+) \(([a-z0-9]+)\): +(.+) => (.*)} $mesg match which addr cid cmd result]} {
                     if {![string match identity* $cmd]} {
                         append actions($tval) $mesg \n
@@ -434 +475 @@
             set from $nodes($fname)
             set addr $started($key)
+
+            if {![info exists nodes($addr-x)]} {
+                unset started(connect$cid-time)
+                unset started(connect$cid-addr)
+                continue
+            }
             set x0 $nodes($from-x)
             set y0 $nodes($from-y)
@@ -558 +605 @@
     pack .diagram.$view -expand yes -fill both
 }
+
+proc test_bids {} {
+    set info [Rappture::foreman::bids]
+    .client.bids.info delete 0 end
+    eval .client.bids.info insert end $info
+}

trunk/p2p/worker.tcl

@@ -21 +21 @@
 
 # handle log file for this worker
+log channel error on
+log channel system on
 log channel debug on
-log channel system on
-
-# set up a server at the first open port above 9101
-set server [Server ::#auto 9101? -servername worker \
-    -onprotocol worker_peers_protocol]
-
-# ======================================================================
-#  OPTIONS
-#  These options get the worker started, but new option settings
-#  some from the authority after this worker first connects.
-# ======================================================================
-set authority ""     ;# file handle for authority connection
+
+proc ::bgerror {err} { log error "ERROR: $err $::errorInfo" }
+
+
 set myaddress "?:?"  ;# address/port for this worker
 
 # set of connections for authority servers
-set options(authority_hosts) 127.0.0.1:9001
+p2p::options register authority_hosts 127.0.0.1:9001
 
 # register with the central authority at this frequency
-set options(time_between_registers) 10000
+p2p::options register time_between_authority_checks 60000
+
+# rebuild the peer-to-peer network at this frequency
+p2p::options register time_between_network_rebuilds 600000
 
 # this worker should try to connect with this many other peers
-set options(max_peer_connections) 4
-
-# number of seconds between each check of system load
-set options(time_between_load_checks) 60000
-
-# ======================================================================
-#  PEERS
-# ======================================================================
-# eventually set to the list of all peers in the network
-set peers(all) ""
-
-# list of peers that we're testing connections with
-set peers(testing) ""
-
-# ======================================================================
-#  SYSTEM LOAD
-# ======================================================================
-# Check system load every so often.  When the load changes
-# significantly, execute a "perftest" run to compute the current
-# number of wonks available to this worker.
-
-set sysload(lastLoad) -1
-set sysload(wonks) -1
-set sysload(measured) -1
-
-after idle worker_load_check
-
-# ======================================================================
-#  PROTOCOL: hubzero:peer/1
-#  This protocol gets used when another worker connects to this one.
-# ======================================================================
-$server protocol hubzero:peer/1
-
-$server define hubzero:peer/1 exception {message} {
-    log system "ERROR: $message"
-}
-
-# ----------------------------------------------------------------------
-#  DIRECTIVE: identity
-#  Used for debugging, so that each client can identify itself by
-#  name to this worker.
-# ----------------------------------------------------------------------
-$server define hubzero:peer/1 identity {name} {
-    variable cid
-    variable handler
-    $handler connectionName $cid $name
-    return ""
-}
-
-# ----------------------------------------------------------------------
-#  DIRECTIVE: ping
-# ----------------------------------------------------------------------
-$server define hubzero:peer/1 ping {} {
-    return "pong"
-}
-
-# ======================================================================
-#  PROTOCOL: hubzero:worker/1
-#  The authority_connect procedure connects this worker to an authority
-#  and begins the authority/worker protocol.  The directives below
-#  handle the incoming (authority->worker) traffic.
-# ======================================================================
-proc authority_connect {} {
-    global authority options
-    log system "connecting to authorities..."
-
-    authority_disconnect
-
-    # scan through list of authorities and try to connect
-    foreach addr [randomize $options(authority_hosts)] {
-        if {[catch {Client ::#auto $addr} result] == 0} {
-            set authority $result
-            break
-        }
-    }
-
-    if {"" == $authority} {
-        error "can't connect to any known authorities"
-    }
-
-    # add handlers to process the incoming commands...
-    $authority protocol hubzero:worker/1
-    $authority define hubzero:worker/1 exception {args} {
-        log system "error from authority: $args"
-    }
-
-    # ------------------------------------------------------------------
-    #  DIRECTIVE: options <key1> <value1> <key2> <value2> ...
+p2p::options register max_peer_connections 4
+
+# workers propagate messages until time-to-live reaches 0
+p2p::options register peer_time_to_live 4
+
+# ======================================================================
+#  PROTOCOL: hubzero:worker<-authority/1
+#
+#  The worker initiates communication with the authority, and the
+#  authority responds by sending these messages.
+# ======================================================================
+p2p::protocol::register hubzero:worker<-authority/1 {
+
+    # ------------------------------------------------------------------
+    #  INCOMING: options <key1> <value1> <key2> <value2> ...
     #  These option settings coming from the authority override the
-    #  option settings built into the client at the top of this
-    #  script.  The authority probably has a more up-to-date list
-    #  of other authorities and better policies for living within
-    #  the network.
-    # ------------------------------------------------------------------
-    $authority define hubzero:worker/1 options {args} {
-        global options server myaddress
+    #  option settings built into the client.  The authority probably
+    #  has a more up-to-date list of other authorities and better
+    #  policies for living within the network.
+    # ------------------------------------------------------------------
+    define options {args} {
+        global server myaddress
 
         array set options $args
+        foreach key [array names options] {
+            catch {p2p::options set $key $options($key)}
+        }
 
         if {[info exists options(ip)]} {
@@ -150 +76 @@
 
     # ------------------------------------------------------------------
-    #  DIRECTIVE: peers <listOfAddresses>
+    #  INCOMING: peers <listOfAddresses>
     #  This message comes in after this worker has sent the "peers"
     #  message to request the current list of peers.  The
@@ -156 +82 @@
     #  worker should contact to enter the p2p network.
     # ------------------------------------------------------------------
-    $authority define hubzero:worker/1 peers {plist} {
+    define peers {plist} {
         global peers
         set peers(all) $plist
-        after idle worker_peers_update
+        after idle {peer-network goto measure}
 
         # now that we've gotten the peers, we're done with the authority
-        authority_disconnect
-    }
-
-    # ------------------------------------------------------------------
-    #  DIRECTIVE: identity
+        authority-connection goto idle
+        return ""
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: identity
     #  Used for debugging, so that each client can identify itself by
     #  name to this worker.
     # ------------------------------------------------------------------
-    $authority define hubzero:worker/1 identity {name} {
+    define identity {name} {
         variable cid
         variable handler
@@ -176 +103 @@
         return ""
     }
-
-    $authority send "protocol hubzero:worker/1"
-    return $authority
-}
-
-proc authority_disconnect {} {
-    global authority
-    if {"" != $authority} {
-        catch {itcl::delete object $authority}
-        set authority ""
-    }
-}
-
-# ======================================================================
-#  USEFUL ROUTINES
-# ======================================================================
-# ----------------------------------------------------------------------
-#  COMMAND:  worker_register
-#
-#  Invoked when this worker first starts up and periodically thereafter
-#  to register the worker with the central authority, and to request
-#  a list of peers that this peer can talk to.
-# ----------------------------------------------------------------------
-proc worker_register {} {
-    global options server
+}
+
+# ======================================================================
+#  PROTOCOL: hubzero:worker<-foreman/1
+#
+#  The foreman initiates communication with a worker, and sends the
+#  various messages supported below.
+# ======================================================================
+p2p::protocol::register hubzero:worker<-foreman/1 {
+
+    # ------------------------------------------------------------------
+    #  INCOMING: solicit
+    #  Foremen send this message to solicit bids for a simulation job.
+    # ------------------------------------------------------------------
+    define solicit {args} {
+log debug "SOLICIT from foreman: $args"
+        variable cid
+        log debug "solicitation request from foreman: $args"
+        eval Solicitation ::#auto $args -connection $cid
+        return ""
+    }
+}
+
+# ======================================================================
+#  PROTOCOL: hubzero:workers<-workerc/1
+#
+#  Workers initiate connections with other workers as peers.  The
+#  following messages are sent by worker clients to the worker server.
+# ======================================================================
+p2p::protocol::register hubzero:workers<-workerc/1 {
+    # ------------------------------------------------------------------
+    #  INCOMING: identity
+    #  Used for debugging, so that each client can identify itself by
+    #  name to this worker.
+    # ------------------------------------------------------------------
+    define identity {name} {
+        variable cid
+        variable handler
+        $handler connectionName $cid $name
+        return ""
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: ping
+    #  If another worker sends "ping", they are trying to measure the
+    #  speed of the connection.  Send back a "pong" message.  If this
+    #  worker is close by, the other worker will stay connected to it.
+    # ------------------------------------------------------------------
+    define ping {} {
+        return "pong"
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: solicit -job info -path hosts -token xxx
+    #  Workers send this message on to their peers to solicit bids
+    #  for a simulation job.
+    # ------------------------------------------------------------------
+    define solicit {args} {
+log debug "SOLICIT from peer: $args"
+        variable cid
+        log debug "solicitation request from peer: $args"
+        eval Solicitation ::#auto $args -connection $cid
+        return ""
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: proffer <token> <details>
+    #  Workers send this message back after a "solicit" request with
+    #  details about what they can offer in terms of CPU power.  When
+    #  a worker has received all replies from its peers, it sends back
+    #  its own proffer message to the client that started the
+    #  solicitation.
+    # ------------------------------------------------------------------
+    define proffer {token details} {
+log debug "PROFFER from peer: $token $details"
+        Solicitation::proffer $token $details
+        return ""
+    }
+}
+
+# ======================================================================
+#  PROTOCOL: hubzero:workerc<-workers/1
+#
+#  The following messages are received by a worker client in response
+#  to the requests that they send to a worker server.
+# ======================================================================
+p2p::protocol::register hubzero:workerc<-workers/1 {
+    # ------------------------------------------------------------------
+    #  INCOMING: identity
+    #  Used for debugging, so that each client can identify itself by
+    #  name to this worker.
+    # ------------------------------------------------------------------
+    define identity {name} {
+        variable cid
+        variable handler
+        $handler connectionName $cid $name
+        return ""
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: pong
+    #  When forming the peer-to-peer network, workers send ping/pong
+    #  messages to one another to measure the latency.
+    # ------------------------------------------------------------------
+    define pong {} {
+        global peers
+        variable handler
+        set now [clock clicks -milliseconds]
+        set delay [expr {$now - $peers(ping-$handler-start)}]
+        set peers(ping-$handler-latency) $delay
+        if {[incr peers(responses)] >= [llength $peers(testing)]} {
+            after cancel {peer-network goto finalize}
+            after idle {peer-network goto finalize}
+        }
+        return ""
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: solicit -job info -path hosts -token xxx
+    #  Workers send this message on to their peers to solicit bids
+    #  for a simulation job.
+    # ------------------------------------------------------------------
+    define solicit {args} {
+log debug "SOLICIT from peer: $args"
+        variable cid
+        log debug "solicitation request from peer: $args"
+        eval Solicitation ::#auto $args -connection $cid
+        return ""
+    }
+
+    # ------------------------------------------------------------------
+    #  INCOMING: proffer <token> <details>
+    #  Workers send this message back after a "solicit" request with
+    #  details about what they can offer in terms of CPU power.  When
+    #  a worker has received all replies from its peers, it sends back
+    #  its own proffer message to the client that started the
+    #  solicitation.
+    # ------------------------------------------------------------------
+    define proffer {token details} {
+log debug "PROFFER from peer: $token $details"
+        Solicitation::proffer $token $details
+        return ""
+    }
+}
+
+# ----------------------------------------------------------------------
+#  COMMAND:  broadcast_to_peers <message> ?<avoidList>?
+#
+#  Used to broadcast a message out to all peers.  The <message> must
+#  be one of the commands defined in the worker protocol.  If there
+#  are no peer connections yet, this command does nothing.  If any
+#  peer appears on the <avoidList>, it is skipped.  Returns the
+#  number of messages sent out, so this peer knows how many replies
+#  to wait for.
+# ----------------------------------------------------------------------
+proc broadcast_to_peers {message {avoidList ""}} {
+    global server peers
+
+    #
+    # Build a list of all peer addresses, so we know who we're
+    # going to send this message out to.
+    #
+    set recipients ""
+    foreach key [array names peers current-*] {
+        set addr [$peers($key) address]
+        if {[lsearch $avoidList $addr] < 0} {
+            lappend recipients $addr
+        }
+    }
+    foreach cid [$server connections hubzero:workers<-workerc/1] {
+        set addr [lindex [$server connectionName $cid] 0]  ;# x.x.x.x (sockN)
+        if {[llength $avoidList] == 0 || [lsearch $avoidList $addr] < 0} {
+            lappend recipients $addr
+        }
+    }
+
+    #
+    # If the message has any @RECIPIENTS fields, replace them
+    # with the list of recipients
+    #
+    regsub -all @RECIPIENTS $message $recipients message
+
+    #
+    # Send the message out to all peers.  Keep a count and double-check
+    # it against the list of recipients generated above.
+    #
+    set nmesgs 0
+
+    # send off to other workers that this one has connected to
+    foreach key [array names peers current-*] {
+        set addr [$peers($key) address]
+        if {[lsearch $avoidList $addr] < 0} {
+            if {[catch {$peers($key) send $message} err] == 0} {
+                incr nmesgs
+            } else {
+                log error "ERROR: broadcast failed to [$peers($key) address]: $result\n  (message was \"$message\")"
+            }
+        }
+    }
+
+    # send off to other workers that connected to this one
+    foreach cid [$server connections hubzero:workers<-workerc/1] {
+        set addr [lindex [$server connectionName $cid] 0]  ;# x.x.x.x (sockN)
+        if {[llength $avoidList] == 0 || [lsearch $avoidList $addr] < 0} {
+            if {[catch {puts $cid $message} result] == 0} {
+                incr nmesgs
+            } else {
+                log error "ERROR: broadcast failed for $cid: $result"
+                log error "  (message was $message)"
+            }
+        }
+    }
+
+    # did we send the right number of messages?
+    if {[llength $recipients] != $nmesgs} {
+        log error "ERROR: sent only $nmesgs messages to peers {$recipients}"
+    }
+
+    return $nmesgs
+}
+
+# ----------------------------------------------------------------------
+#  COMMAND:  worker_got_protocol
+#
+#  Invoked whenever a peer sends their protocol message to this
+#  worker.  Sends the same protocol name back, so the other worker
+#  understands what protocol we're speaking.
+# ----------------------------------------------------------------------
+proc worker_got_protocol {cid protocol} {
+    switch -glob -- $protocol {
+        *<-worker* {
+            puts $cid [list protocol hubzero:workerc<-workers/1]
+        }
+        *<-foreman* {
+            puts $cid [list protocol hubzero:foreman<-worker/1]
+        }
+        DEF* {
+            # do nothing
+        }
+        default {
+            error "don't recognize protocol \"$protocol\""
+        }
+    }
+}
+
+# ----------------------------------------------------------------------
+#  COMMAND:  worker_got_dropped <address>
+#
+#  Invoked whenever an inbound connection to this worker drops.
+#  At some point we should try updating our connections to other
+#  peers, to replace this missing connection.
+# ----------------------------------------------------------------------
+proc worker_got_dropped {addr} {
+    global peers
+    if {[info exists peers(current-$addr)]} {
+        unset peers(current-$addr)
+        after cancel {peer-network goto measure}
+        after 5000 {peer-network goto measure}
+log debug "peer dropped: $addr"
+    }
+}
+
+# ======================================================================
+#  Connect to one of the authorities to get a list of peers.  Then,
+#  sit in the event loop and process events.
+# ======================================================================
+p2p::wonks::init
+
+set server [p2p::server -port 9101? \
+        -protocols {hubzero:workers<-workerc hubzero:worker<-foreman} \
+        -servername worker \
+        -onprotocol worker_got_protocol \
+        -ondisconnect worker_got_dropped]
+
+# ----------------------------------------------------------------------
+#  AUTHORITY CONNECTION
+#
+#  This is the state machine representing the connection to the
+#  authority server.  We start in the "idle" state.  Whenever we
+#  try to move to "connected", we'll open a connection to an authority
+#  and request updated information on workers.  If that connection
+#  fails, we'll get kicked back to "idle" and we'll try again later.
+# ----------------------------------------------------------------------
+StateMachine authority-connection
+authority-connection statedata cnx ""
+
+# sit here whenever we don't have a connection
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+authority-connection state idle -onenter {
+    # have an open connection? then close it
+    if {"" != [statedata cnx]} {
+        catch {itcl::delete object [statedata cnx]}
+        statedata cnx ""
+    }
+    # try to connect again later
+    set delay [p2p::options get time_between_authority_checks]
+    after $delay {authority-connection goto connected}
+} -onleave {
+    after cancel {authority-connection goto connected}
+}
+
+# sit here after we're connected and waiting for a response
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+authority-connection state connected
+
+# when moving to the connected state, make a connection
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+authority-connection transition idle->connected -onchange {
+    global server
 
     # connect to the authority and request a list of peers
-    if {[catch {
-        set client [authority_connect]
-        $client send "listening [$server port]"
-        $client send "peers"
-    } result]} {
-        log system "ERROR: $result"
-    }
-
-    # register again at regular intervals in case the authority
-    # gets restarted in between.
-    after $options(time_between_registers) worker_register
-}
-
-# ----------------------------------------------------------------------
-#  COMMAND:  worker_load_check
-#
-#  Invoked when this worker first starts up and periodically thereafter
-#  to compute the system load available to the worker.  If the system
-#  load has changed significantly, then the "perftest" program is
-#  executed to get a measure of performance available.  This program
-#  returns a number of "wonks" which we report to peers as our available
-#  performance.
-# ----------------------------------------------------------------------
-proc worker_load_check {} {
-    global options sysload dir
-
-    # see if the load has changed significantly
-    set changed 0
-    if {$sysload(lastLoad) < 0} {
-        set changed 1
+    statedata cnx ""
+    foreach addr [randomize [p2p::options get authority_hosts]] {
+        if {[catch {p2p::client -address $addr \
+            -sendprotocol hubzero:authority<-worker/1 \
+            -receiveprotocol hubzero:worker<-authority/1} result] == 0} {
+            statedata cnx $result
+            break
+        }
+    }
+
+    if {"" != [statedata cnx]} {
+        [statedata cnx] send "listening [$server port]"
+        [statedata cnx] send "peers"
     } else {
-        set load [Rappture::sysinfo load5]
-        if {$load < 0.9*$sysload(lastLoad) || $load > 1.1*$sysload(lastLoad)} {
-            set changed 1
-        }
-    }
-
-    if {$changed} {
-        set sysload(lastLoad) [Rappture::sysinfo load5]
-puts "LOAD CHANGED: $sysload(lastLoad) [Rappture::sysinfo freeram freeswap]"
-        set sysload(measured) [clock seconds]
-
-        # Run the program, but don't use exec, since it blocks.
-        # Instead, run it in the background and harvest its output later.
-        set sysload(test) [open "| [file join $dir perftest]" r]
-        fileevent $sysload(test) readable worker_load_results
-    }
-
-    # monitor the system load at regular intervals
-    after $options(time_between_load_checks) worker_load_check
-}
-
-# ----------------------------------------------------------------------
-#  COMMAND:  worker_load_results
-#
-#  Invoked automatically when the "perftest" run finishes.  Reads the
-#  number of wonks reported on standard output and reports them to
-#  other peers.
-# ----------------------------------------------------------------------
-proc worker_load_results {} {
-    global sysload
-
-    if {[catch {read $sysload(test)} msg] == 0} {
-        if {[regexp {[0-9]+} $msg match]} {
-puts "WONKS: $match"
-            set sysload(wonks) $match
-        } else {
-            log system "ERROR: performance test failed: $msg"
-        }
-    }
-    catch {close $sysload(test)}
-    set sysload(test) ""
-}
-
-# ----------------------------------------------------------------------
-#  COMMAND:  worker_peers_update
-#
-#  Invoked when this worker has received a list of peers, and from
-#  time to time thereafter, to establish connections with other peers
-#  in the network.  This worker picks a number of peers randomly from
-#  the list of all available peers, and then pings each of them.
-#  Timing results from the pings are stored away, and when all pings
-#  have completed, this worker picks the best few connections and
-#  keeps those.
-# ----------------------------------------------------------------------
-proc worker_peers_update {} {
-    global options peers myaddress
-    worker_peers_cleanup
+        error "can't connect to any authority\nAUTHORITY LIST: [p2p::options get authority_hosts]"
+    }
+}
+
+# ----------------------------------------------------------------------
+#  PEER-TO-PEER CONNECTIONS
+#
+#  This is the state machine representing the network of worker peers.
+#  We start in the "idle" state.  From time to time we move to the
+#  "measure" state and attempt to establish connections with a set
+#  peers.  We then wait for ping/pong responses and move to "finalize".
+#  When we have all responses, we move to "finalize" and finalize all
+#  connections, and then move back to "idle".
+# ----------------------------------------------------------------------
+StateMachine peer-network
+peer-network statedata cnx ""
+
+# sit here when the peer network is okay
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+peer-network state idle -onenter {
+    # try to connect again later
+    set delay [p2p::options get time_between_network_rebuilds]
+    after $delay {peer-network goto measure}
+} -onleave {
+    after cancel {peer-network goto measure}
+}
+
+# sit here when we need to rebuild the peer-to-peer network
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+peer-network state measure
+
+# when moving to the start state, make connections to a bunch of peers
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+peer-network transition idle->measure -onchange {
+    global server myaddress peers
+
+    #
+    # Get a list of workers already connected to this one
+    # with an inbound connection to the server.
+    #
+    foreach cid [$server connections hubzero:workers<-workerc/1] {
+        set addr [lindex [$server connectionName $cid] 0]
+        set inbound($addr) $cid
+    }
 
     #
     # Pick a random group of peers and try to connect to them.
-    # Start with the existing peers, and then add a random
-    # bunch of others.
-    #
+    # Start with the existing peers to see if their connection
+    # is still favorable, and then add a random bunch of others.
+    #
+    set peers(testing) ""
+    set peers(responses) 0
     foreach key [array names peers current-*] {
         set peer $peers($key)
@@ -307 +490 @@
     # final number we want to talk to.
     #
-    set ntest [expr {10 * $options(max_peer_connections)}]
-    foreach addr [randomize $peers(all) $ntest] {
-        if {$addr == $myaddress} {
+    set maxpeers [p2p::options get max_peer_connections]
+    foreach addr [randomize $peers(all) [expr {10*$maxpeers}]] {
+        #
+        # Avoid connecting to ourself, or to peers that we're
+        # already connected to (either as inbound connections
+        # to the server or as outbound connections to others).
+        #
+        if {$addr == $myaddress
+              || [info exists peers(current-$addr)]
+              || [info exists inbound($addr)]} {
             continue
         }
-        if {![info exists peers(current-$addr)]
-              && [catch {Client ::#auto $addr} peer] == 0} {
-            # open a new connection to this address
-            lappend peers(testing) $peer
-            $peer protocol hubzero:peer/1
-
-            $peer define hubzero:peer/1 exception {message} {
-                log system "ERROR: $message"
-            }
-
-            $peer define hubzero:peer/1 identity {name} {
-                variable cid
-                variable handler
-                $handler connectionName $cid $name
-                return ""
-            }
-
-            # when we get a "pong" back, store the latency
-            $peer define hubzero:peer/1 pong {} {
-                global peers
-                variable handler
-puts "  pong from $handler"
-                set now [clock clicks -milliseconds]
-                set delay [expr {$now - $peers(ping-$handler-start)}]
-                set peers(ping-$handler-latency) $delay
-                incr peers(responses)
-                worker_peers_finalize
-                return ""
-            }
-
-            # start the ping/pong session with the peer
-            $peer send "protocol hubzero:peer/1"
-
-            # send tell this peer our name (for debugging)
-            if {[log channel debug]} {
-                $peer send "identity $myaddress"
-            }
+
+        if {[catch {p2p::client -address $addr \
+            -sendprotocol hubzero:workers<-workerc/1 \
+            -receiveprotocol hubzero:workerc<-workers/1} cnx]} {
+            continue
+        }
+        $cnx send "identity $myaddress"
+        lappend peers(testing) $cnx
+
+        # have enough connections to test?
+        if {[llength $peers(testing)] >= 2*$maxpeers} {
+            break
         }
     }
@@ -355 +520 @@
     # Now, loop through all peers and send a "ping" message.
     #
-    foreach peer $peers(testing) {
-puts "pinging $peer = [$peer address]..."
+    foreach cnx $peers(testing) {
         # mark the time and send a ping to this peer
-        set peers(ping-$peer-start) [clock clicks -milliseconds]
-        $peer send "ping"
+        set peers(ping-$cnx-start) [clock clicks -milliseconds]
+        $cnx send "ping"
     }
 
     # if this test takes too long, just give up
-    after 10000 worker_peers_finalize -force
-}
-
-# ----------------------------------------------------------------------
-#  COMMAND:  worker_peers_finalize ?-force?
-#
-#  Called after worker_peers_update has finished its business to
-#  clean up all of the connections that were open for testing.
-# ----------------------------------------------------------------------
-proc worker_peers_finalize {{option -check}} {
-    global peers options
-    if {$option == "-force" || $peers(responses) == [llength $peers(testing)]} {
-        # build a list:  {peer latency} {peer latency} ...
-        set plist ""
-        foreach obj $peers(testing) {
-            if {[info exists peers(ping-$obj-latency)]} {
-                lappend plist [list $obj $peers(ping-$obj-latency)]
-            }
-        }
-puts "-------------------\nFINALIZING $::myaddress"
-puts "PINGS: $plist"
-
-        # sort the list and extract the top peers
-        set plist [lsort -increasing -index 1 $plist]
-        set plist [lrange $plist 0 [expr {$options(max_peer_connections)-1}]]
-puts "TOP: $plist"
-
-        set current [array names peers current-*]
-puts "  current: $current"
-
-        # transfer the top peers to the "current" list
-        set all ""
-        foreach rec $plist {
-            set peer [lindex $rec 0]
-            set addr [$peer address]
-            set peers(current-$addr) $peer
-            lappend all $addr
-
-            # if it is already on the "current" list, then keep it
-            set i [lsearch $current current-$addr]
-            if {$i >= 0} {
-                set current [lreplace $current $i $i]
-            }
-            set i [lsearch $peers(testing) $peer]
-            if {$i >= 0} {
-                set peers(testing) [lreplace $peers(testing) $i $i]
-            }
-        }
-        log system "connected to peers: $all"
-puts "  new current: [array names peers current-*]"
-puts "  final: $all"
-puts "  leftover: $current $peers(testing)"
-
-        # get rid of old peers that we no longer want to talk to
-        foreach leftover $current {
-            itcl::delete object $peers($leftover)
-            unset peers($leftover)
-puts "  cleaned up $leftover (was on current list)"
-        }
-
-        # clean up after this test
-        worker_peers_cleanup
-    }
-}
-
-# ----------------------------------------------------------------------
-#  COMMAND:  worker_peers_cleanup
-#
-#  Called after worker_peers_update has finished its business to
-#  clean up all of the connections that were open for testing.
-# ----------------------------------------------------------------------
-proc worker_peers_cleanup {} {
+    after 10000 {peer-network goto finalize}
+}
+
+# sit here when we need to rebuild the peer-to-peer network
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+peer-network state finalize -onenter {
+    after cancel {peer-network goto finalize}
+
+    # everything is finalized now, so go to idle
+    after idle {peer-network goto idle}
+}
+
+# when moving to the finalize state, decide on the network of peers
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+peer-network transition measure->finalize -onchange {
     global peers
+
+    set maxpeers [p2p::options get max_peer_connections]
+
+    # build a list:  {peer latency} {peer latency} ...
+    set plist ""
+    foreach obj $peers(testing) {
+        if {[info exists peers(ping-$obj-latency)]} {
+            lappend plist [list $obj $peers(ping-$obj-latency)]
+        }
+    }
+
+    # sort the list and extract the top peers
+    set plist [lsort -increasing -index 1 $plist]
+    set plist [lrange $plist 0 [expr {$maxpeers-1}]]
+
+    set currentpeers [array names peers current-*]
+
+    # transfer the top peers to the "current" list
+    set all ""
+    foreach rec $plist {
+        set peer [lindex $rec 0]
+        set addr [$peer address]
+        set peers(current-$addr) $peer
+        lappend all $addr
+
+        # if it is already on the "current" list, then keep it
+        set i [lsearch $currentpeers current-$addr]
+        if {$i >= 0} {
+            set currentpeers [lreplace $currentpeers $i $i]
+        }
+        set i [lsearch $peers(testing) $peer]
+        if {$i >= 0} {
+            set peers(testing) [lreplace $peers(testing) $i $i]
+        }
+    }
+    log system "connected to peers: $all"
+
+    # get rid of old peers that we no longer want to talk to
+    foreach leftover $currentpeers {
+        itcl::delete object $peers($leftover)
+        unset peers($leftover)
+    }
+
+    # clean up after this test
     foreach obj $peers(testing) {
         catch {itcl::delete object $obj}
-puts "  cleaned up $obj (was on testing list)"
     }
     set peers(testing) ""
@@ -446 +596 @@
     }
     set peers(responses) 0
-
-    after cancel worker_peers_finalize -force
-}
-
-# ----------------------------------------------------------------------
-#  COMMAND:  worker_peers_protocol
-#
-#  Invoked whenever a peer sends their protocol message to this
-#  worker.  Sends the same protocol name back, so the other worker
-#  understands what protocol we're speaking.
-# ----------------------------------------------------------------------
-proc worker_peers_protocol {cid protocol} {
-    if {"DEFAULT" != $protocol} {
-        puts $cid [list protocol $protocol]
-    }
-}
-
-# ======================================================================
-#  Connect to one of the authorities to get a list of peers.  Then,
-#  sit in the event loop and process events.
-# ======================================================================
-after idle worker_register
+}
+
+# ----------------------------------------------------------------------
+#  JOB SOLICITATION
+#
+#  Each worker can receive a "solicit" request, asking for information
+#  about performance and price of peers available to work.  Each worker
+#  sends the request on to its peers, then gathers the information
+#  and sends it back to the client or to the peer requesting the
+#  information.  There can be multiple requests going on at once,
+#  and each may have different job types and return different info,
+#  so the class below helps to watch over each request until it has
+#  completed.
+# ----------------------------------------------------------------------
+itcl::class Solicitation {
+    public variable connection ""
+    public variable job ""
+    public variable path ""
+    public variable avoid ""
+    public variable token ""
+
+    private variable _serial ""
+    private variable _response ""
+    private variable _waitfor 0
+    private variable _timeout ""
+
+    constructor {args} {
+        eval configure $args
+
+        global myaddress
+        lappend path $myaddress
+        lappend avoid $myaddress
+        set _serial [incr counter]
+        set all($_serial) $this
+
+        set delay "idle"  ;# finalize after waiting for responses
+        set ttl [p2p::options get peer_time_to_live]
+        if {[llength $path] < $ttl} {
+            set mesg [list solicit -job $job -path $path -avoid "$avoid @RECIPIENTS" -token $_serial]
+            set _waitfor [broadcast_to_peers $mesg $avoid]
+log debug "WAIT FOR: $_waitfor"
+
+            if {$_waitfor > 0} {
+                # add a delay proportional to ttl + time for wonks measurement
+                set delay [expr {($ttl-[llength $path]-1)*1000 + 3000}]
+            }
+        }
+log debug "TIMEOUT: $delay"
+        set _timeout [after $delay [itcl::code $this finalize]]
+    }
+    destructor {
+        after cancel $_timeout
+        catch {unset all($_serial)}
+    }
+
+    # this adds the info from each proffer to this solicitation
+    method response {details} {
+        set addr [lindex $details 0]
+        append _response $details "\n"
+        if {[incr _waitfor -1] <= 0} {
+log debug "ALL RESPONSES"
+            finalize
+        }
+    }
+
+    # called to finalize when all peers have responded back
+    method finalize {} {
+        global myaddress
+
+        # filter out duplicate info from clients
+        set block ""
+        foreach line [split $_response \n] {
+            set addr [lindex $line 0]
+            if {"" != $addr && ![info exists response($addr)]} {
+                append block $line "\n"
+                set response($addr) 1
+            }
+        }
+        # add details about this client to the message
+        append block "$myaddress -job $job -cost 1 -wonks [p2p::wonks::current]"
+log debug "FINALIZE {$block}"
+
+        # send the composite results back to the caller
+        set cmd [list proffer $token $block]
+        if {[catch {puts $connection $cmd} err]} {
+            log error "ERROR while sending back proffer: $err"
+        }
+        itcl::delete object $this
+    }
+
+    proc proffer {serial details} {
+        if {[info exists all($serial)]} {
+            $all($serial) response $details
+        }
+    }
+
+    common counter 0 ;# generates serial nums for objects
+    common all       ;# maps serial num => solicitation object
+}
+
+# ----------------------------------------------------------------------
 log system "starting..."
+
+after idle {
+    authority-connection goto connected
+}
+
 vwait main-loop