%3 ::nx::Object ::nx::Object __default_accessor __default_method_call_protection __object_configureparameter __resolve_method_path contains copy delete object method delete object property delete object variable destroy_on_cleanup info info info lookup parameters info lookup slots info lookup syntax info lookup variables info object method args info object method body info object method callprotection info object method debug info object method definition info object method definitionhandle info object method deprecated info object method exists info object method handle info object method origin info object method parameters info object method registrationhandle info object method returns info object method submethods info object method syntax info object method type info object slots info object variables info variable definition info variable name info variable parameter move object alias object forward object method object property object variable private protected public qn require namespace require object method require private object method require protected object method require public object method serialize ::acs::Cluster ::acs::Cluster allowed_command broadcast check_state current_server_is_canonical_server current_server_is_dynamic_cluster_peer current_server_locations disconnect_request dynamic_cluster_nodes dynamic_cluster_reconfigure execute external_location incoming_request init is_canonical_server is_configured_server is_current_server join_request last_contact last_request log map_inaddr_any message decode message encode message sign message verify ns_http_send peer_nodes preauth preferred_location qualified_location reachable register_nodes secret secret_configured send send_disconnect_request_to_canonical_server send_dynamic_cluster_reconfigure_request send_join_request_to_canonical_server setup update_node_info ::acs::Cluster->::nx::Object

Class ::acs::Cluster

::acs::Cluster[i] create ... \
           [ -allowed_command (default " set "" unset "" nsv_set "" nsv_unset "" nsv_incr "" nsv_dict "" bgdelivery "" callback "" ns_cache "^ns_cache\s+eval" ns_cache_flush "" util_memoize_flush_regexp_local "" ns_urlspace "" acs::cache_flush_all "" acs::cache_flush_pattern "" lang::message::cache "" ad_parameter_cache_flush_dict "" ::acs::cluster "^::acs::cluster\s+join_request" ::acs::cluster "^::acs::cluster\s+disconnect_request" ") ] \
           [ -allowed_host (default " "127.0.0.1" 1 ") ] \
           [ -myLocation (default "") ] \
           [ -url (default "/acs-cluster-do") ]

Class for managing a cluster of OpenACS nodes
Defined in packages/acs-tcl/tcl/cluster-procs.tcl

Class Relations

  • class: ::nx::Class[i]
  • superclass: ::nx::Object[i]
::nx::Class create ::acs::Cluster \
     -superclass ::nx::Object

Methods (to be applied on instances)

  • broadcast (scripted, public)

     <instance of acs::Cluster[i]> broadcast args [ args... ]

    Send requests to all cluster peers.

    Parameters:
    args (required)

    Testcases:
    No testcase defined.
    if {[ns_ictl epoch] > 0} {
        catch {::throttle do incr ::count(cluster:broadcast)}
    }
    
    # Small optimization for cachingmode "none": no need to
    # send cache flushing requests to nodes, when there is no
    # caching in place.
    #
    if {[ns_config "ns/parameters" cachingmode "per-node"] eq "none"
        && [lindex $args 0] in {
            acs::cache_flush_pattern
            acs::cache_flush_all
            ns_cache}
    } {
        #
        # If caching mode is none, it is expected that all
        # nodes have this parameter set. Therefore, there is no
        # need to communicate cache flushing commands.
        #
        return
    }
    
    if {[nsv_get cluster cluster_peer_nodes locations]} {
        #
        # During startup the throttle thread might not be started,
        # so omit these statistic values
        #
        if {[ns_ictl epoch] > 0} {
            foreach location $locations {
                catch {::throttle do incr ::count(cluster:sent)}
                set t0 [clock clicks -microseconds]
                :send $location {*}$args
                set ms [expr {([clock clicks -microseconds] - $t0)/1000}]
                catch {::throttle do incr ::agg_time(cluster:sent) $ms}
            }
        } else {
            foreach location $locations {
                :send $location {*}$args
            }
        }
    }
  • check_state (scripted, public)

     <instance of acs::Cluster[i]> check_state

    Check the livelyness of the dynamic cluster nodes. This method is intended to be run on the canonical server only, since it might update the DynamicClusterPeers via acs::clusterwide.

    Testcases:
    No testcase defined.
    set autodeleteInterval [parameter::get  -package_id $::acs::kernel_id  -parameter ClusterAutodeleteInterval  -default 2m]
    
    foreach node [:dynamic_cluster_nodes] {
        set last_contact [acs::cluster last_contact $node]
        if {$last_contact ne ""} {
            set seconds [expr {$last_contact/1000}]
            if {[clock seconds]-($last_contact/1000) > [ns_baseunit -time $autodeleteInterval]} {
                ns_log notice "[self] disconnect dynamic node $node due to ClusterAutodeleteInterval"
                :disconnect_request $node
            }
        }
    }
  • current_server_is_canonical_server (scripted, public)

     <instance of acs::Cluster[i]> current_server_is_canonical_server

    Check, if the current server is the canonical_server.

    Testcases:
    No testcase defined.
    if { ![info exists :canonicalServer] || ${:canonicalServer} eq "" } {
        ns_log Error "Your configuration is not correct for server clustering."  "Please ensure that you have the CanonicalServer parameter set correctly."
        return 1
    }
    set result [:is_canonical_server ${:myLocation}]
    # set result 0
    # foreach location ${:myLocations} {
    #     if {[:is_canonical_server $location]} {
    #         set result 1
    #         break
    #     }
    # }
    #:log "current_server_is_canonical_server $result"
    return $result
  • current_server_is_dynamic_cluster_peer (scripted, public)

     <instance of acs::Cluster[i]> current_server_is_dynamic_cluster_peer

    We are a dynamic cluster peer, when we are not the canonical server neither isted in the static server locations.

    Testcases:
    No testcase defined.
    if {${:current_server_is_canonical_server}} {
        return 0
    }
    return [expr {${:myLocation} ni ${:staticServerLocations}}]
  • disconnect_request (scripted, public)

     <instance of acs::Cluster[i]> disconnect_request peerLocation

    Server received a request to disconnect $peerLocation from dynamic cluster nodes.

    Parameters:
    peerLocation (required)

    Testcases:
    No testcase defined.
    return [:dynamic_cluster_reconfigure disconnect [:qualified_location $peerLocation]]
  • dynamic_cluster_nodes (scripted, public)

     <instance of acs::Cluster[i]> dynamic_cluster_nodes

    Convenience function returning the list of dynamic cluster nodes.

    Testcases:
    No testcase defined.
    return [parameter::get  -package_id $::acs::kernel_id  -parameter DynamicClusterPeers]
  • dynamic_cluster_reconfigure (scripted, public)

     <instance of acs::Cluster[i]> dynamic_cluster_reconfigure operation \
        qualifiedLocation

    Reconfigure the cluster via "join" or "disconnect" operation, when running on the canonical server. The result of the reconfiguration is a changed list of DynamicClusterPeers. The method returns a boolean value indicating success.

    Parameters:
    operation (required)
    qualifiedLocation (required)

    Testcases:
    No testcase defined.
    ns_log notice "Cluster reconfigure $operation from '$qualifiedLocation'"
    
    set success 1
    #
    # To be ultra-conservative, we could allow cluster
    # reconfigure operations only on the canonical
    # server. This would require also to alter the
    # acs-admin/cluster page to show the trash icon only when
    # the page is executed on the canonical server.
    #
    if {0 && ![:current_server_is_canonical_server]} {
        ns_log warning "Cluster reconfigure rejected,"  "since it was received by a non-canonical server"
        set success 0
    } else {
        #
        # We know, we are running on the canonical server, and
        # we know that the request is trustworthy.
        #
        ns_log notice "Cluster reconfigure $qualifiedLocation accepted from $qualifiedLocation"
        set dynamicClusterNodes [:dynamic_cluster_nodes]
        switch $operation {
            "join" {
                set dynamicClusterNodes  [lsort -unique [concat $dynamicClusterNodes $qualifiedLocation]]
            }
            "disconnect" {
                set dynamicClusterNodes  [lsearch -inline -all -not -exact $dynamicClusterNodes $qualifiedLocation]
            }
            default {
                ns_log warning "Cluster reconfigure rejected,"  "received invalid operation '$operation'"
                return 0
            }
        }
        #
        # The parameter::set_value operation causes a
        # clusterwide cache-flush for the parameters
        #
        parameter::set_value  -package_id $::acs::kernel_id  -parameter DynamicClusterPeers  -value $dynamicClusterNodes
        ns_log notice "[self] reconfigure $operation leads to DynamicClusterPeers $dynamicClusterNodes"
    }
    return $success
  • incoming_request (scripted, public)

     <instance of acs::Cluster[i]> incoming_request

    We received an incoming request from a cluster peer.

    Testcases:
    No testcase defined.
    catch {::throttle do incr ::count(cluster:received)}
    #ns_log notice "==== [self] incoming_request [ns_conn query]"
    
    ad_try {
        #ns_logctl severity Debug(connchan) on
        #ns_logctl severity Debug(request) on
        #ns_logctl severity Debug(ns:driver) on
        #ns_logctl severity Debug on
        set r [:message decode]
        set receive_timestamp [clock clicks -milliseconds]
        dict with r {
            #
            # We could check here the provided timepstamp and
            # honor only recent requests (protection against
            # replay attacks). However, the allowed requests
            # are non-destructive.
            #
            nsv_set cluster $peer-last-contact $receive_timestamp
            nsv_set cluster $peer-last-request $receive_timestamp
            nsv_incr cluster $peer-count
            ns_log notice "--cluster got cmd='$cmd' from $peer after [expr {$receive_timestamp - $timestamp}]ms"
    
            set result [:execute $r]
        }
    } on error {errorMsg} {
        ns_log notice "--cluster error: $errorMsg"
        ns_return 417 text/plain $errorMsg
    } on ok {r} {
        #ns_log notice "--cluster success $result"
        ns_return 200 text/plain $result
    }
  • is_canonical_server (scripted, public)

     <instance of acs::Cluster[i]> is_canonical_server location

    Check, if provided location belongs to the canonical server specs. The canonical server might listen on multiple protocols, IP addresses and ports.

    Parameters:
    location (required)

    Testcases:
    No testcase defined.
    if { ![info exists :canonicalServer] || ${:canonicalServer} eq "" } {
        ns_log Error "Your configuration is not correct for server clustering."  "Please ensure that you have the CanonicalServer parameter set correctly."
        return 1
    }
    set location [:qualified_location $location]
    set result [expr {$location in ${:canonicalServerLocation}}]
    return $result
  • join_request (scripted, public)

     <instance of acs::Cluster[i]> join_request peerLocation

    Server received a request to join dynamic cluster nodes from $peerLocation. ns_log notice "Server received a join request" ns_log notice "... ns_conn host <[ns_conn host]> peer <[ns_conn peeraddr]>" ns_log notice "... ns_conn port <[ns_conn port]> peerport <[ns_conn peerport]>" ns_log notice "... peerLocation <$peerLocation> qualified [:qualified_location $peerLocation]" set headers [join [lmap {key value} [ns_set array [ns_conn headers]] {set _ "$key: $value\n... "}]] ns_log notice "... headers $headers"

    Parameters:
    peerLocation (required)

    Testcases:
    No testcase defined.
    return [:dynamic_cluster_reconfigure join [:qualified_location $peerLocation]]
  • last_contact (scripted, public)

     <instance of acs::Cluster[i]> last_contact location

    Return the milliseconds since the last contact with the denoted server. If there is no data available, the return values is empty.

    Parameters:
    location (required)

    Testcases:
    No testcase defined.
    if {[nsv_get cluster $location-last-contact clicksms]} {
        return $clicksms
    }
  • last_request (scripted, public)

     <instance of acs::Cluster[i]> last_request location

    Return the milliseconds since the last request from the denoted server. If there is no data available, the return values is empty.

    Parameters:
    location (required)

    Testcases:
    No testcase defined.
    if {[nsv_get cluster $location-last-request clicksms]} {
        return $clicksms
    }
  • preauth (scripted, public)

     <instance of acs::Cluster[i]> preauth args [ args... ]

    Process no more pre-authorization filters for this connection (avoid running of expensive filters). ns_log notice "PREAUTH returns filter_break"

    Parameters:
    args (required)

    Testcases:
    No testcase defined.
    return filter_break
  • qualified_location (scripted, public)

     <instance of acs::Cluster[i]> qualified_location location

    Return a canonical representation of the provided location, where the DNS name is resolved and the protocol and port is always included. When there is no protocol provided, HTTP is assumed. Provide defaults, when no port is included in the passed-in location. Note, that there is no default provided for non-HTTP* locations, so these must contain the port.

    Parameters:
    location (required)

    Testcases:
    No testcase defined.
    set d {port 80 proto http}
    if {[regexp {^([^:]+)://} $location . proto]} {
        if {$proto eq "https"} {
            set d {port 443 proto https}
        }
        set d [dict merge $d [ns_parseurl $location]]
        dict unset d tail
        dict unset d path
    } else {
        set d [dict merge $d [ns_parsehostport $location]]
    }
    set label [dict get $d port]/tcp
    set containerMapping [acs::container mapping]
    # catch {
    #     ns_log notice "check container mapping for $label: $d"
    #     ns_log notice "... internal?[expr {[dict get $d host] eq {host.docker.internal}}]"
    #     ns_log notice "... mapping? [expr {$containerMapping ne {}}]"
    #     ns_log notice "... label?   [dict exists $containerMapping $label]"
    #     ns_log notice "... port=    [dict get $containerMapping $label port]"
    # }
    if {$containerMapping ne ""
        && [dict get $d host] eq "host.docker.internal"
        && [dict exists $containerMapping $label]
        && [dict get $containerMapping $label port] < 32768
    } {
        # Ephemeral ports on Linux are typically 32768-60999
        # https://en.wikipedia.org/wiki/Ephemeral_port
        #ns_log notice "... there is a container mapping for $d -> [dict get $containerMapping $label]"
        set d [dict get $containerMapping $label]
    } else {
        #
        # In theory, an input location might map to multiple
        # values, when e.g., a provided DNS name refers to
        # multiple IP addresses. For now, we just return always a
        # single value.
        #
        # To return all IP addresses, we could use "ns_addrbyhost
        # -all ..." instead.
        #
        dict set d host [ns_addrbyhost [dict get $d host]]
    }
    
    set d [:map_inaddr_any -dict $d]
    dict with d {
        set result [util::join_location -noabbrev -proto $proto -hostname $host -port $port]
    }
    return $result
  • register_nodes (scripted, public)

     <instance of acs::Cluster[i]> register_nodes [ -startup ]

    Register the defined cluster nodes by creating/recreating cluster node objects.

    Switches:
    -startup (optional, defaults to "false")

    Testcases:
    No testcase defined.
    :log ":register_nodes startup $startup"
    
    #
    # Configure base configuration values
    #
    #
    set dynamic_peers [:dynamic_cluster_nodes]
    
    # At startup, when we are running on the canonical server,
    # check, whether the existing dynamic cluster nodes are
    # still reachable. When the canonical server is started
    # before the other cluster nodes, this parameter should be
    # empty. However, when the canonical server is restarted,
    # there might be some of the peer nodes already active.
    #
    if {$startup
        && ${:current_server_is_canonical_server}
        && $dynamic_peers ne ""
    } {
        #
        # When we are starting the canonical server, it resets
        # the potentially pre-existing dynamic nodes unless
        # these are reachable.
        #
        set old_peer_locations $dynamic_peers
        :log "canonical server starts with existing DynamicClusterPeers nodes: $old_peer_locations"
        #
        # Keep the reachable cluster nodes in
        # "DynamicClusterPeers".
        #
        set new_peer_locations {}
        foreach location $old_peer_locations {
            if {[:reachable $location]} {
                lappend new_peer_locations $location
            }
        }
        if {$new_peer_locations ne $old_peer_locations} {
            #
            # Update the DynamicClusterPeers in the database
            # such that the other nodes will pick it up as
            # well.
            #
            :log "updating DynamicClusterPeers to '$new_peer_locations' epoch [ns_ictl epoch]"
            parameter::set_value  -package_id $::acs::kernel_id  -parameter DynamicClusterPeers  -value [lsort $new_peer_locations]
            set dynamic_peers $new_peer_locations
        }
    }
    
    #
    # Determine the peer nodes.
    #
    set cluster_peer_nodes [:peer_nodes $dynamic_peers]
    nsv_set cluster cluster_peer_nodes $cluster_peer_nodes
    #:log "cluster_peer_nodes <$cluster_peer_nodes>"
    
    if {![:is_configured_server ${:myLocations}]} {
        #
        # Current node is not pre-registered.
        #
        ns_log notice "Current host ${:myLocation} is not included in ${:configured_cluster_hosts}"
        if {![:current_server_is_canonical_server]} {
            ns_log notice "... must join at canonical server ${:canonicalServerLocation}"
            :send_join_request_to_canonical_server
        }
    } else {
        #ns_log notice "Current host ${:myLocation} is included in ${:configured_cluster_hosts}"
    }
  • secret_configured (scripted, public)

     <instance of acs::Cluster[i]> secret_configured

    Check, whether the secret for signing messages in the intra-cluster talk is configured. More checks for different secret definition methods might be added.

    Testcases:
    No testcase defined.
    set secret [:secret]
    return [expr {$secret ne ""}]
  • send (scripted, public)

     <instance of acs::Cluster[i]> send [ -delivery delivery ] location \
        args [ args... ]

    Send a command by different means to the cluster node for intra-server talk. Valid delivery methods are - ns_http (for HTTP and HTTPS) - connchan (for HTTP and HTTPS) - udp (plain UDP only)

    Switches:
    -delivery (optional, defaults to "ns_http")
    Parameters:
    location (required)
    args (required)

    Testcases:
    No testcase defined.
    :log "outgoing request to $location // $args"
    set t0 [clock clicks -microseconds]
    switch $delivery {
        #connchan -
        #udp      -
        ns_http   {set result [:${delivery}_send $location {*}$args]}
        default {error "unknown delivery method '$delivery'"}
    }
    ns_log notice "-cluster: $location $args sent"  "total [expr {([clock clicks -microseconds] - $t0)/1000.0}]ms"
    return $result
  • send_disconnect_request_to_canonical_server (scripted, public)

     <instance of acs::Cluster[i]> send_disconnect_request_to_canonical_server

    Send a disconnect request to the canonical server.

    Testcases:
    No testcase defined.
    :send_dynamic_cluster_reconfigure_request disconnect
  • send_join_request_to_canonical_server (scripted, public)

     <instance of acs::Cluster[i]> send_join_request_to_canonical_server

    Send a join request to the canonical server.

    Testcases:
    No testcase defined.
    :send_dynamic_cluster_reconfigure_request join
  • setup (scripted, public)

     <instance of acs::Cluster[i]> setup

    Setup object specific variables. Make sure to call this method, when the called procs are available. Make sure the container support is initialized

    Testcases:
    No testcase defined.
    ::acs::Container create ::acs::container
    #
    # Set the variables controlling the behavior
    #
    set :myLocations [:current_server_locations]
    set :myLocation [:preferred_location ${:myLocations}]
    
    set :canonicalServer [parameter::get -package_id $::acs::kernel_id -parameter CanonicalServer]
    set :canonicalServerLocation [:preferred_location [:qualified_location ${:canonicalServer}]]
    
    set :current_server_is_canonical_server [:current_server_is_canonical_server]
    set :staticServerLocations  [lmap entry [parameter::get -package_id $::acs::kernel_id -parameter ClusterPeerIP] {
            :preferred_location [:qualified_location $entry]
        }]
    
    ns_log notice "[self]: cluster configured to"
    set :myLocations [:current_server_locations]
    set :myLocation [:preferred_location ${:myLocations}]
    ns_log notice "... myLocations                        ${:myLocations}"
    ns_log notice "... myLocation                         ${:myLocation}"
    ns_log notice "... canonicalServer                    ${:canonicalServer}"
    ns_log notice "... canonicalServerLocation            ${:canonicalServerLocation}"
    ns_log notice "... current_server_is_canonical_server ${:current_server_is_canonical_server}"
    ns_log notice "... staticServerLocations              '${:staticServerLocations}'"
  • update_node_info (scripted, public)

     <instance of acs::Cluster[i]> update_node_info

    Update cluster configuration when the when the configuration variables changed, or when nodes become available/unavailable after some time. Typically, this method is called via scheduled procedure every couple of seconds when clustering is enabled.

    Testcases:
    No testcase defined.
    set dynamic_peers [:dynamic_cluster_nodes]
    
    if {!${:current_server_is_canonical_server}} {
        #
        # The current node might be a static or a dynamic
        # peer.  Do we have contact to the canonical_server?
        #
        if {![:reachable ${:canonicalServerLocation}]} {
            #
            # We lost contact to the canonical server. This is
            # for our server not a big problem, since all
            # other peer-to-peer updates will continue to
            # work.
            #
            # During downtime of the canonical server,
            # scheduled procedures (e.g. mail delivery) will
            # be interrupted, and no new servers can register.
            #
            ns_log warning "cluster node lost contact to "  "canonical server: ${:canonicalServerLocation}"
        }
        #
        # Are we an dynamic peer and not listed in
        # dynamic cluster nodes? This might happen in
        # situations, where the canonical server was
        # restarted (or separated for a while).
        #
        if {[:current_server_is_dynamic_cluster_peer]
            && ${:myLocation} ni $dynamic_peers
        } {
            ns_log warning "cluster node is not listed in dynamic peers."  "Must re-join canonical server: ${:canonicalServerLocation}"
            ns_log notice "... myLocation: ${:myLocation}"
            ns_log notice "... dynamic_peers: $dynamic_peers"
            :send_join_request_to_canonical_server
        }
    }
    
    #
    # Update cluster_peer_nodes if necessary
    #
    set oldConfig [lsort [nsv_get cluster cluster_peer_nodes]]
    set newConfig [lsort [:peer_nodes $dynamic_peers]]
    if {$newConfig ne $oldConfig} {
        #
        # The cluster configuration has changed
        #
        ns_log notice "cluster config changed:\nOLD $oldConfig\nNEW $newConfig"
        nsv_set cluster cluster_peer_nodes $newConfig
    }