Class ::nsmcp::SystemLogSummary (public)
::nx::Class ::nsmcp::SystemLogSummary
Defined in /usr/local/ns/tcl/nsmcp/log-summary.tcl
- Testcases:
- No testcase defined.
Source code: :public object method requestMetadata {text redactor} { # # Extract bounded authentication and User-Agent evidence from warning diagnostics. # Retain authentication state without retaining the OpenACS user ID. # set result {authenticationState unknown authenticationSource unavailable} if {[regexp -line {^:[ \t]+(?:GET|POST|HEAD|PUT|DELETE|OPTIONS|PATCH|CONNECT|TRACE) [^\r\n]* user_id ([^ \t\r\n]+)(?: user_agent[^\r\n]*)?[ \t]*$} $text -> id] && [regexp {^[0-9]{1,18}$} $id]} { scan $id %lld value dict set result authenticationState [expr {$value == 0 ? "anonymous" : "authenticated"}] dict set result authenticationSource system-log } return [dict merge $result [::nsmcp::warningUserAgent $text $redactor]] } :public object method emptyOrigins {} { # # Initialize the warning-origin accumulator. # return {recordCount 0 authenticationStates {anonymous 0 authenticated 0 unknown 0} agents {} unknownUserAgentRecords 0 omittedUserAgentRecords 0 truncatedUserAgentRecords 0} } :public object method addOrigin {statsVar metadata capacity} { # # Update the caller's warning-origin accumulator with one record. # upvar 1 $statsVar stats dict incr stats recordCount set auth [dict get $metadata authenticationState] set counts [dict get $stats authenticationStates] dict incr counts $auth dict set stats authenticationStates $counts if {![dict exists $metadata userAgent] || [dict get $metadata userAgent] eq ""} { dict incr stats unknownUserAgentRecords return } set agent [dict get $metadata userAgent] set truncated [dict get $metadata userAgentMayBeTruncated] if {$truncated} { dict incr stats truncatedUserAgentRecords } if {![dict exists $stats agents $agent]} { if {[dict size [dict get $stats agents]] >= $capacity} { dict incr stats omittedUserAgentRecords return } dict set stats agents $agent {count 0 authenticationStates {anonymous 0 authenticated 0 unknown 0} truncatedRecords 0} } set entry [dict get $stats agents $agent] dict incr entry count if {$truncated} { dict incr entry truncatedRecords } set counts [dict get $entry authenticationStates] dict incr counts $auth dict set entry authenticationStates $counts dict set stats agents $agent $entry } :public object method originFields {stats capacity} { # # Encode origin counts and coverage as typed JSON members. # set auth {} dict for {name count} [dict get $stats authenticationStates] { lappend auth $name number $count } set ordered {} dict for {agent entry} [dict get $stats agents] { lappend ordered [list [dict get $entry count] $agent $entry] } set rows {} set index 0 set omittedGroups 0 set omittedRanked 0 foreach pair [lsort -integer -decreasing -index 0 $ordered] { lassign $pair count agent entry if {$index >= 10} { incr omittedGroups incr omittedRanked $count continue } set states {} dict for {name n} [dict get $entry authenticationStates] { lappend states $name number $n } lappend rows $index object [list userAgent string $agent count number $count authenticationStates object $states truncatedRecords number [dict get $entry truncatedRecords] userAgentMayBeTruncated boolean [expr {[dict get $entry truncatedRecords] > 0}]] incr index } set result [list recordCount number [dict get $stats recordCount] authenticationStates object $auth userAgentCounts array $rows userAgentCapacity number $capacity userAgentOutputLimit number 10 omittedUserAgentGroups number $omittedGroups omittedRankedUserAgentRecords number $omittedRanked] foreach name {unknownUserAgentRecords omittedUserAgentRecords truncatedUserAgentRecords} { lappend result $name number [dict get $stats $name] } set aggregationComplete [expr {[dict get $stats unknownUserAgentRecords] == 0 && [dict get $stats omittedUserAgentRecords] == 0 && [dict get $stats truncatedUserAgentRecords] == 0}] set rankingComplete [expr {$aggregationComplete && $omittedRanked == 0}] lappend result userAgentAggregationComplete boolean $aggregationComplete userAgentRankingComplete boolean $rankingComplete userAgentCountsComplete boolean $rankingComplete return $result } :public object method emptyDocumentation {} { # # Initialize counts for missing procedure documentation. # return {recordCount 0 procedures {} unknownProcedureRecords 0 omittedProcedureRecords 0} } :public object method addDocumentation {statsVar procedure timestamp} { # # Count one documentation warning in the caller's accumulator. # upvar 1 $statsVar stats dict incr stats recordCount if {$procedure eq {} || [string length [encoding convertto utf-8 $procedure]]>256} { dict incr stats unknownProcedureRecords } elseif {[dict exists $stats procedures $procedure]} { set entry [dict get $stats procedures $procedure] dict incr entry count dict set entry firstAt [expr {min([dict get $entry firstAt],$timestamp)}] dict set entry lastAt [expr {max([dict get $entry lastAt],$timestamp)}] dict set stats procedures $procedure $entry } elseif {[dict size [dict get $stats procedures]]<128} { dict set stats procedures $procedure [dict create count 1 firstAt $timestamp lastAt $timestamp] } else { dict incr stats omittedProcedureRecords } } :public object method documentationFields {stats} { # # Encode ranked procedure labels and omission counts. # set ordered {} dict for {name entry} [dict get $stats procedures] { lappend ordered [list [dict get $entry count] $name $entry] } set entries {};set index 0;set omittedGroups 0;set omittedRecords 0 foreach item [lsort -integer -decreasing -index 0 [lsort -index 1 $ordered]] { lassign $item count name entry if {$index>=20} { incr omittedGroups incr omittedRecords $count continue } lappend entries $index object [list procedure string $name count number $count firstAt number [dict get $entry firstAt] lastAt number [dict get $entry lastAt]] incr index } set distinct [dict size [dict get $stats procedures]] set complete [expr {[dict get $stats unknownProcedureRecords]==0 && [dict get $stats omittedProcedureRecords]==0}] return [list recordCount number [dict get $stats recordCount] procedures array $entries procedureCapacity number 128 procedureOutputLimit number 20 retainedDistinctProcedures number $distinct distinctProcedures [expr {$complete ? "number" : "null"}] [expr {$complete ? $distinct : {}}] distinctProceduresAtLeast number [expr {$distinct+([dict get $stats omittedProcedureRecords]>0)}] unknownProcedureRecords number [dict get $stats unknownProcedureRecords] omittedProcedureRecords number [dict get $stats omittedProcedureRecords] rankedOmittedGroups number $omittedGroups rankedOmittedRecords number $omittedRecords aggregationComplete boolean $complete rankingComplete boolean [expr {$complete && $omittedGroups==0}] limitations string {Physical missing-documentation records in this severity group, not failed procedure executions or distinct incidents. At most 128 redacted procedure labels are counted and 20 ranked labels returned; names are limited to 256 UTF-8 bytes. Unknown labels, capacity-omitted records and ranked omissions are separate. Names can merge after redaction. Distinct counts are exact only when aggregationComplete; rankings are not global when capacity omissions exist. Counts are not capped at two.}] } :public object method addHost {statsVar value} { # # Count one Host warning in the caller's accumulator. # Preserve case, brackets and ports for bounded, display-safe values. # upvar 1 $statsVar stats dict incr stats recordCount if {[string length $value] > 256 || ![regexp {^(?:[A-Za-z0-9_.-]+(?::[0-9]+)?|\[[0-9A-Fa-f:.]+\](?::[0-9]+)?)$} $value]} { dict incr stats unknownRecords } elseif {[dict exists $stats counts $value] || [dict size [dict get $stats counts]] < 256} { set counts [dict get $stats counts] dict incr counts $value dict set stats counts $counts } else { dict incr stats capacityOmittedRecords } } :public object method hostFields {stats} { # # Encode ranked Host values and omission counts. # set counts [dict get $stats counts] set ranked {} foreach host [lsort [dict keys $counts]] { lappend ranked [list [dict get $counts $host] $host] } set rows {} set index 0 set omitted 0 foreach pair [lsort -integer -decreasing -index 0 $ranked] { lassign $pair count host if {$index < 50} { lappend rows $index object [list value string $host count number $count] incr index } else { incr omitted $count } } set unknown [dict get $stats unknownRecords] set capacity [dict get $stats capacityOmittedRecords] return [list recordCount number [dict get $stats recordCount] values array $rows unknownRecords number $unknown capacityOmittedRecords number $capacity rankedOmittedValues number [expr {[llength $ranked]-$index}] rankedOmittedRecords number $omitted aggregationComplete boolean [expr {$unknown==0 && $capacity==0}] rankingComplete boolean [expr {$unknown==0 && $capacity==0 && $omitted==0}] limitations string {Counts physical invalid or untrusted Host warnings across the selected window. Malformed or unsafe values, clipped records and missing values are unknown. The ranking retains at most 256 distinct values and shows up to 50. Host is a claimed request target. Suppression annotations are accounted for separately.}] } :public object method heading {text} { # # Normalize a warning heading and extract known classification fields. # Keep request values and page titles out of known group keys. # set heading [lindex [split $text \n] 0] regsub {^\[[^\]]+\]\[[^\]]+\]\[[^\]]+\][ \t]+[A-Za-z]+:[ \t]*} $heading {} heading set common {suppressedOccurrences 0} # Strip only recognized trailing suppression annotations. Preserve the # original redacted example; reported omissions never inflate record totals. foreach expression { {[ \t]+\(([0-9]{1,18}) additional messages suppressed(?: since previous log|; see [^\r\n]+)?\)[ \t]*$} {, suppressed ([0-9]{1,18}) repeated notices since last notice[ \t]*$} } { if {[regexp -- $expression $heading -> count]} { scan $count %lld count dict set common suppressedOccurrences $count regsub -- $expression $heading {} heading break } } # Match the fixed emitter prefix before testing arbitrary warning text. if {[regexp -nocase {^(?:Ns_ConnLocation:[ \t]+)?ignore (?:invalid or )?untrusted host header field:[ \t]*(.*)$} $heading -> detail]} { set host {} # Older warnings may append a whitelist suggestion after the value. regexp {^'([^']*)'(?:\.[ \t]+Consider adding this value to 'whitelistedHosts' in the section 'ns/server/[^']+/acs' of your configuration file)?[ \t]*$} $detail -> host return [dict merge $common [dict create category message parameter {} pattern {Host validation: untrusted Host header ignored} hostValue $host]] } set parameter {} foreach expression { {duplicate(?:d)?[ \t]+(?:form[ \t]+)?(?:parameter|variable)[ \t:]+["'<]?([A-Za-z0-9_%.-]+)} {duplicate(?:d)?[ \t]+["']?([A-Za-z0-9_%.-]+)["']?[ \t]+parameter} {supplied[ \t]+(?:two|multiple|[0-9]+)[ \t]+values for[ \t]+["']?([A-Za-z0-9_%.-]+)} {more than one value for[ \t]+["']?([A-Za-z0-9_%.-]+)} } { if {[regexp -nocase -- $expression $heading -> parameter]} { break } } if {$parameter ne ""} { catch { set parameter [ns_urldecode $parameter] } if {[regexp {^[A-Za-z_][A-Za-z0-9_.:-]{0,63}$} $parameter]} { return [dict merge $common [dict create category duplicate-parameter parameter $parameter pattern {Duplicate parameter}]] } } set procedure {} set documentation false foreach expression { {^api_add_to_proc_doc:[ \t]+no proc_doc available for[ \t]+(.+)$} {^(?:.*:[ \t]+)?missing procedure documentation for[ \t]+(.+)$} {^(?:.*:[ \t]+)?no documentation for proc(?:edure)?[ \t]+(.+)$} } { if {[regexp -nocase -- $expression $heading -> procedure]} { set documentation true set procedure [string trim $procedure] break } } if {$documentation || [regexp -nocase {^.*(?:missing procedure documentation|no documentation for proc)} $heading]} { return [dict merge $common [dict create category missing-procedure-documentation parameter {} procedure $procedure pattern {API documentation: missing procedure documentation}]] } if {[regexp {^sched id ([0-9]+): job started late by ([0-9]+(?:\.[0-9]+)?) sec \(flags ([A-Z|]+|none), interval ([0-9]+(?:\.[0-9]+)?) sec\)} $heading -> id delay flags interval]} { set interval [expr {double($interval)}] set timings [dict create nominalLatenessSeconds [expr {double($delay)}]] foreach {label field} { wait-return-late waitReturnLatenessSeconds explicit-mutex-acquisition explicitMutexAcquisitionSeconds dispatch-delay dispatchDelaySeconds } { if {[regexp -- [format {%s ([0-9]+(?:\.[0-9]+)?)s} $label] $heading -> value]} { dict set timings $field [expr {double($value)}] } } if {[regexp {last-nonthread-id -?[0-9]+ duration ([0-9]+(?:\.[0-9]+)?)s} $heading -> value]} { dict set timings lastNonThreadedSeconds [expr {double($value)}] } return [dict merge $common [dict create category scheduler-lateness parameter {} pattern "Scheduler job $id started late (flags $flags, interval ${interval}s)" scheduler [dict create jobId $id flags $flags intervalSeconds $interval] timings $timings]] } if {[regexp -nocase {^last log entry for this thread was repeated ([0-9]{1,18}) times$} $heading -> count]} { return [dict merge $common [dict create category log-repeat-marker parameter {} pattern {Last log entry for this thread was repeated <n> times} repeats $count]] } # Normalize known message templates, not arbitrary quoted application text. # Original redacted examples retain the varying procedure/path/request data. foreach {expression pattern} { {^api_proc_documentation: received invalid proc_name <[^>]*>, try to sanitize$} {API documentation: invalid procedure name; sanitization attempted} {^invalid API proc name .+$} {API documentation: invalid procedure name} {^rp_sources_filter: file ".*" does not exists trying to serve as a normal request$} {Missing resource: rp_sources_filter fallback to normal request} {^REQ BUFFER(?::|[ \t])} {HTTP parser: request-buffer diagnostic} {^someone tried to sneak in an untrusted referrer } {Host validation: untrusted referrer rejected} {^.*x-forwarded-for.*localhost} {Host validation: invalid x-forwarded-for localhost} {^.*(?:unbounded create-entry wait|unbounded.*wait.*create.entry)} {Cache: unbounded create-entry wait} {^.*vhostcertificates.*(?:missing|does not exist|not.*directory)} {Certificate configuration: virtual-host directory unavailable} {^.*(?:missing procedure documentation|no documentation for proc)} {API documentation: procedure documentation unavailable} } { if {[regexp -nocase -- $expression $heading]} { return [dict merge $common [dict create category message parameter {} pattern $pattern]] } } set pattern [string range $heading 0 511] regsub -all {0x[0-9A-Fa-f]+} $pattern {<hex>} pattern regsub -all {\m[0-9]+\M} $pattern {<n>} pattern return [dict merge $common [dict create category message parameter {} pattern $pattern]] } :public object method family {normalized} { # # Classify a normalized heading into a broad warning family. # switch -- [dict get $normalized category] { duplicate-parameter { return duplicate-parameter } scheduler-lateness { return scheduler } log-repeat-marker { return log-repeat-marker } } set heading [dict get $normalized pattern] foreach {family expression} { scheduler {^sched(?: |:)} host-validation {^Host validation:} http-parser {^HTTP parser:} api-documentation {^API documentation:} missing-resource {^Missing resource:} server-runtime {^(?:Cache:|Certificate configuration:)} host-validation {^(?:Ns_ConnLocation: )?(?:ignore.*(?:host header|referrer)|untrusted (?:host|referrer)|someone tried to sneak in an untrusted referrer|Cannot parse provided host header|cannot determine server for request)} tls {^(?:invalid request.*received TLS handshake|.*(?:TLS|SSL|STARTTLS) handshake)} http-parser {^(?:invalid (?:request|ASTERISK)|REQ BUFFER|SockParse:|request (?:line|header (?:line|field)).*(?:long|missing|more than once)|.*request.line.*pars)} name-resolution {^(?:dns:|Ns_?GetHostByAddr|Ns_?GetAddrByHost|getaddrinfo|gethostbyaddr|hostname resolution|reverse.DNS|temporary.*(?:DNS|name.*resolution))} api-documentation {^(?:.*invalid (?:API )?(?:proc|procedure) name|.*(?:proc|procedure) name.*invalid|.*API.*(?:source lookup|documentation)|api_proc_documentation:|.*(?:documentation|doc).*title)} missing-resource {^(?:.*(?:could not|cannot|failed to|missing|not found).*\.(?:js|css|map)|.*(?:missing|non.existent|not found).*resources? (?:path|directory)|.*(?:file|resource).*not found|rp_sources_filter: file)} database {^(?:.*slow.*(?:database|SQL|db operation)|.*(?:database|SQL|db operation).*slow|.*db_.*(?:query|took)|.*(?:search_query|record_view).*took)} request-validation {^(?:ad_page_contract:|.*nonstandard (?:form )?variable|.*invalid (?:UTF.8|form|parameter|signature)|.*(?:index\.vuh|disallowed method|rejected POST))} application {^(?:.*(?:xowiki|Graphviz|record.shape|flat.edge|serializer|cyclic dependency)|.*deprecated.*folder|.*community.*unavailable|no community info)} server-runtime {^(?:.*writing counter is already zero|.*(?:NsWriter|Ns_Writer|writer queue|queue.statistics)|.*unbounded.*create.entry.*wait|.*vhostcertificates)} } { if {[regexp -nocase -- $expression $heading]} { return $family } } return other } :public object method familyCounts {} { # # Initialize counters for all recognized warning families. # return {duplicate-parameter 0 scheduler 0 host-validation 0 tls 0 http-parser 0 name-resolution 0 api-documentation 0 missing-resource 0 database 0 request-validation 0 application 0 server-runtime 0 log-repeat-marker 0 other 0} } :public object method addOverflowSample {sampleVar capacity priority offset fields} { # # Update a bounded sample of records excluded from detailed grouping. # Retain the lowest deterministic CRC-32 priorities for sampling. # upvar 1 $sampleVar sample set entries [dict get $sample entries] if {[llength $entries] >= $capacity && $priority >= [dict get $sample maximum]} { return } lappend entries [list $priority $offset $fields] set entries [lrange [lsort -integer -index 0 $entries] 0 [expr {$capacity-1}]] dict set sample entries $entries dict set sample maximum [lindex $entries end 0] } :public object method overflowExamples {sample} { # # Return retained overflow examples in file order. # set result {} foreach entry [lsort -integer -index 1 [dict get $sample entries]] { lappend result [lindex $entry 2] } return $result }XQL Not present: Generic, PostgreSQL, Oracle
![[i]](/resources/acs-subsite/ZoomIn16.gif)