Class ::github::ActivityMonitor (public)
::nx::Class ::github::ActivityMonitor
Defined in packages/xowiki/tcl/github-activity-monitor-procs.tcl
- Testcases:
- No testcase defined.
Source code: :property {organization OpenACS} :property {api_base "https://api.github.com"} :property {api_token} # Checked by each scheduled sync; no additional scheduler is needed. :property {reconcile_interval 3600} :property {backfill_pages_per_run 2} :property {max_requests_per_run 200} :property {max_seconds_per_run 120} :property {discovery_requests_per_run 20} :property {recent_days 180} # Exact branch names or Tcl glob patterns; empty means no branches. :property {allowed_branches {main oacs-5-10}} #--- get ------------------------------------------------------------------------- :public method get {path {query_args {}}} { # # Issue an API call on GitHub for the configured # organization. # :check_rate_limit :charge_request set url "${:api_base}${path}" # default query arg dict set query per_page 100 foreach {k v} $query_args { dict set query $k $v } set urlDict [ns_parseurl $url] dict set urlDict query [join [lmap {k v} $query {string cat [ns_urlencode -part query $k]=[ns_urlencode -part query $v]}] &] set headers [ns_set create headers Authorization "Bearer ${:api_token}" Accept "application/vnd.github+json" User-Agent "openacs-activity-dashboard/1.0" ] # Do not log the Authorization header. ns_log notice "GitHub GET [ns_joinurl $urlDict]" set timeout 15 if {[info exists :deadline]} { set timeout [expr {min(15, max(1, ${:deadline} - [clock seconds]))}] } set result [ns_http run -timeout $timeout -method GET -headers $headers [ns_joinurl $urlDict]] set status [dict get $result status] set body [dict get $result body] set rate_headers {} if {[dict exists $result headers]} { set response_headers [dict get $result headers] foreach name {x-ratelimit-remaining x-ratelimit-reset retry-after} { dict set rate_headers $name [ns_set iget $response_headers $name] } } :record_rate_limit $status $rate_headers $body if {$status < 200 || $status >= 300} { ns_log Error "GitHub GET $url failed: status $status, body: $body" return -code error -errorcode [list GITHUB HTTP $status] "GitHub API error $status" } return $body } :method check_rate_limit {} { set until [:get_state github_not_before 0] if {[string is wideinteger -strict $until] && $until > [clock seconds]} { set when [clock format $until -gmt 1 -format {%Y-%m-%d %H:%M:%S UTC}] return -code error -errorcode {GITHUB RATE_LIMIT} "GitHub requests paused until $when" } } :method record_rate_limit {status headers body} { set now [clock seconds] set remaining "" set reset "" set retry "" foreach {name variable} { x-ratelimit-remaining remaining x-ratelimit-reset reset retry-after retry } { if {[dict exists $headers $name]} { set $variable [string trim [dict get $headers $name]] } } set exhausted [expr {$remaining eq "0"}] set limited [expr {$exhausted || $status == 429 || ($status == 403 && ($retry ne "" || [regexp -nocase {rate limit|secondary limit|abuse detection} $body]))}] if {!$limited} { if {$status >= 200 && $status < 300 && [:get_state github_rate_backoff 0] ne "0"} { :set_state github_rate_backoff 0 } return } set until 0 if {$exhausted && [string is wideinteger -strict $reset] && $reset > $now} { set until [expr {$reset + 2}] } if {[string is integer -strict $retry] && $retry >= 0} { set until [expr {max($until, $now + $retry + 2)}] } elseif {$retry ne "" && ![catch {clock scan $retry -gmt 1} retry_epoch]} { set until [expr {max($until, $retry_epoch + 2)}] } # Missing/invalid headers, or repeated secondary limits: at least # one minute, doubling on repeated failures up to one hour. if {$until <= $now} { set previous [:get_state github_rate_backoff 0] if {![string is integer -strict $previous] || $previous < 0} {set previous 0} set delay [expr {min(3600, max(60, 2 * $previous))}] set until [expr {$now + $delay}] :set_state github_rate_backoff $delay } set previous [:get_state github_not_before 0] if {[string is wideinteger -strict $previous]} {set until [expr {max($until, $previous)}]} :set_state github_not_before $until ns_log Warning "GitHub: requests paused until [clock format $until -gmt 1 -format {%Y-%m-%d %H:%M:%S UTC}]" } #--- accessing information from activity state ----------------------------------- :method get_state {name {default ""}} { return [xo::dc get_value get_github_activity_state { select value from github_activity_state where name = :name } $default] } :method set_state {name value} { # ns_pg_bind maps an empty Tcl value to SQL NULL. Empty queues # and discovery cursors must be stored as empty text, not NULL. xo::dc dml set_github_activity_state { insert into github_activity_state(name, value) values (:name, coalesce(:value, '')) on conflict (name) do update set value = excluded.value } } #--- fetch_new_events ------------------------------------------------------------ :method fetch_new_events {} { # # Fetch available events since the last processed ID. # Repository reconciliation independently covers missing events. # # Last event we processed, if any set last_event_id [:get_state last_org_event_id ""] #set last_event_id [xo::dc get_value last_id { # select max(event_id) from github_activity #}] set events {} set page 1 set done 0 while {!$done} { set body [:get "/orgs/${:organization}/events" [list page $page]] set batch [::util::json2dict $body] if {[llength $batch] == 0} { break } foreach ev $batch { set ev_id [dict get $ev id] # Events are newest-first; once we reach an older or equal ID, stop. if {$last_event_id ne "" && $ev_id eq $last_event_id} { set done 1 break } if {$last_event_id ne "" && $ev_id < $last_event_id} { ns_log Warning "GitHub: reached page with a smaller ev_id than last_event_id ($ev_id < $last_event_id)" } lappend events $ev } incr page if {$page > 3} { ns_log Warning "GitHub: reached page limit while fetching events; stopping at page $page" break } if {$done} { break } } # We collected in newest-first order; reverse to oldest-first for insertion. return [lreverse $events] } #--- summarize_push_event -------------------------------------------------------- :public method summarize_push_event {ev} { # # Fetch commit details and build a summary for PushEvents. # # @return dict with fields of push event set type [dict get $ev type] if {$type ne "PushEvent"} { return "" } set repo_full [dict get $ev repo name] ;# e.g. "openacs/openacs-core" set ref [dict get $ev payload ref] ;# "refs/heads/main" if {![string match "refs/heads/*" $ref]} { return "" } set branch [string range $ref [string length "refs/heads/"] end] set head_sha [dict get $ev payload head] set created_at [dict get $ev created_at] set actor_login [dict get $ev actor login] if {$repo_full eq "" || $head_sha eq ""} { return "" } # Fetch commit details set commit_body [:get "/repos/$repo_full/commits/$head_sha"] set c [::util::json2dict $commit_body] set commit [dict get $c commit] set author_dict [dict get $commit author] set author_name [dict get $author_dict name] set commit_date [dict get $author_dict date] ;# ISO8601 string set message_full [dict get $commit message] set title [lindex [split $message_full "\n"] 0] set stats [dict get $c stats] set additions [dict get $stats additions] set deletions [dict get $stats deletions] set files [dict get $c files] set files_changed [llength $files] return [list event_id [dict get $ev id] repo $repo_full branch $branch sha [dict get $c sha] author_name $author_name author_login $actor_login commit_date $commit_date created_at $created_at title $title url [dict get $c html_url] files_changed $files_changed additions $additions deletions $deletions raw_payload $commit_body ] } #--- reconciliation and durable backfill jobs ------------------------------------ :method charge_request {} { if {[info exists :requests_left]} { if {${:requests_left} <= 0 || [clock seconds] >= ${:deadline}} { return -code error -errorcode {GITHUB BUDGET} "sync budget exhausted" } incr :requests_left -1 } } :method recent_since {} { return [clock format [expr {[clock seconds] - ${:recent_days} * 86400}] -gmt 1 -format {%Y-%m-%dT%H:%M:%SZ}] } :method discover {jobsVar heads} { upvar 1 $jobsVar jobs set cursor [:get_state repository_discovery {}] set now [clock seconds] if {$cursor eq ""} { if {$now - [:get_state last_repository_reconcile 0] < ${:reconcile_interval}} { return } set cursor [dict create repo_page 1 repos {} branch_page 1] } set allowance [expr {min(${:discovery_requests_per_run}, max(1, ${:max_requests_per_run} / 4))}] set stop [expr {${:requests_left} - $allowance}] while {${:requests_left} > $stop} { set repos [dict get $cursor repos] if {[llength $repos] == 0} { if {[dict exists $cursor repos_done]} { :set_state repository_discovery {} :set_state last_repository_reconcile $now return } set page [dict get $cursor repo_page] set repos [::util::json2dict [:get "/orgs/${:organization}/repos" [list page $page]]] dict incr cursor repo_page dict set cursor repos $repos if {[llength $repos] < 100} {dict set cursor repos_done 1} :set_state repository_discovery $cursor continue } set repo [dict get [lindex $repos 0] full_name] set page [dict get $cursor branch_page] set branches [::util::json2dict [:get "/repos/$repo/branches" [list page $page]]] foreach item $branches { set branch [dict get $item name] set sha [dict get $item commit sha] set key [list $repo $branch] if {![dict exists $heads $key] || [dict get $heads $key] ne $sha} { :queue_backfill jobs $repo $branch $sha } } if {[llength $branches] < 100} { dict set cursor repos [lrange $repos 1 end] dict set cursor branch_page 1 } else { dict incr cursor branch_page } :set_state pending_backfills $jobs :set_state repository_discovery $cursor } } :method branch_enabled {branch} { if {![info exists :allowed_branches]} { set :allowed_branches {main oacs-5-10} } foreach pattern ${:allowed_branches} { if {[string match $pattern $branch]} {return 1} } return 0 } :method queue_backfill {jobsVar repo branch revision} { upvar 1 $jobsVar jobs if {![:branch_enabled $branch]} {return} set key [list $repo $branch] # Preserve progress on a pinned revision. A newer head will be # discovered on the next reconciliation after this job completes. if {![dict exists $jobs $key]} { set heads [:get_state reconciled_branch_heads {}] set base "" if {[dict exists $heads $key]} { set base [dict get $heads $key] } dict set jobs $key [dict create revision $revision next_page 1 base $base since [:recent_since]] } } :public method sync_from_github {} { # An xact-scoped lock stays on the transaction's checked-out DB # connection, is shared across threads/processes, and is released # automatically on commit, rollback or connection loss. ::xo::dc transaction { set locked [::xo::dc get_value github_sync_lock { select case when pg_try_advisory_xact_lock(714230, 1) then 1 else 0 end }] if {$locked} { :sync_locked } else { ns_log Notice "GitHub: sync skipped; another activity writer is running" } } } :method sync_locked {} { if {[catch {:check_rate_limit} message]} { ns_log Notice "GitHub: sync skipped; $message" return } foreach {name default} { reconcile_interval 3600 backfill_pages_per_run 2 max_requests_per_run 200 max_seconds_per_run 120 discovery_requests_per_run 20 recent_days 180 } { if {![info exists :$name]} {set :$name $default} if {![string is integer -strict [set :$name]] || [set :$name] < 1} { error "$name must be a positive integer" } } set :requests_left ${:max_requests_per_run} set :deadline [expr {[clock seconds] + ${:max_seconds_per_run}}] try { :sync_work } finally { unset :requests_left :deadline } } :method sync_work {} { ns_log Notice "GitHub: refresh_activity start" set jobs [:get_state pending_backfills {}] set heads [:get_state reconciled_branch_heads {}] # Convert old unbounded jobs once. Page numbers of the recent # filtered list differ, so restart jobs without a baseline. set removed 0 dict for {key job} $jobs { lassign $key repo branch if {![:branch_enabled $branch]} { dict unset jobs $key incr removed continue } if {![dict exists $job since]} { dict set jobs $key since [:recent_since] if {![dict exists $job base] || [dict get $job base] eq ""} { dict set jobs $key next_page 1 } } } :set_state pending_backfills $jobs if {$removed > 0} { ns_log Notice "GitHub: removed $removed excluded branch job(s) from pending queue" } if {[catch {:discover jobs $heads} message options]} { ns_log Warning "GitHub: discovery deferred: $message" } set new_events {} if {[catch {set new_events [:fetch_new_events]} errorMsg]} { ns_log Error "GitHub: fetching events failed: $errorMsg" } foreach ev $new_events { if {[dict get $ev type] ne "PushEvent"} { continue } set ref [dict get $ev payload ref] if {![string match "refs/heads/*" $ref]} { continue } set repo [dict get $ev repo name] set branch [string range $ref [string length "refs/heads/"] end] set revision [dict get $ev payload head] # Deleted branches have a zero head, with no history to fetch. if {$revision eq "" || [regexp {^0+$} $revision]} { continue } :queue_backfill jobs $repo $branch $revision } # Checkpointing events is safe only after their work is durable. :set_state pending_backfills $jobs if {[llength $new_events] > 0} { :set_state last_org_event_id [dict get [lindex $new_events end] id] } dict for {key job} $jobs { if {${:requests_left} <= 0 || [clock seconds] >= ${:deadline}} {break} lassign $key repo branch set revision [dict get $job revision] # Missing baselines use the job's fixed recent-history window. set base "" if {[dict exists $job base]} {set base [dict get $job base]} if {[catch { set progress [:backfill_locked -repo $repo -branch $branch -revision $revision -base $base -since [dict get $job since] -start_page [dict get $job next_page] -max_pages ${:backfill_pages_per_run}] } errorMsg options]} { # Rotate unfinished/failed jobs so one large branch cannot # monopolize the first requests on every invocation. dict unset jobs $key dict set jobs $key $job :set_state pending_backfills $jobs if {[dict get $options -errorcode] in { {GITHUB BUDGET} {GITHUB RATE_LIMIT} {GITHUB HTTP 403} {GITHUB HTTP 429} }} {break} ns_log Error "GitHub: backfill failed for $repo $branch; retained for retry: $errorMsg" continue } if {[dict get $progress complete]} { dict set heads $key $revision # Save the successful head before removing its retry job. :set_state reconciled_branch_heads $heads dict unset jobs $key } else { dict unset jobs $key dict set job next_page [dict get $progress next_page] dict set job base [dict get $progress base] dict set jobs $key $job } :set_state pending_backfills $jobs } ns_log Notice "GitHub: refresh_activity done; [dict size $jobs] pending backfill job(s)" } #--- next_backfill_event_id ------------------------------------------------------ :method next_backfill_event_id {} { # # Helper for backfill_repo_history. Returns the next # negative event_id to use. # # smallest existing event_id (could be null, positive, or already negative) set min_id [xo::dc get_value min_event_id { select min(event_id) from github_activity }] if {$min_id eq "" || $min_id >= 0} { # no rows, or only positive IDs so far -> start at -1 return -1 } else { # already have negative IDs -> one step lower return [expr {$min_id - 1}] } } #--- backfill_repo_history ------------------------------------------------------- :public method backfill_repo_history {args} { ::xo::dc transaction { set locked [::xo::dc get_value github_backfill_lock { select case when pg_try_advisory_xact_lock(714230, 1) then 1 else 0 end }] if {!$locked} { error "GitHub activity writer already running; retry later" } # Catch inside the transaction so a cooldown learned from an # HTTP failure commits even when this explicit call fails. set code [catch { :check_rate_limit :backfill_locked {*}$args } result options] } if {$code} {return -options $options $result} return $result } :method backfill_locked {-repo {-branch main} {-revision ""} {-base ""} {-since ""} {-start_page 1} {-max_pages 50}} { # # Backfill commit history for a single repo/branch using # /repos/{repo}/commits. Uses synthetic negative event_id values. # # repo e.g. "openacs/openacs-core" # branch e.g. "main" or "oacs-5-10" # if {![string is integer -strict $start_page] || $start_page < 1 || ![string is integer -strict $max_pages] || $max_pages < 1} { error "start_page and max_pages must be positive integers" } if {$revision eq ""} { set revision $branch } ns_log Notice "GitHub backfill_repo_history: $repo ($branch)" # We’ll assign event_id from next negative downwards. set next_event_id [:next_backfill_event_id] set page $start_page set pages_done 0 set done 0 set count 0 while {!$done} { if {$pages_done >= $max_pages} { ns_log notice "GitHub backfill_repo_history: reached" "max_pages=$max_pages for $repo ($branch)" break } # Compare against the successfully reconciled head. Do not # simply break when that SHA appears in a date-ordered commit # list: newly merged side branches may have older timestamps. set batch "" set full_history [expr {$base eq ""}] if {!$full_history} { if {[catch { set comparison [::util::json2dict [:get "/repos/$repo/compare/${base}...${revision}" [list page $page per_page 20]]] } message options]} { if {[dict get $options -errorcode] ne {GITHUB HTTP 404}} { return -options $options $message } # Missing base or unrelated rewritten history: scan # the pinned revision in full. Transient errors retry. set full_history 1 } elseif {[dict get $comparison status] in {ahead identical}} { set batch [dict get $comparison commits] } else { # Diverged or rewound branch: conservatively rescan. set full_history 1 } } if {$full_history} { if {$base ne ""} { # Comparison and full-history page numbers describe # different lists. Restart before persisting fallback. set base "" set page 1 } set query [list sha $revision page $page per_page 20] if {$since ne ""} {lappend query since $since} set body [:get "/repos/$repo/commits" $query] set batch [::util::json2dict $body] } if {[llength $batch] == 0} { ns_log notice "GitHub backfill_repo_history: no more commits" at page $page for $repo ($branch) set done 1 break } # The public entry point owns the transaction and lock. # Avoid nested transaction abort semantics on budget exhaustion. foreach summary $batch { set sha [dict get $summary sha] # Skip duplicates; a commit shared with another branch # does not prove that all older history is present. set exists [xo::dc 0or1row exists_commit { select 1 from github_activity where repo = :repo and sha = :sha limit 1 }] if {$exists} { continue } incr count # # Fetch full commit details (stats, files, etc.), # similar to summarize_push_event. # set commit_body [:get "/repos/$repo/commits/$sha"] set c [::util::json2dict $commit_body] set commit [dict get $c commit] #ns_log notice "commit keys: [dict keys $commit]" ns_log notice "(page $page, count $count) commit.message($count):" [dict get $commit message] set author_dict [dict get $commit author] set author_name [dict get $author_dict name] set commit_date [dict get $author_dict date] set stats [dict get $c stats] set additions [dict get $stats additions] set deletions [dict get $stats deletions] set files [dict get $c files] set files_changed [llength $files] set message_full [dict get $commit message] if {$message_full eq ""} { # Fallback: don’t leave title NULL, DB wants NOT NULL set title "(no commit message)" } else { set title [lindex [split [string trimleft $message_full] \n] 0] } #ns_log notice "commit.title: $title" if {[dict exists $c html_url]} { set url [dict get $c html_url] } else { # construct GitHub web URL as a fallback set url "https://github.com/$repo/commit/$sha" } # Synthetic negative event_id set event_id $next_event_id incr next_event_id -1 set created_at $commit_date ;# for backfill, we just reuse commit time ::xo::dc dml insert_backfill { insert into github_activity ( event_id, repo, branch, sha, author_name, author_login, commit_date, created_at, title, url, files_changed, additions, deletions, raw_payload ) values ( :event_id, :repo, :branch, :sha, :author_name, NULL, :commit_date, :created_at, :title, :url, :files_changed, :additions, :deletions, :commit_body ) on conflict (repo, sha) do nothing } } if {$done} { break } incr page incr pages_done if {[llength $batch] < 20} { set done 1 } } ns_log notice "GitHub backfill_repo_history: done for $repo ($branch)" return [dict create complete $done next_page $page base $base] }XQL Not present: Generic, PostgreSQL, Oracle
![[i]](/resources/acs-subsite/ZoomIn16.gif)