apm_git_build_repository (private)

 apm_git_build_repository [ -debug ] [ -force_fresh ] \
    [ -channels channels ] [ -min_final_version min_final_version ] \
    [ -min_compat_version min_compat_version ]

Defined in packages/acs-admin/tcl/apm-admin-procs.tcl

Rebuild the repository on the local machine. Only useful for the openacs.org site. Adapted from the CVS implementation, which came from Lars' build-repository.tcl page.

Switches:

-debug (optional, boolean, defaults to "0")

Set to 1 to test with only a small subset of packages and branches instead of all of them.

-force_fresh (optional, boolean, defaults to "false")

Force a frech clone of the Git repos.

-channels (optional, defaults to "*")

A string match style pattern. Generate apm files for the matching channels only

-min_final_version (optional, defaults to "5.8.0")

-min_compat_version (optional, defaults to "5.3.0")

Partial Call Graph (max 5 caller/called nodes):

Testcases:

No testcase defined.

Source code:

    #----------------------------------------------------------------------
    # Configuration Settings
    #----------------------------------------------------------------------

    set sep [file separator]

    set cd_helper              [file join $::acs::rootdir bin cd-helper]

    set work_dir               [file join $::acs::rootdir repository-builder]${sep}

    set repository_dir         [file join $::acs::rootdir www repository]${sep}
    set repository_url         /repository/

    set exclude_package_list {}

    set channel_index_template [template::themed_template /packages/acs-admin/www/apm/repository-channel-index]
    set index_template         [template::themed_template /packages/acs-admin/www/apm/repository-index]


    #
    # Make sure workdir exists. Clear it before we start if requested.
    #
    if {$force_fresh_p} {
        file delete -force -- $work_dir
    }

    file mkdir $work_dir

    #----------------------------------------------------------------------
    # Prepare output
    #----------------------------------------------------------------------

    ns_log Debug "Repository: Building Package Repository"

    #----------------------------------------------------------------------
    # Find available channels
    #----------------------------------------------------------------------

    #
    # The core repo is considered the source of truth concerning
    # available channels. We fetch it first.
    #
    apm_git_fetch_repo -path $work_dir -repo openacs-core
    set core_repo_dir ${work_dir}openacs-core

    #
    # Channels that exist both from tags and from branches will be
    # taken from tags.
    #
    # Among tags, the compat one will have precedence over the final
    # one.
    #
    set core_channels [list]
    foreach tag [apm_git_repo_tags -path $core_repo_dir] {
        if {[regexp {^openacs-(.*)-(final|compat)} $tag _ channel type]} {
            if {![dict exists $core_channels $channel] ||
                $type eq "compat"
            } {
                dict set core_channels $channel $tag
            }
        }
    }
    #
    # The latest release branch is special. It will have precedence
    # over the corresponding tag: this way people will get a fresher
    # version.
    #
    set branches [lsort -dictionary [apm_git_repo_branches -path $core_repo_dir]]
    set latest_branch [lindex $branches end]
    foreach branch $branches {
        regsub {^oacs-} $branch {} channel
        if {![dict exists $core_channels $channel] ||
            $branch eq $latest_branch
        } {
            dict set core_channels $channel $branch
        }
    }

    #
    # We don't want to generate a channel for ancient versions of
    # packages. Here we remove those channels that are too old. For
    # some old versions, we will only generate the compat packages.
    #
    foreach {channel branch} $core_channels {
        regsub -all -- - $channel {.} channel_version
        if {([regexp {^.*-final} $branch] &&
             [apm_version_names_compare $channel_version $min_final_version] == -1)
            ||
            [apm_version_names_compare $channel_version $min_compat_version] == -1
        } {
            dict unset core_channels $channel
        }
    }

    #
    # The HEAD channel is always included.
    #
    lappend core_channels HEAD HEAD

    if {$debug_p} {
        #
        # When debugging, only pick the last branch.
        #
        set core_channels [lrange $core_channels end-1 end]
    }

    ns_log notice "Repository channels:" $core_channels

    #
    # The core packages are those included in the openacs-core
    # repository.
    #
    set core_packages_dir ${core_repo_dir}${sep}packages

    set core_packages [list]
    foreach package_folder [glob  -types d  -directory $core_packages_dir *] {
        lappend core_packages [file tail $package_folder]
    }
    ns_log notice "Core packages:" $core_packages

    set non_core_packages_dir ${work_dir}openacs-non-core${sep}
    file mkdir $non_core_packages_dir

    #
    # This is the list of all packages that are not included in the
    # openacs-core repository. We currently maintain this list as
    # hardcoded here. One improvement would be to fetch it from the
    # Git host directly, either via scraping or via API.
    #
    # As long as this does not change, every time a new package is
    # added to the Git mirror, one should also add the corresponding
    # package key to this list.
    #
    set non_core_packages {
        accounts-desk
        accounts-finance
        accounts-ledger
        accounts-payables
        accounts-payroll
        accounts-receivables
        acs-datetime
        acs-events
        acs-interface
        acs-ldap-authentication
        acs-mail
        acs-notification
        acs-object-management
        acs-object-management
        acs-outdated
        acs-person
        address-book
        adserver
        ae-portlet
        ajax-filestorage-ui
        ajax-photoalbum-ui
        ajaxhelper
        ams
        anon-eval
        application-track
        application-track-portlet
        assessment
        assessment-portlet
        attachments
        attendance
        auth-cas
        auth-http
        auth-ldap
        auth-pam
        auth-server
        authorize-gateway
        bboard-portlet
        beehive
        beehive-portlet
        bm-portlet
        bookmarks
        bookshelf
        boomerang
        bootstrap-icons
        bug-tracker
        bulk-mail
        caldav
        calendar
        calendar-includelet
        calendar-portlet
        captcha
        cards
        cards-portlet
        categories
        chat
        chat-includelet
        chat-portlet
        clickthrough
        clipboard
        cms
        cms-news-demo
        connections
        contacts
        contacts-lite
        contacts-portlet
        content-includelet
        content-portlet
        cookie-consent
        courses
        cronjob
        curriculum
        curriculum-central
        curriculum-portlet
        curriculum-tracker
        customer-service
        datamanager
        datamanager-portlet
        dbm
        diagram
        directory
        docker-s6
        dotfolio
        dotfolio-ui
        dotkul
        dotkul-admin
        dotlrn
        dotlrn-admin
        dotlrn-ae
        dotlrn-application-track
        dotlrn-assessment
        dotlrn-attendance
        dotlrn-bboard
        dotlrn-beehive
        dotlrn-bm
        dotlrn-calendar
        dotlrn-cards
        dotlrn-catalog
        dotlrn-chat
        dotlrn-contacts
        dotlrn-content
        dotlrn-curriculum
        dotlrn-datamanager
        dotlrn-dotlrn
        dotlrn-ecommerce
        dotlrn-edit-this-page
        dotlrn-eduwiki
        dotlrn-evaluation
        dotlrn-expense-tracking
        dotlrn-faq
        dotlrn-forums
        dotlrn-fs
        dotlrn-glossar
        dotlrn-homework
        dotlrn-imsld
        dotlrn-invoices
        dotlrn-jabber
        dotlrn-lamsint
        dotlrn-latest
        dotlrn-learning-content
        dotlrn-lorsm
        dotlrn-messages
        dotlrn-mmplayer
        dotlrn-news
        dotlrn-news-aggregator
        dotlrn-photo-album
        dotlrn-portlet
        dotlrn-project-manager
        dotlrn-quota
        dotlrn-random-photo
        dotlrn-recruiting
        dotlrn-research
        dotlrn-static
        dotlrn-survey
        dotlrn-syllabus
        dotlrn-tasks
        dotlrn-user-tracking
        dotlrn-weblogger
        dotlrn-wikipedia
        dotlrn-wps
        dotlrn-xowiki
        dotlrndoc
        download
        dynamic-types
        ec-serial-numbers
        ecommerce
        edit-this-page
        edit-this-page-portlet
        eduwiki
        eduwiki-portlet
        email-handler
        evaluation
        evaluation-portlet
        expense-tracking
        expenses
        ezic-gateway
        fa-icons
        fabrik
        facebook-api
        faq
        faq-portlet
        feed-parser
        file-manager
        file-storage
        file-storage-includelet
        forums
        forums-includelet
        forums-portlet
        fs-portlet
        gatekeeper
        general-comments
        glossar
        glossar-portlet
        glossary
        highcharts
        image-magick
        ims-ent
        imsld
        imsld-portlet
        inventory-control
        invoices
        invoices-portlet
        jabber
        jabber-portlet
        lab-report
        lab-report-central
        lams-conf
        lamsint
        lamsint-portlet
        lars-blogger
        latest
        latest-portlet
        layout-managed-subsite
        layout-manager
        learning-content
        learning-content-portlet
        logger
        lors
        lors-central
        lorsm
        lorsm-includelet
        lorsm-portlet
        mail-tracking
        messages
        messages-portlet
        mmplayer
        mmplayer-portlet
        monitoring
        new-portal
        news
        news-aggregator
        news-aggregator-portlet
        news-includelet
        news-portlet
        notes
        oacs-dav
        oct-election
        online-catalog
        openacs-bootstrap3-theme
        openacs-bootstrap5-theme
        openfts-driver
        organizations
        package-builder
        page
        pages
        payflowpro
        payment-gateway
        photo-album
        photo-album-portlet
        places
        planner
        poll
        postal-address
        postcard
        press
        proctoring-support
        profile-provider
        project-manager
        project-manager-portlet
        quota
        quota-portlet
        random-photo-portlet
        ratings
        recruiting
        recruiting-portlet
        redirect
        ref-currency
        ref-gifi
        ref-itu
        ref-unspec
        ref-us-counties
        ref-us-states
        ref-us-zipcodes
        related-items
        research-portlet
        richtext-ckeditor4
        richtext-ckeditor5
        richtext-tinymce
        richtext-xinha
        robot-detection
        rss-support
        rules
        s5
        sample-gateway
        schema-browser
        scholarship-fund
        scorm-core
        scorm-importer
        scorm-player
        scorm-simple-lms
        shipping-gateway
        shipping-tracking
        simple-survey
        simulation
        site-wide-search
        skin
        sloan-bboard
        soap-db
        soap-gateway
        spam
        spreadsheet
        static-pages
        static-portlet
        survey
        survey-builder-ui
        survey-library
        survey-portlet
        survey-reports
        t-account
        tasks
        tasks-portlet
        telecom-number
        theme-selva
        theme-zen
        timezones
        trackback
        tracker
        tsoap
        user-preferences
        user-profile
        user-tracking
        user-tracking-portlet
        value-based-shipping
        version-control
        views
        weblogger-portlet
        webmail
        webmail-system
        wiki
        wikipedia
        wikipedia-portlet
        workflow
        wp-slim
        wps-portlet
        xcms-ui
        xml-rpc
        xolp
        xooauth
        xotcl-core
        xotcl-request-monitor
        xowf
        xowf-monaco-plugin
        xowiki
        xowiki-includelet
        xowiki-portlet
    }

    if {$debug_p} {
        #
        # When debugging, pick only a subset of all packages.
        #
        set non_core_packages [lrange $non_core_packages 0 10]
    }

    foreach package_key $non_core_packages {
        apm_git_fetch_repo -path $non_core_packages_dir -repo $package_key
    }


    #----------------------------------------------------------------------
    # Read all package .info files, building manifest file
    #----------------------------------------------------------------------

    set update_pretty_date [lc_time_fmt [clock format [clock seconds] -format "%Y-%m-%d %T"] %c]

    foreach {channel branch} $core_channels {
        ns_log Notice "Repository: Channel $channel using branch $branch"

        #
        # Checkout the channel branch on the core repository.
        #
        apm_git_checkout_repo -path $core_repo_dir -branch $branch

        #
        # Try to check out the channel from the non-core packages.
        #
        set branch_packages [list]
        foreach package_key $non_core_packages {
            set package_dir ${non_core_packages_dir}${package_key}
            if {![file isdirectory $package_dir]} {
                ns_log notice "Package '$package_key' was not cloned in '$package_dir', skipping."
                continue
            }

            #
            # Not all packages will have a release branch. Skip the
            # package when the branch is not found.
            #
            if {$branch in [apm_git_repo_channels -path $package_dir]} {
                apm_git_checkout_repo -path $package_dir -branch $branch
                lappend branch_packages $package_key
            }
        }

        #
        # Now collect the info files for all core and non-core
        # packages belonging to this branch.
        #
        set info_files [list]
        foreach package_key $core_packages {
            if {[catch {
                set info_file [apm_package_info_file_path -path $core_packages_dir $package_key]
            } errmsg]} {
                ns_log warning "Cannot find an .info file on '$branch' for core package '$package_key':" $errmsg
                continue
            }

            lappend info_files $info_file
        }
        foreach package_key $branch_packages {
            if {[catch {
                set info_file [apm_package_info_file_path -path $non_core_packages_dir $package_key]
            } errmsg]} {
                ns_log warning "Cannot find an .info file on '$branch' for non.core package '$package_key':" $errmsg
                continue
            }

            lappend info_files $info_file
        }

        # Prepare channel directory
        set channel_dir "${work_dir}repository${sep}${channel}${sep}"
        file mkdir $channel_dir

        set manifest "<manifest>\n"

        template::multirow create packages  package_path package_key version pretty_name  package_type summary description  release_date vendor_url vendor  maturity maturity_text  license license_url download_url

        set packages [list]

        foreach spec_file [lsort $info_files] {

            set package_path [file join {*}[lrange [file split $spec_file] 0 end-1]]
            set package_key [lindex [file split $spec_file] end-1]

            if { $package_key in $exclude_package_list } {
                ns_log Debug "Repository: Package $package_key is on list of packages to exclude - skipping"
                continue
            }

            unset -nocomplain pkg_info

            ad_try {
                array set pkg_info [apm_read_package_info_file $spec_file]

                if { $pkg_info(package.key) in $packages } {
                    ns_log Debug "Repository: Skipping package $package_key, because we already have another version of it"
                } else {
                    lappend packages $pkg_info(package.key)

                    append manifest  "  <package>" \n  "    <package-key>[ns_quotehtml $pkg_info(package.key)]</package-key>\n"  "    <version>[ns_quotehtml $pkg_info(name)]</version>\n"  "    <pretty-name>[ns_quotehtml $pkg_info(package-name)]</pretty-name>\n"  "    <package-type>[ns_quotehtml $pkg_info(package.type)]</package-type>\n"  "    <summary>[ns_quotehtml $pkg_info(summary)]</summary>\n"  "    <description format=\"[ns_quotehtml $pkg_info(description.format)]\">"  [ns_quotehtml $pkg_info(description)] "</description>\n"  "    <release-date>[ns_quotehtml $pkg_info(release-date)]</release-date>\n"  "    <vendor url=\"[ns_quotehtml $pkg_info(vendor.url)]\">"  [ns_quotehtml $pkg_info(vendor)] "</vendor>\n"  "    <license url=\"[ns_quotehtml $pkg_info(license.url)]\">"  [ns_quotehtml $pkg_info(license)] "</license>\n"  "    <maturity>$pkg_info(maturity)</maturity>\n"

                    foreach e $pkg_info(install) {
                        append manifest "    <install package=\"$e\"/>\n"
                    }

                    set apm_file "${channel_dir}${pkg_info(package.key)}-${pkg_info(name)}.apm"
                    ns_log Notice "Repository: Building package $package_key for channel $channel"

                    set files [apm_get_package_files  -all  -include_data_model_files  -all_db_types  -package_key $pkg_info(package.key)  -package_path $package_path]

                    if { [llength $files] == 0 } {
                        ns_log Notice "Repository: No files in package"
                    } else {
                        ns_log Notice "Repository: [llength $files] files in package $pkg_info(package.key) ($channel)"
                        set cmd [list exec [apm_tar_cmd] cf -  2>/dev/null]

                        # The path to the 'packages' directory in the checkout
                        set packages_root_path [file join {*}[lrange [file split $spec_file] 0 end-2]]

                        set fp [ad_opentmpfile tmp_filename]
                        foreach file $files {
                            puts $fp $package_key/$file
                        }
                        close $fp

                        lappend cmd -C $packages_root_path --files-from $tmp_filename

                        lappend cmd "|" [apm_gzip_cmd] -c ">" $apm_file
                        ns_log Notice "Executing: exec $cd_helper $packages_root_path $cmd"
                        if {[catch "exec $cd_helper $packages_root_path $cmd" errmsg]} {
                            ns_log Error "Error during tar in repository creation for file ${channel_dir}$pkg_info(package.key)-$pkg_info(name).apm: \n$errmsg\n$::errorCode,$::errorInfo"
                        }
                        file delete -- $tmp_filename
                    }

                    set apm_url "${repository_url}$channel/$pkg_info(package.key)-$pkg_info(name).apm"

                    template::multirow append packages  $package_path $package_key $pkg_info(name) $pkg_info(package-name)  $pkg_info(package.type) $pkg_info(summary) $pkg_info(description)  $pkg_info(release-date) $pkg_info(vendor.url) $pkg_info(vendor)  $pkg_info(maturity) $pkg_info(maturity_text)  $pkg_info(license)  $pkg_info(license.url) $apm_url

                    append manifest "    <download-url>$apm_url</download-url>\n"
                    foreach elm $pkg_info(provides) {
                        append manifest "    <provides "  "url=\"[ns_quotehtml [lindex $elm 0]]\" "  "version=\"[ns_quotehtml [lindex $elm 1]]\" />\n"
                    }

                    foreach elm $pkg_info(requires) {
                        append manifest "    <requires "  "url=\"[ns_quotehtml [lindex $elm 0]]\" "  "version=\"[ns_quotehtml [lindex $elm 1]]\" />\n"
                    }
                    append manifest "  </package>\n"
                }
            } on error {errorMsg} {
                ns_log Notice "Repository: Error on spec_file $spec_file: $errorMsg\n$::errorInfo\n"
            }
        }

        append manifest "</manifest>\n"

        ns_log Notice "Repository: Writing $channel manifest to ${channel_dir}manifest.xml"
        set fw [open "${channel_dir}manifest.xml" w]
        puts $fw $manifest
        close $fw

        ns_log Notice "Repository: Writing $channel index page to ${channel_dir}index.adp"
        set fw [open "${channel_dir}index.adp" w]
        set packages [lsort $packages]
        puts $fw "<master>\n<property name=\"doc(title)\">OpenACS $channel Compatible Packages</property>\n\n"
        puts $fw "<h1>OpenACS $channel (Git branch $branch)</h1>
           <p>Packages can be installed with the OpenACS Automated Installer on
           your OpenACS site at <code>/acs-admin/install</code>.  Only packages
           potentially compatible with your OpenACS kernel will be shown.</p>
        "
        set category_title(core) "Core Packages"
        set package_keys(core) $core_packages

        set category_title(common-app) "Common Applications"
        set package_keys(common-app) {
            xowiki
            xotcl-request-monitor
            file-storage
            acs-developer-support
            forums
            calendar
            news
            faq
        }

        set category_title(extra) "Extra Packages and Libraries"
        set package_keys(extra) ""
        foreach p $packages {
            if {$p ni $package_keys(core) && $p ni $package_keys(common-app)} {
                lappend package_keys(extra) $p
            }
        }

        foreach category {core common-app extra} {

            template::multirow create pkgs  package_path package_key version pretty_name  package_type summary description  release_date vendor_url vendor  maturity maturity_text  license license_url download_url

            template::multirow foreach packages {
                if {$package_key in $package_keys($category)} {
                    template::multirow append pkgs  $package_path $package_key $version $pretty_name  $package_type $summary $description  $release_date $vendor_url $vendor  $maturity $maturity_text  $license $license_url $download_url
                }
            }

            puts $fw "\n<h2>$category_title($category)</h2>\n"

            puts $fw [template::adp_include $channel_index_template  [list channel $channel &pkgs pkgs update_pretty_date $update_pretty_date]]

        }
        close $fw

        ns_log Notice "Repository:  Channel $channel complete."

    }

    ns_log Notice "Repository: Finishing Repository"

    foreach channel [dict keys $core_channels] {
        if {[regexp {^([1-9][0-9]*)-([0-9]+)$} $channel . major minor]} {
            #
            # *-compat channels: The "patchlevel" of these channels is
            # the highest possible value, higher than the released
            # -final channels.
            #
            set tag_order([format %.3d $major]-[format %.3d $minor]-999) $channel
            set tag_label($channel) "OpenACS $major.$minor"
        } elseif {[regexp {^([1-9][0-9]*)-([0-9]+)-([0-9]+)$} $channel . major minor patch]} {
            #
            # *-final channels: a concrete patchlevel is provided.
            #
            set tag_order([format %.3d $major]-[format %.3d $minor]-[format %.3d $patch]) $channel
            set tag_label($channel) "OpenACS $major.$minor.$patch"
        } else {
            set tag_order(999-999-999) $channel
            set tag_label($channel) "OpenACS $channel"
        }
    }


    # Write the index page
    ns_log Notice "Repository: Writing repository index page to ${work_dir}repository/index.adp"
    template::multirow create channels name tag label
    foreach key [lsort -decreasing [array names tag_order]] {
        set channel $tag_order($key)
        template::multirow append channels $channel [dict get $core_channels $channel] $tag_label($channel)
    }
    set fw [open "${work_dir}repository/index.adp" w]
    puts $fw "<master>\n<property name=\"doc(title)\">OpenACS Package Repository</property>\n\n"
    puts $fw [template::adp_include -- $index_template  [list &channels channels update_pretty_date $update_pretty_date]]
    close $fw

    # Add a redirector for outdated releases
    set fw [open "${work_dir}repository/index.vuh" w]
    puts $fw "ns_returnredirect /repository/"
    close $fw

    # Without the trailing slash
    set work_repository_dirname "${work_dir}repository"
    set repository_dirname [string range $repository_dir 0 end-1]
    set repository_bak "[string range $repository_dir 0 end-1]_bak"

    ns_log Notice "Repository: Moving work repository $work_repository_dirname to live repository dir at <a href=\"/repository\/>$repository_dir</a>\n"

    if { [file exists $repository_bak] } {
        file delete -force -- $repository_bak
    }
    if { [file exists $repository_dirname] } {
        file rename -- $repository_dirname $repository_bak
    }
    file rename -- $work_repository_dirname  $repository_dirname

    ns_log Debug "Repository: DONE"

    return 0

XQL Not present:

PostgreSQL, Oracle

Generic XQL file:

packages/acs-admin/tcl/apm-admin-procs.xql