apm_build_repository (private)

 apm_build_repository [ -debug ] [ -channels channels ] \
    [ -head_channel head_channel ]

Defined in packages/acs-admin/tcl/apm-admin-procs.tcl

Rebuild the repository on the local machine. Only useful for the openacs.org site. Adapted from Lars' build-repository.tcl page.

Switches:

-debug (optional, boolean, defaults to "0")

Set to 1 to test with only a small subset of packages instead of the whole cvs tree.

-channels (optional, defaults to "*")

Generate apm files for the matching channels only

-head_channel (optional, defaults to "HEAD")

The artificial branch label to apply to HEAD. Should be one minor version past the current release.

Returns:

0 for success. Also outputs debug messages to log.

Author:

Lars Pind <lars@collaboraid.biz>

Partial Call Graph (max 5 caller/called nodes):

Testcases:

No testcase defined.

Source code:

    #----------------------------------------------------------------------
    # Configuration Settings
    #----------------------------------------------------------------------

    set cd_helper              [file join $::acs::rootdir bin cd-helper]

    set cvs_command            cvs
    set cvs_root               :pserver:anonymous@cvs.openacs.org:/cvsroot

    set work_dir               [file join $::acs::rootdir repository-builder][file separator]

    set repository_dir         [file join $::acs::rootdir www repository][file separator]
    set repository_url         https://openacs.org/repository/

    set exclude_package_list {}

    set channel_index_template [template::themed_template /packages/acs-admin/www/apm/repository-channel-index]
    set index_template         [template::themed_template /packages/acs-admin/www/apm/repository-index]

    #----------------------------------------------------------------------
    # Prepare output
    #----------------------------------------------------------------------

    ns_log Debug "Repository: Building Package Repository"

    #----------------------------------------------------------------------
    # Find available channels
    #----------------------------------------------------------------------

    # Prepare work dir
    file mkdir $work_dir

    cd $work_dir
    set msg [ exec $cd_helper $work_dir $cvs_command -d $cvs_root -z3 co openacs-4/readme.txt ]
    set output [ exec $cd_helper $work_dir $cvs_command -d $cvs_root -z3 log -h openacs-4/readme.txt ]

    set lines [split $output \n]
    for { set i 0 } { $i < [llength $lines] } { incr i } {
        if { [string trim [lindex $lines $i]] eq "symbolic names:" } {
            incr i
            break
        }
    }

    array set channel_tag [list]
    array set channel_bugfix_version [list]

    for { } { $i < [llength $lines] } { incr i } {
        # Tag lines have the form   tag: cvs-version
        #     openacs-5-0-0-final: 1.25.2.5

        if { ![regexp {^\s+([^:]+):\s+([0-9.]+)} [lindex $lines $i] match tag_name version_name] } {
            break
        }

        # Look for tags named 'openacs-x-y-compat'
        if { [regexp {^openacs-([1-9][0-9]*-[0-9]+)-compat$} $tag_name match oacs_version] } {
            lassign [split $oacs_version "-"] major_version minor_version
            if { $major_version >= 5 && $minor_version >= 3} {
                set channel "${major_version}-${minor_version}"
                ns_log Notice "Repository: Found channel $channel using tag $tag_name"
                set channel_tag($channel) $tag_name
            }
        } elseif { [regexp {^openacs-([1-9][0-9]*-[0-9]+-[0-9]+)-final$} $tag_name match oacs_version] } {
            lassign [split $oacs_version "-"] major_version minor_version patch_version
            #ns_log Notice "Repository: tag <$tag_name> oacs version <$oacs_version> split into /$major_version/$minor_version/$patch_version/"
            if { $major_version >= 5 && $minor_version >= 8} {
                set channel "${major_version}-${minor_version}-$patch_version"
                ns_log Notice "Repository: Found channel $channel using tag $tag_name"
                set channel_tag($channel) $tag_name
            }
        }
    }

    set channel_tag($head_channel) HEAD
    set channel_tag(5-10) oacs-5-10

    ns_log Notice "Repository: Channels are: [array get channel_tag]"


    #----------------------------------------------------------------------
    # Read all package .info files, building manifest file
    #----------------------------------------------------------------------

    # Wipe and re-create the working directory
    file delete -force -- $work_dir
    file mkdir ${work_dir}
    set update_pretty_date [lc_time_fmt [clock format [clock seconds] -format "%Y-%m-%d %T"] %c]

    #cd $work_dir

    foreach channel [lsort -decreasing [array names channel_tag]] {

        if {![string match $channels $channel]} continue
        ns_log Notice "Repository: Channel $channel using tag $channel_tag($channel)"

        # Wipe and re-create the checkout directory
        file delete -force -- "${work_dir}openacs-4"
        file delete -force -- "${work_dir}dotlrn"
        file mkdir "${work_dir}dotlrn/packages"

        # Prepare channel directory
        set channel_dir "${work_dir}repository/$channel/"
        file mkdir $channel_dir

        # Store the list of packages we've seen for this channel, so we don't include the same package twice
        # Seems odd, but we have to do this given the forked packages sitting in /contrib
        set packages [list]

        # Checkout from the tag given by channel_tag($channel)
        if { $debug_p } {
            # Smaller list for debugging purposes
            set checkout_list [list $work_dir $cvs_root openacs-4/packages/acs-core-docs ]
        } else {
            # Full list for real use
            set checkout_list [list  $work_dir $cvs_root openacs-4/packages  $work_dir $cvs_root openacs-4/contrib/packages]
        }

        foreach { cur_work_dir cur_cvs_root cur_module } $checkout_list {
            #cd $cur_work_dir
            set cmd [list exec $cd_helper $cur_work_dir cvs -d $cur_cvs_root -z3 co]
            if { $channel_tag($channel) ne "HEAD" } {
                lappend cmd -r $channel_tag($channel)
            }
            catch { {*}$cmd $cur_module } output
            ns_log Notice "Repository: $cur_module [llength $output] files ($channel_tag($channel))"
        }
        #cd $work_dir

        set manifest "<manifest>\n"

        template::multirow create packages  package_path package_key version pretty_name  package_type summary description  release_date vendor_url vendor  maturity maturity_text  license license_url download_url

        set work_dirs [list ${work_dir}openacs-4/packages ${work_dir}openacs-4/contrib/packages ]
        foreach packages_dir $work_dirs {

            foreach spec_file [lsort [apm_scan_packages $packages_dir]] {

                set package_path [file join {*}[lrange [file split $spec_file] 0 end-1]]
                set package_key [lindex [file split $spec_file] end-1]

                if { $package_key in $exclude_package_list } {
                    ns_log Debug "Repository: Package $package_key is on list of packages to exclude - skipping"
                    continue
                }

                if { [array exists pkg_info] } {
                    array unset pkg_info
                }
                if { [info exists pkg_info] } {
                    unset pkg_info
                }

                ad_try {
                    array set pkg_info [apm_read_package_info_file $spec_file]

                    if { $pkg_info(package.key) in $packages } {
                        ns_log Debug "Repository: Skipping package $package_key, because we already have another version of it"
                    } else {
                        lappend packages $pkg_info(package.key)

                        append manifest  "  <package>" \n  "    <package-key>[ns_quotehtml $pkg_info(package.key)]</package-key>\n"  "    <version>[ns_quotehtml $pkg_info(name)]</version>\n"  "    <pretty-name>[ns_quotehtml $pkg_info(package-name)]</pretty-name>\n"  "    <package-type>[ns_quotehtml $pkg_info(package.type)]</package-type>\n"  "    <summary>[ns_quotehtml $pkg_info(summary)]</summary>\n"  "    <description format=\"[ns_quotehtml $pkg_info(description.format)]\">"  [ns_quotehtml $pkg_info(description)] "</description>\n"  "    <release-date>[ns_quotehtml $pkg_info(release-date)]</release-date>\n"  "    <vendor url=\"[ns_quotehtml $pkg_info(vendor.url)]\">"  [ns_quotehtml $pkg_info(vendor)] "</vendor>\n"  "    <license url=\"[ns_quotehtml $pkg_info(license.url)]\">"  [ns_quotehtml $pkg_info(license)] "</license>\n"  "    <maturity>$pkg_info(maturity)</maturity>\n"

                        foreach e $pkg_info(install) {
                            append manifest "    <install package=\"$e\"/>\n"
                        }

                        set apm_file "${channel_dir}${pkg_info(package.key)}-${pkg_info(name)}.apm"
                        ns_log Notice "Repository: Building package $package_key for channel $channel"

                        set files [apm_get_package_files  -all  -include_data_model_files  -all_db_types  -package_key $pkg_info(package.key)  -package_path $package_path]

                        if { [llength $files] == 0 } {
                            ns_log Notice "Repository: No files in package"
                        } else {
                            ns_log Notice "Repository: [llength $files] files in package $pkg_info(package.key) ($channel)"
                            set cmd [list exec [apm_tar_cmd] cf -  2>/dev/null]

                            # The path to the 'packages' directory in the checkout
                            set packages_root_path [file join {*}[lrange [file split $spec_file] 0 end-2]]

                            set fp [ad_opentmpfile tmp_filename]
                            foreach file $files {
                                puts $fp $package_key/$file
                            }
                            close $fp

                            lappend cmd -C $packages_root_path --files-from $tmp_filename

                            lappend cmd "|" [apm_gzip_cmd] -c ">" $apm_file
                            ns_log Notice "Executing: exec $cd_helper $packages_root_path $cmd"
                            if {[catch "exec $cd_helper $packages_root_path $cmd" errmsg]} {
                                ns_log Error "Error during tar in repository creation for file ${channel_dir}$pkg_info(package.key)-$pkg_info(name).apm: \n$errmsg\n$::errorCode,$::errorInfo"
                            }
                            file delete -- $tmp_filename
                        }

                        set apm_url "${repository_url}$channel/$pkg_info(package.key)-$pkg_info(name).apm"

                        template::multirow append packages  $package_path $package_key $pkg_info(name) $pkg_info(package-name)  $pkg_info(package.type) $pkg_info(summary) $pkg_info(description)  $pkg_info(release-date) $pkg_info(vendor.url) $pkg_info(vendor)  $pkg_info(maturity) $pkg_info(maturity_text)  $pkg_info(license)  $pkg_info(license.url) $apm_url

                        append manifest "    <download-url>$apm_url</download-url>\n"
                        foreach elm $pkg_info(provides) {
                            append manifest "    <provides "  "url=\"[ns_quotehtml [lindex $elm 0]]\" "  "version=\"[ns_quotehtml [lindex $elm 1]]\" />\n"
                        }

                        foreach elm $pkg_info(requires) {
                            append manifest "    <requires "  "url=\"[ns_quotehtml [lindex $elm 0]]\" "  "version=\"[ns_quotehtml [lindex $elm 1]]\" />\n"
                        }
                        append manifest "  </package>\n"
                    }
                } on error {errorMsg} {
                    ns_log Notice "Repository: Error on spec_file $spec_file: $errorMsg\n$::errorInfo\n"
                }
            }
        }
        append manifest "</manifest>\n"

        ns_log Notice "Repository: Writing $channel manifest to ${channel_dir}manifest.xml"
        set fw [open "${channel_dir}manifest.xml" w]
        puts $fw $manifest
        close $fw

        ns_log Notice "Repository: Writing $channel index page to ${channel_dir}index.adp"
        set fw [open "${channel_dir}index.adp" w]
        set packages [lsort $packages]
        puts $fw "<master>\n<property name=\"doc(title)\">OpenACS $channel Compatible Packages</property>\n\n"
        puts $fw "<h1>OpenACS $channel (CVS tag $channel_tag($channel))</h1>
           <p>Packages can be installed with the OpenACS Automated Installer on
           your OpenACS site at <code>/acs-admin/install</code>.  Only packages
           potentially compatible with your OpenACS kernel will be shown.</p>
        "
        set category_title(core) "Core Packages"
        set package_keys(core) {
            acs-admin
            acs-api-browser
            acs-authentication
            acs-automated-testing
            acs-bootstrap-installer
            acs-content-repository
            acs-core-docs
            acs-kernel
            acs-lang
            acs-mail-lite
            acs-messaging
            acs-reference
            acs-service-contract
            acs-subsite
            acs-tcl
            acs-templating
            ref-timezones
            acs-translations
            intermedia-driver
            openacs-default-theme
            notifications
            search
            tsearch2-driver
        }
        set category_title(common-app) "Common Applications"
        set package_keys(common-app) {
            xowiki
            xotcl-request-monitor
            file-storage
            acs-developer-support
            forums
            calendar
            news
            faq
        }
        set category_title(extra) "Extra Packages and Libraries"
        set package_keys(extra) ""
        foreach p $packages {
            if {$p ni $package_keys(core) && $p ni $package_keys(common-app)} {
                lappend package_keys(extra) $p
            }
        }

        foreach category {core common-app extra} {

            template::multirow create pkgs  package_path package_key version pretty_name  package_type summary description  release_date vendor_url vendor  maturity maturity_text  license license_url download_url

            template::multirow foreach packages {
                if {$package_key in $package_keys($category)} {
                    template::multirow append pkgs  $package_path $package_key $version $pretty_name  $package_type $summary $description  $release_date $vendor_url $vendor  $maturity $maturity_text  $license $license_url $download_url
                }
            }

            puts $fw "\n<h2>$category_title($category)</h2>\n"

            puts $fw [template::adp_include $channel_index_template  [list channel $channel &pkgs pkgs update_pretty_date $update_pretty_date]]

        }
        close $fw

        ns_log Notice "Repository:  Channel $channel complete."

    }

    ns_log Notice "Repository: Finishing Repository"

    foreach channel [array names channel_tag] {
        if {[regexp {^([1-9][0-9]*)-([0-9]+)$} $channel . major minor]} {
            #
            # *-compat channels: The "patchlevel" of these channels is
            # the highest possible value, higher than the released
            # -final channels.
            #
            set tag_order([format %.3d $major]-[format %.3d $minor]-999) $channel
            set tag_label($channel) "OpenACS $major.$minor"
        } elseif {[regexp {^([1-9][0-9]*)-([0-9]+)-([0-9]+)$} $channel . major minor patch]} {
            #
            # *-final channels: a concrete patchlevel is provided.
            #
            set tag_order([format %.3d $major]-[format %.3d $minor]-[format %.3d $patch]) $channel
            set tag_label($channel) "OpenACS $major.$minor.$patch"
        } else {
            set tag_order(999-999-999) $channel
            set tag_label($channel) "OpenACS $channel"
        }
    }


    # Write the index page
    ns_log Notice "Repository: Writing repository index page to ${work_dir}repository/index.adp"
    template::multirow create channels name tag label
    foreach key [lsort -decreasing [array names tag_order]] {
        set channel $tag_order($key)
        template::multirow append channels $channel $channel_tag($channel) $tag_label($channel)
    }
    set fw [open "${work_dir}repository/index.adp" w]
    puts $fw "<master>\n<property name=\"doc(title)\">OpenACS Package Repository</property>\n\n"
    puts $fw [template::adp_include -- $index_template  [list &channels channels update_pretty_date $update_pretty_date]]
    close $fw

    # Add a redirector for outdated releases
    set fw [open "${work_dir}repository/index.vuh" w]
    puts $fw "ns_returnredirect /repository/"
    close $fw

    # Without the trailing slash
    set work_repository_dirname "${work_dir}repository"
    set repository_dirname [string range $repository_dir 0 end-1]
    set repository_bak "[string range $repository_dir 0 end-1]_bak"

    ns_log Notice "Repository: Moving work repository $work_repository_dirname to live repository dir at <a href=\"/repository\/>$repository_dir</a>\n"

    if { [file exists $repository_bak] } {
        file delete -force -- $repository_bak
    }
    if { [file exists $repository_dirname] } {
        file rename -- $repository_dirname $repository_bak
    }
    file rename -- $work_repository_dirname  $repository_dirname

    ns_log Debug "Repository: DONE"

    return 0

XQL Not present:

PostgreSQL, Oracle

Generic XQL file:

packages/acs-admin/tcl/apm-admin-procs.xql