Annotation of pkgsrc/mk/scripts/binpkg-cache, revision 1.22

1.1       dmcmahil    1: #!/bin/sh
                      2: #
1.22    ! maya        3: # $NetBSD: binpkg-cache,v 1.21 2010/07/21 12:29:46 spz Exp $
1.1       dmcmahil    4: #
                      5: # Script for generating a cache file with information about
                      6: # all binary packages contained in a directory.
                      7: #
1.7       dmcmahil    8: # Copyright (c) 2005, 2006 The NetBSD Foundation, Inc.
1.1       dmcmahil    9: # All rights reserved.
                     10: #
                     11: # This code is derived from software contributed to The NetBSD Foundation
                     12: # by Dan McMahill.
                     13: #
                     14: # Redistribution and use in source and binary forms, with or without
                     15: # modification, are permitted provided that the following conditions
                     16: # are met:
                     17: # 1. Redistributions of source code must retain the above copyright
                     18: #    notice, this list of conditions and the following disclaimer.
                     19: # 2. Redistributions in binary form must reproduce the above copyright
                     20: #    notice, this list of conditions and the following disclaimer in the
                     21: #    documentation and/or other materials provided with the distribution.
                     22: #
                     23: # THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
                     24: # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
                     25: # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
                     26: # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
                     27: # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
                     28: # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
                     29: # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
                     30: # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
                     31: # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
                     32: # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
                     33: # POSSIBILITY OF SUCH DAMAGE.
                     34: #
                     35: 
                     36: 
                     37: TMPDIR=${TMPDIR:-/tmp}
                     38: PACKAGES=${PACKAGES:-/usr/pkgsrc/packages/}
                     39: AWK=${AWK:-awk}
                     40: CMP=${CMP:-cmp}
1.3       dmcmahil   41: FIND=${FIND:-find}
1.1       dmcmahil   42: GREP=${GREP:-grep}
1.11      dmcmahil   43: GZIP_CMD=${GZIP_CMD:-gzip}
1.19      spz        44: BZIP2=${BZIP2:-bzip2}
1.1       dmcmahil   45: PKG_INFO=${PKG_INFO:-pkg_info}
                     46: PKG_SUFX=${PKG_SUFX:-.tgz}
                     47: SED=${SED:-sed}
                     48: SORT=${SORT:-sort}
                     49: STAT=${STAT:-stat}
                     50: 
                     51: cachefile=.pkgcache
1.11      dmcmahil   52: summaryfile=pkg_summary
1.9       dmcmahil   53: cacheversion=20050428
1.1       dmcmahil   54: 
                     55: prompt="----> "
                     56: tab="      "
                     57: 
                     58: tmpd=${TMPDIR}/pkg-cache.$$
                     59: mkdir -m 0700 ${tmpd}
                     60: if test $? -ne 0 ; then
                     61:        echo "ERROR:  Could not create temporary directory ${tmpd}"
                     62:        echo "Either you do not have write permission to ${tmpd} or"
                     63:        echo "${tmpd} already exists"
                     64:        exit 1
                     65: fi
1.10      dmcmahil   66: all_dirs=${tmpd}/all_dirs
1.1       dmcmahil   67: 
                     68: prog=$0
                     69: 
                     70: usage(){
                     71:        echo "$prog - Generates cache files for each directory containing binary"
                     72:        echo "        packages.  This cache file can then be used by the README.html"
                     73:        echo "        generation code to avoid having to call pkg_info(1) over and over"
1.13      dmcmahil   74:        echo "        on the same binary package.  In addition, pkg_summary.gz files for"
                     75:        echo "        use by the pkgtools/pkg_chk pacakge may be generated."
1.1       dmcmahil   76:        echo " "
1.13      dmcmahil   77:        echo "Usage:      $prog [-d|--debug] [-v|--verbose] [-s|--summary] [-p|--packages <dir>]"
1.1       dmcmahil   78:        echo " "
                     79:        echo "            $prog -h|--help"
                     80:        echo " "
                     81:        echo "            $prog -V|--version"
                     82:        echo " "
                     83:        echo "The options supported by $prog are: "
                     84:        echo " "
                     85:        echo "  -d|--debug            Enables debugging output"
                     86:        echo " "
1.13      dmcmahil   87:        echo "  -F|--force            Forces a rebuild of the cache files even if they are"
                     88:        echo "                        otherwise up to date."
                     89:        echo " "
1.1       dmcmahil   90:        echo "  -h|--help             Displays this help message"
                     91:        echo " "
                     92:        echo "  -p|--packages <dir>   Specifies the top level directory to be searched"
                     93:        echo "                        for binary packages."
                     94:        echo " "
1.13      dmcmahil   95:        echo "  -s|--summary          Enables the creation of pkg_summary.gz files in each"
                     96:        echo "                        directory containing binary packages.  The pkg_summary.gz"
                     97:        echo "                        file is used by the pkgtools/pkg_chk package."
                     98:        echo " "
                     99:        echo "  -v|--verbose          Enables verbose output from the script."
                    100:        echo " "
                    101:        echo "  -V|--version          Displays the version of this script and exits."
1.1       dmcmahil  102:        echo " "
1.7       dmcmahil  103:        echo "Returns 0 on success, 1 on errors, 2 if the packages"
                    104:        echo "directory does not exist."
                    105:        echo " "
1.1       dmcmahil  106:        echo "Example:    $prog -v --packages /usr/pkgsrc/packages"
                    107:        echo " "
                    108: }
                    109: 
                    110: clean_and_exit0(){
                    111:        rm -fr ${tmpd}
                    112:        exit 0
                    113: }
                    114: 
                    115: clean_and_exit1(){
                    116:        rm -fr ${tmpd}
                    117:        exit 1
                    118: }
                    119: 
1.7       dmcmahil  120: clean_and_exit2(){
                    121:        rm -fr ${tmpd}
                    122:        exit 2
                    123: }
                    124: 
1.1       dmcmahil  125: all_cache_files=""
                    126: 
1.4       dmcmahil  127: #############################################################################
                    128: #
                    129: # process_binpkg_dir()
                    130: #
                    131: # Process a directory by checking to see if a cache file exists.  If the
                    132: # cache file exists, make sure it is up to date.  If the file does not
                    133: # exist, create one.
                    134: #
                    135: # also keep track of this directory so it can be added to the master
                    136: # cache.
                    137: #
                    138: 
1.1       dmcmahil  139: process_binpkg_dir(){
1.10      dmcmahil  140:        rdir=`${GREP} "^${d} " ${all_dirs} | ${AWK} '{print $2}'`
1.1       dmcmahil  141:        need_update=no
                    142:        if test -f ${d}/${cachefile} ; then
1.3       dmcmahil  143:                stale_entries=`${FIND} ${d} -type f -name \*${PKG_SUFX} -newer ${d}/${cachefile} -print`
1.1       dmcmahil  144: 
                    145:                # FIX_ME
                    146:                #
                    147:                # We also should find cache entries for files which no longer exist
                    148:                # and nuke them.  Right now we simply declare the entire cache out
                    149:                # of date.  Once we implement incremental updates to the cache,
                    150:                # we need to remove the entries but not mark the entire cache as
                    151:                # bad.
1.17      rillig    152:                $if_debug echo "      Checking for cache entries with no corresponding pkg."
1.1       dmcmahil  153:                # get the list of what pkgs belong in the cache
                    154:                rm -f ${tmpd}/pkg_list ${tmpd}/cache_pkg_list
1.3       dmcmahil  155:                ${FIND} ${d}/ -name \*${PKG_SUFX} -print | \
1.10      dmcmahil  156:                        ${SED} -e "s;^${d}/*;${rdir}/;g" -e 's;//;/;g' | \
1.3       dmcmahil  157:                        ${SORT} > ${tmpd}/pkg_list
1.1       dmcmahil  158: 
                    159:                # and get the list of what is in the cache
                    160:                ${AWK} '/pkgcache_begin/ {gsub(/pkgcache_begin[ \t]*/, ""); print}' \
                    161:                        ${d}/${cachefile} | ${SORT} > ${tmpd}/cache_pkg_list
                    162: 
                    163:                if ${CMP} -s ${tmpd}/pkg_list ${tmpd}/cache_pkg_list ; then
1.17      rillig    164:                        $if_debug echo "      No extra cache entries in ${d}/${cachefile}"
1.1       dmcmahil  165:                else
1.17      rillig    166:                        $if_debug echo "Package list:"
                    167:                        $if_debug cat  ${tmpd}/pkg_list
                    168:                        $if_debug echo "Cache list:"
                    169:                        $if_debug cat ${tmpd}/cache_pkg_list
1.1       dmcmahil  170:                        echo "      Entries found in ${d}/${cachefile} but no packages found"
                    171:                        need_update=yes
                    172:                fi
                    173:        else
                    174:                stale_entries=""
                    175:        fi
                    176: 
                    177:        if test "X${force}" = "Xyes" -o "X${need_update}" = "Xyes" ; then
                    178:                need_update=yes
                    179:                echo "${tab}Forcing rebuild of cache ${d}/${cachefile}."
                    180:        elif test ! -f ${d}/${cachefile} ; then
                    181:                need_update=yes
                    182:                echo "${tab}Missing cache file.  ${d}/${cachefile} will be generated."
                    183:        elif test -n "${stale_entries}" ; then
                    184:                need_update=yes
                    185:                echo "${tab}Stale cache file.  ${d}/${cachefile} will be regenerated."
                    186:        else
                    187:                ${GREP} "pkgcache_version ${cacheversion}" ${d}/${cachefile} >/dev/null 2>&1
                    188:                if test $? -ne 0 ; then
                    189:                        need_update=yes
1.5       dmcmahil  190:                        echo "${tab}Invalid version cache file.  ${d}/${cachefile} will be regenerated."
                    191:                        echo "Need version ${cacheversion} but the file has"
                    192:                        ${GREP} "^pkgcache_version " ${d}/${cachefile}
1.1       dmcmahil  193:                else
1.21      spz       194:                        $if_verbose echo "${tab}Cache file ${d}/${cachefile} is up to date."
1.1       dmcmahil  195:                fi
                    196:        fi
                    197: 
1.14      martti    198:        if test "${build_summary}" = "yes" -a ! -f ${d}/${summaryfile}.gz ; then
1.11      dmcmahil  199:                echo "${tab}Summary file ${d}/${summaryfile}.gz is missing and will be created."
                    200:                need_update=yes
                    201:        fi
                    202:        if test "${build_summary}" = "yes" -a ${d}/${summaryfile}.gz -ot ${d}/${cachefile} ; then
1.18      spz       203:                echo "${tab}Summary file ${d}/${summaryfile}.gz is out of date and will be regenerated."
1.11      dmcmahil  204:                need_update=yes
                    205:        fi
                    206: 
1.1       dmcmahil  207:        # FIX_ME
1.14      martti    208:        # We should use stale_entries in a way where we only update the
1.1       dmcmahil  209:        # cache file entries corresponding to these if we're rebuilding
                    210:        # due to stale entries.  That should save a good bit of time.
1.14      martti    211:        #
1.1       dmcmahil  212:        if test "X${need_update}" = "Xyes" ; then
                    213:                echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
1.12      dmcmahil  214:                rm -f ${tmpd}/${summaryfile}
                    215:                touch ${tmpd}/${summaryfile}
1.1       dmcmahil  216:                for f in ${d}/*${PKG_SUFX} ; do
1.10      dmcmahil  217:                        fn=`grep "^${d} " ${all_dirs} | ${AWK} '{print $2}'`"/"`basename ${f}`
1.17      rillig    218:                        $if_debug echo "     Adding ${fn} (${f}) to the cache"
1.1       dmcmahil  219:                        echo " " >> ${tmpd}/${cachefile}
                    220:                        # stat(1) needs to be added to the bootstrap kit
                    221:                        # first if we want to use it here
                    222:                        #eval $(${STAT} -s ${f} 2>/dev/null)
                    223:                        echo "pkgcache_begin ${fn}" >> ${tmpd}/${cachefile}
                    224:                        #echo "pkgcache_mtime=${st_mtime}" >> ${tmpd}/${cachefile}
1.11      dmcmahil  225: 
1.17      rillig    226:                        $if_debug echo "${PKG_INFO} -q -B ${f}"
1.11      dmcmahil  227:                        ${PKG_INFO} -q -B ${f} >> ${tmpd}/${cachefile}
                    228: 
                    229:                        if test "${build_summary}" = "yes" ; then
1.17      rillig    230:                                $if_debug echo "${PKG_INFO} -X ${f}"
1.11      dmcmahil  231:                                ${PKG_INFO} -X ${f} >> ${tmpd}/${summaryfile}
1.1       dmcmahil  232:                        fi
                    233:                        echo "pkgcache_end ${fn}" >> ${tmpd}/${cachefile}
                    234:                done
1.18      spz       235:                if test -f ${d}/${cachefile} ; then
                    236:                        rm -f ${d}/${cachefile}
                    237:                fi
1.4       dmcmahil  238:                mv -f ${tmpd}/${cachefile} ${d}/${cachefile}
1.2       dmcmahil  239:                if test $? -ne 0 ; then
1.4       dmcmahil  240:                        echo "********** WARNING **********"
1.2       dmcmahil  241:                        echo "move of ${tmpd}/${cachefile} to ${d}/${cachefile} failed!"
                    242:                        echo "Perhaps you do not have write permissions to ${d}?"
1.4       dmcmahil  243:                        echo "This directory will be dropped from the master cache file."
                    244:                        echo "********** WARNING **********"
                    245:                        return
1.2       dmcmahil  246:                fi
1.15      martti    247: 
1.11      dmcmahil  248:                if test "${build_summary}" = "yes" ; then
1.18      spz       249:                        if test -f ${d}/${summaryfile}.gz ; then
                    250:                                rm -f ${d}/${summaryfile}.gz
                    251:                        fi
1.11      dmcmahil  252:                        cat ${tmpd}/${summaryfile} | ${GZIP_CMD} > ${d}/${summaryfile}.gz
                    253:                        if test $? -ne 0 ; then
                    254:                                echo "********** WARNING **********"
                    255:                                echo "${GZIP_CMD} of ${tmpd}/${summaryfile} to ${d}/${summaryfile}.gz failed!"
                    256:                                echo "Perhaps you do not have write permissions to ${d}?"
                    257:                                echo "********** WARNING **********"
                    258:                                return
                    259:                        fi
1.19      spz       260:                        # if it's there, update it, otherwise don't bother
                    261:                        if test -f ${d}/${summaryfile}.bz2 ; then
1.20      spz       262:                                rm -f ${d}/${summaryfile}.bz2
1.19      spz       263:                                cat ${tmpd}/${summaryfile} | ${BZIP2} > ${d}/${summaryfile}.bz2
                    264:                                if test $? -ne 0 ; then
                    265:                                        echo "********** WARNING **********"
                    266:                                        echo "${BZIP2} of ${tmpd}/${summaryfile} to ${d}/${summaryfile}.bz2 failed!"
                    267:                                        echo "Perhaps you do not have write permissions to ${d}?"
                    268:                                        echo "********** WARNING **********"
                    269:                                        return
                    270:                                fi
                    271:                        fi
1.11      dmcmahil  272:                fi
                    273: 
1.1       dmcmahil  274:        fi
                    275: 
1.4       dmcmahil  276:        # if we got here, then this directory should have a good cache file in
                    277:        # it and we should be able to add it to the master cache file
                    278:        all_cache_files="${all_cache_files} ${d}/${cachefile}"
1.1       dmcmahil  279: 
                    280: }
                    281: 
                    282: process_cache_files(){
                    283:        echo "${prompt}Checking master cache file ${PACKAGES}/${cachefile}"
                    284:        echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
                    285:        if test -n "${all_cache_files}" ; then
                    286:                for c in ${all_cache_files} ; do
                    287:                        echo "pkgcache_cachefile ${c}" >> ${tmpd}/${cachefile}
                    288:                done
                    289:        fi
                    290:        if test ! -f ${PACKAGES}/${cachefile} ; then
                    291:                echo "${tab}Creating master cache file ${PACKAGES}/${cachefile}"
1.4       dmcmahil  292:                mv -f ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
1.2       dmcmahil  293:                if test $? -ne 0 ; then
1.4       dmcmahil  294:                        echo "********** ERROR **********"
1.2       dmcmahil  295:                        echo "move of ${tmpd}/${cachefile} to ${PACKAGES}/${cachefile} failed!"
                    296:                        echo "Perhaps you do not have write permissions to ${PACKAGES}?"
1.4       dmcmahil  297:                        echo "********** ERROR **********"
1.2       dmcmahil  298:                        clean_and_exit1
                    299:                fi
                    300:        elif ${CMP} -s ${tmpd}/${cachefile} ${PACKAGES}/${cachefile} ; then
1.1       dmcmahil  301:                echo "${tab}Master cache file ${PACKAGES}/${cachefile} is up to date"
                    302:        else
                    303:                echo "${tab}Updating master cache file ${PACKAGES}/${cachefile}"
1.4       dmcmahil  304:                mv -f ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
1.2       dmcmahil  305:                if test $? -ne 0 ; then
1.4       dmcmahil  306:                        echo "********** ERROR **********"
1.2       dmcmahil  307:                        echo "move of ${tmpd}/${cachefile} to ${PACKAGES}/${cachefile} failed!"
                    308:                        echo "Perhaps you do not have write permissions to ${PACKAGES}?"
1.4       dmcmahil  309:                        echo "********** ERROR **********"
1.2       dmcmahil  310:                        clean_and_exit1
                    311:                fi
1.1       dmcmahil  312:        fi
                    313: }
                    314: 
                    315: ######################################################################
                    316: #
                    317: #  Handle command line options
                    318: #
                    319: ######################################################################
                    320: 
1.17      rillig    321: if_debug=:             # either ":" or ""
1.21      spz       322: if_verbose=:           # either ":" or ""
1.1       dmcmahil  323: force=no
1.11      dmcmahil  324: build_summary=no
1.1       dmcmahil  325: 
                    326: while
                    327:        test -n "$1"
                    328: do
                    329:        case "$1" in
                    330: 
                    331:                # Turn on debugging
                    332:                -d|--debug)
1.17      rillig    333:                        if_debug=""
1.21      spz       334:                        if_verbose=""
1.1       dmcmahil  335:                        shift
                    336:                        ;;
                    337: 
                    338:                # Force a rebuilde of the cache
                    339:                -F|--force)
                    340:                        force=yes
                    341:                        shift
                    342:                        ;;
                    343:                # Help
                    344:                -h|--help)
                    345:                        usage
                    346:                        exit 0
                    347:                        ;;
                    348: 
                    349:                # Use the specified packages directory
                    350:                -p|--packages)
                    351:                        PACKAGES=$2
                    352:                        shift 2
                    353:                        ;;
                    354: 
1.11      dmcmahil  355:                # Build the pkg_summary.gz files
                    356:                -s|--summary)
                    357:                        build_summary=yes
                    358:                        shift
                    359:                        ;;
                    360: 
1.1       dmcmahil  361:                # Version
                    362:                -V|--version)
                    363:                        ${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s:  Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
                    364:                        exit 0
                    365:                        ;;
                    366: 
1.17      rillig    367:                # Turn on verbose output, but not as noisy as debug
1.1       dmcmahil  368:                -v|--verbose)
1.21      spz       369:                        if_verbose=""
1.1       dmcmahil  370:                        shift
                    371:                        ;;
                    372: 
                    373:                -*) echo "$prog:  ERROR:  $1 is not a valid option"
                    374:                        usage
                    375:                        clean_and_exit1
                    376:                        ;;
                    377: 
1.14      martti    378:                *)
1.1       dmcmahil  379:                        break
                    380:                        ;;
                    381: 
                    382:        esac
                    383: done
                    384: 
1.6       dmcmahil  385: if test $# -ne 0 ; then
                    386:        echo "$0:  $* is invalid"
                    387:        usage
                    388:        clean_and_exit1
                    389: fi
                    390: 
                    391: if test ! -d ${PACKAGES} ; then
                    392:        echo "Packages directory ${PACKAGES} seems to be missing"
1.7       dmcmahil  393:        clean_and_exit2
1.6       dmcmahil  394: fi
                    395: 
1.14      martti    396: # put a trailing / after ${PACKAGES} in case ${PACKAGES} is
1.1       dmcmahil  397: # a link.
1.10      dmcmahil  398: 
                    399: # pass 1, we find all directories under PACKAGES.  Note that this
                    400: # may contain some directories more than once depending on what sort
                    401: # of soft links may be in place
                    402: rm -f ${all_dirs}.tmp
1.8       dmcmahil  403: for d in `${FIND} ${PACKAGES}/ -type d -follow -print` ; do
1.10      dmcmahil  404:        cname=`(cd ${d} && pwd -P)`
                    405:        rname=`echo ${d} | ${SED} "s;^${PACKAGES}/*;;g"`
                    406:        echo "${cname} ${rname}" >> ${all_dirs}.tmp
                    407: done
                    408: ${SORT} -u -k1,1 ${all_dirs}.tmp > ${all_dirs}
1.17      rillig    409: $if_debug echo "Full directory list:"
                    410: $if_debug cat ${all_dirs}.tmp
                    411: $if_debug echo "Unique directory list:"
                    412: $if_debug cat ${all_dirs}
1.10      dmcmahil  413: 
                    414: for d in `${AWK} '{print $1}' ${all_dirs}` ; do
1.17      rillig    415:        $if_debug echo "${prompt}Processing directory ${d}"
1.1       dmcmahil  416:        is_pkg_dir=no
                    417:        for f in ${d}/*${PKG_SUFX} ; do
                    418:                if test -f "${f}" -a ! -h "${f}" ; then
                    419:                        ${PKG_INFO} ${f} >/dev/null 2>&1
                    420:                        if test $? -eq 0 ; then
                    421:                                is_pkg_dir=yes
                    422:                                break
                    423:                        fi
                    424:                fi
                    425:        done
                    426:        if test "X${is_pkg_dir}" = "Xyes" ; then
1.21      spz       427:                $if_verbose echo "${prompt}Checking cache in ${d}"
1.1       dmcmahil  428:                process_binpkg_dir
                    429:        else
1.17      rillig    430:                $if_debug echo "${prompt}no binary packages in ${d}"
1.1       dmcmahil  431:        fi
1.15      martti    432: 
1.1       dmcmahil  433: done
                    434: 
                    435: process_cache_files
                    436: 
                    437: clean_and_exit0

CVSweb <webmaster@jp.NetBSD.org>