Annotation of pkgsrc/mk/scripts/binpkg-cache, revision 1.8

1.1       dmcmahil    1: #!/bin/sh
                      2: #
1.8     ! dmcmahil    3: # $NetBSD: binpkg-cache,v 1.7 2006/01/05 22:19:42 dmcmahill Exp $
1.1       dmcmahil    4: #
                      5: # Script for generating a cache file with information about
                      6: # all binary packages contained in a directory.
                      7: #
1.7       dmcmahil    8: # Copyright (c) 2005, 2006 The NetBSD Foundation, Inc.
1.1       dmcmahil    9: # All rights reserved.
                     10: #
                     11: # This code is derived from software contributed to The NetBSD Foundation
                     12: # by Dan McMahill.
                     13: #
                     14: # Redistribution and use in source and binary forms, with or without
                     15: # modification, are permitted provided that the following conditions
                     16: # are met:
                     17: # 1. Redistributions of source code must retain the above copyright
                     18: #    notice, this list of conditions and the following disclaimer.
                     19: # 2. Redistributions in binary form must reproduce the above copyright
                     20: #    notice, this list of conditions and the following disclaimer in the
                     21: #    documentation and/or other materials provided with the distribution.
                     22: # 3. All advertising materials mentioning features or use of this software
                     23: #    must display the following acknowledgement:
                     24: #        This product includes software developed by the NetBSD
                     25: #        Foundation, Inc. and its contributors.
                     26: # 4. Neither the name of The NetBSD Foundation nor the names of its
                     27: #    contributors may be used to endorse or promote products derived
                     28: #    from this software without specific prior written permission.
                     29: #
                     30: # THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
                     31: # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
                     32: # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
                     33: # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
                     34: # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
                     35: # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
                     36: # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
                     37: # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
                     38: # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
                     39: # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
                     40: # POSSIBILITY OF SUCH DAMAGE.
                     41: #
                     42: 
                     43: 
                     44: TMPDIR=${TMPDIR:-/tmp}
                     45: PACKAGES=${PACKAGES:-/usr/pkgsrc/packages/}
                     46: AWK=${AWK:-awk}
                     47: CMP=${CMP:-cmp}
1.3       dmcmahil   48: FIND=${FIND:-find}
1.1       dmcmahil   49: GREP=${GREP:-grep}
                     50: PKG_INFO=${PKG_INFO:-pkg_info}
                     51: PKG_SUFX=${PKG_SUFX:-.tgz}
                     52: SED=${SED:-sed}
                     53: SORT=${SORT:-sort}
                     54: STAT=${STAT:-stat}
                     55: 
                     56: cachefile=.pkgcache
1.8     ! dmcmahil   57: cacheversion=20060203
1.1       dmcmahil   58: 
                     59: prompt="----> "
                     60: tab="      "
                     61: 
                     62: tmpd=${TMPDIR}/pkg-cache.$$
                     63: mkdir -m 0700 ${tmpd}
                     64: if test $? -ne 0 ; then
                     65:        echo "ERROR:  Could not create temporary directory ${tmpd}"
                     66:        echo "Either you do not have write permission to ${tmpd} or"
                     67:        echo "${tmpd} already exists"
                     68:        exit 1
                     69: fi
                     70: 
                     71: prog=$0
                     72: 
                     73: usage(){
                     74:        echo "$prog - Generates cache files for each directory containing binary"
                     75:        echo "        packages.  This cache file can then be used by the README.html"
                     76:        echo "        generation code to avoid having to call pkg_info(1) over and over"
                     77:        echo "        on the same binary package."
                     78:        echo " "
                     79:        echo "Usage:      $prog [-d|--debug] [-v|--verbose] [-p|--packages <dir>]"
                     80:        echo " "
                     81:        echo "            $prog -h|--help"
                     82:        echo " "
                     83:        echo "            $prog -V|--version"
                     84:        echo " "
                     85:        echo "The options supported by $prog are: "
                     86:        echo " "
                     87:        echo "  -d|--debug            Enables debugging output"
                     88:        echo " "
                     89:        echo "  -f|--database <file>  Writes the database into file specified by <file>"
                     90:        echo " "
                     91:        echo "  -h|--help             Displays this help message"
                     92:        echo " "
                     93:        echo "  -p|--packages <dir>   Specifies the top level directory to be searched"
                     94:        echo "                        for binary packages."
                     95:        echo " "
                     96:        echo "  -v|--version          Displays the version of this script and exits."
                     97:        echo " "
1.7       dmcmahil   98:        echo "Returns 0 on success, 1 on errors, 2 if the packages"
                     99:        echo "directory does not exist."
                    100:        echo " "
1.1       dmcmahil  101:        echo "Example:    $prog -v --packages /usr/pkgsrc/packages"
                    102:        echo " "
                    103: }
                    104: 
                    105: clean_and_exit0(){
                    106:        rm -fr ${tmpd}
                    107:        exit 0
                    108: }
                    109: 
                    110: clean_and_exit1(){
                    111:        rm -fr ${tmpd}
                    112:        exit 1
                    113: }
                    114: 
1.7       dmcmahil  115: clean_and_exit2(){
                    116:        rm -fr ${tmpd}
                    117:        exit 2
                    118: }
                    119: 
1.1       dmcmahil  120: all_cache_files=""
                    121: 
1.4       dmcmahil  122: #############################################################################
                    123: #
                    124: # process_binpkg_dir()
                    125: #
                    126: # Process a directory by checking to see if a cache file exists.  If the
                    127: # cache file exists, make sure it is up to date.  If the file does not
                    128: # exist, create one.
                    129: #
                    130: # also keep track of this directory so it can be added to the master
                    131: # cache.
                    132: #
                    133: 
1.1       dmcmahil  134: process_binpkg_dir(){
                    135:        need_update=no
                    136:        if test -f ${d}/${cachefile} ; then
1.3       dmcmahil  137:                stale_entries=`${FIND} ${d} -type f -name \*${PKG_SUFX} -newer ${d}/${cachefile} -print`
1.1       dmcmahil  138: 
                    139:                # FIX_ME
                    140:                #
                    141:                # We also should find cache entries for files which no longer exist
                    142:                # and nuke them.  Right now we simply declare the entire cache out
                    143:                # of date.  Once we implement incremental updates to the cache,
                    144:                # we need to remove the entries but not mark the entire cache as
                    145:                # bad.
                    146:                if test "X${DEBUG}" = "Xyes" ; then
                    147:                        echo "      Checking for cache entries with no corresponding pkg."
                    148:                fi
                    149:                # get the list of what pkgs belong in the cache
                    150:                rm -f ${tmpd}/pkg_list ${tmpd}/cache_pkg_list
1.3       dmcmahil  151:                ${FIND} ${d}/ -name \*${PKG_SUFX} -print | \
                    152:                        ${SED} -e "s;^${PACKAGES}/*;;g" -e 's;//;/;g' | \
                    153:                        ${SORT} > ${tmpd}/pkg_list
1.1       dmcmahil  154: 
                    155:                # and get the list of what is in the cache
                    156:                ${AWK} '/pkgcache_begin/ {gsub(/pkgcache_begin[ \t]*/, ""); print}' \
                    157:                        ${d}/${cachefile} | ${SORT} > ${tmpd}/cache_pkg_list
                    158: 
                    159:                if ${CMP} -s ${tmpd}/pkg_list ${tmpd}/cache_pkg_list ; then
                    160:                        if test "X${DEBUG}" = "Xyes" ; then
                    161:                                echo "      No extra cache entries in ${d}/${cachefile}"
                    162:                        fi
                    163:                else
                    164:                        echo "      Entries found in ${d}/${cachefile} but no packages found"
                    165:                        need_update=yes
                    166:                fi
                    167:        else
                    168:                stale_entries=""
                    169:        fi
                    170: 
                    171:        if test "X${force}" = "Xyes" -o "X${need_update}" = "Xyes" ; then
                    172:                need_update=yes
                    173:                echo "${tab}Forcing rebuild of cache ${d}/${cachefile}."
                    174:        elif test ! -f ${d}/${cachefile} ; then
                    175:                need_update=yes
                    176:                echo "${tab}Missing cache file.  ${d}/${cachefile} will be generated."
                    177:        elif test -n "${stale_entries}" ; then
                    178:                need_update=yes
                    179:                echo "${tab}Stale cache file.  ${d}/${cachefile} will be regenerated."
                    180:        else
                    181:                ${GREP} "pkgcache_version ${cacheversion}" ${d}/${cachefile} >/dev/null 2>&1
                    182:                if test $? -ne 0 ; then
                    183:                        need_update=yes
1.5       dmcmahil  184:                        echo "${tab}Invalid version cache file.  ${d}/${cachefile} will be regenerated."
                    185:                        echo "Need version ${cacheversion} but the file has"
                    186:                        ${GREP} "^pkgcache_version " ${d}/${cachefile}
1.1       dmcmahil  187:                else
                    188:                        echo "${tab}Cache file ${d}/${cachefile} is up to date."
                    189:                fi
                    190:        fi
                    191: 
                    192:        # FIX_ME
                    193:        # We should use stale_entries in a way where we only update the 
                    194:        # cache file entries corresponding to these if we're rebuilding
                    195:        # due to stale entries.  That should save a good bit of time.
                    196:        # 
                    197:        if test "X${need_update}" = "Xyes" ; then
                    198:                echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
                    199:                for f in ${d}/*${PKG_SUFX} ; do
                    200:                        fn=`echo $f | ${SED} "s;^${PACKAGES}/*;;g"`
                    201:                        if test "X${DEBUG}" = "Xyes" ; then
                    202:                                echo "     Adding ${fn} (${f}) to the cache"
                    203:                        fi
                    204:                        echo " " >> ${tmpd}/${cachefile}
                    205:                        # stat(1) needs to be added to the bootstrap kit
                    206:                        # first if we want to use it here
                    207:                        #eval $(${STAT} -s ${f} 2>/dev/null)
                    208:                        echo "pkgcache_begin ${fn}" >> ${tmpd}/${cachefile}
                    209:                        #echo "pkgcache_mtime=${st_mtime}" >> ${tmpd}/${cachefile}
                    210:                        if test "X${DEBUG}" = "Xyes" ; then
                    211:                                echo "${PKG_INFO} -q -B ${f}"
                    212:                        fi
1.8     ! dmcmahil  213:                        # The output of pkg_info contains the dependency information first
        !           214:                        # (from the -n) followed by all the build definitions (-B).
        !           215:                        # To make the dependency information easier to parse later, we
        !           216:                        # want to prefix it with DEPENDS+=.  Right now the only way I see
        !           217:                        # to figure out when the DEPENDS section ends in the pkg_info output
        !           218:                        # is by looking for a blank line.
        !           219:                        ${PKG_INFO} -q -Bn ${f} | \
        !           220:                                ${AWK} '/^$/ {x=1; print; next;} x==0 {printf("DEPENDS+=");} {print}' \
        !           221:                                >> ${tmpd}/${cachefile}
1.1       dmcmahil  222:                        echo "pkgcache_end ${fn}" >> ${tmpd}/${cachefile}
                    223:                done
1.4       dmcmahil  224:                mv -f ${tmpd}/${cachefile} ${d}/${cachefile}
1.2       dmcmahil  225:                if test $? -ne 0 ; then
1.4       dmcmahil  226:                        echo "********** WARNING **********"
1.2       dmcmahil  227:                        echo "move of ${tmpd}/${cachefile} to ${d}/${cachefile} failed!"
                    228:                        echo "Perhaps you do not have write permissions to ${d}?"
1.4       dmcmahil  229:                        echo "This directory will be dropped from the master cache file."
                    230:                        echo "********** WARNING **********"
                    231:                        return
1.2       dmcmahil  232:                fi
1.1       dmcmahil  233:        fi
                    234: 
1.4       dmcmahil  235:        # if we got here, then this directory should have a good cache file in
                    236:        # it and we should be able to add it to the master cache file
                    237:        all_cache_files="${all_cache_files} ${d}/${cachefile}"
1.1       dmcmahil  238: 
                    239: }
                    240: 
                    241: process_cache_files(){
                    242:        echo "${prompt}Checking master cache file ${PACKAGES}/${cachefile}"
                    243:        echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
                    244:        if test -n "${all_cache_files}" ; then
                    245:                for c in ${all_cache_files} ; do
                    246:                        echo "pkgcache_cachefile ${c}" >> ${tmpd}/${cachefile}
                    247:                done
                    248:        fi
                    249:        if test ! -f ${PACKAGES}/${cachefile} ; then
                    250:                echo "${tab}Creating master cache file ${PACKAGES}/${cachefile}"
1.4       dmcmahil  251:                mv -f ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
1.2       dmcmahil  252:                if test $? -ne 0 ; then
1.4       dmcmahil  253:                        echo "********** ERROR **********"
1.2       dmcmahil  254:                        echo "move of ${tmpd}/${cachefile} to ${PACKAGES}/${cachefile} failed!"
                    255:                        echo "Perhaps you do not have write permissions to ${PACKAGES}?"
1.4       dmcmahil  256:                        echo "********** ERROR **********"
1.2       dmcmahil  257:                        clean_and_exit1
                    258:                fi
                    259:        elif ${CMP} -s ${tmpd}/${cachefile} ${PACKAGES}/${cachefile} ; then
1.1       dmcmahil  260:                echo "${tab}Master cache file ${PACKAGES}/${cachefile} is up to date"
                    261:        else
                    262:                echo "${tab}Updating master cache file ${PACKAGES}/${cachefile}"
1.4       dmcmahil  263:                mv -f ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
1.2       dmcmahil  264:                if test $? -ne 0 ; then
1.4       dmcmahil  265:                        echo "********** ERROR **********"
1.2       dmcmahil  266:                        echo "move of ${tmpd}/${cachefile} to ${PACKAGES}/${cachefile} failed!"
                    267:                        echo "Perhaps you do not have write permissions to ${PACKAGES}?"
1.4       dmcmahil  268:                        echo "********** ERROR **********"
1.2       dmcmahil  269:                        clean_and_exit1
                    270:                fi
1.1       dmcmahil  271:        fi
                    272: }
                    273: 
                    274: ######################################################################
                    275: #
                    276: #  Handle command line options
                    277: #
                    278: ######################################################################
                    279: 
                    280: DEBUG=no
                    281: verbose=no
                    282: force=no
                    283: 
                    284: while
                    285:        test -n "$1"
                    286: do
                    287:        case "$1" in
                    288: 
                    289:                # Turn on debugging
                    290:                -d|--debug)
                    291:                        DEBUG=yes
                    292:                        verbose=yes
                    293:                        shift
                    294:                        ;;
                    295: 
                    296:                # Force a rebuilde of the cache
                    297:                -F|--force)
                    298:                        force=yes
                    299:                        shift
                    300:                        ;;
                    301:                # Help
                    302:                -h|--help)
                    303:                        usage
                    304:                        exit 0
                    305:                        ;;
                    306: 
                    307:                # Use the specified packages directory
                    308:                -p|--packages)
                    309:                        PACKAGES=$2
                    310:                        shift 2
                    311:                        ;;
                    312: 
                    313:                # Version
                    314:                -V|--version)
                    315:                        ${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s:  Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
1.8     ! dmcmahil  316:                        echo "Generating version ${cacheversion} of the cache files."
1.1       dmcmahil  317:                        exit 0
                    318:                        ;;
                    319: 
                    320:                # Turn on verbose output, but not as noisy as DEBUG
                    321:                -v|--verbose)
                    322:                        verbose=yes
                    323:                        shift
                    324:                        ;;
                    325: 
                    326:                -*) echo "$prog:  ERROR:  $1 is not a valid option"
                    327:                        usage
                    328:                        clean_and_exit1
                    329:                        ;;
                    330: 
                    331:                *) 
                    332:                        break
                    333:                        ;;
                    334: 
                    335:        esac
                    336: done
                    337: 
1.6       dmcmahil  338: if test $# -ne 0 ; then
                    339:        echo "$0:  $* is invalid"
                    340:        usage
                    341:        clean_and_exit1
                    342: fi
                    343: 
                    344: if test ! -d ${PACKAGES} ; then
                    345:        echo "Packages directory ${PACKAGES} seems to be missing"
1.7       dmcmahil  346:        clean_and_exit2
1.6       dmcmahil  347: fi
                    348: 
1.1       dmcmahil  349: # put a trailing / after ${PACKAGES} in case ${PACKAGES} is 
                    350: # a link.
1.8     ! dmcmahil  351: for d in `${FIND} ${PACKAGES}/ -type d -follow -print` ; do
1.1       dmcmahil  352:        if test "X${DEBUG}" = "Xyes" ; then
                    353:                echo "${prompt}Processing directory ${d}"
                    354:        fi
                    355:        is_pkg_dir=no
                    356:        for f in ${d}/*${PKG_SUFX} ; do
                    357:                if test -f "${f}" -a ! -h "${f}" ; then
                    358:                        ${PKG_INFO} ${f} >/dev/null 2>&1
                    359:                        if test $? -eq 0 ; then
                    360:                                is_pkg_dir=yes
                    361:                                break
                    362:                        fi
                    363:                fi
                    364:        done
                    365:        if test "X${is_pkg_dir}" = "Xyes" ; then
                    366:                echo "${prompt}Checking cache in ${d}"
                    367:                process_binpkg_dir
                    368:        else
                    369:                if test "X${DEBUG}" = "Xyes" ; then
                    370:                        echo "${prompt}no binary packages in ${d}"
                    371:                fi
                    372:        fi
                    373:                
                    374: done
                    375: 
                    376: process_cache_files
                    377: 
                    378: clean_and_exit0
                    379: 

CVSweb <webmaster@jp.NetBSD.org>