Annotation of pkgsrc/mk/scripts/binpkg-cache, revision 1.6

1.1       dmcmahil    1: #!/bin/sh
                      2: #
1.6     ! dmcmahil    3: # $NetBSD: binpkg-cache,v 1.5 2005/05/24 20:56:47 dmcmahill Exp $
1.1       dmcmahil    4: #
                      5: # Script for generating a cache file with information about
                      6: # all binary packages contained in a directory.
                      7: #
                      8: # Copyright (c) 2005 The NetBSD Foundation, Inc.
                      9: # All rights reserved.
                     10: #
                     11: # This code is derived from software contributed to The NetBSD Foundation
                     12: # by Dan McMahill.
                     13: #
                     14: # Redistribution and use in source and binary forms, with or without
                     15: # modification, are permitted provided that the following conditions
                     16: # are met:
                     17: # 1. Redistributions of source code must retain the above copyright
                     18: #    notice, this list of conditions and the following disclaimer.
                     19: # 2. Redistributions in binary form must reproduce the above copyright
                     20: #    notice, this list of conditions and the following disclaimer in the
                     21: #    documentation and/or other materials provided with the distribution.
                     22: # 3. All advertising materials mentioning features or use of this software
                     23: #    must display the following acknowledgement:
                     24: #        This product includes software developed by the NetBSD
                     25: #        Foundation, Inc. and its contributors.
                     26: # 4. Neither the name of The NetBSD Foundation nor the names of its
                     27: #    contributors may be used to endorse or promote products derived
                     28: #    from this software without specific prior written permission.
                     29: #
                     30: # THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
                     31: # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
                     32: # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
                     33: # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
                     34: # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
                     35: # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
                     36: # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
                     37: # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
                     38: # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
                     39: # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
                     40: # POSSIBILITY OF SUCH DAMAGE.
                     41: #
                     42: 
                     43: 
                     44: TMPDIR=${TMPDIR:-/tmp}
                     45: PACKAGES=${PACKAGES:-/usr/pkgsrc/packages/}
                     46: AWK=${AWK:-awk}
                     47: CMP=${CMP:-cmp}
1.3       dmcmahil   48: FIND=${FIND:-find}
1.1       dmcmahil   49: GREP=${GREP:-grep}
                     50: PKG_INFO=${PKG_INFO:-pkg_info}
                     51: PKG_SUFX=${PKG_SUFX:-.tgz}
                     52: SED=${SED:-sed}
                     53: SORT=${SORT:-sort}
                     54: STAT=${STAT:-stat}
                     55: 
                     56: cachefile=.pkgcache
                     57: cacheversion=20050428
                     58: 
                     59: prompt="----> "
                     60: tab="      "
                     61: 
                     62: tmpd=${TMPDIR}/pkg-cache.$$
                     63: mkdir -m 0700 ${tmpd}
                     64: if test $? -ne 0 ; then
                     65:        echo "ERROR:  Could not create temporary directory ${tmpd}"
                     66:        echo "Either you do not have write permission to ${tmpd} or"
                     67:        echo "${tmpd} already exists"
                     68:        exit 1
                     69: fi
                     70: 
                     71: prog=$0
                     72: 
                     73: usage(){
                     74:        echo "$prog - Generates cache files for each directory containing binary"
                     75:        echo "        packages.  This cache file can then be used by the README.html"
                     76:        echo "        generation code to avoid having to call pkg_info(1) over and over"
                     77:        echo "        on the same binary package."
                     78:        echo " "
                     79:        echo "Usage:      $prog [-d|--debug] [-v|--verbose] [-p|--packages <dir>]"
                     80:        echo " "
                     81:        echo "            $prog -h|--help"
                     82:        echo " "
                     83:        echo "            $prog -V|--version"
                     84:        echo " "
                     85:        echo "The options supported by $prog are: "
                     86:        echo " "
                     87:        echo "  -d|--debug            Enables debugging output"
                     88:        echo " "
                     89:        echo "  -f|--database <file>  Writes the database into file specified by <file>"
                     90:        echo " "
                     91:        echo "  -h|--help             Displays this help message"
                     92:        echo " "
                     93:        echo "  -p|--packages <dir>   Specifies the top level directory to be searched"
                     94:        echo "                        for binary packages."
                     95:        echo " "
                     96:        echo "  -v|--version          Displays the version of this script and exits."
                     97:        echo " "
                     98:        echo "Example:    $prog -v --packages /usr/pkgsrc/packages"
                     99:        echo " "
                    100: }
                    101: 
                    102: clean_and_exit0(){
                    103:        rm -fr ${tmpd}
                    104:        exit 0
                    105: }
                    106: 
                    107: clean_and_exit1(){
                    108:        rm -fr ${tmpd}
                    109:        exit 1
                    110: }
                    111: 
                    112: all_cache_files=""
                    113: 
1.4       dmcmahil  114: #############################################################################
                    115: #
                    116: # process_binpkg_dir()
                    117: #
                    118: # Process a directory by checking to see if a cache file exists.  If the
                    119: # cache file exists, make sure it is up to date.  If the file does not
                    120: # exist, create one.
                    121: #
                    122: # also keep track of this directory so it can be added to the master
                    123: # cache.
                    124: #
                    125: 
1.1       dmcmahil  126: process_binpkg_dir(){
                    127:        need_update=no
                    128:        if test -f ${d}/${cachefile} ; then
1.3       dmcmahil  129:                stale_entries=`${FIND} ${d} -type f -name \*${PKG_SUFX} -newer ${d}/${cachefile} -print`
1.1       dmcmahil  130: 
                    131:                # FIX_ME
                    132:                #
                    133:                # We also should find cache entries for files which no longer exist
                    134:                # and nuke them.  Right now we simply declare the entire cache out
                    135:                # of date.  Once we implement incremental updates to the cache,
                    136:                # we need to remove the entries but not mark the entire cache as
                    137:                # bad.
                    138:                if test "X${DEBUG}" = "Xyes" ; then
                    139:                        echo "      Checking for cache entries with no corresponding pkg."
                    140:                fi
                    141:                # get the list of what pkgs belong in the cache
                    142:                rm -f ${tmpd}/pkg_list ${tmpd}/cache_pkg_list
1.3       dmcmahil  143:                ${FIND} ${d}/ -name \*${PKG_SUFX} -print | \
                    144:                        ${SED} -e "s;^${PACKAGES}/*;;g" -e 's;//;/;g' | \
                    145:                        ${SORT} > ${tmpd}/pkg_list
1.1       dmcmahil  146: 
                    147:                # and get the list of what is in the cache
                    148:                ${AWK} '/pkgcache_begin/ {gsub(/pkgcache_begin[ \t]*/, ""); print}' \
                    149:                        ${d}/${cachefile} | ${SORT} > ${tmpd}/cache_pkg_list
                    150: 
                    151:                if ${CMP} -s ${tmpd}/pkg_list ${tmpd}/cache_pkg_list ; then
                    152:                        if test "X${DEBUG}" = "Xyes" ; then
                    153:                                echo "      No extra cache entries in ${d}/${cachefile}"
                    154:                        fi
                    155:                else
                    156:                        echo "      Entries found in ${d}/${cachefile} but no packages found"
                    157:                        need_update=yes
                    158:                fi
                    159:        else
                    160:                stale_entries=""
                    161:        fi
                    162: 
                    163:        if test "X${force}" = "Xyes" -o "X${need_update}" = "Xyes" ; then
                    164:                need_update=yes
                    165:                echo "${tab}Forcing rebuild of cache ${d}/${cachefile}."
                    166:        elif test ! -f ${d}/${cachefile} ; then
                    167:                need_update=yes
                    168:                echo "${tab}Missing cache file.  ${d}/${cachefile} will be generated."
                    169:        elif test -n "${stale_entries}" ; then
                    170:                need_update=yes
                    171:                echo "${tab}Stale cache file.  ${d}/${cachefile} will be regenerated."
                    172:        else
                    173:                ${GREP} "pkgcache_version ${cacheversion}" ${d}/${cachefile} >/dev/null 2>&1
                    174:                if test $? -ne 0 ; then
                    175:                        need_update=yes
1.5       dmcmahil  176:                        echo "${tab}Invalid version cache file.  ${d}/${cachefile} will be regenerated."
                    177:                        echo "Need version ${cacheversion} but the file has"
                    178:                        ${GREP} "^pkgcache_version " ${d}/${cachefile}
1.1       dmcmahil  179:                else
                    180:                        echo "${tab}Cache file ${d}/${cachefile} is up to date."
                    181:                fi
                    182:        fi
                    183: 
                    184:        # FIX_ME
                    185:        # We should use stale_entries in a way where we only update the 
                    186:        # cache file entries corresponding to these if we're rebuilding
                    187:        # due to stale entries.  That should save a good bit of time.
                    188:        # 
                    189:        if test "X${need_update}" = "Xyes" ; then
                    190:                echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
                    191:                for f in ${d}/*${PKG_SUFX} ; do
                    192:                        fn=`echo $f | ${SED} "s;^${PACKAGES}/*;;g"`
                    193:                        if test "X${DEBUG}" = "Xyes" ; then
                    194:                                echo "     Adding ${fn} (${f}) to the cache"
                    195:                        fi
                    196:                        echo " " >> ${tmpd}/${cachefile}
                    197:                        # stat(1) needs to be added to the bootstrap kit
                    198:                        # first if we want to use it here
                    199:                        #eval $(${STAT} -s ${f} 2>/dev/null)
                    200:                        echo "pkgcache_begin ${fn}" >> ${tmpd}/${cachefile}
                    201:                        #echo "pkgcache_mtime=${st_mtime}" >> ${tmpd}/${cachefile}
                    202:                        if test "X${DEBUG}" = "Xyes" ; then
                    203:                                echo "${PKG_INFO} -q -B ${f}"
                    204:                        fi
                    205:                        ${PKG_INFO} -q -B ${f} >> ${tmpd}/${cachefile}
                    206:                        echo "pkgcache_end ${fn}" >> ${tmpd}/${cachefile}
                    207:                done
1.4       dmcmahil  208:                mv -f ${tmpd}/${cachefile} ${d}/${cachefile}
1.2       dmcmahil  209:                if test $? -ne 0 ; then
1.4       dmcmahil  210:                        echo "********** WARNING **********"
1.2       dmcmahil  211:                        echo "move of ${tmpd}/${cachefile} to ${d}/${cachefile} failed!"
                    212:                        echo "Perhaps you do not have write permissions to ${d}?"
1.4       dmcmahil  213:                        echo "This directory will be dropped from the master cache file."
                    214:                        echo "********** WARNING **********"
                    215:                        return
1.2       dmcmahil  216:                fi
1.1       dmcmahil  217:        fi
                    218: 
1.4       dmcmahil  219:        # if we got here, then this directory should have a good cache file in
                    220:        # it and we should be able to add it to the master cache file
                    221:        all_cache_files="${all_cache_files} ${d}/${cachefile}"
1.1       dmcmahil  222: 
                    223: }
                    224: 
                    225: process_cache_files(){
                    226:        echo "${prompt}Checking master cache file ${PACKAGES}/${cachefile}"
                    227:        echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
                    228:        if test -n "${all_cache_files}" ; then
                    229:                for c in ${all_cache_files} ; do
                    230:                        echo "pkgcache_cachefile ${c}" >> ${tmpd}/${cachefile}
                    231:                done
                    232:        fi
                    233:        if test ! -f ${PACKAGES}/${cachefile} ; then
                    234:                echo "${tab}Creating master cache file ${PACKAGES}/${cachefile}"
1.4       dmcmahil  235:                mv -f ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
1.2       dmcmahil  236:                if test $? -ne 0 ; then
1.4       dmcmahil  237:                        echo "********** ERROR **********"
1.2       dmcmahil  238:                        echo "move of ${tmpd}/${cachefile} to ${PACKAGES}/${cachefile} failed!"
                    239:                        echo "Perhaps you do not have write permissions to ${PACKAGES}?"
1.4       dmcmahil  240:                        echo "********** ERROR **********"
1.2       dmcmahil  241:                        clean_and_exit1
                    242:                fi
                    243:        elif ${CMP} -s ${tmpd}/${cachefile} ${PACKAGES}/${cachefile} ; then
1.1       dmcmahil  244:                echo "${tab}Master cache file ${PACKAGES}/${cachefile} is up to date"
                    245:        else
                    246:                echo "${tab}Updating master cache file ${PACKAGES}/${cachefile}"
1.4       dmcmahil  247:                mv -f ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
1.2       dmcmahil  248:                if test $? -ne 0 ; then
1.4       dmcmahil  249:                        echo "********** ERROR **********"
1.2       dmcmahil  250:                        echo "move of ${tmpd}/${cachefile} to ${PACKAGES}/${cachefile} failed!"
                    251:                        echo "Perhaps you do not have write permissions to ${PACKAGES}?"
1.4       dmcmahil  252:                        echo "********** ERROR **********"
1.2       dmcmahil  253:                        clean_and_exit1
                    254:                fi
1.1       dmcmahil  255:        fi
                    256: }
                    257: 
                    258: ######################################################################
                    259: #
                    260: #  Handle command line options
                    261: #
                    262: ######################################################################
                    263: 
                    264: DEBUG=no
                    265: verbose=no
                    266: force=no
                    267: 
                    268: while
                    269:        test -n "$1"
                    270: do
                    271:        case "$1" in
                    272: 
                    273:                # Turn on debugging
                    274:                -d|--debug)
                    275:                        DEBUG=yes
                    276:                        verbose=yes
                    277:                        shift
                    278:                        ;;
                    279: 
                    280:                # Force a rebuilde of the cache
                    281:                -F|--force)
                    282:                        force=yes
                    283:                        shift
                    284:                        ;;
                    285:                # Help
                    286:                -h|--help)
                    287:                        usage
                    288:                        exit 0
                    289:                        ;;
                    290: 
                    291:                # Use the specified packages directory
                    292:                -p|--packages)
                    293:                        PACKAGES=$2
                    294:                        shift 2
                    295:                        ;;
                    296: 
                    297:                # Version
                    298:                -V|--version)
                    299:                        ${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s:  Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
                    300:                        exit 0
                    301:                        ;;
                    302: 
                    303:                # Turn on verbose output, but not as noisy as DEBUG
                    304:                -v|--verbose)
                    305:                        verbose=yes
                    306:                        shift
                    307:                        ;;
                    308: 
                    309:                -*) echo "$prog:  ERROR:  $1 is not a valid option"
                    310:                        usage
                    311:                        clean_and_exit1
                    312:                        ;;
                    313: 
                    314:                *) 
                    315:                        break
                    316:                        ;;
                    317: 
                    318:        esac
                    319: done
                    320: 
1.6     ! dmcmahil  321: if test $# -ne 0 ; then
        !           322:        echo "$0:  $* is invalid"
        !           323:        usage
        !           324:        clean_and_exit1
        !           325: fi
        !           326: 
        !           327: if test ! -d ${PACKAGES} ; then
        !           328:        echo "Packages directory ${PACKAGES} seems to be missing"
        !           329:        clean_and_exit1
        !           330: fi
        !           331: 
1.1       dmcmahil  332: # put a trailing / after ${PACKAGES} in case ${PACKAGES} is 
                    333: # a link.
1.3       dmcmahil  334: for d in `${FIND} ${PACKAGES}/ -type d -print` ; do
1.1       dmcmahil  335:        if test "X${DEBUG}" = "Xyes" ; then
                    336:                echo "${prompt}Processing directory ${d}"
                    337:        fi
                    338:        is_pkg_dir=no
                    339:        for f in ${d}/*${PKG_SUFX} ; do
                    340:                if test -f "${f}" -a ! -h "${f}" ; then
                    341:                        ${PKG_INFO} ${f} >/dev/null 2>&1
                    342:                        if test $? -eq 0 ; then
                    343:                                is_pkg_dir=yes
                    344:                                break
                    345:                        fi
                    346:                fi
                    347:        done
                    348:        if test "X${is_pkg_dir}" = "Xyes" ; then
                    349:                echo "${prompt}Checking cache in ${d}"
                    350:                process_binpkg_dir
                    351:        else
                    352:                if test "X${DEBUG}" = "Xyes" ; then
                    353:                        echo "${prompt}no binary packages in ${d}"
                    354:                fi
                    355:        fi
                    356:                
                    357: done
                    358: 
                    359: process_cache_files
                    360: 
                    361: clean_and_exit0
                    362: 

CVSweb <webmaster@jp.NetBSD.org>