Annotation of pkgsrc/mk/scripts/binpkg-cache, revision 1.22
1.1 dmcmahil 1: #!/bin/sh
2: #
1.22 ! maya 3: # $NetBSD: binpkg-cache,v 1.21 2010/07/21 12:29:46 spz Exp $
1.1 dmcmahil 4: #
5: # Script for generating a cache file with information about
6: # all binary packages contained in a directory.
7: #
1.7 dmcmahil 8: # Copyright (c) 2005, 2006 The NetBSD Foundation, Inc.
1.1 dmcmahil 9: # All rights reserved.
10: #
11: # This code is derived from software contributed to The NetBSD Foundation
12: # by Dan McMahill.
13: #
14: # Redistribution and use in source and binary forms, with or without
15: # modification, are permitted provided that the following conditions
16: # are met:
17: # 1. Redistributions of source code must retain the above copyright
18: # notice, this list of conditions and the following disclaimer.
19: # 2. Redistributions in binary form must reproduce the above copyright
20: # notice, this list of conditions and the following disclaimer in the
21: # documentation and/or other materials provided with the distribution.
22: #
23: # THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
24: # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
25: # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
26: # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
27: # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28: # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29: # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30: # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31: # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32: # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33: # POSSIBILITY OF SUCH DAMAGE.
34: #
35:
36:
37: TMPDIR=${TMPDIR:-/tmp}
38: PACKAGES=${PACKAGES:-/usr/pkgsrc/packages/}
39: AWK=${AWK:-awk}
40: CMP=${CMP:-cmp}
1.3 dmcmahil 41: FIND=${FIND:-find}
1.1 dmcmahil 42: GREP=${GREP:-grep}
1.11 dmcmahil 43: GZIP_CMD=${GZIP_CMD:-gzip}
1.19 spz 44: BZIP2=${BZIP2:-bzip2}
1.1 dmcmahil 45: PKG_INFO=${PKG_INFO:-pkg_info}
46: PKG_SUFX=${PKG_SUFX:-.tgz}
47: SED=${SED:-sed}
48: SORT=${SORT:-sort}
49: STAT=${STAT:-stat}
50:
51: cachefile=.pkgcache
1.11 dmcmahil 52: summaryfile=pkg_summary
1.9 dmcmahil 53: cacheversion=20050428
1.1 dmcmahil 54:
55: prompt="----> "
56: tab=" "
57:
58: tmpd=${TMPDIR}/pkg-cache.$$
59: mkdir -m 0700 ${tmpd}
60: if test $? -ne 0 ; then
61: echo "ERROR: Could not create temporary directory ${tmpd}"
62: echo "Either you do not have write permission to ${tmpd} or"
63: echo "${tmpd} already exists"
64: exit 1
65: fi
1.10 dmcmahil 66: all_dirs=${tmpd}/all_dirs
1.1 dmcmahil 67:
68: prog=$0
69:
70: usage(){
71: echo "$prog - Generates cache files for each directory containing binary"
72: echo " packages. This cache file can then be used by the README.html"
73: echo " generation code to avoid having to call pkg_info(1) over and over"
1.13 dmcmahil 74: echo " on the same binary package. In addition, pkg_summary.gz files for"
75: echo " use by the pkgtools/pkg_chk pacakge may be generated."
1.1 dmcmahil 76: echo " "
1.13 dmcmahil 77: echo "Usage: $prog [-d|--debug] [-v|--verbose] [-s|--summary] [-p|--packages <dir>]"
1.1 dmcmahil 78: echo " "
79: echo " $prog -h|--help"
80: echo " "
81: echo " $prog -V|--version"
82: echo " "
83: echo "The options supported by $prog are: "
84: echo " "
85: echo " -d|--debug Enables debugging output"
86: echo " "
1.13 dmcmahil 87: echo " -F|--force Forces a rebuild of the cache files even if they are"
88: echo " otherwise up to date."
89: echo " "
1.1 dmcmahil 90: echo " -h|--help Displays this help message"
91: echo " "
92: echo " -p|--packages <dir> Specifies the top level directory to be searched"
93: echo " for binary packages."
94: echo " "
1.13 dmcmahil 95: echo " -s|--summary Enables the creation of pkg_summary.gz files in each"
96: echo " directory containing binary packages. The pkg_summary.gz"
97: echo " file is used by the pkgtools/pkg_chk package."
98: echo " "
99: echo " -v|--verbose Enables verbose output from the script."
100: echo " "
101: echo " -V|--version Displays the version of this script and exits."
1.1 dmcmahil 102: echo " "
1.7 dmcmahil 103: echo "Returns 0 on success, 1 on errors, 2 if the packages"
104: echo "directory does not exist."
105: echo " "
1.1 dmcmahil 106: echo "Example: $prog -v --packages /usr/pkgsrc/packages"
107: echo " "
108: }
109:
110: clean_and_exit0(){
111: rm -fr ${tmpd}
112: exit 0
113: }
114:
115: clean_and_exit1(){
116: rm -fr ${tmpd}
117: exit 1
118: }
119:
1.7 dmcmahil 120: clean_and_exit2(){
121: rm -fr ${tmpd}
122: exit 2
123: }
124:
1.1 dmcmahil 125: all_cache_files=""
126:
1.4 dmcmahil 127: #############################################################################
128: #
129: # process_binpkg_dir()
130: #
131: # Process a directory by checking to see if a cache file exists. If the
132: # cache file exists, make sure it is up to date. If the file does not
133: # exist, create one.
134: #
135: # also keep track of this directory so it can be added to the master
136: # cache.
137: #
138:
1.1 dmcmahil 139: process_binpkg_dir(){
1.10 dmcmahil 140: rdir=`${GREP} "^${d} " ${all_dirs} | ${AWK} '{print $2}'`
1.1 dmcmahil 141: need_update=no
142: if test -f ${d}/${cachefile} ; then
1.3 dmcmahil 143: stale_entries=`${FIND} ${d} -type f -name \*${PKG_SUFX} -newer ${d}/${cachefile} -print`
1.1 dmcmahil 144:
145: # FIX_ME
146: #
147: # We also should find cache entries for files which no longer exist
148: # and nuke them. Right now we simply declare the entire cache out
149: # of date. Once we implement incremental updates to the cache,
150: # we need to remove the entries but not mark the entire cache as
151: # bad.
1.17 rillig 152: $if_debug echo " Checking for cache entries with no corresponding pkg."
1.1 dmcmahil 153: # get the list of what pkgs belong in the cache
154: rm -f ${tmpd}/pkg_list ${tmpd}/cache_pkg_list
1.3 dmcmahil 155: ${FIND} ${d}/ -name \*${PKG_SUFX} -print | \
1.10 dmcmahil 156: ${SED} -e "s;^${d}/*;${rdir}/;g" -e 's;//;/;g' | \
1.3 dmcmahil 157: ${SORT} > ${tmpd}/pkg_list
1.1 dmcmahil 158:
159: # and get the list of what is in the cache
160: ${AWK} '/pkgcache_begin/ {gsub(/pkgcache_begin[ \t]*/, ""); print}' \
161: ${d}/${cachefile} | ${SORT} > ${tmpd}/cache_pkg_list
162:
163: if ${CMP} -s ${tmpd}/pkg_list ${tmpd}/cache_pkg_list ; then
1.17 rillig 164: $if_debug echo " No extra cache entries in ${d}/${cachefile}"
1.1 dmcmahil 165: else
1.17 rillig 166: $if_debug echo "Package list:"
167: $if_debug cat ${tmpd}/pkg_list
168: $if_debug echo "Cache list:"
169: $if_debug cat ${tmpd}/cache_pkg_list
1.1 dmcmahil 170: echo " Entries found in ${d}/${cachefile} but no packages found"
171: need_update=yes
172: fi
173: else
174: stale_entries=""
175: fi
176:
177: if test "X${force}" = "Xyes" -o "X${need_update}" = "Xyes" ; then
178: need_update=yes
179: echo "${tab}Forcing rebuild of cache ${d}/${cachefile}."
180: elif test ! -f ${d}/${cachefile} ; then
181: need_update=yes
182: echo "${tab}Missing cache file. ${d}/${cachefile} will be generated."
183: elif test -n "${stale_entries}" ; then
184: need_update=yes
185: echo "${tab}Stale cache file. ${d}/${cachefile} will be regenerated."
186: else
187: ${GREP} "pkgcache_version ${cacheversion}" ${d}/${cachefile} >/dev/null 2>&1
188: if test $? -ne 0 ; then
189: need_update=yes
1.5 dmcmahil 190: echo "${tab}Invalid version cache file. ${d}/${cachefile} will be regenerated."
191: echo "Need version ${cacheversion} but the file has"
192: ${GREP} "^pkgcache_version " ${d}/${cachefile}
1.1 dmcmahil 193: else
1.21 spz 194: $if_verbose echo "${tab}Cache file ${d}/${cachefile} is up to date."
1.1 dmcmahil 195: fi
196: fi
197:
1.14 martti 198: if test "${build_summary}" = "yes" -a ! -f ${d}/${summaryfile}.gz ; then
1.11 dmcmahil 199: echo "${tab}Summary file ${d}/${summaryfile}.gz is missing and will be created."
200: need_update=yes
201: fi
202: if test "${build_summary}" = "yes" -a ${d}/${summaryfile}.gz -ot ${d}/${cachefile} ; then
1.18 spz 203: echo "${tab}Summary file ${d}/${summaryfile}.gz is out of date and will be regenerated."
1.11 dmcmahil 204: need_update=yes
205: fi
206:
1.1 dmcmahil 207: # FIX_ME
1.14 martti 208: # We should use stale_entries in a way where we only update the
1.1 dmcmahil 209: # cache file entries corresponding to these if we're rebuilding
210: # due to stale entries. That should save a good bit of time.
1.14 martti 211: #
1.1 dmcmahil 212: if test "X${need_update}" = "Xyes" ; then
213: echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
1.12 dmcmahil 214: rm -f ${tmpd}/${summaryfile}
215: touch ${tmpd}/${summaryfile}
1.1 dmcmahil 216: for f in ${d}/*${PKG_SUFX} ; do
1.10 dmcmahil 217: fn=`grep "^${d} " ${all_dirs} | ${AWK} '{print $2}'`"/"`basename ${f}`
1.17 rillig 218: $if_debug echo " Adding ${fn} (${f}) to the cache"
1.1 dmcmahil 219: echo " " >> ${tmpd}/${cachefile}
220: # stat(1) needs to be added to the bootstrap kit
221: # first if we want to use it here
222: #eval $(${STAT} -s ${f} 2>/dev/null)
223: echo "pkgcache_begin ${fn}" >> ${tmpd}/${cachefile}
224: #echo "pkgcache_mtime=${st_mtime}" >> ${tmpd}/${cachefile}
1.11 dmcmahil 225:
1.17 rillig 226: $if_debug echo "${PKG_INFO} -q -B ${f}"
1.11 dmcmahil 227: ${PKG_INFO} -q -B ${f} >> ${tmpd}/${cachefile}
228:
229: if test "${build_summary}" = "yes" ; then
1.17 rillig 230: $if_debug echo "${PKG_INFO} -X ${f}"
1.11 dmcmahil 231: ${PKG_INFO} -X ${f} >> ${tmpd}/${summaryfile}
1.1 dmcmahil 232: fi
233: echo "pkgcache_end ${fn}" >> ${tmpd}/${cachefile}
234: done
1.18 spz 235: if test -f ${d}/${cachefile} ; then
236: rm -f ${d}/${cachefile}
237: fi
1.4 dmcmahil 238: mv -f ${tmpd}/${cachefile} ${d}/${cachefile}
1.2 dmcmahil 239: if test $? -ne 0 ; then
1.4 dmcmahil 240: echo "********** WARNING **********"
1.2 dmcmahil 241: echo "move of ${tmpd}/${cachefile} to ${d}/${cachefile} failed!"
242: echo "Perhaps you do not have write permissions to ${d}?"
1.4 dmcmahil 243: echo "This directory will be dropped from the master cache file."
244: echo "********** WARNING **********"
245: return
1.2 dmcmahil 246: fi
1.15 martti 247:
1.11 dmcmahil 248: if test "${build_summary}" = "yes" ; then
1.18 spz 249: if test -f ${d}/${summaryfile}.gz ; then
250: rm -f ${d}/${summaryfile}.gz
251: fi
1.11 dmcmahil 252: cat ${tmpd}/${summaryfile} | ${GZIP_CMD} > ${d}/${summaryfile}.gz
253: if test $? -ne 0 ; then
254: echo "********** WARNING **********"
255: echo "${GZIP_CMD} of ${tmpd}/${summaryfile} to ${d}/${summaryfile}.gz failed!"
256: echo "Perhaps you do not have write permissions to ${d}?"
257: echo "********** WARNING **********"
258: return
259: fi
1.19 spz 260: # if it's there, update it, otherwise don't bother
261: if test -f ${d}/${summaryfile}.bz2 ; then
1.20 spz 262: rm -f ${d}/${summaryfile}.bz2
1.19 spz 263: cat ${tmpd}/${summaryfile} | ${BZIP2} > ${d}/${summaryfile}.bz2
264: if test $? -ne 0 ; then
265: echo "********** WARNING **********"
266: echo "${BZIP2} of ${tmpd}/${summaryfile} to ${d}/${summaryfile}.bz2 failed!"
267: echo "Perhaps you do not have write permissions to ${d}?"
268: echo "********** WARNING **********"
269: return
270: fi
271: fi
1.11 dmcmahil 272: fi
273:
1.1 dmcmahil 274: fi
275:
1.4 dmcmahil 276: # if we got here, then this directory should have a good cache file in
277: # it and we should be able to add it to the master cache file
278: all_cache_files="${all_cache_files} ${d}/${cachefile}"
1.1 dmcmahil 279:
280: }
281:
282: process_cache_files(){
283: echo "${prompt}Checking master cache file ${PACKAGES}/${cachefile}"
284: echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
285: if test -n "${all_cache_files}" ; then
286: for c in ${all_cache_files} ; do
287: echo "pkgcache_cachefile ${c}" >> ${tmpd}/${cachefile}
288: done
289: fi
290: if test ! -f ${PACKAGES}/${cachefile} ; then
291: echo "${tab}Creating master cache file ${PACKAGES}/${cachefile}"
1.4 dmcmahil 292: mv -f ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
1.2 dmcmahil 293: if test $? -ne 0 ; then
1.4 dmcmahil 294: echo "********** ERROR **********"
1.2 dmcmahil 295: echo "move of ${tmpd}/${cachefile} to ${PACKAGES}/${cachefile} failed!"
296: echo "Perhaps you do not have write permissions to ${PACKAGES}?"
1.4 dmcmahil 297: echo "********** ERROR **********"
1.2 dmcmahil 298: clean_and_exit1
299: fi
300: elif ${CMP} -s ${tmpd}/${cachefile} ${PACKAGES}/${cachefile} ; then
1.1 dmcmahil 301: echo "${tab}Master cache file ${PACKAGES}/${cachefile} is up to date"
302: else
303: echo "${tab}Updating master cache file ${PACKAGES}/${cachefile}"
1.4 dmcmahil 304: mv -f ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
1.2 dmcmahil 305: if test $? -ne 0 ; then
1.4 dmcmahil 306: echo "********** ERROR **********"
1.2 dmcmahil 307: echo "move of ${tmpd}/${cachefile} to ${PACKAGES}/${cachefile} failed!"
308: echo "Perhaps you do not have write permissions to ${PACKAGES}?"
1.4 dmcmahil 309: echo "********** ERROR **********"
1.2 dmcmahil 310: clean_and_exit1
311: fi
1.1 dmcmahil 312: fi
313: }
314:
315: ######################################################################
316: #
317: # Handle command line options
318: #
319: ######################################################################
320:
1.17 rillig 321: if_debug=: # either ":" or ""
1.21 spz 322: if_verbose=: # either ":" or ""
1.1 dmcmahil 323: force=no
1.11 dmcmahil 324: build_summary=no
1.1 dmcmahil 325:
326: while
327: test -n "$1"
328: do
329: case "$1" in
330:
331: # Turn on debugging
332: -d|--debug)
1.17 rillig 333: if_debug=""
1.21 spz 334: if_verbose=""
1.1 dmcmahil 335: shift
336: ;;
337:
338: # Force a rebuilde of the cache
339: -F|--force)
340: force=yes
341: shift
342: ;;
343: # Help
344: -h|--help)
345: usage
346: exit 0
347: ;;
348:
349: # Use the specified packages directory
350: -p|--packages)
351: PACKAGES=$2
352: shift 2
353: ;;
354:
1.11 dmcmahil 355: # Build the pkg_summary.gz files
356: -s|--summary)
357: build_summary=yes
358: shift
359: ;;
360:
1.1 dmcmahil 361: # Version
362: -V|--version)
363: ${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s: Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
364: exit 0
365: ;;
366:
1.17 rillig 367: # Turn on verbose output, but not as noisy as debug
1.1 dmcmahil 368: -v|--verbose)
1.21 spz 369: if_verbose=""
1.1 dmcmahil 370: shift
371: ;;
372:
373: -*) echo "$prog: ERROR: $1 is not a valid option"
374: usage
375: clean_and_exit1
376: ;;
377:
1.14 martti 378: *)
1.1 dmcmahil 379: break
380: ;;
381:
382: esac
383: done
384:
1.6 dmcmahil 385: if test $# -ne 0 ; then
386: echo "$0: $* is invalid"
387: usage
388: clean_and_exit1
389: fi
390:
391: if test ! -d ${PACKAGES} ; then
392: echo "Packages directory ${PACKAGES} seems to be missing"
1.7 dmcmahil 393: clean_and_exit2
1.6 dmcmahil 394: fi
395:
1.14 martti 396: # put a trailing / after ${PACKAGES} in case ${PACKAGES} is
1.1 dmcmahil 397: # a link.
1.10 dmcmahil 398:
399: # pass 1, we find all directories under PACKAGES. Note that this
400: # may contain some directories more than once depending on what sort
401: # of soft links may be in place
402: rm -f ${all_dirs}.tmp
1.8 dmcmahil 403: for d in `${FIND} ${PACKAGES}/ -type d -follow -print` ; do
1.10 dmcmahil 404: cname=`(cd ${d} && pwd -P)`
405: rname=`echo ${d} | ${SED} "s;^${PACKAGES}/*;;g"`
406: echo "${cname} ${rname}" >> ${all_dirs}.tmp
407: done
408: ${SORT} -u -k1,1 ${all_dirs}.tmp > ${all_dirs}
1.17 rillig 409: $if_debug echo "Full directory list:"
410: $if_debug cat ${all_dirs}.tmp
411: $if_debug echo "Unique directory list:"
412: $if_debug cat ${all_dirs}
1.10 dmcmahil 413:
414: for d in `${AWK} '{print $1}' ${all_dirs}` ; do
1.17 rillig 415: $if_debug echo "${prompt}Processing directory ${d}"
1.1 dmcmahil 416: is_pkg_dir=no
417: for f in ${d}/*${PKG_SUFX} ; do
418: if test -f "${f}" -a ! -h "${f}" ; then
419: ${PKG_INFO} ${f} >/dev/null 2>&1
420: if test $? -eq 0 ; then
421: is_pkg_dir=yes
422: break
423: fi
424: fi
425: done
426: if test "X${is_pkg_dir}" = "Xyes" ; then
1.21 spz 427: $if_verbose echo "${prompt}Checking cache in ${d}"
1.1 dmcmahil 428: process_binpkg_dir
429: else
1.17 rillig 430: $if_debug echo "${prompt}no binary packages in ${d}"
1.1 dmcmahil 431: fi
1.15 martti 432:
1.1 dmcmahil 433: done
434:
435: process_cache_files
436:
437: clean_and_exit0
CVSweb <webmaster@jp.NetBSD.org>