Annotation of pkgsrc/pkgtools/pkg_rolling-replace/files/pkg_rolling-replace.sh, revision 1.15

1.1       gdt         1: #!/bin/sh
                      2: 
1.15    ! tnn         3: # $NetBSD: pkg_rolling-replace.sh,v 1.14 2007/08/17 01:19:45 gdt Exp $
1.1       gdt         4: #<license>
                      5: # Copyright (c) 2006 BBN Technologies Corp.  All rights reserved.
                      6: #
                      7: # Redistribution and use in source and binary forms, with or without
                      8: # modification, are permitted provided that the following conditions
                      9: # are met:
                     10: # 1. Redistributions of source code must retain the above copyright
                     11: #    notice, this list of conditions and the following disclaimer.
                     12: # 2. Redistributions in binary form must reproduce the above copyright
                     13: #    notice, this list of conditions and the following disclaimer in the
                     14: #    documentation and/or other materials provided with the distribution.
                     15: # 3. Neither the name of BBN Technologies nor the names of its contributors
                     16: #    may be used to endorse or promote products derived from this software
                     17: #    without specific prior written permission.
                     18: #
                     19: # THIS SOFTWARE IS PROVIDED BY BBN TECHNOLOGIES AND CONTRIBUTORS ``AS IS''
                     20: # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
                     21: # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
                     22: # ARE DISCLAIMED.  IN NO EVENT SHALL BBN TECHNOLOGIES OR CONTRIBUTORS BE
                     23: # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
                     24: # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
                     25: # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
                     26: # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
                     27: # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
                     28: # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
                     29: # POSSIBILITY OF SUCH DAMAGE.
                     30: #
                     31: # Effort sponsored in part by the Defense Advanced Research Projects
                     32: # Agency (DARPA) and the Department of the Interior National Business
                     33: # Center under agreement number NBCHC050166.
                     34: #</license>
                     35: 
                     36: # REVIEW NOTATION LEGEND
                     37: #   XXX marks things that are broken/kludgy
                     38: #   TODO marks things that need to be done
                     39: #   FIXED marks items fixed, needs review and deletion of FIXED comment
                     40: 
                     41: # Items to be completed for initial release:
                     42: 
                     43: # TODO: move bulk of help into a real man page.
                     44: 
                     45: # Items for future work (some imply changes in pkgsrc, not necessarily here)
                     46: #
                     47: #   Make a pass over all packages for build depends, and rememeber
                     48: # which packages have been so checked.  Check newly unsafe packages.
                     49: # The current lazy evaluation can perhaps fail to do things in the
                     50: # right order.  Alternatively, explain why the current scheme is safe
                     51: # and preferred.
                     52: #
                     53: #   Consider expansion to update via binary packages.  This raises
                     54: # interesting questions about ABIs and binary builds
                     55: #
                     56: #   Deal with conflicts in a smarter way, perhaps more automatically.
                     57: # This is hard because just removing packages is not ok.
                     58: #
                     59: #   Handle foo being split into foo and foo-share.  Now, make replace
                     60: # of foo pulls in foo-share which conflicts with foo.  This needlessly
                     61: # loses.
                     62: #
                     63: 
1.3       tnn        64: # Substituted by pkgsrc at pre-configure time.
1.4       tnn        65: MAKE="@MAKE@"
1.15    ! tnn        66: AWK="@AWK@"
1.3       tnn        67: 
1.4       tnn        68: test -z "$MAKECONF" && MAKECONF="@MAKECONF@"
1.7       tnn        69: test -f "$MAKECONF" && test -z "$PKGSRCDIR" && PKGSRCDIR="` \
                     70:     printf '.include "%s"\n_print_pkgsrcdir:\n\t@echo "${PKGSRCDIR}"\n' \
1.8       tnn        71:     "$MAKECONF" | "$MAKE" -f - BSD_PKG_MK=1 _print_pkgsrcdir`"
1.3       tnn        72: test -z "$PKGSRCDIR" && PKGSRCDIR=/usr/pkgsrc
1.12      tnn        73: test -z "$PKG_CHK" && PKG_CHK="@PKG_CHK@"
                     74: test -z "$PKG_INFO" && PKG_INFO="@PKG_INFO_CMD@"
1.1       gdt        75: 
                     76: unset PKG_PATH || true  #or pkgsrc makefiles will complain
                     77: 
                     78: usage()
                     79: {
                     80:     echo "Usage: pkg_rolling-replace [opts]
                     81:         -h         This help
                     82:         -n         Don't actually do make replace
1.13      tnn        83:         -r         Just replace, don't create binary packages
                     84:         -s         Replace even if the ABIs are still compatible ("strict")
1.1       gdt        85:         -u         Update mismatched packages
                     86:         -v         Verbose
                     87:         -x <pkg>   exclude <pkg> from update check
                     88: 
                     89: pkg_rolling-replace does 'make replace' on one package at a time,
                     90: tsorting the packages being replaced according to their
                     91: interdependencies, which avoids most duplicate rebuilds.
                     92: 
                     93: pkg_rolling-replace can be used in one of two ways:
                     94: 
                     95:     - 'make replace' is unsafe in that, if the replaced package's ABI
                     96:       changes, its dependent packages may break.  If this happens, run
                     97:       'pkg_rolling-replace' (no arguments) to rebuild them against the
                     98:       new version.
                     99: 
                    100:     - 'pkg_chk -u' will delete all your mismatched (outdated)
                    101:       packages, then reinstall them one at a time, leaving you without
                    102:       those packages in the meantime.  'pkg_rolling-replace -u' will
                    103:       instead upgrade them in place, allowing you to keep using your
                    104:       system in the meantime (maybe...if you're lucky...because
                    105:       pkg_rolling-replace replaces the \"deepest\" dependency first,
                    106:       things could still break if that happens to be a fundamental
                    107:       library whose ABI has changed).
                    108: "
                    109:     exit 1
                    110: }
                    111: 
                    112: # pkg_rolling-replace's output is intermingled with pkgsrc make
                    113: # output.  Thus, we use a different prefix.  pkgsrc uses '===>' for
                    114: # major steps, and '=>' for minor ones.  Thus, we use 'RR>'.
                    115: 
                    116: OPI='RR>'
                    117: OPC='rr>' # continuation
                    118: 
                    119: 
                    120: # Echo the names of packages needing updates, versions stripped.  This
                    121: # has been tested with pkg_chk 1.76.  Older versions are not
                    122: # supported.  Newer versions may or may not work (patches welcome).
                    123: check_packages_mismatched()
                    124: {
1.12      tnn       125:     ${PKG_CHK} -u -q | while read line; do
1.1       gdt       126:         # duplicate output of pkg_chk to stderr (bypass $(...) or `...`)
1.11      tnn       127:         echo "${OPC} $line" 1>&2
1.1       gdt       128:        # Look for the first thing that looks like pkg-version rather
                    129:        # than category/pkg and remove the version.
                    130:         for word in $line; do
1.5       tnn       131:             if [ "$(echo $word | egrep '^[^/]+-[0-9][^-/]*$')" ]; then
                    132:                 echo $word | sed 's/-[0-9][^-]*$//'
1.1       gdt       133:                 break  #done with this line
                    134:             fi
                    135:         done
                    136:     done
                    137: }
                    138: 
                    139: # usage: check_packages_w_flag flag
                    140: # echo packages with flag=YES
                    141: check_packages_w_flag()
                    142: {
                    143:     _flag=$1; shift
1.12      tnn       144:     for pkgver in $(${PKG_INFO} -e '*'); do
                    145:         if ${PKG_INFO} -Bq $pkgver \
1.1       gdt       146:                 | egrep "^$_flag=[Yy][Ee][Ss]" > /dev/null; then
1.5       tnn       147:             echo $pkgver | sed 's/-[0-9][^-]*$//'
1.1       gdt       148:         fi
                    149:     done
                    150: }
                    151: 
                    152: # echo dep->pkg edges for all installed packages
                    153: depgraph_installed()
                    154: {
1.15    ! tnn       155:        ${PKG_INFO} -N '*' | ${AWK} '                                   \
        !           156:                /^Information for/ {                                    \
        !           157:                        pkg=$3; sub("-[0-9][^-]*:$", "", pkg);          \
        !           158:                        print pkg" "pkg;                                \
        !           159:                        state=1;                                        \
        !           160:                }                                                       \
        !           161:                /^./ {                                                  \
        !           162:                        if (state == 2) {                               \
        !           163:                                dep=$1; sub("-[0-9][^-]*$", "", dep);   \
        !           164:                                print dep" "pkg;                        \
        !           165:                        }                                               \
        !           166:                }                                                       \
        !           167:                /^Built using/ {                                        \
        !           168:                        state=2                                         \
        !           169:                }                                                       \
        !           170:        '
1.1       gdt       171: }
                    172: 
                    173: # usage: who_requires pkg --in-graph DEPGRAPH
                    174: who_requires()
                    175: {
                    176:     _target=$1; shift
                    177:     test "$1" = '--in-graph' && shift || return $((false))
                    178: 
                    179:     while [ $# -gt 0 ]; do
                    180:         _dep=$1; _pkg=$2; shift 2;
                    181:         if [ $_dep = $_target -a $_dep != $_pkg ]; then
                    182:             echo $_pkg
                    183:         fi
                    184:     done
                    185: }
                    186: 
                    187: # usage: is_member x list
                    188: # return true (0) if x appears in list, false (1) otherwise
                    189: is_member()
                    190: {
                    191:     _x="$1"; shift;
                    192:     while [ $# -gt 0 -a "$1" != "$_x" ]; do
                    193:         shift
                    194:     done
                    195: 
                    196:     test $# -gt 0
                    197: }
                    198: 
                    199: # usage: exclude list1 --from list2
                    200: # echo list2 with items from list1 excluded
                    201: exclude()
                    202: {
                    203:     _excl=
                    204:     while [ $# -gt 0 -a $1 != "--from" ]; do
                    205:         _excl="$_excl $1"
                    206:         shift
                    207:     done
                    208:     if [ $# -gt 0 ]; then
                    209:         shift  #discard '--from'
                    210:     fi
                    211:     while [ $# -gt 0 ]; do
                    212:         is_member $1 $_excl || echo $1
                    213:         shift
                    214:     done
                    215: }
                    216: 
                    217: # usage: uniquify list
                    218: # just does sort -u on list (equiv. to sort | uniq)
                    219: # CAUTION: reorders list
                    220: uniqify()
                    221: {
                    222:     for _x in $*; do echo $_x; done | sort -u
                    223: }
                    224: 
                    225: verbose()
                    226: {
                    227:     if [ -n "$opt_v" ]; then
                    228:         echo "$@"
                    229:     fi
                    230: }
                    231: 
                    232: vsleep()
                    233: {
                    234:     if [ -n "$opt_v" ]; then
                    235:         sleep $1
                    236:     fi
                    237: }
                    238: 
1.13      tnn       239: abort()
                    240: {
                    241:        echo "*** $1"
                    242:        echo "*** Please read the errors listed above, fix the problem,"
                    243:        echo "*** then re-run pkg_rolling-replace to continue."
                    244:        exit 1
                    245: }
                    246: 
1.1       gdt       247: ######################################################################
                    248: ##
                    249: ## main()
                    250: ##
                    251: 
                    252: EXCLUDE=
                    253: 
1.13      tnn       254: args=$(getopt hnursvx: $*)
1.1       gdt       255: if [ $? -ne 0 ]; then
                    256:     opt_h=1
                    257: fi
                    258: set -- $args
                    259: while [ $# -gt 0 ]; do
                    260:     case "$1" in
                    261:         -h) opt_h=1 ;;
                    262:         -n) opt_n=1 ;;
1.13      tnn       263:         -r) opt_r=1 ;;
                    264:         -s) opt_s=1 ;;
1.1       gdt       265:         -u) opt_u=1 ;;
                    266:         -v) opt_v=1 ;;
                    267:         -x) EXCLUDE="$EXCLUDE $(echo $2 | sed 's/,/ /g')" ; shift ;;
                    268:         --) shift; break ;;
                    269:     esac
                    270:     shift
                    271: done
                    272: 
                    273: if [ -n "$opt_h" ]; then
                    274:     usage
                    275: fi
                    276: 
1.13      tnn       277: if [ -n "$opt_s" ]; then
                    278:     UNSAFE_VAR=unsafe_depends_strict
                    279: else
                    280:     UNSAFE_VAR=unsafe_depends
                    281: fi
                    282: 
1.1       gdt       283: MISMATCH_TODO=
                    284: if [ -n "$opt_u" ]; then
                    285:     echo "${OPI} Checking for mismatched installed packages using pkg_chk"
                    286:     MISMATCH_TODO=$(check_packages_mismatched)
                    287:     echo "${OPI} Excluding the following mismatched packages:"
                    288:     echo "${OPC} EXCLUDE=[$EXCLUDE]"
                    289:     MISMATCH_TODO=$(exclude $EXCLUDE --from $MISMATCH_TODO)
                    290: fi
                    291: 
                    292: echo "${OPI} Checking for rebuild-requested installed packages (rebuild=YES)"
                    293: REBUILD_TODO=$(check_packages_w_flag 'rebuild')
                    294: 
1.13      tnn       295: echo "${OPI} Checking for unsafe installed packages (${UNSAFE_VAR}=YES)"
                    296: UNSAFE_TODO=$(check_packages_w_flag ${UNSAFE_VAR})
1.1       gdt       297: 
                    298: # DEPGRAPH_INSTALLED is rebuilt each round.  DEPGRAPH_SRC will collect
                    299: # edges that we discover using 'make show-depends', but that weren't
                    300: # listed as depends by the installed version of a package, and
                    301: # DEPENDS_CHECKED lists packages for which we've already done that
                    302: # check.
                    303: DEPGRAPH_INSTALLED=
                    304: DEPGRAPH_SRC=
                    305: DEPENDS_CHECKED=
                    306: 
                    307: verbose "${OPI} Packages to rebuild:"
                    308: verbose "${OPC} MISMATCH_TODO=[$(echo $MISMATCH_TODO)]"  #strip newlines
                    309: verbose "${OPC} REBUILD_TODO=[$(echo $REBUILD_TODO)]"
                    310: verbose "${OPC} UNSAFE_TODO=[$(echo $UNSAFE_TODO)]"
                    311: vsleep 2
                    312: 
                    313: REPLACE_TODO=$(uniqify $MISMATCH_TODO $REBUILD_TODO $UNSAFE_TODO)
                    314: depgraph_built=0
                    315: 
                    316: while [ -n "$REPLACE_TODO" ]; do
                    317:     # don't rebuild depgraph if we continued from new-depends step below
                    318:     if [ "$depgraph_built" -eq 0 ]; then
                    319:         echo "${OPI} Building dependency graph for installed packages"
                    320:         DEPGRAPH_INSTALLED=$(depgraph_installed)
                    321:         depgraph_built=1
                    322:     fi
                    323: 
                    324:     echo "${OPI} Tsorting dependency graph"
                    325:     TSORTED=$(echo $DEPGRAPH_INSTALLED $DEPGRAPH_SRC | tsort)
                    326:     for pkg in $TSORTED; do
                    327:         if is_member $pkg $REPLACE_TODO; then
                    328:             break;
                    329:         fi
                    330:     done
1.15    ! tnn       331:     pkgdir=$(${PKG_INFO} -Bq $pkg | ${AWK} -F= '/PKGPATH=/{print $2}' | sed -e 's/^ //')
1.1       gdt       332:     echo "${OPI} Selecting $pkg ($pkgdir) as next package to replace"
                    333:     sleep 1
                    334: 
                    335:     # Newer versions in pkgsrc sometimes have more depends than are
                    336:     # recorded for the installed version (this is entirely to be
                    337:     # expected as packages are split, renamed, and add new features).
                    338:     # When this happens, add the new edges to the graph and re-tsort.
                    339:     # Don't re-tsort if the new edges were already installed once
                    340:     # (otherwise we'll go into a loop).
                    341: 
                    342:     # XXX I'm not sure that DEPENDS_CHECKED is necessary, or whether
                    343:     # simply ignoring new deps that aren't in $REPLACE_TODO would have
                    344:     # been sufficient.  The DEPENDS_CHECKED approach causes one extra
                    345:     # tsort in some cases, which isn't overly expensive.
                    346: 
                    347:     # XXX After tsorting and choosing what to 'make replace', we
                    348:     # filter out packages that aren't already installed and marked for
                    349:     # replacement.  This is safe, since uninstalled depends will be
                    350:     # pulled in automatically by 'make replace'; but in rare cases it
                    351:     # might be non-optimal, leading to a duplicate rebuild that we
                    352:     # would have avoided had we selected the uninstalled package in
                    353:     # tsorted order and run 'make install' on it.  This seems like
                    354:     # such a rare case that the added complexity isn't worth it.
                    355: 
                    356:     if ! is_member $pkg $DEPENDS_CHECKED; then
                    357:        echo "${OPI} Checking if $pkg has new depends..."
1.12      tnn       358:        OLD_DEPENDS=$(${PKG_INFO} -Nq $pkg | sed 's/-[0-9][^-]*$//')
1.1       gdt       359:        NEW_DEPENDS=
                    360:        cd "$PKGSRCDIR/$pkgdir"
                    361:        bdeps=$(${MAKE} show-depends VARNAME=BUILD_DEPENDS)
                    362:        rdeps=$(${MAKE} show-depends)
                    363:        for depver in $bdeps $rdeps; do
1.10      tnn       364:            dep=$(echo $depver | sed -e 's/[:[].*$/0/' -e 's/[<>]=/-/' \
                    365:                -e 's/-[0-9][^-]*$//')
1.1       gdt       366:            if ! is_member $dep $OLD_DEPENDS $NEW_DEPENDS; then
                    367:                NEW_DEPENDS="$NEW_DEPENDS $dep"
                    368:                DEPGRAPH_SRC="$DEPGRAPH_SRC $dep $pkg"
                    369:            fi
                    370:        done
                    371:        DEPENDS_CHECKED="$DEPENDS_CHECKED $pkg"
                    372:        if [ -n "$NEW_DEPENDS" ]; then
                    373:            echo "${OPI} $pkg has the following new depends (need to re-tsort):"
                    374:            echo "${OPC} [$(echo $NEW_DEPENDS)]"
                    375:            sleep 2
                    376:            continue
                    377:        fi
                    378:     fi
                    379: 
                    380:     # Do make replace, with clean before, and package and clean afterwards.
1.12      tnn       381:     echo "${OPI} Replacing $(${PKG_INFO} -e $pkg)"
1.13      tnn       382:     fail=
                    383:     cmd="cd \"$PKGSRCDIR/$pkgdir\" && ${MAKE} clean && ${MAKE} replace || fail=1"
                    384:     if [ -n "$opt_n" ]; then
                    385:        echo "${OPI} Would run: $cmd"
                    386:     else
                    387:        eval "$cmd"
                    388:        [ -z "$fail" ] || abort "'make replace' failed for package $pkg."
                    389:     fi
1.1       gdt       390:     if [ -z "$opt_n" ]; then
1.13      tnn       391:        [ -z "$(${PKG_INFO} -Q unsafe_depends_strict $pkg)" ] || \
                    392:            abort "package $pkg still has unsafe_depends_strict."
                    393:        [ -z "$(${PKG_INFO} -Q unsafe_depends $pkg)" ] || \
                    394:            abort "package $pkg still has unsafe_depends."
                    395:        [ -z "$(${PKG_INFO} -Q rebuild $pkg)" ] || \
                    396:            abort "package $pkg is still requested to be rebuilt."
1.1       gdt       397:     fi
1.13      tnn       398:     if [ -z "$opt_r" ]; then
                    399:        echo "${OPI} Packaging $(${PKG_INFO} -e $pkg)"
                    400:            cmd="${MAKE} package && ${MAKE} clean || fail=1"
                    401:        if [ -n "$opt_n" ]; then
                    402:            echo "${OPI} Would run: $cmd"
                    403:        else
                    404:            eval "$cmd"
                    405:            [ -z "$fail" ] || abort "'make package' failed for package $pkg."
                    406:        fi
1.1       gdt       407:     fi
                    408:     sleep 1
                    409: 
                    410:     # remove just-replaced package from all *_TODO lists
                    411:     MISMATCH_TODO=$(exclude $pkg --from $MISMATCH_TODO)
                    412:     REBUILD_TODO=$(exclude $pkg --from $REBUILD_TODO)
                    413:     UNSAFE_TODO=$(exclude $pkg --from $UNSAFE_TODO)
                    414: 
1.13      tnn       415:     echo "${OPI} Re-checking for unsafe installed packages (${UNSAFE_VAR}=YES)"
1.1       gdt       416:     if [ -n "$opt_n" ]; then
                    417:        # With -n, the replace didn't happen, and thus the packages that would
                    418:        # have been marked unsafe_depends=YES were not.  Add the set that
                    419:        # would have been marked so we can watch what pkg_rolling-replace
                    420:        # would have done (approximately).
                    421:         UNSAFE_TODO=$(uniqify $UNSAFE_TODO \
                    422:             $(who_requires $pkg --in-graph $DEPGRAPH_INSTALLED))
                    423:         sleep 1
                    424:     else
1.13      tnn       425:         UNSAFE_TODO=$(check_packages_w_flag ${UNSAFE_VAR})
1.1       gdt       426:     fi
                    427: 
                    428:     verbose "${OPI} Packages to rebuild:"
                    429:     verbose "${OPC} MISMATCH_TODO=[$(echo $MISMATCH_TODO)]"  #strips newlines
                    430:     verbose "${OPC} REBUILD_TODO=[$(echo $REBUILD_TODO)]"
                    431:     verbose "${OPC} UNSAFE_TODO=[$(echo $UNSAFE_TODO)]"
                    432:     vsleep 4
                    433: 
                    434:     REPLACE_TODO=$(uniqify $MISMATCH_TODO $REBUILD_TODO $UNSAFE_TODO)
                    435:     depgraph_built=0
                    436: done
                    437: 
                    438: echo "${OPI} No more packages to replace; done."

CVSweb <webmaster@jp.NetBSD.org>