Return to cdgen.awk CVS log

Up to [cvs.NetBSD.org] / pkgsrc / pkgtools / cdpack / files

Update to cdpack-1.9.  This version teaches cdpack how to add packages
ending in .tbz as well as .tgz.  Should address the problem reported
in PR pkg/40515.  I am explicitly not using the mk.conf settings
for PKG_SUFX though because at run time we may or may not have a valid
mk.conf on hand that corresponds to how the packages were built.

#!@AWK@ -f
# $NetBSD: cdgen.awk,v 1.7 2009/02/20 05:16:51 dmcmahill Exp $
#
# Copyright (c) 2001, 2002, 2003, 2005, 2009 Dan McMahill, All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#	This product includes software developed by Dan McMahill
# 4. The name of the author may not be used to endorse or promote
#    products derived from this software without specific prior written
#    permission.
#
# THIS SOFTWARE IS PROVIDED BY DAN MCMAHILL
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

BEGIN {

# all of the PKG_SUFX values which are currently allowed.  This is hardcoded into
# pkg_create(1) and friends.

    allsf[1] = ".tgz";
    allsf[2] = ".tbz";

# ARGV[0] = program name
# ARGV[1] = packages directory (/usr/pkgsrc/packages/All)
# ARGV[2] = output directory
# ARGV[3] = Depends tree file.  Has packages in tsort(1) input format.
# ARGV[4] = Exclude file.  Packages listed here are excluded.
# ARGV[5] = Depends order file.  Has packages in tsort(1) output format.
# ARGV[6] = CD list.  This script leaves a list of the CD directories in this file
# ARGV[7] = dup flag.  "dup=yes" for package duplication, "dup=no" for no duplication. 
# ARGV[8] = verbose flag.  "verbose=yes" for verbose output
# ARGV[9] = xtra_size.  How many kB are needed per CD for common files

    reqnum = 12;
    if (ARGC != reqnum){
	printf("cdgen.awk:  wrong number of arguments (got %d, wanted %d)\n", ARGC, reqnum);
	usage();
	exit(1);
    }

    prog     = "cdgen.awk";	
    packages = ARGV[1];
    cddir    = ARGV[2];
    deptree  = ARGV[3];
    exclude  = ARGV[4];
    order    = ARGV[5];
    cdlist   = ARGV[6];
    xtra_size= ARGV[10];
    other_size= ARGV[11];

    if (ARGV[7] ~ "dup=yes"){
	dup=1;
    }
    else{
	dup=0;
    }
    if (ARGV[8] ~ "verbose=yes"){
	verbose=1;
    }
    if (ARGV[9] ~ "dvd=yes"){
	dvd=1;
    }
    else{
	dvd=0;
    }

#
# Make sure all required directories and files exist
#
    cmd="test -d " packages ;
    if(system(cmd) != 0){
	printf("%s:  packages directory \"%s\" does not exist\n",prog,packages);
	usage();
	exit(1);
    }

    cmd="test -d " cddir ;
    if(system(cmd) != 0){
	printf("%s:  cd image directory \"%s\" does not exist\n",prog,cddir);
	usage();
	exit(1);
    }

    cmd="test -f " deptree ;
    if(system(cmd) != 0){
	printf("%s:  dependstree file \"%s\" does not exist\n",prog,deptree);
	usage();
	exit(1);
    }
    
    cmd="test -f " exclude ;
    if(system(cmd) != 0){
	printf("%s:  exclude file \"%s\" does not exist\n",prog,exclude);
	usage();
	exit(1);
    }
    
    cmd="test -f " order ;
    if(system(cmd) != 0){
	printf("%s:  build order file \"%s\" does not exist\n",prog,order);
	usage();
	exit(1);
    }
    
    
    "date" | getline now;
    printf("%s starting %28s\n",prog,now);
    
#
# Read in the build order.  This gives the list of all possible 
# packages (note that some may actually not be available as binary
# packages).  The order is such that a package earlier in the list
# will never depend on a package lower on the list. 
# 

    printf("Reading the depends order from \"%s\".\n",order);
    n=1;
    while(getline < order > 0){
	pkgorder[n] = $1;
	n++;
    }
    close(order);
    npkgs = n-1;
    printf("%d packages to go on CD-ROM/DVD!\n",npkgs);

#
# Read in the list of excluded packages
#
    printf("Reading list of packages to be excluded\n");
    while(getline < exclude > 0){
	excludes[$1] = 1 ;
    }
    close(exclude);

#
# Read in the depends tree and flatten it.
#
    printf("Reading depends tree\n");
    while(getline < deptree > 0){
	if ($2 in topdepends)
	    topdepends[$2] = topdepends[$2] " " $1 " " ;
	else
	    topdepends[$2] = " " $1 " " ;
    }
    close(deptree);

# Now recurse the tree to give a flattened depends list for each pkg
    printf("Flattening depends tree\n");
    for (toppkg in topdepends){
	find_all_depends(toppkg);
    }


#
# Next get the sizes of each of the binary pacakges if they
# exist.  For those which don't exist, make a note of that.
# getsize() returns size in kB if the file exists, 0 otherwise.
#
    printf("Loading binary package sizes\n");
    nonzero=0;
    totsize=0;
    for (n in pkgorder){
	pkgsize[pkgorder[n]] = getsize(pkgorder[n]);
	if(pkgsize[pkgorder[n]] > 0){
	    nonzero++;
	}
	else{
	    printf("WARNING:  binary package \"%s\" has zero size\n",pkgorder[n]);
	    if(!verbose) {
		verbose = 1;
		getsize(pkgorder[n]);
		verbose = 0;
	    }
	}
	totsize = totsize + pkgsize[pkgorder[n]];
    }
    printf("%d binary packages are available (%g Mb)!\n",nonzero,totsize/1024);

#
# Now we need to figure out which binary packages go on each
# CD.  The way we'll do this is start taking packages in the order
# specified by 'pkgorder' until the first CD is full, then move
# to the second, etc.  This way, we never have to reinsert a CD
# while installing pkgs.
#
    printf("Figuring out which packages go on each CD/DVD\n");

# maximum kB for binary pkgs per CD.
    if( dvd ) 
	maxcd = 1024 * 4300;
    else
	maxcd = 1024 * 620;

    if( verbose ) printf("Maximum image size = %g Mb\n", maxcd/1024);
#
# no package duplication
#
    if (!dup){
	cdn=1;
	pkgn=0;
	cdtot[cdn]=xtra_size;
	cdpkgs[cdn]=0;
	
	for (n=1; n<=npkgs ; n=n+1){
	    if (verbose) printf("Processing: %s\n",pkgorder[n]);
	    if (pkgorder[n] in excludes) {
		if (verbose) printf("Skipping excluded package: %s\n",pkgorder[n]);
	    }
	    else {
# only process the package if it exists.
		if (pkgsize[pkgorder[n]] > 0){
		    if (cdtot[cdn] < (maxcd-pkgsize[pkgorder[n]]) ){
			cdtot[cdn] = cdtot[cdn] +pkgsize[pkgorder[n]];
			cdcontents[cdn":"pkgorder[n]] = 1;
			pkgn = pkgn + 1;
		    }
		    else{
# the CD is full
			printf("disk number %d is full (%g Mb)\n",cdn,
			       cdtot[cdn]/1024);
			cdpkgs[cdn] = pkgn;
# increment the CD counter
			cdn = cdn + 1;
			pkgn = 1;
			cdtot[cdn] = xtra_size + pkgsize[pkgorder[n]];
			cdcontents[cdn":"pkgorder[n]] = 1;
		    }
		}
	    }
	}
	cdpkgs[cdn] = pkgn;
	
# see if the extra files will fit on the last CD
	if ( (cdtot[cdn] + other_size) < maxcd ){
	    printf("disk number %d is partially full (%g Mb)\n",cdn,
		   cdtot[cdn]/1024);
	}
	else{
	    printf("disk number %d is full (%g Mb)\n",cdn,
		   cdtot[cdn]/1024);
	    cdn = cdn + 1;
	    cdtot[cdn] = other_size;
	    cdpkgs[cdn] = 0;
	    printf("disk number %d is partially full (%g Mb)\n",cdn,
		   cdtot[cdn]/1024);
	}
	
    }
    
#
# We will duplicate some packages to eliminate inter-CD dependencies.
#
    else{
	cdn=1;
	pkgn=0;
# initialize the size count for the current CD with the extras that we
# are putting on all CD's
	cdtot[cdn]=xtra_size;
	cdpkgs[cdn]=0;
	n=npkgs;
	while (n > 0){
	    if (verbose) printf("Begin processing %s\n",pkgorder[n]);
	    if ( !pkg_done[pkgorder[n]]){
		size_needed = pkgsize[pkgorder[n]];
		deps_needed = 0;
		split(alldepends[pkgorder[n]],pkgdeps);
		for (dep in pkgdeps){
		    if (verbose) printf("   Examining dependency: %s\n",pkgdeps[dep]);
		    if (pkgdeps[dep] in excludes) {
			if (verbose) printf("   Skipping excluded dependency in count: %s\n",pkgdeps[dep]);
		    }
		    else {
			if(!cdcontents[cdn":"pkgdeps[dep]]){
			    size_needed = size_needed + pkgsize[pkgdeps[dep]];
			    deps_needed++;
			}
		    }
		}
		if (cdtot[cdn] + size_needed < maxcd){
		    if (verbose) printf("   Processing %s\n",pkgorder[n]);
		    if (pkgorder[n] in excludes) {
			if (verbose) printf("   Skipping excluded package in packing: %s\n",pkgorder[n]);
		    }
		    else {
			cdcontents[cdn":"pkgorder[n]] = 1;
		    }
		    pkg_done[pkgorder[n]] = 1;
		    if (verbose) printf("   Marked %s as processed\n",pkgorder[n]);
		    for (dep in pkgdeps){
			if (pkgdeps[dep] in excludes) {
			    if (verbose) printf("   Skipping excluded dependency in packing: %s\n",pkgdeps[dep]);
			}
			else {
			    cdcontents[cdn":"pkgdeps[dep]] = 1;
			    pkg_done[pkgdeps[dep]] = 1;
			    if (verbose) printf("   Marked dependency pkg %s as processed\n",pkgdeps[dep]);
			}
		    }
		    cdtot[cdn] = cdtot[cdn] + size_needed;
		    if (pkgorder[n] in excludes) {
# don't include this one in the count if its excluded
			if (verbose) printf("   Added  %d dependencies to the image\n",deps_needed);
			pkgn = pkgn + deps_needed;
		    }
		    else {
			if (verbose) printf("   Added %s plus %d dependencies to the image\n",pkgorder[n],deps_needed);
			pkgn = pkgn + 1 + deps_needed;
		    }
		    n--;
		}
		else{
# the CD is full
		    printf("disk number %d is full (%g Mb)\n",cdn,
			   cdtot[cdn]/1024);
		    cdpkgs[cdn] = pkgn;
		    cdn++;
		    cdtot[cdn] = xtra_size;
		    pkgn = 0;
		}
	    }
	    else{
# we've already done this pkg
		if (verbose) printf("   %s has already been processed\n",pkgorder[n]);
		n--;
	    }
	    
	}
	cdpkgs[cdn] = pkgn;

# see if the extra files will fit on the last CD
	if ( (cdtot[cdn] + other_size) < maxcd ){
	    printf("disk number %d is partially full (%g Mb)\n",cdn,
		   cdtot[cdn]/1024);
	}
	else{
	    printf("disk number %d is full (%g Mb)\n",cdn,
		   cdtot[cdn]/1024);
	    cdn = cdn + 1;
	    cdtot[cdn] = other_size;
	    cdpkgs[cdn] = 0;
	    printf("disk number %d is partially full (%g Mb)\n",cdn,
		   cdtot[cdn]/1024);
	}
    }

# remember how many cd's there are
    ncd=cdn;

# print some stats
    tot_ex=0;
    for (ex in excludes){
	tot_ex++;
    }
    if (dup){
	tot_cdpkgs=0;
	for (cdn=1; cdn<=ncd; cdn=cdn+1){
	    tot_cdpkgs = tot_cdpkgs + cdpkgs[cdn];
	}
	printf("CD/DVD images with package duplication resulted in %d packages total\n",tot_cdpkgs);
	printf("This is an increase of %d over the base %d packages\n",tot_cdpkgs-(npkgs-tot_ex),npkgs-tot_ex);
    }
	
    printf("%d packages out of %d have been excluded due to redistribution restrictions\n",tot_ex,npkgs);
#
# Next, create a subdirectory for each CD and populate the directory
# with links to the actual binary pkgs
#
    printf("Creating subdirectories for each CD/DVD and populating it with links\n");
    printf("to the binary packages and other required files.\n");
    for (cdn=1; cdn<=ncd; cdn=cdn+1){
	printf("----------- CD/DVD #%d ----------\n",cdn);
	printf("      %3d binary packages\n",cdpkgs[cdn]);
	outdir=cddir "/disk" cdn "/packages/All";
	cmd="test -d " outdir;
	if(system(cmd) == 0){
	    printf("%10s:  output dir \"%s\" already exists.\n",prog,outdir);
	    printf("%10s   please remove it.\n","");
	    exit(1);
	}
	
# create output directory
	cmd = "mkdir -p " outdir;
	do_system(cmd);
	
# populate it with the symlinks to the binary packages
	for (n=1; n<=npkgs; n++){
	    if (cdcontents[cdn":"pkgorder[n]]){
		for(i in allsf) {
		    cmd="test -f " packages "/" pkgorder[n] allsf[i];
		    if( (r = system(cmd)) == 0){
			cmd = "cd " outdir " && ln -s " packages "/" ;
			cmd = cmd pkgorder[n] allsf[i];
			do_system(cmd);
		    }
		}
	    }
	}

# add it to the cd list
	printf("disk%d\n", cdn) > cdlist ;
	
    }

    close(cdlist);

    "date" | getline now;
    printf("%s finished on %28s\n",prog,now);

    printf("\n");

    exit 0
	} # BEGIN

# This is a bit of a hack, but in the event that we have both a
# foo-1.3.tgz and a foo-1.3.tbz, we currently take them both.
# I'm not sure how we avoid this.  One option is a flag to 
# cdpack that says to prefer the .tbz over the .tgz for example.
# To implement that, the cdpack.sh shell script would need some
# updating as well as this function.  Grep for ".tgz" in
# pkgsrc/pkgutils/cdpack/files/* and also grep for allsf in this file
  
function getsize(name,cmd,sz,i,pkgsfx,r){
    sz=0;
    for(i in allsf) {
	pkgsfx=allsf[i];
	cmd="test -f " packages "/" name pkgsfx;
	if( verbose ) {
	    printf("getsize(%s):  Checking for suffix \"%s\" with:\n", name, pkgsfx);
	    printf("    %s\n", cmd);
	}
	if( (r = system(cmd)) == 0){
	    cmd="du -k " packages "/" name pkgsfx;
	    cmd | getline ;
	    close(cmd);
	    if( verbose ) {
		printf("    \"%s\" gave %s\n", cmd, $0);
	    }
	    sz=sz + $1;
	} else {
	    if( verbose ) {
		printf("    command returned %d\n", r);
	    }
	}
    }

    return(sz);	
}

function do_system(cmd,rv){
    rv = system(cmd);
    if (rv != 0){
	printf("Error:  system(\"%s\") returned %d\n",cmd,rv);
	exit(1);
    }
    return(0);
}

#
# show usage
#
function usage(){
    printf("\nUsage:\n\n");
    printf("%10s -- Create directories for building binary package ISO images\n",
	   prog,prog);
    printf("%10s    packages images dependstree order cdlist\n","");
    printf("\n");
    printf("\n");

}


function find_all_depends(pkg,pkgreg,i,deps){
# if we find the package already has been fully depended
# then return the depends list
    if (pkg in alldepends){
#	printf("\t%s is already depended.\n",pkg);
	return(alldepends[pkg]);	
    }

# if we find the package listed in its own depends list, then
# return an empty list if we're going down the depends tree.
# When a package lists itself in the depends tree file, it simply
# is a place holder and means the package has no depends.  However
# other packages may still depend upon it, so we need to keep looking.
    if ( topdepends[pkg]~reg2str(pkg) ){
#	printf("\t%s depends on its self.\n",pkg);
	alldepends[pkg] = " ";
	return(alldepends[pkg]);	
    }

# otherwise recursively gather depends that each of the depends
# has
    pkgreg=reg2str(pkg);
    split(topdepends[pkg],deps);
    i=1;
    alldepends[pkg] = " ";
    while ( i in deps ){
# don't add ourselves to the list (a possibility when going up the tree)
	if (" "deps[i]" "!~pkgreg){
	    alldepends[pkg] = alldepends[pkg] " " deps[i] " " find_all_depends(deps[i]);
	}
	i=i+1;
    }
    alldepends[pkg] = uniq(alldepends[pkg]);
    return(alldepends[pkg]);	
}

#
# take a string which has special characters like '+' in it and
# escape them.  Also put a space before and after since that's how
# we'll distinguish things like gnome from gnome-libs
#
function reg2str(reg){
    gsub(/\+/,"\\\+",reg);
    gsub(/\./,"\\\.",reg);
    reg = " "reg" ";
    return(reg);
}

#
# take the depends lists and uniq them.
#
function uniq(list,deps,i,ulist){
    
# split out the depends
    split(list,deps);

    i=1;
    ulist = " ";
    while (i in deps){
	if (ulist !~reg2str(deps[i])){
	    ulist = ulist deps[i]" ";
	}
	i++;
    }
    return(ulist);
}