ipkg-build mkipkg : speedup (avoid grepping over controlfile by loading into array)

SpeciapMTFiles tothreaded : convert control files to -mt.control

4 files changed, 205 insertions, 18 deletions

diff --git a/scripts/SpecialMTFiles b/scripts/SpecialMTFiles
new file mode 100644
index 0000000..456550c
--- a/dev/null
+++ b/scripts/SpecialMTFiles
@@ -0,0 +1,14 @@
+ConvertSpecialFiles() {
+       local f
+       for f in "$*"
+       do
+         case "$f" in
+           *libqte*)
+             echo ${f//qte/qte-mt}
+             ;;
+           *)
+             echo $f
+             ;;
+         esac
+       done
+}
diff --git a/scripts/ipkg-build b/scripts/ipkg-build
index c6af056..b661bbf 100755
--- a/scripts/ipkg-build
+++ b/scripts/ipkg-build
@@ -1,201 +1,242 @@
 #!/bin/bash
 
 # ipkg-build -- construct a .ipk from a directory
 # Carl Worth <cworth@east.isi.edu>
 # based on a script by Steve Redler IV, steve@sr-tech.com 5-21-2001
 set -e
 
-ipkg_extract_value() {
-	sed -e "s/^[^:]*:[[:space:]]*//"
+#declare array for values and fields found in control file
+declare -a AllValues AllFields
+
+collect_values() {
+      local i line
+
+      i=0
+      while read line
+      do
+        AllFields[$i]=${line%%:*}
+        AllValues[$i]=`echo ${line#*:}` # echo to remove spaces
+        let i=i+1
+      done < ${pkg_dir}/${CONTROL}/control
 }
 
 required_field() {
+        local field i
 	field=$1
 
-	value=`grep "^$field:" < $CONTROL/control | ipkg_extract_value`
-	if [ -z "$value" ]; then
+        i=0
+        for af in "${AllFields[@]}"
+        do
+          if [ "$field" = "$af" ]
+          then
+            # return value
+            echo "${AllValues[$i]}"
+            return 0;
+          fi
+          let i=i+1
+        done
+        # not found
 		echo "*** Error: $CONTROL/control is missing field $field" >&2
 		return 1
-	fi
-	echo $value
-	return 0
 }
 
+#ipkg_extract_value() {
+#	sed -e "s/^[^:]*:[[:space:]]*//"
+#}
+#
+#required_field() {
+#	field=$1
+#
+#	value=`grep "^$field:" < $CONTROL/control | ipkg_extract_value`
+#	if [ -z "$value" ]; then
+#		echo "*** Error: $CONTROL/control is missing field $field" >&2
+#		return 1
+#	fi
+#	echo $value
+#	return 0
+#}
+
 pkg_appears_sane() {
 	local pkg_dir=$1
 
 	local owd=`pwd`
 	cd $pkg_dir
 
 	PKG_ERROR=0
 
 	tilde_files=`find . -name '*~'`
 	if [ -n "$tilde_files" ]; then
 		echo "*** Warning: The following files have names ending in '~'.
 You probably want to remove them: " >&2
 		ls -ld $tilde_files
 		echo >&2
 	fi
 
 	large_uid_files=`find . -uid +99`
 
 	if [ "$ogargs" = "" ]  && [ -n "$large_uid_files" ]; then
 		echo "*** Warning: The following files have a UID greater than 99.
 You probably want to chown these to a system user: " >&2
 		ls -ld $large_uid_files
 		echo >&2
 	fi
 	    
 
 	if [ ! -f "$CONTROL/control" ]; then
 		echo "*** Error: Control file $pkg_dir/$CONTROL/control not found." >&2
 		cd $owd
 		return 1
 	fi
 
 	pkg=`required_field Package`
 	[ "$?" -ne 0 ] && PKG_ERROR=1
 
 	version=`required_field Version | sed 's/Version://; s/^.://g;'`
 	[ "$?" -ne 0 ] && PKG_ERROR=1
 
 	arch=`required_field Architecture`
 	[ "$?" -ne 0 ] && PKG_ERROR=1
 
 	required_field Maintainer >/dev/null
 	[ "$?" -ne 0 ] && PKG_ERROR=1
 
 	required_field Description >/dev/null
 	[ "$?" -ne 0 ] && PKG_ERROR=1
 
 	section=`required_field Section`
 	[ "$?" -ne 0 ] && PKG_ERROR=1
 	if [ -z "$section" ]; then
 	    echo "The Section field should have one of the following values:" >&2
 	    echo "admin, base, comm, editors, extras, games, graphics, kernel, libs, misc, net, text, web, x11" >&2
 	fi
 
 	priority=`required_field Priority`
 	[ "$?" -ne 0 ] && PKG_ERROR=1
 	if [ -z "$priority" ]; then
 	    echo "The Priority field should have one of the following values:" >&2
 	    echo "required, important, standard, optional, extra." >&2
 	    echo "If you don't know which priority value you should be using, then use \`optional'" >&2
 	fi
 
 	if echo $pkg | grep '[^a-z0-9.+-]'; then
 		echo "*** Error: Package name $name contains illegal characters, (other than [a-z0-9.+-])" >&2
 		PKG_ERROR=1;
 	fi
 
 	local bad_fields=`sed -ne 's/^\([^[:space:]][^:[:space:]]\+[[:space:]]\+\)[^:].*/\1/p' < $CONTROL/control | sed -e 's/\\n//'`
 	if [ -n "$bad_fields" ]; then
 		bad_fields=`echo $bad_fields`
 		echo "*** Error: The following fields in $CONTROL/control are missing a ':'" >&2
 		echo "	$bad_fields" >&2
 		echo "ipkg-build: This may be due to a missing initial space for a multi-line field value" >&2
 		PKG_ERROR=1
 	fi
 
 	for script in $CONTROL/preinst $CONTROL/postinst $CONTROL/prerm $CONTROL/postrm; do
 		if [ -f $script -a ! -x $script ]; then
 			echo "*** Error: package script $script is not executable" >&2
 			PKG_ERROR=1
 		fi
 	done
 
 	if [ -f $CONTROL/conffiles ]; then
 		for cf in `cat $CONTROL/conffiles`; do
 			if [ ! -f ./$cf ]; then
 				echo "*** Error: $CONTROL/conffiles mentions conffile $cf which does not exist" >&2
 				PKG_ERROR=1
 			fi
 		done
 	fi
 
 	cd $owd
 	return $PKG_ERROR
 }
 
 ###
 # ipkg-build "main"
 ###
 ogargs=""
 outer=ar
 usage="Usage: $0 [-c] [-o owner] [-g group] <pkg_directory> [<destination_directory>]"
 while getopts "cg:o:" opt; do
     case $opt in
 	o ) owner=$OPTARG
 	    ogargs="--owner=$owner"
 	    ;;
 	g ) group=$OPTARG
 	    ogargs="$ogargs --group=$group"
 	    ;;
         c ) outer=tar
             ;;
 	\? ) 	echo $usage  >&2
 	esac
 done
 
+# maks sure that tar produces headers compatible with busybox format
+# debian needs this
+ogargs="$ogargs --format=oldgnu"
 
 shift $(($OPTIND - 1))
 
 # continue on to process additional arguments
 
 case $# in
 1)
 	dest_dir=.
 	;;
 2)
 	dest_dir=$2
 	;;
 *)
 	echo echo $usage >&2
 	exit 1 
 	;;
 esac
 
 pkg_dir=$1
 
 if [ ! -d $pkg_dir ]; then
 	echo "*** Error: Directory $pkg_dir does not exist" >&2
 	exit 1
 fi
 
 # CONTROL is second so that it takes precedence
 CONTROL=
 [ -d $pkg_dir/DEBIAN ] && CONTROL=DEBIAN
 [ -d $pkg_dir/CONTROL ] && CONTROL=CONTROL
 if [ -z "$CONTROL" ]; then
 	echo "*** Error: Directory $pkg_dir has no CONTROL subdirectory." >&2
 	exit 1
 fi
 
+# load values 
+collect_values 
+
 if ! pkg_appears_sane $pkg_dir; then
 	echo >&2
 	echo "ipkg-build: Please fix the above errors and try again." >&2
 	exit 1
 fi
 
 tmp_dir=$dest_dir/IPKG_BUILD.$$
 mkdir $tmp_dir
 
 tar $ogargs -C $pkg_dir -czf $tmp_dir/data.tar.gz . --exclude=$CONTROL
 tar $ogargs -C $pkg_dir/$CONTROL -czf $tmp_dir/control.tar.gz .
 
 echo "2.0" > $tmp_dir/debian-binary
 
 pkg_file=$dest_dir/${pkg}_${version}_${arch}.ipk
 here_dir=$PWD
 ## tar -C $tmp_dir -czf $pkg_file ./debian-binary ./data.tar.gz ./control.tar.gz
 rm -f $pkg_file
 if [ "$outer" = "ar" ] ; then
   cd $tmp_dir ; ar crf $here_dir/$pkg_file ./debian-binary ./data.tar.gz ./control.tar.gz
 else
   cd $tmp_dir ; tar -zcf $here_dir/$pkg_file ./debian-binary ./data.tar.gz ./control.tar.gz
 fi
 cd $here_dir
 rm $tmp_dir/debian-binary $tmp_dir/data.tar.gz $tmp_dir/control.tar.gz
 rmdir $tmp_dir
 
 echo "Packaged contents of $pkg_dir into $pkg_file"
diff --git a/scripts/mkipkg b/scripts/mkipkg
index 43bd3a3..a336371 100755
--- a/scripts/mkipkg
+++ b/scripts/mkipkg
@@ -1,388 +1,388 @@
 #!/bin/sh
 # vim: et sw=4
 
 # TODO: srcdir!=builddir
 
 usage()
 {
     echo "usage: $self --destdir=[destination installation directory]"
     echo "usage: $self [destination installation directory]"
     exit 1
 }
 
 self=mkipkg
 destdir=
 strip=strip
 control=
 builddir=
 srcdir=
 prefix=
 preinst=
 postinst=
 prerm=
 postrm=
 mkfsjffs2=
 ipkgbuild=ipkg-build
 buildversion=
 subst=
 user=root
 group=root
 filesubst=
 
 oldpwd=
 
 for option
 do
     case "$option" in
         -*=*)
             arg=`echo "$option" | sed 's/[-_a-zA-Z0-9]*=//'`
             ;;
     esac
 
     case "$option" in
         --destdir=*)
             destdir=$arg
             ;;
         --strip=*)
             strip=$arg
             ;;
        --control=*)
             control=$arg
             ;;
        --preinst=*)
             preinst=$arg
             ;;
        --postinst=*)
             postinst=$arg
             ;;
        --prerm=*)
             prerm=$arg
             ;;
        --postrm=*)
             postrm=$arg
             ;;
        --builddir=*)
             builddir=$arg
             ;;
        --srcdir=*)
             srcdir=$arg
             ;;
        --prefix=*)
             prefix=$arg
             ;;
        --mkfsjffs2=*)
             mkfsjffs2=$arg
             ;;
        --ipkgbuild=*)
             ipkgbuild=$arg
             ;;
        --buildversion=*)
             buildversion=$arg
             ;;
        --subst=*)
             subst=$arg
             ;;
        --user=*)
             user=$arg
             ;;
        --group=*)
             group=$arg
             ;;
        --filesubst=*)
             filesubst=$arg
             ;;
        --classic)
             classic=-c
             ;;
        -*)
             usage
             ;;
        *)
             destdir=$option
             ;;
     esac
 done
 
 if [ -z "$control" ]; then
     control=$destdir/CONTROL/control
 fi
 if [ -z "$prerm" ]; then
     prerm=$destdir/CONTROL/prerm
 fi
 if [ -z "$postrm" ]; then
     postrm=$destdir/CONTROL/postrm
 fi
 if [ -z "$preinst" ]; then
     preinst=$destdir/CONTROL/preinst
 fi
 if [ -z "$postinst" ]; then
     postinst=$destdir/CONTROL/postinst
 fi
 
 # remove leading slash from prefix (to fix globbing)
 if [ -n "$prefix" ]; then
     prefix=`echo $prefix | sed -e "s,/\(.*\),\\1,"`
 fi
 
 if [ -z "$destdir" ]; then
     usage
 fi
 
 if [ ! -r $control ]; then
    echo "$self: cannot find $control, exiting..."
    exit 1
 fi
 
 if [ -z "`which $ipkgbuild 2>/dev/null`" ]; then
     echo "$self: cannot find ipkg-build, exiting..."
     exit 1
 fi
 
 findFile()
 {
     local path=
     if [ $# = 1 ]; then
         find $1 -type f -o -type b -o -type c -o -type l
         find $1 -type d -a -empty
     else
         find . -type f -o -type b -o -type c -o -type l | \
 	    sed -e "s,\./\(.*\),\\1,g"
         find . -type d -a -empty
     fi
 }
 
 _pushd() { oldpwd=`pwd`; cd $1; }
 _popd() { cd $oldpwd; }
 
 setVar()
 {
     eval "$1='$2'"
 }
 
 expandMaskToList()
 {
     local _list=`echo $1`
     local _tmpFileList=
     for f in $_list; do
         if [ -d $f ]; then
             f="`findFile $f`"
         fi
         _tmpFileList="`eval echo $f` $_tmpFileList"
     done
     setVar $2 "$_tmpFileList"
 }
 
 createFileList()
 {
     local excludeMask
     local includeMask
     local includemaskpresent=0
     local excludemaskpresent=0
 
-    if (cat $1|grep -q ^FileExcludeMask); then
+    if (grep -q ^FileExcludeMask $1); then
         excludemaskpresent=1
         excludeMask=$(eval echo '"'$(sed -n -e "s,^FileExcludeMask: *,,p" $1)'"')
     fi
-    if (cat $1|grep -q ^FileIncludeMask); then
+    if (grep -q ^FileIncludeMask $1); then
         includemaskpresent=1
         includeMask=$(eval echo '"'$(sed -n -e "s,^FileIncludeMask: *,,p" $1)'"')
     else
-        if (cat $1|grep -q ^Files:); then
+        if (grep -q ^Files: $1); then
             includemaskpresent=1
             includeMask=$(eval echo '"'$(sed -n -e "s,^Files: *,,p" $1)'"')
         fi
     fi
  
     _pushd $destdir
     
-    excludeMask="$excludeMask CONTROL/* usr/share/*"
+    declare -a excludeMaskArray
+    excludeMaskArray=( $excludeMask )
+    excludeMaskArray=( "${excludeMaskArray[@]}" "CONTROL/*" "usr/share/*" )
 
     if [ $includemaskpresent != 1 ]; then
         includeMask="."
     fi
 
     if [ -z "$includeMask" ]; then
         setVar $2 ""
         _popd
         return 0
     fi
 
     expandMaskToList "$includeMask" _fileList
 
-    _excludeList=
-    if [ -n "$excludeMask" ]; then
-        expandMaskToList "$excludeMask" _excludeList
-        _excludeList="$_excludeList `find -name \*CVS\* -o -name \*SCCS\*`"
-    fi
+    excludeMaskArray=( "${excludeMaskArray[@]}" "*/CVS*" "*/SCCS*" )
 
     local realFileList=
     local missing=0
     for file in $_fileList; do
         local containedInList=0
-        for i in $_excludeList; do
-            if [ $file = $i ]; then
+        for i in "${excludeMaskArray[@]}"; do
+            if [[ $file == $i ]]; then
                 containedInList=1
+                break;
             fi
         done
+
         if [ $containedInList = 0 ]; then
             if ! [ -e $file -o -L $file ]; then
                 echo "$self: $file not found"
                 missing=1
             fi
             realFileList=$file" $realFileList"
         fi
     done
 
     _popd
 
     if [ $missing = 1 ]; then
         return 1
     fi
 
     setVar $2 "$realFileList"
 }
 
 stripFile()
 {
     if [ -f $1 -a -x $1 ]; then
         $strip --strip-all $1
     fi
 }
 
 substFile()
 {
     local oldfile=$1
 
     if [ ! -e $2 ]; then return 1; fi
 
     if [ -e $oldfile -o -L $oldfile ]; then
         newfile=`echo $oldfile|sed -f $2|sed -e's,^/,,g'`
         olddir=`dirname $oldfile`
         base=`basename $oldfile`
         newdir=`dirname $newfile`
 
 #        echo >&2 moving $oldfile to $newfile
 
         if [ "$newdir" = "$olddir" ]; then
             return 0
         fi
 
         mkdir -p $newdir
         mv $olddir/$base $newfile
         rmdir -p $olddir 2>/dev/null
     fi
 }
 
 stripFiles()
 {
     for f in $1; do
         stripFile ./$f
     done
 }
 
 substFiles()
 {
     for f in $1; do
         substFile ./$f
     done
 }
 
 substAndStripFiles()
 {
     for f in $1; do
         stripFile ./$f
         substFile ./$f $2
     done
 }
 
 installScript()
 {
     if [ -n "$1" -a -f "$1" ]; then
         destfile=`basename $1`
         filetype=`echo $destfile|cut -d. -f2`
         if [ -n "$filetype" ]; then destfile=$filetype; fi
         if [ -n "$subst" ]; then
             sed -f $subst < $1 > $ctrldir/$destfile
         else
             cat $1 > $ctrldir/$destfile
         fi
         chmod +x $ctrldir/$destfile
     fi
 }
 
 tempDir=/tmp/`basename $self`.$$
 mkdir -p $tempDir
 if [ $? != 0 ]; then
     echo "$self: cannot create $tempDir, exiting..."
     rm -rf $tempDir
     exit 1
 fi
 
 ctrldir=$tempDir/CONTROL
 
 mkdir -p $ctrldir
 
 if [ ! -e $subst ] || [ -z "$subst" ]; then
     cat $control > $ctrldir/control.new
 else
     sed -f $subst < $control > $ctrldir/control.new
 fi
 
 createFileList $ctrldir/control.new ipkgFileList
 
 if [ "$?" != "0" ]; then
     echo "$self: ERROR: missing files, not building $control"
     rm -rf $tempDir
     exit 1
 fi
 
 
 cat $ctrldir/control.new | egrep -v '^(Files|FileExcludeMask|FileIncludeMask):' > $ctrldir/control
 rm -f $ctrldir/control.new
 
 ( cd $destdir && tar -cf - $ipkgFileList 2>/dev/null ) | ( cd $tempDir && tar -xf - 2>/dev/null )
 
 if [ -z "$filesubst" ]; then
     ( cd $tempDir && stripFiles "$ipkgFileList" )
 else
     if ! (echo $filesubst|grep -q '^/'); then
         filesubst="$oldpwd/$filesubst"
     fi
     ( cd $tempDir && substAndStripFiles "$ipkgFileList" $filesubst )
 fi
 
 # removing CVS directories
 find $tempDir -name CVS -a -type d -print0 | xargs -0 rm -rf
 
 path="`echo "$PATH" | sed -e "s/\:/ /g"`"
 if [ -z "$mkfsjffs2" ]; then
     for i in $path; do
       if [ -x "$i/mkfs.jffs2" ]; then
         mkfsjffs2="$i/mkfs.jffs2"
         break
       fi
     done
 fi
 
 if [ -z "$mkfsjffs2" ]; then
     echo "$self: WARNING: no mkjfs.jffs2 found in path. Falling back to using du"
     echo "for size calculation. mkfs.jffs2 is recommended for size calculation"
     echo "as it calculates the real package size on the compressed file system,"
     echo "in contrast to du calculating the uncompressed size!"
     buildsize=`du -h -s $tempDir | awk '{print $1}'`
 else
     buildsize=`$mkfsjffs2 -r $tempDir | wc -c`
 fi
 
 if [ "$buildsize" != "0" ]; then
     echo "Installed-Size: $buildsize" >> $ctrldir/control
 fi
 
 installScript $preinst
 installScript $postinst
 installScript $prerm
 installScript $posrm
 
 $ipkgbuild $classic -o $user -g $group $tempDir
 
 rm -rf $tempDir
diff --git a/scripts/tothreaded b/scripts/tothreaded
new file mode 100755
index 0000000..d9f8488
--- a/dev/null
+++ b/scripts/tothreaded
@@ -0,0 +1,132 @@
+#!/bin/sh
+
+# PURPOSE :
+#
+# this script converts a non-threaded control file to a threaded one
+# by extending appropriate names with -mt as extension
+#
+# eg abc.control becoms abc-mt.control
+#
+
+#
+# make sure that the depends expression has enough spaces
+# expression can contian : , ( ) || && 
+#
+tokenize() {
+    sed "s/,/ & /g" | sed "s/)/ & /g" | sed "s/(/ & /g" |  sed "s/|/ & /g" | sed "s/&/ & /g"
+}
+
+#
+# function converts package name to threaded equivalend IF the
+# package file HAS a threaded version
+#
+findthreadedequiv() {
+      local isin i
+      for i in $*
+      do
+        isin=`grep "^$i\$" "$ALLTHREADEDPKGSFILE"`
+        if [ -z "$isin" ]
+        then
+          # no threaded package
+          echo -n "$i "
+        else
+          # threaded package
+          echo -n "${isin}-mt " 
+        fi
+      done
+      echo 
+}
+
+#
+# signature of binary files
+# currently obsolete
+#
+# ISBINARY="*ELF*LSB*"
+
+usage() {
+      echo "Usage : tothreaded <controlfile> <ALLPackages file>"
+      exit 2
+}
+
+. scripts/SpecialMTFiles
+
+#
+# get the name of the controlfile to check for threading
+#
+if [ -z "$1" ]
+then
+    usage
+fi
+controlfile=$1
+shift
+
+case $controlfile in
+  *-mt.control)
+    #already threaded
+    echo $controlfile
+    exit 0;
+    ;;
+esac
+
+#
+# file containing list of all known threaded packages
+#
+if [ -z "$1" ]
+then
+    usage
+fi
+ALLTHREADEDPKGSFILE=$1
+shift
+
+#
+# strip out the name of the package
+#
+packagename=${controlfile##*/}  # path
+packagename=${packagename%.control} # extension
+
+#
+# generate new control file
+#
+newcontrolfile=${controlfile/\.control/-mt\.control}
+
+#
+# read all lines in original control file
+#
+while read line
+do
+  case $line in
+    # convert some files to threaded equivalent
+    "Files:"*)
+       files=${line#Files:}
+       # thread-converted files
+       T_files=`ConvertSpecialFiles "$files"`
+       echo "Files: $T_files"
+       ;;
+     "Package: "*)
+       T_package=`findthreadedequiv ${line#Package: }`
+       echo "Package: ${T_package}"
+       ;;
+     "Depends: "*)
+       depends=`echo "${line#Depends: }" | tokenize`
+       T_depends=`findthreadedequiv ${depends}`
+       echo "Depends: $T_depends"
+       ;;
+     "Provides: "*)
+       T_provides=`findthreadedequiv ${line#Provides: }`
+       echo "Provides: $T_provides"
+       ;;
+     "Conflicts: "*)
+       conflicts=`echo "${line#Conflicts: }" | tokenize`
+       T_conflicts=`findthreadedequiv ${conflicts}`
+       echo "Conflicts: $T_conflicts"
+       ;;
+     *":"*) 
+       echo "$line"
+       ;;
+     *) # al other lines
+       echo " $line"
+       ;;
+  esac
+done < $controlfile > $newcontrolfile
+
+echo $newcontrolfile