Utilize shell's IFS instead of forking ~6 processes to

handle splitting input files on a '|'.  This greatly
reduces the time taken to process several databases
during the update process.

Additionally add some more debug logging.

1 files changed, 70 insertions, 48 deletions

diff --git a/usr.sbin/portsnap/portsnap/portsnap.sh b/usr.sbin/portsnap/portsnap/portsnap.sh
index e715ce8..95ecc2a 100644
--- a/usr.sbin/portsnap/portsnap/portsnap.sh
+++ b/usr.sbin/portsnap/portsnap/portsnap.sh
@@ -570,14 +570,16 @@ fetch_metadata_sanity() {
 
 # Take a list of ${oldhash}|${newhash} and output a list of needed patches
 fetch_make_patchlist() {
-	grep -vE "^([0-9a-f]{64})\|\1$" | 
-		while read LINE; do
-			X=`echo ${LINE} | cut -f 1 -d '|'`
-			Y=`echo ${LINE} | cut -f 2 -d '|'`
-			if [ -f "files/${Y}.gz" ]; then continue; fi
-			if [ ! -f "files/${X}.gz" ]; then continue; fi
-			echo "${LINE}"
+	IFS='|'
+	echo "" 1>${QUIETREDIR}
+	grep -vE "^([0-9a-f]{64})\|\1$" |
+		while read X Y; do
+			printf "Processing: $X $Y ...\r" 1>${QUIETREDIR}
+			if [ -f "files/${Y}.gz" -o ! -f "files/${X}.gz" ]; then continue; fi
+			echo "${X}|${Y}"
 		done
+	echo "" 1>${QUIETREDIR}
+	IFS=
 }
 
 # Print user-friendly progress statistics
@@ -692,9 +694,8 @@ fetch_update() {
 
 # Attempt to apply metadata patches
 	echo -n "Applying metadata patches... "
-	while read LINE; do
-		X=`echo ${LINE} | cut -f 1 -d '|'`
-		Y=`echo ${LINE} | cut -f 2 -d '|'`
+	IFS='|'
+	while read X Y; do
 		if [ ! -f "${X}-${Y}.gz" ]; then continue; fi
 		gunzip -c < ${X}-${Y}.gz > diff
 		gunzip -c < files/${X}.gz > OLD
@@ -707,6 +708,7 @@ fetch_update() {
 		fi
 		rm -f diff OLD NEW ${X}-${Y}.gz ptmp
 	done < patchlist 2>${QUIETREDIR}
+	IFS=
 	echo "done."
 
 # Update metadata without patches
@@ -724,16 +726,19 @@ fetch_update() {
 	    2>${QUIETREDIR}
 
 	while read Y; do
+		echo -n "Verifying ${Y}... " 1>${QUIETREDIR}
 		if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then
 			mv ${Y}.gz files/${Y}.gz
 		else
 			echo "metadata is corrupt."
 			return 1
 		fi
+		echo "ok." 1>${QUIETREDIR}
 	done < filelist
 	echo "done."
 
 # Extract the index
+	echo -n "Extracting index... " 1>${QUIETREDIR}
 	gunzip -c files/`look INDEX tINDEX.new |
 	    cut -f 2 -d '|'`.gz > INDEX.new
 	fetch_index_sanity || return 1
@@ -754,8 +759,10 @@ fetch_update() {
 	fi
 
 # Generate a list of wanted ports patches
+	echo -n "Generating list of wanted patches..." 1>${QUIETREDIR}
 	join -t '|' -o 1.2,2.2 INDEX INDEX.new |
 	    fetch_make_patchlist > patchlist
+	echo " done." 1>${QUIETREDIR}
 
 # Attempt to fetch ports patches
 	echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
@@ -766,11 +773,18 @@ fetch_update() {
 	echo "done."
 
 # Attempt to apply ports patches
-	echo -n "Applying patches... "
-	while read LINE; do
-		X=`echo ${LINE} | cut -f 1 -d '|'`
-		Y=`echo ${LINE} | cut -f 2 -d '|'`
-		if [ ! -f "${X}-${Y}" ]; then continue; fi
+	PATCHCNT=`wc -l patchlist`
+	echo "Applying patches... "
+	IFS='|'
+	I=0
+	while read X Y; do
+		I=$(($I + 1))
+		F="${X}-${Y}"
+		if [ ! -f "${F}" ]; then
+			printf "  Skipping ${F} (${I} of ${PATCHCNT}).\r"
+			continue;
+		fi
+		echo "  Processing ${F}..." 1>${QUIETREDIR}
 		gunzip -c < files/${X}.gz > OLD
 		${BSPATCH} OLD NEW ${X}-${Y}
 		if [ `${SHA256} -q NEW` = ${Y} ]; then
@@ -779,6 +793,7 @@ fetch_update() {
 		fi
 		rm -f diff OLD NEW ${X}-${Y}
 	done < patchlist 2>${QUIETREDIR}
+	IFS=
 	echo "done."
 
 # Update ports without patches
@@ -795,7 +810,10 @@ fetch_update() {
 	    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
 	    2>${QUIETREDIR}
 
+	I=0
 	while read Y; do
+		I=$(($I + 1))
+		printf "   Processing ${Y} (${I} of ${PATCHCNT}).\r" 1>${QUIETREDIR}
 		if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then
 			mv ${Y}.gz files/${Y}.gz
 		else
@@ -877,6 +895,7 @@ extract_metadata() {
 
 # Do the actual work involved in "extract"
 extract_run() {
+	local IFS='|'
 	mkdir -p ${PORTSDIR} || return 1
 
 	if !
@@ -886,7 +905,7 @@ extract_run() {
 			grep -vE "${REFUSE}" ${WORKDIR}/INDEX
 		else
 			cat ${WORKDIR}/INDEX
-		fi | tr '|' ' ' | while read FILE HASH; do
+		fi | while read FILE HASH; do
 		echo ${PORTSDIR}/${FILE}
 		if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then
 			echo "files/${HASH}.gz not found -- snapshot corrupt."
@@ -916,35 +935,8 @@ extract_run() {
 	extract_indices
 }
 
-# Do the actual work involved in "update"
-update_run() {
-	if ! [ -z "${INDEXONLY}" ]; then
-		extract_indices >/dev/null || return 1
-		return 0
-	fi
-
-	if sort ${WORKDIR}/INDEX |
-	    cmp -s ${PORTSDIR}/.portsnap.INDEX -; then
-		echo "Ports tree is already up to date."
-		return 0
-	fi
-
-# If we are REFUSEing to touch certain directories, don't remove files
-# from those directories (even if they are out of date)
-	echo -n "Removing old files and directories... "
-	if ! [ -z "${REFUSE}" ]; then 
-		sort ${WORKDIR}/INDEX |
-		    comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
-		    grep -vE "${REFUSE}" |
-		    lam -s "${PORTSDIR}/" - |
-		    sed -e 's|/$||' | xargs rm -rf
-	else
-		sort ${WORKDIR}/INDEX |
-		    comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
-		    lam -s "${PORTSDIR}/" - |
-		    sed -e 's|/$||' | xargs rm -rf
-	fi
-	echo "done."
+update_run_exract() {
+	local IFS='|'
 
 # Install new files
 	echo "Extracting new files:"
@@ -955,9 +947,7 @@ update_run() {
 			sort ${WORKDIR}/INDEX
 		fi |
 	    comm -13 ${PORTSDIR}/.portsnap.INDEX - |
-	    while read LINE; do
-		FILE=`echo ${LINE} | cut -f 1 -d '|'`
-		HASH=`echo ${LINE} | cut -f 2 -d '|'`
+	    while read FILE HASH; do
 		echo ${PORTSDIR}/${FILE}
 		if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then
 			echo "files/${HASH}.gz not found -- snapshot corrupt."
@@ -977,7 +967,39 @@ update_run() {
 	done; then
 		return 1
 	fi
+}
+
+# Do the actual work involved in "update"
+update_run() {
+	if ! [ -z "${INDEXONLY}" ]; then
+		extract_indices >/dev/null || return 1
+		return 0
+	fi
+
+	if sort ${WORKDIR}/INDEX |
+	    cmp -s ${PORTSDIR}/.portsnap.INDEX -; then
+		echo "Ports tree is already up to date."
+		return 0
+	fi
+
+# If we are REFUSEing to touch certain directories, don't remove files
+# from those directories (even if they are out of date)
+	echo -n "Removing old files and directories... "
+	if ! [ -z "${REFUSE}" ]; then 
+		sort ${WORKDIR}/INDEX |
+		    comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
+		    grep -vE "${REFUSE}" |
+		    lam -s "${PORTSDIR}/" - |
+		    sed -e 's|/$||' | xargs rm -rf
+	else
+		sort ${WORKDIR}/INDEX |
+		    comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
+		    lam -s "${PORTSDIR}/" - |
+		    sed -e 's|/$||' | xargs rm -rf
+	fi
+	echo "done."
 
+	update_run_exract || return 1
 	extract_metadata
 	extract_indices
 }