Move snapshot related functions to builder_common.sh and leave build_snapshots.sh a simple script that update repo and call build.sh, without need to know anything about build.conf variables

1 files changed, 33 insertions, 298 deletions

diff --git a/tools/build_snapshots.sh b/tools/build_snapshots.sh
index 1791b68..8bc975f 100755
--- a/tools/build_snapshots.sh
+++ b/tools/build_snapshots.sh
@@ -35,6 +35,8 @@ export BUILDER_ROOT=$(realpath "${BUILDER_TOOLS}/..")
 
 NO_UPLOAD=""
 LOOPED_SNAPSHOTS=""
+export minsleepvalue=${minsleepvalue:-"28800"}
+export maxsleepvalue=${maxsleepvalue:-"86400"}
 
 # Handle command line arguments
 while test "$1" != "" ; do
@@ -42,60 +44,24 @@ while test "$1" != "" ; do
 	--noupload|-u)
 		NO_UPLOAD="-u"
 		;;
-	--looped)
+	--looped|-l)
 		LOOPED_SNAPSHOTS="true"
 	esac
 	shift
 done
 
-# Source ${PRODUCT_NAME} / FreeSBIE variables
-# *** DO NOT SOURCE BUILDER_COMMON.SH!
-# *** IT WILL BREAK EVERYTHING FOR 
-# *** SOME UNKNOWN LAYERING REASON.
-# *** 04/07/2008, 11/04/2009                      
-echo ">>> Execing build.conf"
-. ${BUILDER_TOOLS}/builder_defaults.sh
-
-if [ -z "${RSYNCIP}" -a -z "${NO_UPLOAD}" ]; then
-	echo ">>> ERROR: RSYNCIP is not defined"
-	exit 1
-fi
-
-if [ -z "${RSYNCUSER}" -a -z "${NO_UPLOAD}" ]; then
-	echo ">>> ERROR: RSYNCUSER is not defined"
-	exit 1
-fi
-
-if [ -z "${RSYNCPATH}" -a -z "${NO_UPLOAD}" ]; then
-	echo ">>> ERROR: RSYNCPATH is not defined"
-	exit 1
-fi
-
-if [ -z "${RSYNCLOGS}" -a -z "${NO_UPLOAD}" ]; then
-	echo ">>> ERROR: RSYNCLOGS is not defined"
-	exit 1
-fi
-
 # Keeps track of how many time builder has looped
 BUILDCOUNTER=0
 
-# Local variables that are used by builder scripts
-STAGINGAREA=${SCRATCHDIR}/staging
-RSYNCKBYTELIMIT="248000"
-
-export SNAPSHOTSLOGFILE=${SNAPSHOTSLOGFILE:-"$SCRATCHDIR/snapshots-build.log"}
-export SNAPSHOTSLASTUPDATE=${SNAPSHOTSLASTUPDATE:-"$SCRATCHDIR/snapshots-lastupdate.log"}
-
-# Ensure directories exist
-mkdir -p $STAGINGAREA
-
-echo "" > $SNAPSHOTSLOGFILE
-echo "" > $SNAPSHOTSLASTUPDATE
+git_last_commit() {
+	export CURRENT_COMMIT=$(git -C ${BUILDER_ROOT} log -1 --format='%H')
+	export CURRENT_AUTHOR=$(git -C ${BUILDER_ROOT} log -1 --format='%an')
+}
 
 # This routine is called in between runs. We
 # will sleep for a bit and check for new commits
 # in between sleeping for short durations.
-sleep_between_runs() {
+snapshots_sleep_between_runs() {
 	COUNTER=0
 	while [ $COUNTER -lt $maxsleepvalue ]; do
 		sleep 60
@@ -103,280 +69,49 @@ sleep_between_runs() {
 		git -C "${BUILDER_ROOT}" pull -q
 		git_last_commit
 		if [ "${LAST_COMMIT}" != "${CURRENT_COMMIT}" ]; then
-			update_status ">>> New commit: $CURRENT_AUTHOR - $CURRENT_COMMIT .. No longer sleepy."
+			${BUILDER_ROOT}/build.sh --snapshot-update-status ">>> New commit: $CURRENT_AUTHOR - $CURRENT_COMMIT .. No longer sleepy."
 			COUNTER=$(($maxsleepvalue + 60))
 			export LAST_COMMIT="${CURRENT_COMMIT}"
 		fi
 		COUNTER=$(($COUNTER + 60))
 	done
 	if [ $COUNTER -ge $maxsleepvalue ]; then
-		update_status ">>> Sleep timer expired. Restarting build."
+		${BUILDER_ROOT}/build.sh --snapshot-update-status ">>> Sleep timer expired. Restarting build."
 		maxsleepvalue=0
 		COUNTER=0
 	fi
 }
 
-# This routine is called to write out to stdout
-# a string. The string is appended to $SNAPSHOTSLOGFILE
-# and we scp the log file to the builder host if
-# needed for the real time logging functions.
-update_status() {
-	if [ "$1" = "" ]; then
-		return
-	fi
-	echo $1
-	echo "`date` -|- $1" >> $SNAPSHOTSLOGFILE
-	if [ -z "${NO_UPLOAD}" ]; then
-		LU=`cat $SNAPSHOTSLASTUPDATE`
-		CT=`date "+%H%M%S"`
-		# Only update every minute
-		if [ "$LU" != "$CT" ]; then 
-			ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCLOGS}"
-			scp -q $SNAPSHOTSLOGFILE ${RSYNCUSER}@${RSYNCIP}:${RSYNC_LOGS}/build.log
-			date "+%H%M%S" > $SNAPSHOTSLASTUPDATE
-		fi
-	fi
-}
+git_last_commit
 
-# Copy the current log file to $filename.old on
-# the snapshot www server (real time logs)
-rotate_logfile() {
-	if [ -n "$MASTER_BUILDER_SSH_LOG_DEST" -a -z "${NO_UPLOAD}" ]; then
-		scp -q $SNAPSHOTSLOGFILE ${RSYNCUSER}@${RSYNCIP}:${RSYNC_LOGS}/build.log.old
-	fi
+# Main builder loop
+while [ /bin/true ]; do
+	BUILDCOUNTER=$((${BUILDCOUNTER}+1))
 
-	# Cleanup log file
-	echo "" > $SNAPSHOTSLOGFILE
-}
-
-dobuilds() {
-	# Build images
-	(cd ${BUILDER_ROOT} && ./build.sh --flash-size '1g 2g 4g' "iso memstick memstickserial memstickadi fullupdate nanobsd nanobsd-vga")
-	# Copy files
-	copy_to_staging_iso_updates
-	copy_to_staging_nanobsd '1g 2g 4g'
-}
-
-copy_to_staging_nanobsd() {
-	for NANOTYPE in nanobsd nanobsd-vga; do
-		for FILESIZE in ${1}; do
-			FILENAMEFULL="${PRODUCT_NAME}-${PRODUCT_VERSION}-${FILESIZE}-${TARGET}-${NANOTYPE}${TIMESTAMP_SUFFIX}.img.gz"
-			FILENAMEUPGRADE="${PRODUCT_NAME}-${PRODUCT_VERSION}-${FILESIZE}-${TARGET}-${NANOTYPE}-upgrade${TIMESTAMP_SUFFIX}.img.gz"
-			mkdir -p $STAGINGAREA/nanobsd
-			mkdir -p $STAGINGAREA/nanobsdupdates
-
-			cp $IMAGES_FINAL_DIR/$FILENAMEFULL $STAGINGAREA/nanobsd/ 2>/dev/null
-			cp $IMAGES_FINAL_DIR/$FILENAMEUPGRADE $STAGINGAREA/nanobsdupdates 2>/dev/null
-
-			if [ -f $STAGINGAREA/nanobsd/$FILENAMEFULL ]; then
-				md5 $STAGINGAREA/nanobsd/$FILENAMEFULL > $STAGINGAREA/nanobsd/$FILENAMEFULL.md5 2>/dev/null
-				sha256 $STAGINGAREA/nanobsd/$FILENAMEFULL > $STAGINGAREA/nanobsd/$FILENAMEFULL.sha256 2>/dev/null
-			fi
-			if [ -f $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE ]; then
-				md5 $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE > $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE.md5 2>/dev/null
-				sha256 $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE > $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE.sha256 2>/dev/null
-			fi
-
-			# Copy NanoBSD auto update:
-			if [ -f $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE ]; then
-				cp $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE $STAGINGAREA/latest-${NANOTYPE}-$FILESIZE.img.gz 2>/dev/null
-				sha256 $STAGINGAREA/latest-${NANOTYPE}-$FILESIZE.img.gz > $STAGINGAREA/latest-${NANOTYPE}-$FILESIZE.img.gz.sha256 2>/dev/null
-				# NOTE: Updates need a file with output similar to date output
-				# Use the file generated at start of dobuilds() to be consistent on times
-				cp $BUILTDATESTRINGFILE $STAGINGAREA/version-${NANOTYPE}-$FILESIZE
-			fi
-		done
+	${BUILDER_ROOT}/build.sh --clean-builder | while read LINE; do
+		${BUILDER_ROOT}/build.sh --snapshot-update-status "${LINE}"
 	done
-}
-
-copy_to_staging_iso_updates() {
-	# Copy ISOs
-	md5 ${ISOPATH}.gz > ${ISOPATH}.md5
-	sha256 ${ISOPATH}.gz > ${ISOPATH}.sha256
-	cp ${ISOPATH}* $STAGINGAREA/ 2>/dev/null
-
-	# Copy memstick items
-	md5 ${MEMSTICKPATH}.gz > ${MEMSTICKPATH}.md5
-	sha256 ${MEMSTICKPATH}.gz > ${MEMSTICKPATH}.sha256
-	cp ${MEMSTICKPATH}* $STAGINGAREA/ 2>/dev/null
-
-	md5 ${MEMSTICKSERIALPATH}.gz > ${MEMSTICKSERIALPATH}.md5
-	sha256 ${MEMSTICKSERIALPATH}.gz > ${MEMSTICKSERIALPATH}.sha256
-	cp ${MEMSTICKSERIALPATH}* $STAGINGAREA/ 2>/dev/null
-
-	md5 ${MEMSTICKADIPATH}.gz > ${MEMSTICKADIPATH}.md5
-	sha256 ${MEMSTICKADIPATH}.gz > ${MEMSTICKADIPATH}.sha256
-	cp ${MEMSTICKADIPATH}* $STAGINGAREA/ 2>/dev/null
-
-	md5 ${UPDATES_TARBALL_FILENAME} > ${UPDATES_TARBALL_FILENAME}.md5
-	sha256 ${UPDATES_TARBALL_FILENAME} > ${UPDATES_TARBALL_FILENAME}.sha256
-	cp ${UPDATES_TARBALL_FILENAME}* $STAGINGAREA/ 2>/dev/null
-	# NOTE: Updates need a file with output similar to date output
-	# Use the file generated at start of dobuilds() to be consistent on times
-	if [ -z "${_IS_RELEASE}" ]; then
-		cp $BUILTDATESTRINGFILE $STAGINGAREA/version 2>/dev/null
-	fi
-}
-
-scp_files() {
-	if [ -z "${RSYNC_COPY_ARGUMENTS:-}" ]; then
-		RSYNC_COPY_ARGUMENTS="-ave ssh --timeout=60 --bwlimit=${RSYNCKBYTELIMIT}" #--bwlimit=50
-	fi
-	update_status ">>> Copying files to ${RSYNCIP}"
-
-	rm -f $SCRATCHDIR/ssh-snapshots*
-
-	# Ensure directory(s) are available
-	ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/livecd_installer"
-	ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/updates"
-	ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/nanobsd"
-	if [ -d $STAGINGAREA/virtualization ]; then
-		ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/virtualization"
-	fi
-	ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/.updaters"
-	# ensure permissions are correct for r+w
-	ssh ${RSYNCUSER}@${RSYNCIP} "chmod -R ug+rw /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/."
-	ssh ${RSYNCUSER}@${RSYNCIP} "chmod -R ug+rw ${RSYNCPATH}/."
-	ssh ${RSYNCUSER}@${RSYNCIP} "chmod -R ug+rw ${RSYNCPATH}/*/."
-	rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/${PRODUCT_NAME}-*iso* \
-		${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/livecd_installer/
-	rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/${PRODUCT_NAME}-memstick* \
-		${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/livecd_installer/
-	rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/${PRODUCT_NAME}-*Update* \
-		${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/updates/
-	rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/nanobsd/* \
-		${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/nanobsd/
-	rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/nanobsdupdates/* \
-		${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/updates/
-	if [ -d $STAGINGAREA/virtualization ]; then
-		rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/virtualization/* \
-			${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/virtualization/
-	fi
-
-	# Rather than copy these twice, use ln to link to the latest one.
-
-	ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest.tgz"
-	ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest.tgz.sha256"
-
-	LATESTFILENAME="`ls $UPDATESDIR/*.tgz | grep Full | grep -v md5 | grep -v sha256 | tail -n1`"
-	LATESTFILENAME=`basename ${LATESTFILENAME}`
-	ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${LATESTFILENAME} \
-		${RSYNCPATH}/.updaters/latest.tgz"
-	ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${LATESTFILENAME}.sha256 \
-		${RSYNCPATH}/.updaters/latest.tgz.sha256"
 
-	for i in 1g 2g 4g
-	do
-		ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz"
-		ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz.sha256"
-		ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz"
-		ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz.sha256"
-
-		FILENAMEUPGRADE="${PRODUCT_NAME}-${PRODUCT_VERSION}-${i}-${TARGET}-nanobsd-upgrade${TIMESTAMP_SUFFIX}.img.gz"
-		ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE} \
-			${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz"
-		ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE}.sha256 \
-			${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz.sha256"
-
-		FILENAMEUPGRADE="${PRODUCT_NAME}-${PRODUCT_VERSION}-${i}-${TARGET}-nanobsd-vga-upgrade${TIMESTAMP_SUFFIX}.img.gz"
-		ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE} \
-			${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz"
-		ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE}.sha256 \
-			${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz.sha256"
+	${BUILDER_ROOT}/build.sh ${NO_UPLOAD} --flash-size '1g 2g 4g' --snapshots | while read LINE; do
+		${BUILDER_ROOT}/build.sh --snapshot-update-status "${LINE}"
 	done
 
-	rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/version* \
-		${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/.updaters
-	update_status ">>> Finished copying files."
-}
-
-cleanup_builds() {
-	# Remove prior builds
-	update_status ">>> Cleaning up after prior builds..."
-	rm -rf $STAGINGAREA/*
-	rm -rf $IMAGES_FINAL_DIR/*
-	(cd ${BUILDER_ROOT} && ./build.sh --clean-builder)
-}
-
-build_loop_operations() {
-	update_status ">>> Starting build loop operations"
-	# --- Items we need to run for a complete build run ---
-	# Cleanup prior builds
-	cleanup_builds
-	# Update pkgs if necessary
-	if pkg update -r ${PRODUCT_NAME} >/dev/null 2>&1; then
-		update_status ">>> Updating builder packages... "
-		pkg upgrade -r ${PRODUCT_NAME} -y -q >/dev/null 2>&1
-	fi
-	# Do the builds
-	dobuilds
-	# SCP files to snapshot web hosting area
-	if [ -z "${NO_UPLOAD}" ]; then
-		scp_files
+	if [ -z "${LOOPED_SNAPSHOTS}" ]; then
+		# only one build required, exiting
+		exit
 	fi
-	# Alert the world that we have some snapshots ready.
-	update_status ">>> Builder run is complete."
-}
-
-if [ -z "${LOOPED_SNAPSHOTS}" ]; then
-	build_loop_operations
-else
-	# Main builder loop
-	while [ /bin/true ]; do
-		BUILDCOUNTER=`expr $BUILDCOUNTER + 1`
-		update_status ">>> Starting builder run #${BUILDCOUNTER}..."
-
-		# Launch the snapshots builder script and pipe its
-		# contents to the while loop so we can record the 
-		# script progress in real time to the public facing
-		# snapshot server (${RSYNCIP}).
-		( build_loop_operations ) | while read LINE
-		do
-			update_status "$LINE"
-		done
 
-		export minsleepvalue=28800
-		export maxsleepvalue=86400
+	# Initialize variables that keep track of last commit
+	[ -z "${LAST_COMMIT}" ] \
+		&& export LAST_COMMIT=${CURRENT_COMMIT}
 
-		# Initialize variables that keep track of last commit
-		[ -z "${LAST_COMMIT}" ] \
-			&& export LAST_COMMIT="$(git -C ${BUILDER_ROOT} log | head -n1 | cut -d' ' -f2)"
+	${BUILDER_ROOT}/build.sh --snapshot-update-status ">>> Sleeping for at least $minsleepvalue, at most $maxsleepvalue in between snapshot builder runs.  Last known commit ${LAST_COMMIT}"
+	${BUILDER_ROOT}/build.sh --snapshot-update-status ">>> Freezing build process at $(date)."
+	sleep $minsleepvalue
+	${BUILDER_ROOT}/build.sh --snapshot-update-status ">>> Thawing build process and resuming checks for pending commits at $(date)."
 
-		update_status ">>> Sleeping for at least $minsleepvalue, at most $maxsleepvalue in between snapshot builder runs.  Last known commit ${LAST_COMMIT}"
-		update_status ">>> Freezing build process at `date`."
-		sleep $minsleepvalue
-		update_status ">>> Thawing build process and resuming checks for pending commits at `date`."
-
-		# Count some sheep or wait until a new commit turns up 
-		# for one days time.  We will wake up if a new commit
-		# is detected during sleepy time.
-		sleep_between_runs $maxsleepvalue
-
-		# If REBOOT_AFTER_SNAPSHOT_RUN is defined reboot
-		# the box after the run. 
-		if [ ! -z "${REBOOT_AFTER_SNAPSHOT_RUN:-}" ]; then
-			update_status ">>> Rebooting `hostname` due to \$REBOOT_AFTER_SNAPSHOT_RUN"
-			shutdown -r now
-			kill $$
-		fi
-		# Rotate log file (.old)
-		rotate_logfile
-
-		# Set a common DATESTRING for the build if not set from builder_defaults.sh.
-		# Rely on builder_defaults.sh doing the right job the first time included from this script.
-		# NOTE: This is needed to have autoupdate detect a new version.
-		# Override it here to have continuous builds with proper labels
-		rm -f $DATESTRINGFILE
-		rm -f $BUILTDATESTRINGFILE
-		unset TIMESTAMP_SUFFIX
-		unset DATESTRING
-		unset BUILTDATESTRING
-		unset ISOPATH
-		unset MEMSTICKPATH
-		unset MEMSTICKSERIALPATH
-		unset MEMSTICKADIPATH
-		unset UPDATES_TARBALL_FILENAME
-		# builder_defaults.sh will set variables with correct timestamp
-		. ${BUILDER_TOOLS}/builder_defaults.sh
-	done
-fi
+	# Count some sheep or wait until a new commit turns up
+	# for one days time.  We will wake up if a new commit
+	# is detected during sleepy time.
+	snapshots_sleep_between_runs $maxsleepvalue
+done