Move snapshot related functions to builder_common.sh and leave build_snapshots.sh a simple script that update repo and call build.sh, without need to know anything about build.conf variables

1 files changed, 167 insertions, 0 deletions

diff --git a/tools/builder_common.sh b/tools/builder_common.sh
index 733f3d6..0ecbe58 100644
--- a/tools/builder_common.sh
+++ b/tools/builder_common.sh
@@ -950,6 +950,7 @@ clean_builder() {
 
 	echo -n ">>> Cleaning previously built images..."
 	rm -rf $IMAGES_FINAL_DIR/*
+	rm -rf $STAGINGAREA/*
 	echo "Done!"
 
 	if [ -z "${NO_CLEAN_FREEBSD_SRC}" ]; then
@@ -1896,3 +1897,169 @@ poudriere_bulk() {
 		fi
 	done
 }
+
+# This routine is called to write out to stdout
+# a string. The string is appended to $SNAPSHOTSLOGFILE
+# and we scp the log file to the builder host if
+# needed for the real time logging functions.
+snapshots_update_status() {
+	if [ -z "${SNAPSHOTS}" -o -z "$1" ]; then
+		return
+	fi
+	echo $1
+	echo "`date` -|- $1" >> $SNAPSHOTSLOGFILE
+	if [ -z "${DO_NOT_UPLOAD}" -a -n "${RSYNCIP}" ]; then
+		LU=`cat $SNAPSHOTSLASTUPDATE`
+		CT=`date "+%H%M%S"`
+		# Only update every minute
+		if [ "$LU" != "$CT" ]; then
+			ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCLOGS}"
+			scp -q $SNAPSHOTSLOGFILE ${RSYNCUSER}@${RSYNCIP}:${RSYNC_LOGS}/build.log
+			date "+%H%M%S" > $SNAPSHOTSLASTUPDATE
+		fi
+	fi
+}
+
+# Copy the current log file to $filename.old on
+# the snapshot www server (real time logs)
+snapshots_rotate_logfile() {
+	if [ -n "$MASTER_BUILDER_SSH_LOG_DEST" -a -z "${DO_NOT_UPLOAD}" ]; then
+		scp -q $SNAPSHOTSLOGFILE ${RSYNCUSER}@${RSYNCIP}:${RSYNC_LOGS}/build.log.old
+	fi
+
+	# Cleanup log file
+	echo "" > $SNAPSHOTSLOGFILE
+}
+
+snapshots_copy_to_staging_nanobsd() {
+	for NANOTYPE in nanobsd nanobsd-vga; do
+		for FILESIZE in ${1}; do
+			FILENAMEFULL="${PRODUCT_NAME}-${PRODUCT_VERSION}-${FILESIZE}-${TARGET}-${NANOTYPE}${TIMESTAMP_SUFFIX}.img.gz"
+			FILENAMEUPGRADE="${PRODUCT_NAME}-${PRODUCT_VERSION}-${FILESIZE}-${TARGET}-${NANOTYPE}-upgrade${TIMESTAMP_SUFFIX}.img.gz"
+			mkdir -p $STAGINGAREA/nanobsd
+			mkdir -p $STAGINGAREA/nanobsdupdates
+
+			cp $IMAGES_FINAL_DIR/$FILENAMEFULL $STAGINGAREA/nanobsd/ 2>/dev/null
+			cp $IMAGES_FINAL_DIR/$FILENAMEUPGRADE $STAGINGAREA/nanobsdupdates 2>/dev/null
+
+			if [ -f $STAGINGAREA/nanobsd/$FILENAMEFULL ]; then
+				md5 $STAGINGAREA/nanobsd/$FILENAMEFULL > $STAGINGAREA/nanobsd/$FILENAMEFULL.md5 2>/dev/null
+				sha256 $STAGINGAREA/nanobsd/$FILENAMEFULL > $STAGINGAREA/nanobsd/$FILENAMEFULL.sha256 2>/dev/null
+			fi
+			if [ -f $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE ]; then
+				md5 $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE > $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE.md5 2>/dev/null
+				sha256 $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE > $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE.sha256 2>/dev/null
+			fi
+
+			# Copy NanoBSD auto update:
+			if [ -f $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE ]; then
+				cp $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE $STAGINGAREA/latest-${NANOTYPE}-$FILESIZE.img.gz 2>/dev/null
+				sha256 $STAGINGAREA/latest-${NANOTYPE}-$FILESIZE.img.gz > $STAGINGAREA/latest-${NANOTYPE}-$FILESIZE.img.gz.sha256 2>/dev/null
+				# NOTE: Updates need a file with output similar to date output
+				# Use the file generated at start of snapshots_dobuilds() to be consistent on times
+				cp $BUILTDATESTRINGFILE $STAGINGAREA/version-${NANOTYPE}-$FILESIZE
+			fi
+		done
+	done
+}
+
+snapshots_copy_to_staging_iso_updates() {
+	# Copy ISOs
+	md5 ${ISOPATH}.gz > ${ISOPATH}.md5
+	sha256 ${ISOPATH}.gz > ${ISOPATH}.sha256
+	cp ${ISOPATH}* $STAGINGAREA/ 2>/dev/null
+
+	# Copy memstick items
+	md5 ${MEMSTICKPATH}.gz > ${MEMSTICKPATH}.md5
+	sha256 ${MEMSTICKPATH}.gz > ${MEMSTICKPATH}.sha256
+	cp ${MEMSTICKPATH}* $STAGINGAREA/ 2>/dev/null
+
+	md5 ${MEMSTICKSERIALPATH}.gz > ${MEMSTICKSERIALPATH}.md5
+	sha256 ${MEMSTICKSERIALPATH}.gz > ${MEMSTICKSERIALPATH}.sha256
+	cp ${MEMSTICKSERIALPATH}* $STAGINGAREA/ 2>/dev/null
+
+	md5 ${MEMSTICKADIPATH}.gz > ${MEMSTICKADIPATH}.md5
+	sha256 ${MEMSTICKADIPATH}.gz > ${MEMSTICKADIPATH}.sha256
+	cp ${MEMSTICKADIPATH}* $STAGINGAREA/ 2>/dev/null
+
+	md5 ${UPDATES_TARBALL_FILENAME} > ${UPDATES_TARBALL_FILENAME}.md5
+	sha256 ${UPDATES_TARBALL_FILENAME} > ${UPDATES_TARBALL_FILENAME}.sha256
+	cp ${UPDATES_TARBALL_FILENAME}* $STAGINGAREA/ 2>/dev/null
+	# NOTE: Updates need a file with output similar to date output
+	# Use the file generated at start of snapshots_dobuilds() to be consistent on times
+	if [ -z "${_IS_RELEASE}" ]; then
+		cp $BUILTDATESTRINGFILE $STAGINGAREA/version 2>/dev/null
+	fi
+}
+
+snapshots_scp_files() {
+	if [ -z "${RSYNC_COPY_ARGUMENTS:-}" ]; then
+		RSYNC_COPY_ARGUMENTS="-ave ssh --timeout=60 --bwlimit=${RSYNCKBYTELIMIT}" #--bwlimit=50
+	fi
+	snapshots_update_status ">>> Copying files to ${RSYNCIP}"
+
+	rm -f $SCRATCHDIR/ssh-snapshots*
+
+	# Ensure directory(s) are available
+	ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/livecd_installer"
+	ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/updates"
+	ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/nanobsd"
+	if [ -d $STAGINGAREA/virtualization ]; then
+		ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/virtualization"
+	fi
+	ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/.updaters"
+	# ensure permissions are correct for r+w
+	ssh ${RSYNCUSER}@${RSYNCIP} "chmod -R ug+rw /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/."
+	ssh ${RSYNCUSER}@${RSYNCIP} "chmod -R ug+rw ${RSYNCPATH}/."
+	ssh ${RSYNCUSER}@${RSYNCIP} "chmod -R ug+rw ${RSYNCPATH}/*/."
+	rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/${PRODUCT_NAME}-*iso* \
+		${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/livecd_installer/
+	rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/${PRODUCT_NAME}-memstick* \
+		${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/livecd_installer/
+	rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/${PRODUCT_NAME}-*Update* \
+		${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/updates/
+	rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/nanobsd/* \
+		${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/nanobsd/
+	rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/nanobsdupdates/* \
+		${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/updates/
+	if [ -d $STAGINGAREA/virtualization ]; then
+		rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/virtualization/* \
+			${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/virtualization/
+	fi
+
+	# Rather than copy these twice, use ln to link to the latest one.
+
+	ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest.tgz"
+	ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest.tgz.sha256"
+
+	LATESTFILENAME="`ls $UPDATESDIR/*.tgz | grep Full | grep -v md5 | grep -v sha256 | tail -n1`"
+	LATESTFILENAME=`basename ${LATESTFILENAME}`
+	ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${LATESTFILENAME} \
+		${RSYNCPATH}/.updaters/latest.tgz"
+	ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${LATESTFILENAME}.sha256 \
+		${RSYNCPATH}/.updaters/latest.tgz.sha256"
+
+	for i in "${FLASH_SIZE}"
+	do
+		ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz"
+		ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz.sha256"
+		ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz"
+		ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz.sha256"
+
+		FILENAMEUPGRADE="${PRODUCT_NAME}-${PRODUCT_VERSION}-${i}-${TARGET}-nanobsd-upgrade${TIMESTAMP_SUFFIX}.img.gz"
+		ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE} \
+			${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz"
+		ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE}.sha256 \
+			${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz.sha256"
+
+		FILENAMEUPGRADE="${PRODUCT_NAME}-${PRODUCT_VERSION}-${i}-${TARGET}-nanobsd-vga-upgrade${TIMESTAMP_SUFFIX}.img.gz"
+		ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE} \
+			${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz"
+		ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE}.sha256 \
+			${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz.sha256"
+	done
+
+	rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/version* \
+		${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/.updaters
+	snapshots_update_status ">>> Finished copying files."
+}