diff options
| -rwxr-xr-x | build.sh | 71 | ||||
| -rwxr-xr-x | tools/build_snapshots.sh | 331 | ||||
| -rw-r--r-- | tools/builder_common.sh | 167 | ||||
| -rw-r--r-- | tools/builder_defaults.sh | 8 |
4 files changed, 276 insertions, 301 deletions
@@ -65,6 +65,7 @@ usage() { echo " --build-kernels - build all configured kernels" echo " --build-kernel argument - build specified kernel. Example --build-kernel KERNEL_NAME" echo " --install-extra-kernels argument - Put extra kernel(s) under /kernel image directory. Example --install-extra-kernels KERNEL_NAME_WRAP" + echo " --snapshots - Build snapshots and upload them to RSYNCIP" echo " --enable-memorydisks - This will put stage_dir and iso_dir as MFS filesystems" echo " --disable-memorydisks - Will just teardown these filesystems created by --enable-memorydisks" echo " --setup-poudriere - Install poudriere and create necessary jails and ports tree" @@ -85,6 +86,7 @@ unset _USE_OLD_DATESTRING unset pfPORTTOBUILD unset IMAGETYPE unset DO_NOT_UPLOAD +unset SNAPSHOTS BUILDACTION="images" # Maybe use options for nocleans etc? @@ -130,6 +132,10 @@ while test "$1" != ""; do fi export INSTALL_EXTRA_KERNELS="${1}" ;; + --snapshots) + export SNAPSHOTS=1 + IMAGETYPE="all" + ;; --build-kernel) BUILDACTION="buildkernel" shift @@ -193,6 +199,11 @@ while test "$1" != ""; do [ -n "${1}" ] \ && var_to_print="${1}" ;; + --snapshot-update-status) + shift + [ -n "${1}" ] \ + && snapshot_status_message="${1}" + ;; *) usage esac @@ -211,6 +222,13 @@ if [ -n "${var_to_print}" ]; then exit 0 fi +# Update snapshot status and exit +if [ -n "${snapshot_status_message}" ]; then + export SNAPSHOTS=1 + snapshots_update_status "${snapshot_status_message}" + exit 0 +fi + # This should be run first launch @@ -232,7 +250,7 @@ case $BUILDACTION in printflags) print_flags ;; - images) + images|snapshots) # It will be handled below ;; updatesources) @@ -269,6 +287,28 @@ if [ "${BUILDACTION}" != "images" ]; then exit 0 fi +if [ -n "${SNAPSHOTS}" -a -z "${DO_NOT_UPLOAD}" ]; then + if [ -z "${RSYNCIP}" -a -z "${DO_NOT_UPLOAD}" ]; then + echo ">>> ERROR: RSYNCIP is not defined" + exit 1 + fi + + if [ -z "${RSYNCUSER}" -a -z "${DO_NOT_UPLOAD}" ]; then + echo ">>> ERROR: RSYNCUSER is not defined" + exit 1 + fi + + if [ -z "${RSYNCPATH}" -a -z "${DO_NOT_UPLOAD}" ]; then + echo ">>> ERROR: RSYNCPATH is not defined" + exit 1 + fi + + if [ -z "${RSYNCLOGS}" -a -z "${DO_NOT_UPLOAD}" ]; then + echo ">>> ERROR: RSYNCLOGS is not defined" + exit 1 + fi +fi + if [ $# -gt 1 ]; then echo "ERROR: Too many arguments given." echo @@ -291,6 +331,20 @@ fi echo ">>> Building image type(s): ${_IMAGESTOBUILD}" +if [ -n "${SNAPSHOTS}" ]; then + echo "" > $SNAPSHOTSLOGFILE + echo "" > $SNAPSHOTSLASTUPDATE + + snapshots_rotate_logfile + + snapshots_update_status ">>> Starting snapshot build operations" + + if pkg update -r ${PRODUCT_NAME} >/dev/null 2>&1; then + snapshots_update_status ">>> Updating builder packages... " + pkg upgrade -r ${PRODUCT_NAME} -y -q >/dev/null 2>&1 + fi +fi + if [ -z "${_SKIP_REBUILD_PRESTAGE}" ]; then [ -n "${CORE_PKG_TMP}" -a -d "${CORE_PKG_TMP}" ] \ && rm -rf ${CORE_PKG_TMP} @@ -364,8 +418,19 @@ done echo ">>> NOTE: waiting for jobs: `jobs -l` to finish..." wait -echo ">>> ${IMAGES_FINAL_DIR} now contains:" -ls -lah ${IMAGES_FINAL_DIR} +if [ -n "${SNAPSHOTS}" ]; then + snapshots_copy_to_staging_iso_updates + snapshots_copy_to_staging_nanobsd "${FLASH_SIZE}" + # SCP files to snapshot web hosting area + if [ -z "${DO_NOT_UPLOAD}" ]; then + snapshots_scp_files + fi + # Alert the world that we have some snapshots ready. + snapshots_update_status ">>> Builder run is complete." +else + echo ">>> ${IMAGES_FINAL_DIR} now contains:" + ls -lah ${IMAGES_FINAL_DIR} +fi set -e # Run final finish routines diff --git a/tools/build_snapshots.sh b/tools/build_snapshots.sh index 1791b68..8bc975f 100755 --- a/tools/build_snapshots.sh +++ b/tools/build_snapshots.sh @@ -35,6 +35,8 @@ export BUILDER_ROOT=$(realpath "${BUILDER_TOOLS}/..") NO_UPLOAD="" LOOPED_SNAPSHOTS="" +export minsleepvalue=${minsleepvalue:-"28800"} +export maxsleepvalue=${maxsleepvalue:-"86400"} # Handle command line arguments while test "$1" != "" ; do @@ -42,60 +44,24 @@ while test "$1" != "" ; do --noupload|-u) NO_UPLOAD="-u" ;; - --looped) + --looped|-l) LOOPED_SNAPSHOTS="true" esac shift done -# Source ${PRODUCT_NAME} / FreeSBIE variables -# *** DO NOT SOURCE BUILDER_COMMON.SH! -# *** IT WILL BREAK EVERYTHING FOR -# *** SOME UNKNOWN LAYERING REASON. -# *** 04/07/2008, 11/04/2009 -echo ">>> Execing build.conf" -. ${BUILDER_TOOLS}/builder_defaults.sh - -if [ -z "${RSYNCIP}" -a -z "${NO_UPLOAD}" ]; then - echo ">>> ERROR: RSYNCIP is not defined" - exit 1 -fi - -if [ -z "${RSYNCUSER}" -a -z "${NO_UPLOAD}" ]; then - echo ">>> ERROR: RSYNCUSER is not defined" - exit 1 -fi - -if [ -z "${RSYNCPATH}" -a -z "${NO_UPLOAD}" ]; then - echo ">>> ERROR: RSYNCPATH is not defined" - exit 1 -fi - -if [ -z "${RSYNCLOGS}" -a -z "${NO_UPLOAD}" ]; then - echo ">>> ERROR: RSYNCLOGS is not defined" - exit 1 -fi - # Keeps track of how many time builder has looped BUILDCOUNTER=0 -# Local variables that are used by builder scripts -STAGINGAREA=${SCRATCHDIR}/staging -RSYNCKBYTELIMIT="248000" - -export SNAPSHOTSLOGFILE=${SNAPSHOTSLOGFILE:-"$SCRATCHDIR/snapshots-build.log"} -export SNAPSHOTSLASTUPDATE=${SNAPSHOTSLASTUPDATE:-"$SCRATCHDIR/snapshots-lastupdate.log"} - -# Ensure directories exist -mkdir -p $STAGINGAREA - -echo "" > $SNAPSHOTSLOGFILE -echo "" > $SNAPSHOTSLASTUPDATE +git_last_commit() { + export CURRENT_COMMIT=$(git -C ${BUILDER_ROOT} log -1 --format='%H') + export CURRENT_AUTHOR=$(git -C ${BUILDER_ROOT} log -1 --format='%an') +} # This routine is called in between runs. We # will sleep for a bit and check for new commits # in between sleeping for short durations. -sleep_between_runs() { +snapshots_sleep_between_runs() { COUNTER=0 while [ $COUNTER -lt $maxsleepvalue ]; do sleep 60 @@ -103,280 +69,49 @@ sleep_between_runs() { git -C "${BUILDER_ROOT}" pull -q git_last_commit if [ "${LAST_COMMIT}" != "${CURRENT_COMMIT}" ]; then - update_status ">>> New commit: $CURRENT_AUTHOR - $CURRENT_COMMIT .. No longer sleepy." + ${BUILDER_ROOT}/build.sh --snapshot-update-status ">>> New commit: $CURRENT_AUTHOR - $CURRENT_COMMIT .. No longer sleepy." COUNTER=$(($maxsleepvalue + 60)) export LAST_COMMIT="${CURRENT_COMMIT}" fi COUNTER=$(($COUNTER + 60)) done if [ $COUNTER -ge $maxsleepvalue ]; then - update_status ">>> Sleep timer expired. Restarting build." + ${BUILDER_ROOT}/build.sh --snapshot-update-status ">>> Sleep timer expired. Restarting build." maxsleepvalue=0 COUNTER=0 fi } -# This routine is called to write out to stdout -# a string. The string is appended to $SNAPSHOTSLOGFILE -# and we scp the log file to the builder host if -# needed for the real time logging functions. -update_status() { - if [ "$1" = "" ]; then - return - fi - echo $1 - echo "`date` -|- $1" >> $SNAPSHOTSLOGFILE - if [ -z "${NO_UPLOAD}" ]; then - LU=`cat $SNAPSHOTSLASTUPDATE` - CT=`date "+%H%M%S"` - # Only update every minute - if [ "$LU" != "$CT" ]; then - ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCLOGS}" - scp -q $SNAPSHOTSLOGFILE ${RSYNCUSER}@${RSYNCIP}:${RSYNC_LOGS}/build.log - date "+%H%M%S" > $SNAPSHOTSLASTUPDATE - fi - fi -} +git_last_commit -# Copy the current log file to $filename.old on -# the snapshot www server (real time logs) -rotate_logfile() { - if [ -n "$MASTER_BUILDER_SSH_LOG_DEST" -a -z "${NO_UPLOAD}" ]; then - scp -q $SNAPSHOTSLOGFILE ${RSYNCUSER}@${RSYNCIP}:${RSYNC_LOGS}/build.log.old - fi +# Main builder loop +while [ /bin/true ]; do + BUILDCOUNTER=$((${BUILDCOUNTER}+1)) - # Cleanup log file - echo "" > $SNAPSHOTSLOGFILE -} - -dobuilds() { - # Build images - (cd ${BUILDER_ROOT} && ./build.sh --flash-size '1g 2g 4g' "iso memstick memstickserial memstickadi fullupdate nanobsd nanobsd-vga") - # Copy files - copy_to_staging_iso_updates - copy_to_staging_nanobsd '1g 2g 4g' -} - -copy_to_staging_nanobsd() { - for NANOTYPE in nanobsd nanobsd-vga; do - for FILESIZE in ${1}; do - FILENAMEFULL="${PRODUCT_NAME}-${PRODUCT_VERSION}-${FILESIZE}-${TARGET}-${NANOTYPE}${TIMESTAMP_SUFFIX}.img.gz" - FILENAMEUPGRADE="${PRODUCT_NAME}-${PRODUCT_VERSION}-${FILESIZE}-${TARGET}-${NANOTYPE}-upgrade${TIMESTAMP_SUFFIX}.img.gz" - mkdir -p $STAGINGAREA/nanobsd - mkdir -p $STAGINGAREA/nanobsdupdates - - cp $IMAGES_FINAL_DIR/$FILENAMEFULL $STAGINGAREA/nanobsd/ 2>/dev/null - cp $IMAGES_FINAL_DIR/$FILENAMEUPGRADE $STAGINGAREA/nanobsdupdates 2>/dev/null - - if [ -f $STAGINGAREA/nanobsd/$FILENAMEFULL ]; then - md5 $STAGINGAREA/nanobsd/$FILENAMEFULL > $STAGINGAREA/nanobsd/$FILENAMEFULL.md5 2>/dev/null - sha256 $STAGINGAREA/nanobsd/$FILENAMEFULL > $STAGINGAREA/nanobsd/$FILENAMEFULL.sha256 2>/dev/null - fi - if [ -f $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE ]; then - md5 $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE > $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE.md5 2>/dev/null - sha256 $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE > $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE.sha256 2>/dev/null - fi - - # Copy NanoBSD auto update: - if [ -f $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE ]; then - cp $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE $STAGINGAREA/latest-${NANOTYPE}-$FILESIZE.img.gz 2>/dev/null - sha256 $STAGINGAREA/latest-${NANOTYPE}-$FILESIZE.img.gz > $STAGINGAREA/latest-${NANOTYPE}-$FILESIZE.img.gz.sha256 2>/dev/null - # NOTE: Updates need a file with output similar to date output - # Use the file generated at start of dobuilds() to be consistent on times - cp $BUILTDATESTRINGFILE $STAGINGAREA/version-${NANOTYPE}-$FILESIZE - fi - done + ${BUILDER_ROOT}/build.sh --clean-builder | while read LINE; do + ${BUILDER_ROOT}/build.sh --snapshot-update-status "${LINE}" done -} - -copy_to_staging_iso_updates() { - # Copy ISOs - md5 ${ISOPATH}.gz > ${ISOPATH}.md5 - sha256 ${ISOPATH}.gz > ${ISOPATH}.sha256 - cp ${ISOPATH}* $STAGINGAREA/ 2>/dev/null - - # Copy memstick items - md5 ${MEMSTICKPATH}.gz > ${MEMSTICKPATH}.md5 - sha256 ${MEMSTICKPATH}.gz > ${MEMSTICKPATH}.sha256 - cp ${MEMSTICKPATH}* $STAGINGAREA/ 2>/dev/null - - md5 ${MEMSTICKSERIALPATH}.gz > ${MEMSTICKSERIALPATH}.md5 - sha256 ${MEMSTICKSERIALPATH}.gz > ${MEMSTICKSERIALPATH}.sha256 - cp ${MEMSTICKSERIALPATH}* $STAGINGAREA/ 2>/dev/null - - md5 ${MEMSTICKADIPATH}.gz > ${MEMSTICKADIPATH}.md5 - sha256 ${MEMSTICKADIPATH}.gz > ${MEMSTICKADIPATH}.sha256 - cp ${MEMSTICKADIPATH}* $STAGINGAREA/ 2>/dev/null - - md5 ${UPDATES_TARBALL_FILENAME} > ${UPDATES_TARBALL_FILENAME}.md5 - sha256 ${UPDATES_TARBALL_FILENAME} > ${UPDATES_TARBALL_FILENAME}.sha256 - cp ${UPDATES_TARBALL_FILENAME}* $STAGINGAREA/ 2>/dev/null - # NOTE: Updates need a file with output similar to date output - # Use the file generated at start of dobuilds() to be consistent on times - if [ -z "${_IS_RELEASE}" ]; then - cp $BUILTDATESTRINGFILE $STAGINGAREA/version 2>/dev/null - fi -} - -scp_files() { - if [ -z "${RSYNC_COPY_ARGUMENTS:-}" ]; then - RSYNC_COPY_ARGUMENTS="-ave ssh --timeout=60 --bwlimit=${RSYNCKBYTELIMIT}" #--bwlimit=50 - fi - update_status ">>> Copying files to ${RSYNCIP}" - - rm -f $SCRATCHDIR/ssh-snapshots* - - # Ensure directory(s) are available - ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/livecd_installer" - ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/updates" - ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/nanobsd" - if [ -d $STAGINGAREA/virtualization ]; then - ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/virtualization" - fi - ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/.updaters" - # ensure permissions are correct for r+w - ssh ${RSYNCUSER}@${RSYNCIP} "chmod -R ug+rw /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/." - ssh ${RSYNCUSER}@${RSYNCIP} "chmod -R ug+rw ${RSYNCPATH}/." - ssh ${RSYNCUSER}@${RSYNCIP} "chmod -R ug+rw ${RSYNCPATH}/*/." - rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/${PRODUCT_NAME}-*iso* \ - ${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/livecd_installer/ - rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/${PRODUCT_NAME}-memstick* \ - ${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/livecd_installer/ - rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/${PRODUCT_NAME}-*Update* \ - ${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/updates/ - rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/nanobsd/* \ - ${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/nanobsd/ - rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/nanobsdupdates/* \ - ${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/updates/ - if [ -d $STAGINGAREA/virtualization ]; then - rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/virtualization/* \ - ${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/virtualization/ - fi - - # Rather than copy these twice, use ln to link to the latest one. - - ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest.tgz" - ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest.tgz.sha256" - - LATESTFILENAME="`ls $UPDATESDIR/*.tgz | grep Full | grep -v md5 | grep -v sha256 | tail -n1`" - LATESTFILENAME=`basename ${LATESTFILENAME}` - ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${LATESTFILENAME} \ - ${RSYNCPATH}/.updaters/latest.tgz" - ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${LATESTFILENAME}.sha256 \ - ${RSYNCPATH}/.updaters/latest.tgz.sha256" - for i in 1g 2g 4g - do - ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz" - ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz.sha256" - ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz" - ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz.sha256" - - FILENAMEUPGRADE="${PRODUCT_NAME}-${PRODUCT_VERSION}-${i}-${TARGET}-nanobsd-upgrade${TIMESTAMP_SUFFIX}.img.gz" - ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE} \ - ${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz" - ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE}.sha256 \ - ${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz.sha256" - - FILENAMEUPGRADE="${PRODUCT_NAME}-${PRODUCT_VERSION}-${i}-${TARGET}-nanobsd-vga-upgrade${TIMESTAMP_SUFFIX}.img.gz" - ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE} \ - ${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz" - ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE}.sha256 \ - ${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz.sha256" + ${BUILDER_ROOT}/build.sh ${NO_UPLOAD} --flash-size '1g 2g 4g' --snapshots | while read LINE; do + ${BUILDER_ROOT}/build.sh --snapshot-update-status "${LINE}" done - rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/version* \ - ${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/.updaters - update_status ">>> Finished copying files." -} - -cleanup_builds() { - # Remove prior builds - update_status ">>> Cleaning up after prior builds..." - rm -rf $STAGINGAREA/* - rm -rf $IMAGES_FINAL_DIR/* - (cd ${BUILDER_ROOT} && ./build.sh --clean-builder) -} - -build_loop_operations() { - update_status ">>> Starting build loop operations" - # --- Items we need to run for a complete build run --- - # Cleanup prior builds - cleanup_builds - # Update pkgs if necessary - if pkg update -r ${PRODUCT_NAME} >/dev/null 2>&1; then - update_status ">>> Updating builder packages... " - pkg upgrade -r ${PRODUCT_NAME} -y -q >/dev/null 2>&1 - fi - # Do the builds - dobuilds - # SCP files to snapshot web hosting area - if [ -z "${NO_UPLOAD}" ]; then - scp_files + if [ -z "${LOOPED_SNAPSHOTS}" ]; then + # only one build required, exiting + exit fi - # Alert the world that we have some snapshots ready. - update_status ">>> Builder run is complete." -} - -if [ -z "${LOOPED_SNAPSHOTS}" ]; then - build_loop_operations -else - # Main builder loop - while [ /bin/true ]; do - BUILDCOUNTER=`expr $BUILDCOUNTER + 1` - update_status ">>> Starting builder run #${BUILDCOUNTER}..." - - # Launch the snapshots builder script and pipe its - # contents to the while loop so we can record the - # script progress in real time to the public facing - # snapshot server (${RSYNCIP}). - ( build_loop_operations ) | while read LINE - do - update_status "$LINE" - done - export minsleepvalue=28800 - export maxsleepvalue=86400 + # Initialize variables that keep track of last commit + [ -z "${LAST_COMMIT}" ] \ + && export LAST_COMMIT=${CURRENT_COMMIT} - # Initialize variables that keep track of last commit - [ -z "${LAST_COMMIT}" ] \ - && export LAST_COMMIT="$(git -C ${BUILDER_ROOT} log | head -n1 | cut -d' ' -f2)" + ${BUILDER_ROOT}/build.sh --snapshot-update-status ">>> Sleeping for at least $minsleepvalue, at most $maxsleepvalue in between snapshot builder runs. Last known commit ${LAST_COMMIT}" + ${BUILDER_ROOT}/build.sh --snapshot-update-status ">>> Freezing build process at $(date)." + sleep $minsleepvalue + ${BUILDER_ROOT}/build.sh --snapshot-update-status ">>> Thawing build process and resuming checks for pending commits at $(date)." - update_status ">>> Sleeping for at least $minsleepvalue, at most $maxsleepvalue in between snapshot builder runs. Last known commit ${LAST_COMMIT}" - update_status ">>> Freezing build process at `date`." - sleep $minsleepvalue - update_status ">>> Thawing build process and resuming checks for pending commits at `date`." - - # Count some sheep or wait until a new commit turns up - # for one days time. We will wake up if a new commit - # is detected during sleepy time. - sleep_between_runs $maxsleepvalue - - # If REBOOT_AFTER_SNAPSHOT_RUN is defined reboot - # the box after the run. - if [ ! -z "${REBOOT_AFTER_SNAPSHOT_RUN:-}" ]; then - update_status ">>> Rebooting `hostname` due to \$REBOOT_AFTER_SNAPSHOT_RUN" - shutdown -r now - kill $$ - fi - # Rotate log file (.old) - rotate_logfile - - # Set a common DATESTRING for the build if not set from builder_defaults.sh. - # Rely on builder_defaults.sh doing the right job the first time included from this script. - # NOTE: This is needed to have autoupdate detect a new version. - # Override it here to have continuous builds with proper labels - rm -f $DATESTRINGFILE - rm -f $BUILTDATESTRINGFILE - unset TIMESTAMP_SUFFIX - unset DATESTRING - unset BUILTDATESTRING - unset ISOPATH - unset MEMSTICKPATH - unset MEMSTICKSERIALPATH - unset MEMSTICKADIPATH - unset UPDATES_TARBALL_FILENAME - # builder_defaults.sh will set variables with correct timestamp - . ${BUILDER_TOOLS}/builder_defaults.sh - done -fi + # Count some sheep or wait until a new commit turns up + # for one days time. We will wake up if a new commit + # is detected during sleepy time. + snapshots_sleep_between_runs $maxsleepvalue +done diff --git a/tools/builder_common.sh b/tools/builder_common.sh index 733f3d6..0ecbe58 100644 --- a/tools/builder_common.sh +++ b/tools/builder_common.sh @@ -950,6 +950,7 @@ clean_builder() { echo -n ">>> Cleaning previously built images..." rm -rf $IMAGES_FINAL_DIR/* + rm -rf $STAGINGAREA/* echo "Done!" if [ -z "${NO_CLEAN_FREEBSD_SRC}" ]; then @@ -1896,3 +1897,169 @@ poudriere_bulk() { fi done } + +# This routine is called to write out to stdout +# a string. The string is appended to $SNAPSHOTSLOGFILE +# and we scp the log file to the builder host if +# needed for the real time logging functions. +snapshots_update_status() { + if [ -z "${SNAPSHOTS}" -o -z "$1" ]; then + return + fi + echo $1 + echo "`date` -|- $1" >> $SNAPSHOTSLOGFILE + if [ -z "${DO_NOT_UPLOAD}" -a -n "${RSYNCIP}" ]; then + LU=`cat $SNAPSHOTSLASTUPDATE` + CT=`date "+%H%M%S"` + # Only update every minute + if [ "$LU" != "$CT" ]; then + ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCLOGS}" + scp -q $SNAPSHOTSLOGFILE ${RSYNCUSER}@${RSYNCIP}:${RSYNC_LOGS}/build.log + date "+%H%M%S" > $SNAPSHOTSLASTUPDATE + fi + fi +} + +# Copy the current log file to $filename.old on +# the snapshot www server (real time logs) +snapshots_rotate_logfile() { + if [ -n "$MASTER_BUILDER_SSH_LOG_DEST" -a -z "${DO_NOT_UPLOAD}" ]; then + scp -q $SNAPSHOTSLOGFILE ${RSYNCUSER}@${RSYNCIP}:${RSYNC_LOGS}/build.log.old + fi + + # Cleanup log file + echo "" > $SNAPSHOTSLOGFILE +} + +snapshots_copy_to_staging_nanobsd() { + for NANOTYPE in nanobsd nanobsd-vga; do + for FILESIZE in ${1}; do + FILENAMEFULL="${PRODUCT_NAME}-${PRODUCT_VERSION}-${FILESIZE}-${TARGET}-${NANOTYPE}${TIMESTAMP_SUFFIX}.img.gz" + FILENAMEUPGRADE="${PRODUCT_NAME}-${PRODUCT_VERSION}-${FILESIZE}-${TARGET}-${NANOTYPE}-upgrade${TIMESTAMP_SUFFIX}.img.gz" + mkdir -p $STAGINGAREA/nanobsd + mkdir -p $STAGINGAREA/nanobsdupdates + + cp $IMAGES_FINAL_DIR/$FILENAMEFULL $STAGINGAREA/nanobsd/ 2>/dev/null + cp $IMAGES_FINAL_DIR/$FILENAMEUPGRADE $STAGINGAREA/nanobsdupdates 2>/dev/null + + if [ -f $STAGINGAREA/nanobsd/$FILENAMEFULL ]; then + md5 $STAGINGAREA/nanobsd/$FILENAMEFULL > $STAGINGAREA/nanobsd/$FILENAMEFULL.md5 2>/dev/null + sha256 $STAGINGAREA/nanobsd/$FILENAMEFULL > $STAGINGAREA/nanobsd/$FILENAMEFULL.sha256 2>/dev/null + fi + if [ -f $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE ]; then + md5 $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE > $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE.md5 2>/dev/null + sha256 $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE > $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE.sha256 2>/dev/null + fi + + # Copy NanoBSD auto update: + if [ -f $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE ]; then + cp $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE $STAGINGAREA/latest-${NANOTYPE}-$FILESIZE.img.gz 2>/dev/null + sha256 $STAGINGAREA/latest-${NANOTYPE}-$FILESIZE.img.gz > $STAGINGAREA/latest-${NANOTYPE}-$FILESIZE.img.gz.sha256 2>/dev/null + # NOTE: Updates need a file with output similar to date output + # Use the file generated at start of snapshots_dobuilds() to be consistent on times + cp $BUILTDATESTRINGFILE $STAGINGAREA/version-${NANOTYPE}-$FILESIZE + fi + done + done +} + +snapshots_copy_to_staging_iso_updates() { + # Copy ISOs + md5 ${ISOPATH}.gz > ${ISOPATH}.md5 + sha256 ${ISOPATH}.gz > ${ISOPATH}.sha256 + cp ${ISOPATH}* $STAGINGAREA/ 2>/dev/null + + # Copy memstick items + md5 ${MEMSTICKPATH}.gz > ${MEMSTICKPATH}.md5 + sha256 ${MEMSTICKPATH}.gz > ${MEMSTICKPATH}.sha256 + cp ${MEMSTICKPATH}* $STAGINGAREA/ 2>/dev/null + + md5 ${MEMSTICKSERIALPATH}.gz > ${MEMSTICKSERIALPATH}.md5 + sha256 ${MEMSTICKSERIALPATH}.gz > ${MEMSTICKSERIALPATH}.sha256 + cp ${MEMSTICKSERIALPATH}* $STAGINGAREA/ 2>/dev/null + + md5 ${MEMSTICKADIPATH}.gz > ${MEMSTICKADIPATH}.md5 + sha256 ${MEMSTICKADIPATH}.gz > ${MEMSTICKADIPATH}.sha256 + cp ${MEMSTICKADIPATH}* $STAGINGAREA/ 2>/dev/null + + md5 ${UPDATES_TARBALL_FILENAME} > ${UPDATES_TARBALL_FILENAME}.md5 + sha256 ${UPDATES_TARBALL_FILENAME} > ${UPDATES_TARBALL_FILENAME}.sha256 + cp ${UPDATES_TARBALL_FILENAME}* $STAGINGAREA/ 2>/dev/null + # NOTE: Updates need a file with output similar to date output + # Use the file generated at start of snapshots_dobuilds() to be consistent on times + if [ -z "${_IS_RELEASE}" ]; then + cp $BUILTDATESTRINGFILE $STAGINGAREA/version 2>/dev/null + fi +} + +snapshots_scp_files() { + if [ -z "${RSYNC_COPY_ARGUMENTS:-}" ]; then + RSYNC_COPY_ARGUMENTS="-ave ssh --timeout=60 --bwlimit=${RSYNCKBYTELIMIT}" #--bwlimit=50 + fi + snapshots_update_status ">>> Copying files to ${RSYNCIP}" + + rm -f $SCRATCHDIR/ssh-snapshots* + + # Ensure directory(s) are available + ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/livecd_installer" + ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/updates" + ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/nanobsd" + if [ -d $STAGINGAREA/virtualization ]; then + ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/virtualization" + fi + ssh ${RSYNCUSER}@${RSYNCIP} "mkdir -p ${RSYNCPATH}/.updaters" + # ensure permissions are correct for r+w + ssh ${RSYNCUSER}@${RSYNCIP} "chmod -R ug+rw /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/." + ssh ${RSYNCUSER}@${RSYNCIP} "chmod -R ug+rw ${RSYNCPATH}/." + ssh ${RSYNCUSER}@${RSYNCIP} "chmod -R ug+rw ${RSYNCPATH}/*/." + rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/${PRODUCT_NAME}-*iso* \ + ${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/livecd_installer/ + rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/${PRODUCT_NAME}-memstick* \ + ${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/livecd_installer/ + rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/${PRODUCT_NAME}-*Update* \ + ${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/updates/ + rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/nanobsd/* \ + ${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/nanobsd/ + rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/nanobsdupdates/* \ + ${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/updates/ + if [ -d $STAGINGAREA/virtualization ]; then + rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/virtualization/* \ + ${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/virtualization/ + fi + + # Rather than copy these twice, use ln to link to the latest one. + + ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest.tgz" + ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest.tgz.sha256" + + LATESTFILENAME="`ls $UPDATESDIR/*.tgz | grep Full | grep -v md5 | grep -v sha256 | tail -n1`" + LATESTFILENAME=`basename ${LATESTFILENAME}` + ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${LATESTFILENAME} \ + ${RSYNCPATH}/.updaters/latest.tgz" + ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${LATESTFILENAME}.sha256 \ + ${RSYNCPATH}/.updaters/latest.tgz.sha256" + + for i in "${FLASH_SIZE}" + do + ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz" + ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz.sha256" + ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz" + ssh ${RSYNCUSER}@${RSYNCIP} "rm -f ${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz.sha256" + + FILENAMEUPGRADE="${PRODUCT_NAME}-${PRODUCT_VERSION}-${i}-${TARGET}-nanobsd-upgrade${TIMESTAMP_SUFFIX}.img.gz" + ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE} \ + ${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz" + ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE}.sha256 \ + ${RSYNCPATH}/.updaters/latest-nanobsd-${i}.img.gz.sha256" + + FILENAMEUPGRADE="${PRODUCT_NAME}-${PRODUCT_VERSION}-${i}-${TARGET}-nanobsd-vga-upgrade${TIMESTAMP_SUFFIX}.img.gz" + ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE} \ + ${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz" + ssh ${RSYNCUSER}@${RSYNCIP} "ln -s ${RSYNCPATH}/updates/${FILENAMEUPGRADE}.sha256 \ + ${RSYNCPATH}/.updaters/latest-nanobsd-vga-${i}.img.gz.sha256" + done + + rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/version* \ + ${RSYNCUSER}@${RSYNCIP}:${RSYNCPATH}/.updaters + snapshots_update_status ">>> Finished copying files." +} diff --git a/tools/builder_defaults.sh b/tools/builder_defaults.sh index c9231dc..770262e 100644 --- a/tools/builder_defaults.sh +++ b/tools/builder_defaults.sh @@ -318,3 +318,11 @@ export UPDATES_TARBALL_FILENAME=${UPDATES_TARBALL_FILENAME:-"${UPDATESDIR}/${PRO export RSYNCUSER=${RSYNCUSER:-"snapshots"} export RSYNCPATH=${RSYNCPATH:-"/usr/local/www/snapshots/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}"} export RSYNCLOGS=${RSYNCLOGS:-"/usr/local/www/snapshots/logs/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/${TARGET}"} +export RSYNCKBYTELIMIT=${RSYNCKBYTELIMIT:-"248000"} + +# staging area used on snapshots build +STAGINGAREA=${STAGINGAREA:-"${SCRATCHDIR}/staging"} +mkdir -p ${STAGINGAREA} + +export SNAPSHOTSLOGFILE=${SNAPSHOTSLOGFILE:-"${SCRATCHDIR}/snapshots-build.log"} +export SNAPSHOTSLASTUPDATE=${SNAPSHOTSLASTUPDATE:-"${SCRATCHDIR}/snapshots-lastupdate.log"} |
