From 6f73c36226d8583b90ddcff50850701c06de2ee2 Mon Sep 17 00:00:00 2001 From: Renato Botelho Date: Tue, 25 Aug 2015 08:21:15 -0300 Subject: Import build scripts from pfSense-tools repo --- tools/build_snapshots.sh | 389 ++++ tools/builder_common.sh | 2062 ++++++++++++++++++++ tools/builder_defaults.sh | 299 +++ tools/conf/ovf/pfSense.ovf | 136 ++ tools/conf/pfPorts/make.conf | 125 ++ tools/conf/pfPorts/poudriere_bulk | 47 + tools/conf/pfPorts/poudriere_bulk.arm.armv6 | 1 + tools/scripts/generate-privdefs.php | 216 ++ tools/scripts/update_package_pfPorts.php | 449 +++++ tools/set_version.sh | 127 ++ tools/templates/core_pkg/base/exclude_files | 26 + tools/templates/core_pkg/base/metadir/+DESC | 3 + tools/templates/core_pkg/base/metadir/+MANIFEST | 15 + tools/templates/core_pkg/base/pkg-plist | 2 + .../core_pkg/default-config-serial/metadir/+DESC | 3 + .../default-config-serial/metadir/+INSTALL | 15 + .../default-config-serial/metadir/+MANIFEST | 15 + .../core_pkg/default-config-serial/pkg-plist | 2 + .../core_pkg/default-config/metadir/+DESC | 3 + .../core_pkg/default-config/metadir/+INSTALL | 15 + .../core_pkg/default-config/metadir/+MANIFEST | 15 + tools/templates/core_pkg/default-config/pkg-plist | 1 + tools/templates/core_pkg/kernel/metadir/+DEINSTALL | 19 + tools/templates/core_pkg/kernel/metadir/+DESC | 3 + tools/templates/core_pkg/kernel/metadir/+MANIFEST | 15 + tools/templates/pkg-deinstall.in | 3 + tools/templates/pkg-install.in | 7 + 27 files changed, 4013 insertions(+) create mode 100755 tools/build_snapshots.sh create mode 100644 tools/builder_common.sh create mode 100644 tools/builder_defaults.sh create mode 100755 tools/conf/ovf/pfSense.ovf create mode 100644 tools/conf/pfPorts/make.conf create mode 100644 tools/conf/pfPorts/poudriere_bulk create mode 100644 tools/conf/pfPorts/poudriere_bulk.arm.armv6 create mode 100755 tools/scripts/generate-privdefs.php create mode 100755 tools/scripts/update_package_pfPorts.php create mode 100755 tools/set_version.sh create mode 100644 tools/templates/core_pkg/base/exclude_files create mode 100644 tools/templates/core_pkg/base/metadir/+DESC create mode 100644 tools/templates/core_pkg/base/metadir/+MANIFEST create mode 100644 tools/templates/core_pkg/base/pkg-plist create mode 100644 tools/templates/core_pkg/default-config-serial/metadir/+DESC create mode 100644 tools/templates/core_pkg/default-config-serial/metadir/+INSTALL create mode 100644 tools/templates/core_pkg/default-config-serial/metadir/+MANIFEST create mode 100644 tools/templates/core_pkg/default-config-serial/pkg-plist create mode 100644 tools/templates/core_pkg/default-config/metadir/+DESC create mode 100644 tools/templates/core_pkg/default-config/metadir/+INSTALL create mode 100644 tools/templates/core_pkg/default-config/metadir/+MANIFEST create mode 100644 tools/templates/core_pkg/default-config/pkg-plist create mode 100644 tools/templates/core_pkg/kernel/metadir/+DEINSTALL create mode 100644 tools/templates/core_pkg/kernel/metadir/+DESC create mode 100644 tools/templates/core_pkg/kernel/metadir/+MANIFEST create mode 100644 tools/templates/pkg-deinstall.in create mode 100644 tools/templates/pkg-install.in (limited to 'tools') diff --git a/tools/build_snapshots.sh b/tools/build_snapshots.sh new file mode 100755 index 0000000..e1fe313 --- /dev/null +++ b/tools/build_snapshots.sh @@ -0,0 +1,389 @@ +#!/bin/sh +# +# build_snapshots.sh +# +# Copyright (c) 2007-2015 Electric Sheep Fencing, LLC +# All rights reserved +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE pfSense PROJECT ``AS IS'' AND ANY +# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE pfSense PROJECT OR +# ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +# OF THE POSSIBILITY OF SUCH DAMAGE. +# + +if [ ! -f ./build.conf ]; then + echo "You must first run ./set_version.sh !" + exit 1 +fi + +NO_UPLOAD="" +LOOPED_SNAPSHOTS="" + +# Handle command line arguments +while test "$1" != "" ; do + case $1 in + --noupload|-u) + NO_UPLOAD="-u" + ;; + --looped) + LOOPED_SNAPSHOTS="true" + esac + shift +done + +# Source ${PRODUCT_NAME} / FreeSBIE variables +# *** DO NOT SOURCE BUILDER_COMMON.SH! +# *** IT WILL BREAK EVERYTHING FOR +# *** SOME UNKNOWN LAYERING REASON. +# *** 04/07/2008, 11/04/2009 +echo ">>> Execing build.conf" +. ./builder_defaults.sh + +# Keeps track of how many time builder has looped +BUILDCOUNTER=0 + +# Local variables that are used by builder scripts +STAGINGAREA=/tmp/staging +RSYNCIP="snapshots.pfsense.org" +RSYNCKBYTELIMIT="248000" + +export SNAPSHOTSLOGFILE=${SNAPSHOTSLOGFILE:-"$SCRATCHDIR/snapshots-build.log"} +export SNAPSHOTSLASTUPDATE=${SNAPSHOTSLASTUPDATE:-"$SCRATCHDIR/snapshots-lastupdate.log"} +export MASTER_BUILDER_SSH_LOG_DEST=${MASTER_BUILDER_SSH_LOG_DEST:-snapshots@${RSYNCIP}:/usr/local/www/snapshots/logs/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/${TARGET}/build.log} + +# Ensure directories exist +mkdir -p $STAGINGAREA + +echo "" > $SNAPSHOTSLOGFILE +echo "" > $SNAPSHOTSLASTUPDATE + +git_last_commit() { + if [ -d "${1}/.git" ]; then + (cd ${1} && git fetch origin && git rebase) >/dev/null + CURRENT_COMMIT=$(cd ${1} && git log -1 --format='%H') + echo "$CURRENT_COMMIT" + fi +} + +# This routine is called in between runs. We +# will sleep for a bit and check for new commits +# in between sleeping for short durations. +sleep_between_runs() { + COUNTER=0 + while [ $COUNTER -lt $maxsleepvalue ]; do + sleep 60 + PRODUCT_CURRENT_COMMIT=$(git_last_commit "${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}") + if [ "${PRODUCT_LAST_COMMIT}" != "${PRODUCT_CURRENT_COMMIT}" ]; then + update_status ">>> New commit: $CURRENT_AUTHOR - $PRODUCT_CURRENT_COMMIT .. No longer sleepy." + COUNTER=$(($maxsleepvalue + 60)) + export PRODUCT_LAST_COMMIT="${PRODUCT_CURRENT_COMMIT}" + fi + TOOLS_CURRENT_COMMIT=$(git_last_commit "${BUILDER_TOOLS}") + if [ "${TOOLS_LAST_COMMIT}" != "${TOOLS_CURRENT_COMMIT}" ]; then + update_status ">>> New commit: $CURRENT_AUTHOR - $TOOLS_CURRENT_COMMIT .. No longer sleepy." + COUNTER=$(($maxsleepvalue + 60)) + export TOOLS_LAST_COMMIT="${TOOLS_CURRENT_COMMIT}" + fi + COUNTER=$(($COUNTER + 60)) + done + if [ $COUNTER -ge $maxsleepvalue ]; then + update_status ">>> Sleep timer expired. Restarting build." + maxsleepvalue=0 + COUNTER=0 + fi +} + +# This routine is called to write out to stdout +# a string. The string is appended to $SNAPSHOTSLOGFILE +# and we scp the log file to the builder host if +# needed for the real time logging functions. +update_status() { + if [ "$1" = "" ]; then + return + fi + echo $1 + echo "`date` -|- $1" >> $SNAPSHOTSLOGFILE + if [ -n "$MASTER_BUILDER_SSH_LOG_DEST" -a -z "${NO_UPLOAD}" ]; then + LU=`cat $SNAPSHOTSLASTUPDATE` + CT=`date "+%H%M%S"` + # Only update every minute + if [ "$LU" != "$CT" ]; then + scp -q $SNAPSHOTSLOGFILE $MASTER_BUILDER_SSH_LOG_DEST + date "+%H%M%S" > $SNAPSHOTSLASTUPDATE + fi + fi +} + +# Copy the current log file to $filename.old on +# the snapshot www server (real time logs) +rotate_logfile() { + if [ -n "$MASTER_BUILDER_SSH_LOG_DEST" -a -z "${NO_UPLOAD}" ]; then + scp -q $SNAPSHOTSLOGFILE $MASTER_BUILDER_SSH_LOG_DEST.old + fi + + # Cleanup log file + echo "" > $SNAPSHOTSLOGFILE +} + +dobuilds() { + + cd $BUILDER_SCRIPTS + # Build images + ./build.sh --flash-size '1g 2g 4g' "iso memstick memstickserial memstickadi fullupdate nanobsd nanobsd-vga" + # Copy files + copy_to_staging_iso_updates + copy_to_staging_nanobsd '1g 2g 4g' +} + +copy_to_staging_nanobsd() { + cd $BUILDER_SCRIPTS + + for NANOTYPE in nanobsd nanobsd-vga; do + for FILESIZE in ${1}; do + FILENAMEFULL="${PRODUCT_NAME}-${PRODUCT_VERSION}-${FILESIZE}-${TARGET}-${NANOTYPE}-${DATESTRING}.img.gz" + FILENAMEUPGRADE="${PRODUCT_NAME}-${PRODUCT_VERSION}-${FILESIZE}-${TARGET}-${NANOTYPE}-upgrade-${DATESTRING}.img.gz" + mkdir -p $STAGINGAREA/nanobsd + mkdir -p $STAGINGAREA/nanobsdupdates + + cp $MAKEOBJDIRPREFIXFINAL/$FILENAMEFULL $STAGINGAREA/nanobsd/ 2>/dev/null + cp $MAKEOBJDIRPREFIXFINAL/$FILENAMEUPGRADE $STAGINGAREA/nanobsdupdates 2>/dev/null + + if [ -f $STAGINGAREA/nanobsd/$FILENAMEFULL ]; then + md5 $STAGINGAREA/nanobsd/$FILENAMEFULL > $STAGINGAREA/nanobsd/$FILENAMEFULL.md5 2>/dev/null + sha256 $STAGINGAREA/nanobsd/$FILENAMEFULL > $STAGINGAREA/nanobsd/$FILENAMEFULL.sha256 2>/dev/null + fi + if [ -f $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE ]; then + md5 $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE > $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE.md5 2>/dev/null + sha256 $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE > $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE.sha256 2>/dev/null + fi + + # Copy NanoBSD auto update: + if [ -f $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE ]; then + cp $STAGINGAREA/nanobsdupdates/$FILENAMEUPGRADE $STAGINGAREA/latest-${NANOTYPE}-$FILESIZE.img.gz 2>/dev/null + sha256 $STAGINGAREA/latest-${NANOTYPE}-$FILESIZE.img.gz > $STAGINGAREA/latest-${NANOTYPE}-$FILESIZE.img.gz.sha256 2>/dev/null + # NOTE: Updates need a file with output similar to date output + # Use the file generated at start of dobuilds() to be consistent on times + cp $BUILTDATESTRINGFILE $STAGINGAREA/version-${NANOTYPE}-$FILESIZE + fi + done + done +} + +copy_to_staging_iso_updates() { + cd $BUILDER_SCRIPTS + + # Copy ISOs + md5 ${ISOPATH}.gz > ${ISOPATH}.md5 + sha256 ${ISOPATH}.gz > ${ISOPATH}.sha256 + cp ${ISOPATH}* $STAGINGAREA/ 2>/dev/null + + # Copy memstick items + md5 ${MEMSTICKPATH}.gz > ${MEMSTICKPATH}.md5 + sha256 ${MEMSTICKPATH}.gz > ${MEMSTICKPATH}.sha256 + cp ${MEMSTICKPATH}* $STAGINGAREA/ 2>/dev/null + + md5 ${MEMSTICKSERIALPATH}.gz > ${MEMSTICKSERIALPATH}.md5 + sha256 ${MEMSTICKSERIALPATH}.gz > ${MEMSTICKSERIALPATH}.sha256 + cp ${MEMSTICKSERIALPATH}* $STAGINGAREA/ 2>/dev/null + + md5 ${MEMSTICKADIPATH}.gz > ${MEMSTICKADIPATH}.md5 + sha256 ${MEMSTICKADIPATH}.gz > ${MEMSTICKADIPATH}.sha256 + cp ${MEMSTICKADIPATH}* $STAGINGAREA/ 2>/dev/null + + md5 ${UPDATES_TARBALL_FILENAME} > ${UPDATES_TARBALL_FILENAME}.md5 + sha256 ${UPDATES_TARBALL_FILENAME} > ${UPDATES_TARBALL_FILENAME}.sha256 + cp ${UPDATES_TARBALL_FILENAME}* $STAGINGAREA/ 2>/dev/null + # NOTE: Updates need a file with output similar to date output + # Use the file generated at start of dobuilds() to be consistent on times + cp $BUILTDATESTRINGFILE $STAGINGAREA/version 2>/dev/null +} + +scp_files() { + cd $BUILDER_SCRIPTS + if [ -z "${RSYNC_COPY_ARGUMENTS:-}" ]; then + RSYNC_COPY_ARGUMENTS="-ave ssh --timeout=60 --bwlimit=${RSYNCKBYTELIMIT}" #--bwlimit=50 + fi + update_status ">>> Copying files to ${RSYNCIP}" + if [ ! -f /usr/local/bin/rsync ]; then + update_status ">>> Could not find rsync, installing from ports..." + (cd /usr/ports/net/rsync && make install clean) + fi + rm -f $SCRATCHDIR/ssh-snapshots* + + # Ensure directory(s) are available + ssh snapshots@${RSYNCIP} "mkdir -p /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/livecd_installer" + ssh snapshots@${RSYNCIP} "mkdir -p /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/updates" + ssh snapshots@${RSYNCIP} "mkdir -p /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/nanobsd" + if [ -d $STAGINGAREA/virtualization ]; then + ssh snapshots@${RSYNCIP} "mkdir -p /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/virtualization" + fi + ssh snapshots@${RSYNCIP} "mkdir -p /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/.updaters" + # ensure permissions are correct for r+w + ssh snapshots@${RSYNCIP} "chmod -R ug+rw /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/." + ssh snapshots@${RSYNCIP} "chmod -R ug+rw /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/." + ssh snapshots@${RSYNCIP} "chmod -R ug+rw /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/*/." + rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/${PRODUCT_NAME}-*iso* \ + snapshots@${RSYNCIP}:/usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/livecd_installer/ + rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/${PRODUCT_NAME}-memstick* \ + snapshots@${RSYNCIP}:/usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/livecd_installer/ + rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/${PRODUCT_NAME}-*Update* \ + snapshots@${RSYNCIP}:/usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/updates/ + rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/nanobsd/* \ + snapshots@${RSYNCIP}:/usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/nanobsd/ + rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/nanobsdupdates/* \ + snapshots@${RSYNCIP}:/usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/updates/ + if [ -d $STAGINGAREA/virtualization ]; then + rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/virtualization/* \ + snapshots@${RSYNCIP}:/usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/virtualization/ + fi + + # Rather than copy these twice, use ln to link to the latest one. + + ssh snapshots@${RSYNCIP} "rm -f /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/.updaters/latest.tgz" + ssh snapshots@${RSYNCIP} "rm -f /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/.updaters/latest.tgz.sha256" + + LATESTFILENAME="`ls $UPDATESDIR/*.tgz | grep Full | grep -v md5 | grep -v sha256 | tail -n1`" + LATESTFILENAME=`basename ${LATESTFILENAME}` + ssh snapshots@${RSYNCIP} "ln -s /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/updates/${LATESTFILENAME} \ + /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/.updaters/latest.tgz" + ssh snapshots@${RSYNCIP} "ln -s /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/updates/${LATESTFILENAME}.sha256 \ + /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/.updaters/latest.tgz.sha256" + + for i in 1g 2g 4g + do + ssh snapshots@${RSYNCIP} "rm -f /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/.updaters/latest-nanobsd-${i}.img.gz" + ssh snapshots@${RSYNCIP} "rm -f /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/.updaters/latest-nanobsd-${i}.img.gz.sha256" + ssh snapshots@${RSYNCIP} "rm -f /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/.updaters/latest-nanobsd-vga-${i}.img.gz" + ssh snapshots@${RSYNCIP} "rm -f /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/.updaters/latest-nanobsd-vga-${i}.img.gz.sha256" + + FILENAMEUPGRADE="${PRODUCT_NAME}-${PRODUCT_VERSION}-${i}-${TARGET}-nanobsd-upgrade-${DATESTRING}.img.gz" + ssh snapshots@${RSYNCIP} "ln -s /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/updates/${FILENAMEUPGRADE} \ + /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/.updaters/latest-nanobsd-${i}.img.gz" + ssh snapshots@${RSYNCIP} "ln -s /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/updates/${FILENAMEUPGRADE}.sha256 \ + /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/.updaters/latest-nanobsd-${i}.img.gz.sha256" + + FILENAMEUPGRADE="${PRODUCT_NAME}-${PRODUCT_VERSION}-${i}-${TARGET}-nanobsd-vga-upgrade-${DATESTRING}.img.gz" + ssh snapshots@${RSYNCIP} "ln -s /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/updates/${FILENAMEUPGRADE} \ + /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/.updaters/latest-nanobsd-vga-${i}.img.gz" + ssh snapshots@${RSYNCIP} "ln -s /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/updates/${FILENAMEUPGRADE}.sha256 \ + /usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/.updaters/latest-nanobsd-vga-${i}.img.gz.sha256" + done + + rsync $RSYNC_COPY_ARGUMENTS $STAGINGAREA/version* \ + snapshots@${RSYNCIP}:/usr/local/www/snapshots/FreeBSD_${FREEBSD_PARENT_BRANCH}/${TARGET}/${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}/.updaters + update_status ">>> Finished copying files." +} + +cleanup_builds() { + cd $BUILDER_SCRIPTS + # Remove prior builds + update_status ">>> Cleaning up after prior builds..." + rm -rf $STAGINGAREA/* + rm -f $UPDATESDIR/* # Keep updates dir slimmed down + rm -rf $MAKEOBJDIRPREFIXFINAL/* + ./build.sh --clean-builder +} + +build_loop_operations() { + cd $BUILDER_SCRIPTS + update_status ">>> Starting build loop operations" + # --- Items we need to run for a complete build run --- + # Cleanup prior builds + cleanup_builds + # Update pkgs if necessary + if pkg update -r ${PRODUCT_NAME} >/dev/null 2>&1; then + update_status ">>> Updating builder packages... " + pkg upgrade -r ${PRODUCT_NAME} -y -q >/dev/null 2>&1 + fi + # Do the builds + dobuilds + # SCP files to snapshot web hosting area + if [ -z "${NO_UPLOAD}" ]; then + scp_files + fi + # Alert the world that we have some snapshots ready. + update_status ">>> Builder run is complete." +} + +if [ -z "${LOOPED_SNAPSHOTS}" ]; then + build_loop_operations +else + # Main builder loop + while [ /bin/true ]; do + BUILDCOUNTER=`expr $BUILDCOUNTER + 1` + update_status ">>> Starting builder run #${BUILDCOUNTER}..." + + # Launch the snapshots builder script and pipe its + # contents to the while loop so we can record the + # script progress in real time to the public facing + # snapshot server (${RSYNCIP}). + ( build_loop_operations ) | while read LINE + do + update_status "$LINE" + done + + export minsleepvalue=28800 + export maxsleepvalue=86400 + + # Initialize variables that keep track of last commit + [ -z "${PRODUCT_LAST_COMMIT}" ] \ + && export PRODUCT_LAST_COMMIT="$(cd "${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}" && git log | head -n1 | cut -d' ' -f2)" + [ -z "${TOOLS_LAST_COMMIT}" ] \ + && export TOOLS_LAST_COMMIT="$(cd "${BUILDER_SCRIPTS}" && git log | head -n1 | cut -d' ' -f2)" + + update_status ">>> Sleeping for at least $minsleepvalue, at most $maxsleepvalue in between snapshot builder runs. Last known commit ${PRODUCT_LAST_COMMIT}/${TOOLS_LAST_COMMIT}" + update_status ">>> Freezing build process at `date`." + sleep $minsleepvalue + update_status ">>> Thawing build process and resuming checks for pending commits at `date`." + + # Count some sheep or wait until a new commit turns up + # for one days time. We will wake up if a new commit + # is detected during sleepy time. + sleep_between_runs $maxsleepvalue + + # If REBOOT_AFTER_SNAPSHOT_RUN is defined reboot + # the box after the run. + if [ ! -z "${REBOOT_AFTER_SNAPSHOT_RUN:-}" ]; then + update_status ">>> Rebooting `hostname` due to \$REBOOT_AFTER_SNAPSHOT_RUN" + shutdown -r now + kill $$ + fi + # Rotate log file (.old) + rotate_logfile + + # Set a common DATESTRING for the build if not set from builder_defaults.sh. + # Rely on builder_defaults.sh doing the right job the first time included from this script. + # NOTE: This is needed to have autoupdate detect a new version. + # Override it here to have continuous builds with proper labels + rm -f $DATESTRINGFILE + rm -f $BUILTDATESTRINGFILE + unset DATESTRING + unset BUILTDATESTRING + unset ISOPATH + unset MEMSTICKPATH + unset MEMSTICKSERIALPATH + unset MEMSTICKADIPATH + unset UPDATES_TARBALL_FILENAME + # builder_defaults.sh will set variables with correct timestamp + . ./builder_defaults.sh + done +fi diff --git a/tools/builder_common.sh b/tools/builder_common.sh new file mode 100644 index 0000000..8ff0b39 --- /dev/null +++ b/tools/builder_common.sh @@ -0,0 +1,2062 @@ +#!/bin/sh +# +# builder_common.sh +# +# Copyright (c) 2004-2015 Electric Sheep Fencing, LLC +# Copyright (C) 2014 Ermal Luçi +# All rights reserved. +# +# NanoBSD portions of the code +# Copyright (c) 2005 Poul-Henning Kamp. +# and copied from nanobsd.sh +# All rights reserved. +# +# FreeSBIE portions of the code +# Copyright (c) 2005 Dario Freni +# and copied from FreeSBIE project +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE pfSense PROJECT ``AS IS'' AND ANY +# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE pfSense PROJECT OR +# ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +# OF THE POSSIBILITY OF SUCH DAMAGE. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# + +if [ "$MAKEOBJDIRPREFIXFINAL" != "" ]; then + mkdir -p $MAKEOBJDIRPREFIXFINAL +else + echo "MAKEOBJDIRPREFIXFINAL is not defined" + print_error_pfS +fi + +lc() { + echo "${1}" | tr '[[:upper:]]' '[[:lower:]]' +} + +git_last_commit() { + if [ -d "${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}/.git" ]; then + CURRENT_COMMIT=$(cd ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} && git log -1 --format='%H') + CURRENT_AUTHOR=$(cd ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} && git log -1 --format='%an') + echo ">>> Last known commit $CURRENT_AUTHOR - $CURRENT_COMMIT" + echo "$CURRENT_COMMIT" > $SCRATCHDIR/build_commit_info.txt + else + echo ">>> WARNING: git repo is not cloned! Previous commit functions disabled." + fi +} + +# Create core pkg (base, kernel) +core_pkg_create() { + local _template="${1}" + local _flavor="${2}" + local _version="${3}" + local _root="${4}" + + [ -d "${CORE_PKG_TMP}" ] \ + && rm -rf ${CORE_PKG_TMP} + + local _templates_path=${BUILDER_SCRIPTS}/templates/core_pkg/${_template} + local _template_metadir=${_templates_path}/metadir + local _metadir=${CORE_PKG_TMP}/${_template}_metadir + + if [ ! -d ${_template_metadir} ]; then + echo "ERROR: Template dir not found for pkg ${_template}" + exit + fi + + mkdir -p ${CORE_PKG_TMP} + + cp -r ${_template_metadir} ${_metadir} + + local _manifest=${_metadir}/+MANIFEST + local _plist=${CORE_PKG_TMP}/${_template}_plist + local _exclude_plist=${CORE_PKG_TMP}/${_template}_exclude_plist + + if [ -f "${_templates_path}/pkg-plist" ]; then + cp ${_templates_path}/pkg-plist ${_plist} + else + (cd ${_root} && find . -type f -or -type l | sed 's,^.,,' | sort -u) > ${_plist} + fi + + if [ -f "${_templates_path}/exclude_plist" ]; then + cp ${_templates_path}/exclude_plist ${_exclude_plist} + else + touch ${_exclude_plist} + fi + + sed \ + -i '' \ + -e "s,%%PRODUCT_NAME%%,${PRODUCT_NAME},g" \ + -e "s,%%PRODUCT_URL%%,${PRODUCT_URL},g" \ + -e "s,%%FLAVOR%%,${_flavor},g" \ + -e "s,%%VERSION%%,${_version},g" \ + ${_metadir}/* \ + ${_plist} \ + ${exclude_plist} + + if [ -f "${_exclude_plist}" ]; then + sort -u ${_exclude_plist} > ${_plist}.exclude + mv ${_plist} ${_plist}.tmp + comm -23 ${_plist}.tmp ${_plist}.exclude > ${_plist} + rm -f ${_plist}.tmp ${plist}.exclude + fi + + mkdir -p ${CORE_PKG_PATH} + if ! pkg create -o ${CORE_PKG_PATH} -p ${_plist} -r ${_root} -m ${_metadir}; then + echo ">>> ERROR: Error building package ${_template} ${_flavor}" + print_error_pfS + fi +} + +# This routine will output that something went wrong +print_error_pfS() { + echo + echo "####################################" + echo "Something went wrong, check errors!" >&2 + echo "####################################" + echo + echo "NOTE: a lot of times you can run './build.sh --clean-builder' to resolve." + echo + if [ "$1" != "" ]; then + echo $1 + fi + [ -n "${LOGFILE:-}" ] && \ + echo "Log saved on ${LOGFILE}" && \ + tail -n20 ${LOGFILE} >&2 + echo + echo "Press enter to continue." + read ans + kill $$ + exit 1 +} + +prestage_on_ram_setup() { + _AVAIL_MEM=$(($(sysctl -n hw.usermem) / 1024 / 1024)) + if [ $_AVAIL_MEM -lt 2000 ]; then + echo ">>> Builder has less than 2GiB RAM skipping memory disks" + return + else + echo "######################################################################################" + echo + echo ">>> Builder has more than 1.4GiB RAM enabling memory disks" + echo ">>> WARNING: Remember to remove these memory disks by running $0 --disable-memorydisks" + echo + echo "######################################################################################" + fi + + if df /dev/ufs/prestagebacking >/dev/null 2>&1; then + echo ">>> Detected preexisting memory disk enabled for ${STAGE_CHROOT_DIR}." + else + mdconfig -a -t swap -u 10001 -s ${MEMORYDISK_SIZE} + mkdir -p ${STAGE_CHROOT_DIR} + newfs -L prestagebacking -U /dev/md10001 + mount /dev/ufs/prestagebacking ${STAGE_CHROOT_DIR} + fi + + if df /dev/ufs/finalstagebacking >/dev/null 2>&1; then + echo ">>> Detected preexisting memory disk enabled for ${FINAL_CHROOT_DIR}." + else + mdconfig -a -t swap -u 10002 -s ${MEMORYDISK_SIZE} + mkdir -p ${FINAL_CHROOT_DIR} + newfs -L finalstagebacking -U /dev/md10002 + mount /dev/ufs/finalstagebacking ${FINAL_CHROOT_DIR} + fi +} + +prestage_on_ram_cleanup() { + if [ -c /dev/md10001 ]; then + if [ -d ${STAGE_CHROOT_DIR} ]; then + umount ${STAGE_CHROOT_DIR} + fi + mdconfig -d -u 10001 + fi + if [ -c /dev/md10002 ]; then + if [ -d ${FINAL_CHROOT_DIR} ]; then + umount ${FINAL_CHROOT_DIR} + fi + mdconfig -d -u 10002 + fi +} + +# This routine will verify that the kernel has been +# installed OK to the staging area. +ensure_kernel_exists() { + if [ ! -f "$1/boot/kernel/kernel.gz" ]; then + echo ">>> ERROR: Could not locate $1/boot/kernel.gz" + print_error_pfS + fi + KERNEL_SIZE=$(stat -f "%z" $1/boot/kernel/kernel.gz) + if [ "$KERNEL_SIZE" -lt 3500 ]; then + echo ">>> ERROR: Kernel $1/boot/kernel.gz appears to be smaller than it should be: $KERNEL_SIZE" + print_error_pfS + fi +} + +get_pkg_name() { + echo "${PRODUCT_NAME}-${1}-${CORE_PKG_VERSION}" +} + +# This routine builds all related kernels +build_all_kernels() { + # Set KERNEL_BUILD_PATH if it has not been set + if [ -z "${KERNEL_BUILD_PATH}" ]; then + KERNEL_BUILD_PATH=$SCRATCHDIR/kernels + echo ">>> KERNEL_BUILD_PATH has not been set. Setting to ${KERNEL_BUILD_PATH}!" + fi + + [ -d "${KERNEL_BUILD_PATH}" ] \ + && rm -rf ${KERNEL_BUILD_PATH} + + # Build embedded kernel + for BUILD_KERNEL in $BUILD_KERNELS; do + unset KERNCONF + unset KERNEL_DESTDIR + unset KERNELCONF + unset KERNEL_NAME + export KERNCONF=$BUILD_KERNEL + export KERNEL_DESTDIR="$KERNEL_BUILD_PATH/$BUILD_KERNEL" + export KERNELCONF="$SRCDIR/sys/${TARGET}/conf/$BUILD_KERNEL" + export KERNEL_NAME=${BUILD_KERNEL} + + LOGFILE="${BUILDER_LOGS}/kernel.${KERNCONF}.${TARGET}.log" + echo ">>> Building $BUILD_KERNEL kernel." | tee -a ${LOGFILE} + + if [ ! -e "${SRCDIR}/sys/${TARGET}/conf/${BUILD_KERNEL}" ]; then + echo ">>> ERROR: Could not find $KERNELCONF" + print_error_pfS + fi + + if [ -n "${NO_BUILDKERNEL:-}" -a -f "${CORE_PKG_PATH}/$(get_pkg_name kernel-${KERNEL_NAME}).txz" ]; then + echo ">>> NO_BUILDKERNEL set, skipping build" | tee -a ${LOGFILE} + continue + fi + + export SRC_CONF=${SRC_CONF} + buildkernel + + echo ">>> Staging $BUILD_KERNEL kernel..." | tee -a ${LOGFILE} + installkernel + + ensure_kernel_exists $KERNEL_DESTDIR + + echo -n ">>> Creating pkg of $KERNEL_NAME kernel to staging area..." | tee -a ${LOGFILE} + core_pkg_create kernel ${KERNEL_NAME} ${CORE_PKG_VERSION} ${KERNEL_DESTDIR} + + rm -rf $KERNEL_DESTDIR 2>&1 1>/dev/null + + echo ".Done" | tee -a ${LOGFILE} + done +} + +install_default_kernel() { + if [ -z "${1}" ]; then + echo ">>> ERROR: install_default_kernel called without a kernel config name"| tee -a ${LOGFILE} + print_error_pfS + fi + + export KERNEL_NAME="${1}" + + echo -n ">>> Installing kernel to be used by image ${KERNEL_NAME}..." | tee -a ${LOGFILE} + + # Copy kernel package to chroot, otherwise pkg won't find it to install + if ! pkg_chroot_add ${FINAL_CHROOT_DIR} kernel-${KERNEL_NAME}; then + echo ">>> ERROR: Error installing kernel package $(get_pkg_name kernel-${KERNEL_NAME}).txz" | tee -a ${LOGFILE} + print_error_pfS + fi + + # Lock kernel to avoid user end up removing it for any reason + pkg_chroot ${FINAL_CHROOT_DIR} lock -q -y $(get_pkg_name kernel-${KERNEL_NAME}) + + if [ ! -f $FINAL_CHROOT_DIR/boot/kernel/kernel.gz ]; then + echo ">>> ERROR: No kernel installed on $FINAL_CHROOT_DIR and the resulting image will be unusable. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + mkdir -p $FINAL_CHROOT_DIR/pkgs + if [ -z "${2}" -o -n "${INSTALL_EXTRA_KERNELS}" ]; then + cp ${CORE_PKG_PATH}/$(get_pkg_name kernel-${KERNEL_NAME}).txz $FINAL_CHROOT_DIR/pkgs + if [ -n "${INSTALL_EXTRA_KERNELS}" ]; then + for _EXTRA_KERNEL in $INSTALL_EXTRA_KERNELS; do + _EXTRA_KERNEL_PATH=${CORE_PKG_PATH}/$(get_pkg_name kernel-${_EXTRA_KERNEL}).txz + if [ -f "${_EXTRA_KERNEL_PATH}" ]; then + echo -n ". adding ${_EXTRA_KERNEL_PATH} on image /pkgs folder" + cp ${_EXTRA_KERNEL_PATH} $FINAL_CHROOT_DIR/pkgs + else + echo ">>> ERROR: Requested kernel $(get_pkg_name kernel-${_EXTRA_KERNEL}).txz was not found to be put on image /pkgs folder!" + print_error_pfS + fi + done + fi + fi + echo "Done." | tee -a ${LOGFILE} + + unset KERNEL_NAME +} + +# Items that need to be fixed up that are +# specific to nanobsd builds +cust_fixup_nanobsd() { + local _NANO_WITH_VGA="${1}" + + echo ">>> Fixing up NanoBSD Specific items..." | tee -a ${LOGFILE} + + echo "nanobsd" > $FINAL_CHROOT_DIR/etc/platform + + local BOOTCONF=${FINAL_CHROOT_DIR}/boot.config + local LOADERCONF=${FINAL_CHROOT_DIR}/boot/loader.conf + + if [ "${_NANO_WITH_VGA}" = "nanobsd" ]; then + # Tell loader to use serial console early. + echo "-S115200 -h" >> ${BOOTCONF} + else + # Empty file to identify nanobsd_vga images + touch ${FINAL_CHROOT_DIR}/etc/nano_use_vga.txt + fi + + # Remove old console options if present. + [ -f "${LOADERCONF}" ] \ + && sed -i "" -Ee "/(console|boot_multicons|boot_serial|hint.uart)/d" ${LOADERCONF} + # Activate serial console+video console in loader.conf + echo 'autoboot_delay="5"' >> ${LOADERCONF} + echo 'loader_color="NO"' >> ${LOADERCONF} + echo 'beastie_disable="YES"' >> ${LOADERCONF} + echo 'boot_serial="YES"' >> ${LOADERCONF} + echo 'console="comconsole"' >> ${LOADERCONF} + echo 'comconsole_speed="115200"' >> ${LOADERCONF} +} + +# Creates a full update file +create_Full_update_tarball() { + mkdir -p $UPDATESDIR + + customize_stagearea_for_image "fullupdate" + install_default_kernel ${DEFAULT_KERNEL} + + rm -rf ${FINAL_CHROOT_DIR}/cf + rm -rf ${FINAL_CHROOT_DIR}/conf + rm -f ${FINAL_CHROOT_DIR}/etc/rc.conf + rm -f ${FINAL_CHROOT_DIR}/etc/pwd.db 2>/dev/null + rm -f ${FINAL_CHROOT_DIR}/etc/group 2>/dev/null + rm -f ${FINAL_CHROOT_DIR}/etc/spwd.db 2>/dev/null + rm -f ${FINAL_CHROOT_DIR}/etc/passwd 2>/dev/null + rm -f ${FINAL_CHROOT_DIR}/etc/master.passwd 2>/dev/null + rm -f ${FINAL_CHROOT_DIR}/etc/fstab 2>/dev/null + rm -f ${FINAL_CHROOT_DIR}/etc/bogons 2>/dev/null + # Remove loader.conf and friends. Ticket #560 + rm ${FINAL_CHROOT_DIR}/boot/loader.conf 2>/dev/null + rm ${FINAL_CHROOT_DIR}/boot/loader.conf.local 2>/dev/null + + echo ">>> Creating ${UPDATES_TARBALL_FILENAME} ..." | tee -a ${LOGFILE} + tar --exclude=./dev -czPf ${UPDATES_TARBALL_FILENAME} -C ${FINAL_CHROOT_DIR} . +} + +# Outputs various set variables aka env +print_flags() { + + echo + printf " Product version: %s\n" $PRODUCT_VERSION + printf " BASE_DIR: %s\n" $BASE_DIR + printf " Stage DIR: %s\n" $STAGE_CHROOT_DIR + printf " Updates dir: %s\n" $UPDATESDIR + printf " Image Preparation Stage DIR: %s\n" $FINAL_CHROOT_DIR + printf " Source DIR: %s\n" $SRCDIR + printf " FreeBSD repository: %s\n" $FREEBSD_REPO_BASE + printf " FreeBSD-src branch: %s\n" $FREEBSD_BRANCH + printf " FreeBSD original branch: %s\n" $FREEBSD_PARENT_BRANCH + printf " BUILD_KERNELS: %s\n" $BUILD_KERNELS + printf " Git Repository: %s\n" $GIT_REPO_URL + printf " Git Branch or Tag: %s\n" $GIT_REPO_BRANCH_OR_TAG + printf " MODULES_OVERRIDE: %s\n" $MODULES_OVERRIDE + printf " OVADISKSIZE: %s\n" $OVADISKSIZE + printf " OVABLOCKSIZE: %s\n" $OVABLOCKSIZE + printf " OVA_FIRST_PART_SIZE: %s\n" $OVA_FIRST_PART_SIZE + printf " OVA_SWAP_PART_SIZE: %s\n" $OVA_SWAP_PART_SIZE + printf " OVFFILE: %s\n" $OVFFILE + printf " OVFVMDK: %s\n" $OVFVMDK + printf " OVFCERT: %s\n" $OVFCERT + printf " SRC_CONF: %s\n" $SRC_CONF + printf " ISOPATH: %s\n" $ISOPATH + printf " MEMSTICKPATH: %s\n" $MEMSTICKPATH + printf " UPDATES_TARBALL_FILENAME: %s\n" $UPDATES_TARBALL_FILENAME +if [ -n "$SHOW_ENV" ]; then + for LINE in $(env | egrep -v '(terminal|PASS|NAME|USER|SSH|GROUP|HOST)'); do + echo "SHOW_ENV: $LINE" + done +fi + echo +} + +# This updates the product sources +update_product_repository() { + if [ ! -d "${GIT_REPO_DIR}" ]; then + echo ">>> Creating ${GIT_REPO_DIR}" + mkdir -p ${GIT_REPO_DIR} + fi + + echo ">>> Using GIT to checkout ${GIT_REPO_BRANCH_OR_TAG}" + + # There is already a cloned repo, test if branch is correct + if [ -d "${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}/.git" ]; then + if ! (cd ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} && git rev-parse --verify ${GIT_REPO_BRANCH_OR_TAG} >/dev/null 2>&1); then + echo -n ">>> ${PRODUCT_NAME} git repo is at wrong branch, removing it... " + rm -rf ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} + echo "Done!" + fi + fi + + if [ ! -d "${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}/.git" ]; then + echo -n ">>> Cloning ${GIT_REPO_URL} (${GIT_REPO_BRANCH_OR_TAG})... " + (git clone --depth 1 --single-branch --branch ${GIT_REPO_BRANCH_OR_TAG} ${GIT_REPO_URL} ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}) 2>&1 | \ + egrep -B3 -A3 -wi -E '(error)|fatal' + if [ ! -d "${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}/conf.default" ]; then + echo "Failed!" + echo + echo "!!!! An error occurred while checking out ${PRODUCT_NAME}" + echo " Could not locate ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}/conf.default" + echo + print_error_pfS + fi + echo "Done!" + else + # It is necessary for the case when a tag has moved + local TAG=$(cd ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} && git tag) + if [ "${TAG}" = "${GIT_REPO_BRANCH_OR_TAG}" ]; then + RESET_TARGET="${GIT_REPO_BRANCH_OR_TAG}" + else + RESET_TARGET="origin/${GIT_REPO_BRANCH_OR_TAG}" + fi + + # Fetch an update of the repository + if ! (cd ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} && \ + git fetch origin; \ + git reset --hard ${RESET_TARGET}; \ + git clean -fxd ) >/dev/null 2>&1 + then + echo ">>> ERROR: Something went wrong while checking out GIT." + print_error_pfS + fi + fi +} + +# This builds FreeBSD (make buildworld) +# Imported from FreeSBIE +make_world() { + + LOGFILE=${BUILDER_LOGS}/buildworld.${TARGET} + if [ -n "${NO_BUILDWORLD:-}" ]; then + echo ">>> NO_BUILDWORLD set, skipping build" | tee -a ${LOGFILE} + return + fi + + # Set SRC_CONF variable if it's not already set. + if [ -z "${SRC_CONF:-}" ]; then + echo ">>> SRC_CONF is unset make sure this is what you want!" | tee -a ${LOGFILE} + else + echo ">>> Setting SRC_CONF to $SRC_CONF" | tee -a ${LOGFILE} + fi + + # Set default parameters + export MAKE_ARGS="${MAKEJ_WORLD:-} __MAKE_CONF=${MAKE_CONF} SRCCONF=${SRC_CONF} TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH}" + + echo ">>> LOGFILE set to $LOGFILE." | tee -a ${LOGFILE} + makeargs="${MAKE_ARGS}" + echo ">>> Building world for ${TARGET} architecture... (Starting - $(LC_ALL=C date))" | tee -a ${LOGFILE} + echo ">>> Builder is running the command: env LOCAL_ITOOLS=\"${EXTRA_TOOLS}\" script -aq $LOGFILE make -C ${SRCDIR} -DNO_CLEAN ${makeargs:-} buildworld" | tee -a ${LOGFILE} + (env LOCAL_ITOOLS="${EXTRA_TOOLS}" script -aq $LOGFILE make -C ${SRCDIR} -DNO_CLEAN ${makeargs:-} buildworld || print_error_pfS;) | egrep '^>>>' | tee -a ${LOGFILE} + echo ">>> Building world for ${TARGET} architecture... (Finished - $(LC_ALL=C date))" | tee -a ${LOGFILE} + + LOGFILE=${BUILDER_LOGS}/installworld.${TARGET} + echo ">>> LOGFILE set to $LOGFILE." | tee -a ${LOGFILE} + # Create if cleaned up + mkdir -p ${STAGE_CHROOT_DIR} + makeargs="${MAKE_ARGS} DESTDIR=${STAGE_CHROOT_DIR} WITHOUT_TOOLCHAIN=1" + echo ">>> Installing world for ${TARGET} architecture... (Starting - $(LC_ALL=C date))" | tee -a ${LOGFILE} + echo ">>> Builder is running the command: env LOCAL_ITOOLS=\"${EXTRA_TOOLS}\" script -aq $LOGFILE make -C ${SRCDIR} ${makeargs:-} installworld" | tee -a ${LOGFILE} + (env LOCAL_ITOOLS="${EXTRA_TOOLS}" script -aq $LOGFILE make -C ${SRCDIR} ${makeargs:-} installworld || print_error_pfS;) | egrep '^>>>' | tee -a ${LOGFILE} + echo ">>> Installing world for ${TARGET} architecture... (Finished - $(LC_ALL=C date))" | tee -a ${LOGFILE} + + makeargs="${MAKE_ARGS} DESTDIR=${STAGE_CHROOT_DIR}" + echo ">>> Distribution world for ${TARGET} architecture... (Starting - $(LC_ALL=C date))" | tee -a ${LOGFILE} + echo ">>> Builder is running the command: script -aq $LOGFILE make -C ${SRCDIR} ${makeargs:-} distribution " | tee -a ${LOGFILE} + (script -aq $LOGFILE make -C ${SRCDIR} ${makeargs:-} distribution || print_error_pfS;) | egrep '^>>>' | tee -a ${LOGFILE} + echo ">>> Distribution world for ${TARGET} architecture... (Finished - $(LC_ALL=C date))" | tee -a ${LOGFILE} + + [ -d "${STAGE_CHROOT_DIR}/usr/local/bin" ] \ + || mkdir -p ${STAGE_CHROOT_DIR}/usr/local/bin + makeargs="${MAKE_ARGS} DESTDIR=${STAGE_CHROOT_DIR}" + echo ">>> Building and installing crypto tools and athstats for ${TARGET} architecture... (Starting - $(LC_ALL=C date))" | tee -a ${LOGFILE} + echo ">>> Builder is running the command: script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/crypto ${makeargs:-} clean all install " | tee -a ${LOGFILE} + (script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/crypto ${makeargs:-} clean all install || print_error_pfS;) | egrep '^>>>' | tee -a ${LOGFILE} + echo ">>> Builder is running the command: script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/ath/athstats ${makeargs:-} clean" | tee -a ${LOGFILE} + (script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/ath/athstats ${makeargs:-} clean || print_error_pfS;) | egrep '^>>>' | tee -a ${LOGFILE} + echo ">>> Builder is running the command: script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/ath/athstats ${makeargs:-} all" | tee -a ${LOGFILE} + (script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/ath/athstats ${makeargs:-} all || print_error_pfS;) | egrep '^>>>' | tee -a ${LOGFILE} + echo ">>> Builder is running the command: script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/ath/athstats ${makeargs:-} install" | tee -a ${LOGFILE} + (script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/ath/athstats ${makeargs:-} install || print_error_pfS;) | egrep '^>>>' | tee -a ${LOGFILE} + echo ">>> Building and installing crypto tools and athstats for ${TARGET} architecture... (Finished - $(LC_ALL=C date))" | tee -a ${LOGFILE} + + unset makeargs +} + +# This routine originated in nanobsd.sh +nanobsd_set_flash_details () { + a1=$(echo $1 | tr '[:upper:]' '[:lower:]') + + # Source: + # SanDisk CompactFlash Memory Card + # Product Manual + # Version 10.9 + # Document No. 20-10-00038 + # April 2005 + # Table 2-7 + # NB: notice math error in SDCFJ-4096-388 line. + # + case "${a1}" in + 1024|1024m|1024mb|1g) + NANO_MEDIASIZE=$((997129216/512)) + ;; + 2048|2048m|2048mb|2g) + NANO_MEDIASIZE=$((1989999616/512)) + ;; + 4096|4096m|4096mb|4g) + NANO_MEDIASIZE=$((3989999616/512)) + ;; + 8192|8192m|8192mb|8g) + NANO_MEDIASIZE=$((7989999616/512)) + ;; + 16384|16384m|16384mb|16g) + NANO_MEDIASIZE=$((15989999616/512)) + ;; + *) + echo "Unknown Flash capacity" + exit 2 + ;; + esac + + NANO_HEADS=16 + NANO_SECTS=63 + + echo ">>> [nanoo] $1" + echo ">>> [nanoo] NANO_MEDIASIZE: $NANO_MEDIASIZE" + echo ">>> [nanoo] NANO_HEADS: $NANO_HEADS" + echo ">>> [nanoo] NANO_SECTS: $NANO_SECTS" + echo ">>> [nanoo] NANO_BOOT0CFG: $NANO_BOOT0CFG" +} + +# This routine originated in nanobsd.sh +create_nanobsd_diskimage () { + if [ -z "${1}" ]; then + echo ">>> ERROR: Type of image has not been specified" + print_error_pfS + fi + if [ -z "${2}" ]; then + echo ">>> ERROR: Size of image has not been specified" + print_error_pfS + fi + + if [ "${1}" = "nanobsd" ]; then + # It's serial + export NANO_BOOTLOADER="boot/boot0sio" + elif [ "${1}" = "nanobsd-vga" ]; then + # It's vga + export NANO_BOOTLOADER="boot/boot0" + else + echo ">>> ERROR: Type of image to create unknown" + print_error_pfS + fi + + if [ -z "${2}" ]; then + echo ">>> ERROR: Media size(s) not specified." + print_error_pfS + fi + + if [ -z "${2}" ]; then + echo ">>> ERROR: FLASH_SIZE is not set." + print_error_pfS + fi + + LOGFILE=${BUILDER_LOGS}/${1}.${TARGET} + # Prepare folder to be put in image + customize_stagearea_for_image "${1}" + install_default_kernel ${DEFAULT_KERNEL} "no" + + # Must be run after customize_stagearea_for_image + cust_fixup_nanobsd ${1} + + for _NANO_MEDIASIZE in ${2}; do + if [ -z "${_NANO_MEDIASIZE}" ]; then + continue; + fi + + echo ">>> building NanoBSD(${1}) disk image with size ${_NANO_MEDIASIZE} for platform (${TARGET})..." | tee -a ${LOGFILE} + echo "" > $BUILDER_LOGS/nanobsd_cmds.sh + + IMG="${MAKEOBJDIRPREFIXFINAL}/${PRODUCT_NAME}-${PRODUCT_VERSION}-${_NANO_MEDIASIZE}-${TARGET}-${1}-${DATESTRING}.img" + IMGUPDATE="${MAKEOBJDIRPREFIXFINAL}/${PRODUCT_NAME}-${PRODUCT_VERSION}-${_NANO_MEDIASIZE}-${TARGET}-${1}-upgrade-${DATESTRING}.img" + + nanobsd_set_flash_details ${_NANO_MEDIASIZE} + + # These are defined in FlashDevice and on builder_default.sh + echo $NANO_MEDIASIZE \ + $NANO_IMAGES \ + $NANO_SECTS \ + $NANO_HEADS \ + $NANO_CODESIZE \ + $NANO_CONFSIZE \ + $NANO_DATASIZE | +awk ' +{ + printf "# %s\n", $0 + + # size of cylinder in sectors + cs = $3 * $4 + + # number of full cylinders on media + cyl = int ($1 / cs) + + # output fdisk geometry spec, truncate cyls to 1023 + if (cyl <= 1023) + print "g c" cyl " h" $4 " s" $3 + else + print "g c" 1023 " h" $4 " s" $3 + + if ($7 > 0) { + # size of data partition in full cylinders + dsl = int (($7 + cs - 1) / cs) + } else { + dsl = 0; + } + + # size of config partition in full cylinders + csl = int (($6 + cs - 1) / cs) + + if ($5 == 0) { + # size of image partition(s) in full cylinders + isl = int ((cyl - dsl - csl) / $2) + } else { + isl = int (($5 + cs - 1) / cs) + } + + # First image partition start at second track + print "p 1 165 " $3, isl * cs - $3 + c = isl * cs; + + # Second image partition (if any) also starts offset one + # track to keep them identical. + if ($2 > 1) { + print "p 2 165 " $3 + c, isl * cs - $3 + c += isl * cs; + } + + # Config partition starts at cylinder boundary. + print "p 3 165 " c, csl * cs + c += csl * cs + + # Data partition (if any) starts at cylinder boundary. + if ($7 > 0) { + print "p 4 165 " c, dsl * cs + } else if ($7 < 0 && $1 > c) { + print "p 4 165 " c, $1 - c + } else if ($1 < c) { + print "Disk space overcommitted by", \ + c - $1, "sectors" > "/dev/stderr" + exit 2 + } + + # Force slice 1 to be marked active. This is necessary + # for booting the image from a USB device to work. + print "a 1" +} + ' > ${MAKEOBJDIRPREFIXFINAL}/_.fdisk + + MNT=${MAKEOBJDIRPREFIXFINAL}/_.mnt + mkdir -p ${MNT} + + dd if=/dev/zero of=${IMG} bs=${NANO_SECTS}b \ + count=0 seek=$((${NANO_MEDIASIZE}/${NANO_SECTS})) 2>&1 >> ${LOGFILE} + + MD=$(mdconfig -a -t vnode -f ${IMG} -x ${NANO_SECTS} -y ${NANO_HEADS}) + trap "mdconfig -d -u ${MD}; return" 1 2 15 EXIT + + fdisk -i -f ${MAKEOBJDIRPREFIXFINAL}/_.fdisk ${MD} 2>&1 >> ${LOGFILE} + fdisk ${MD} 2>&1 >> ${LOGFILE} + + boot0cfg -B -b ${FINAL_CHROOT_DIR}/${NANO_BOOTLOADER} ${NANO_BOOT0CFG} ${MD} 2>&1 >> ${LOGFILE} + + # Create first image + bsdlabel -m i386 -w -B -b ${FINAL_CHROOT_DIR}/boot/boot ${MD}s1 2>&1 >> ${LOGFILE} + bsdlabel -m i386 ${MD}s1 2>&1 >> ${LOGFILE} + local _label=$(lc ${PRODUCT_NAME}) + newfs -L ${_label}0 ${NANO_NEWFS} /dev/${MD}s1a 2>&1 >> ${LOGFILE} + mount /dev/ufs/${_label}0 ${MNT} + if [ $? -ne 0 ]; then + echo ">>> ERROR: Something wrong happened during mount of first slice image creation. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + # Consider the unmounting as well + trap "umount /dev/ufs/${_label}0; mdconfig -d -u ${MD}; return" 1 2 15 EXIT + + clone_directory_contents ${FINAL_CHROOT_DIR} ${MNT} + + # Set NanoBSD image size + echo "${_NANO_MEDIASIZE}" > ${MNT}/etc/nanosize.txt + rm -f $MNT/cf/conf/* 2>/dev/null + + echo "/dev/ufs/${_label}0 / ufs ro,sync,noatime 1 1" > ${MNT}/etc/fstab + if [ $NANO_CONFSIZE -gt 0 ] ; then + echo "/dev/ufs/cf /cf ufs ro,sync,noatime 1 1" >> ${MNT}/etc/fstab + fi + + umount ${MNT} + sync + # Restore the original trap + trap "mdconfig -d -u ${MD}; return" 1 2 15 EXIT + + # Setting NANO_IMAGES to 1 and NANO_INIT_IMG2 will tell + # NanoBSD to only create one partition. We default to 2 + # partitions in case anything happens to the first the + # operator can boot from the 2nd and should be OK. + + # Before just going to use dd for duplicate think! + # The images are created as sparse so lets take advantage + # of that by just exec some commands. + if [ $NANO_IMAGES -gt 1 -a $NANO_INIT_IMG2 -gt 0 ] ; then + # Duplicate to second image (if present) + echo ">>> Creating NanoBSD second slice by duplicating first slice." | tee -a ${LOGFILE} + # Create second image + dd if=/dev/${MD}s1 of=/dev/${MD}s2 conv=sparse bs=64k 2>&1 >> ${LOGFILE} + tunefs -L ${_label}1 /dev/${MD}s2a 2>&1 >> ${LOGFILE} + mount /dev/ufs/${_label}1 ${MNT} + if [ $? -ne 0 ]; then + echo ">>> ERROR: Something wrong happened during mount of second slice image creation. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + # Consider the unmounting as well + trap "umount /dev/ufs/${_label}1; mdconfig -d -u ${MD}; return" 1 2 15 EXIT + + echo "/dev/ufs/${_label}1 / ufs ro,sync,noatime 1 1" > ${MNT}/etc/fstab + if [ $NANO_CONFSIZE -gt 0 ] ; then + echo "/dev/ufs/cf /cf ufs ro,sync,noatime 1 1" >> ${MNT}/etc/fstab + fi + + umount ${MNT} + # Restore the trap back + trap "mdconfig -d -u ${MD}; return" 1 2 15 EXIT + fi + + # Create Data slice, if any. + # Note the changing of the variable to NANO_CONFSIZE + # from NANO_DATASIZE. We also added glabel support + # and populate the Product configuration from the /cf + # directory located in FINAL_CHROOT_DIR + if [ $NANO_CONFSIZE -gt 0 ] ; then + echo ">>> Creating /cf area to hold config.xml" + newfs -L cf ${NANO_NEWFS} /dev/${MD}s3 2>&1 >> ${LOGFILE} + # Mount data partition and copy contents of /cf + # Can be used later to create custom default config.xml while building + mount /dev/ufs/cf ${MNT} + if [ $? -ne 0 ]; then + echo ">>> ERROR: Something wrong happened during mount of cf slice image creation. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + # Consider the unmounting as well + trap "umount /dev/ufs/cf; mdconfig -d -u ${MD}; return" 1 2 15 EXIT + + clone_directory_contents ${FINAL_CHROOT_DIR}/cf ${MNT} + + umount ${MNT} + # Restore the trap back + trap "mdconfig -d -u ${MD}; return" 1 2 15 EXIT + else + ">>> [nanoo] NANO_CONFSIZE is not set. Not adding a /conf partition.. You sure about this??" | tee -a ${LOGFILE} + fi + + echo ">>> [nanoo] Creating NanoBSD upgrade file from first slice..." | tee -a ${LOGFILE} + dd if=/dev/${MD}s1 of=$IMGUPDATE conv=sparse bs=64k 2>&1 >> ${LOGFILE} + + mdconfig -d -u $MD + # Restore default action + trap "-" 1 2 15 EXIT + + # Check each image and ensure that they are over + # 3 megabytes. If either image is under 20 megabytes + # in size then error out. + IMGSIZE=$(stat -f "%z" ${IMG}) + IMGUPDATESIZE=$(stat -f "%z" ${IMGUPDATE}) + CHECKSIZE="20040710" + if [ "$IMGSIZE" -lt "$CHECKSIZE" ]; then + echo ">>> ERROR: Something went wrong when building NanoBSD. The image size is under 20 megabytes!" | tee -a ${LOGFILE} + print_error_pfS + fi + if [ "$IMGUPDATESIZE" -lt "$CHECKSIZE" ]; then + echo ">>> ERROR: Something went wrong when building NanoBSD upgrade image. The image size is under 20 megabytes!" | tee -a ${LOGFILE} + print_error_pfS + fi + + # Wrap up the show, Johnny + echo ">>> NanoBSD Image completed for size: $_NANO_MEDIASIZE." | tee -a ${LOGFILE} + + gzip -f $IMG & + gzip -f $IMGUPDATE & + done + + unset IMG + unset IMGUPDATE + unset IMGUPDATESIZE + unset IMGSIZE + + ls -lah $MAKEOBJDIRPREFIXFINAL +} + +# This routine creates a ova image that contains +# a ovf and vmdk file. These files can be imported +# right into vmware or virtual box. +# (and many other emulation platforms) +# http://www.vmware.com/pdf/ovf_whitepaper_specification.pdf +create_ova_image() { + # XXX create a .ovf php creator that you can pass: + # 1. populatedSize + # 2. license + # 3. product name + # 4. version + # 5. number of network interface cards + # 6. allocationUnits + # 7. capacity + # 8. capacityAllocationUnits + + LOGFILE=${BUILDER_LOGS}/ova.${TARGET}.log + + # Prepare folder to be put in image + customize_stagearea_for_image "ova" + install_default_kernel ${DEFAULT_KERNEL} "no" + + ova_prereq_check + ova_remove_old_tmp_files + ova_setup_ovf_file + ova_create_raw_backed_file + /bin/echo -n ">>> Creating mdconfig image ${OVFPATH}/${OVFVMDK}.raw... " | tee -a ${LOGFILE} + MD=$(mdconfig -a -t vnode -f ${OVFPATH}/${OVFVMDK}.raw) + # Just in case + trap "mdconfig -d -u ${MD}" 1 2 15 EXIT + + # comment out if using pc-sysinstall + ova_partition_gpart $MD + ova_mount_mnt $MD + ova_cpdup_files + ova_setup_platform_specific # after cpdup + ova_calculate_mnt_size + ova_umount_mnt $MD + # Restore default action + trap "-" 1 2 15 EXIT + ova_umount_mdconfig $MD + # We use vbox because it compresses the vmdk on export + ova_create_vbox_image + # We repack the file with a more universal xml file that + # works in both virtual box and esx server + ova_repack_vbox_image +} + +ova_repack_vbox_image() { + BUILDPLATFORM=$(uname -p) + POPULATEDSIZE=$(du -d0 -m $FINAL_CHROOT_DIR | awk '{ print \$1 }') + POPULATEDSIZEBYTES=$(echo "${POPULATEDSIZE}*1024^2" | bc) + REFERENCESSIZE=$(stat -f "%z" ${OVFPATH}/${OVFVMDK}) + echo ">>> Setting REFERENCESSIZE to ${REFERENCESSIZE}..." | tee -a ${LOGFILE} + file_search_replace REFERENCESSIZE ${REFERENCESSIZE} ${OVFPATH}/${PRODUCT_NAME}.ovf + echo ">>> Setting POPULATEDSIZEBYTES to ${POPULATEDSIZEBYTES}..." | tee -a ${LOGFILE} + # OperatingSystemSection (${PRODUCT_NAME}.ovf) + # 42 FreeBSD 32-Bit + # 78 FreeBSD 64-Bit + if [ "$BUILDPLATFORM" = "i386" ]; then + file_search_replace '"101"' '"42"' ${OVFPATH}/${PRODUCT_NAME}.ovf + file_search_replace 'FreeBSD XX-Bit' 'FreeBSD' ${OVFPATH}/${PRODUCT_NAME}.ovf + fi + if [ "$BUILDPLATFORM" = "amd64" ]; then + file_search_replace '"101"' '"78"' ${OVFPATH}/${PRODUCT_NAME}.ovf + file_search_replace 'FreeBSD XX-Bit' 'FreeBSD 64-Bit' ${OVFPATH}/${PRODUCT_NAME}.ovf + fi + file_search_replace DISKSECTIONPOPULATEDSIZE $POPULATEDSIZEBYTES ${OVFPATH}/${PRODUCT_NAME}.ovf + # 10737254400 = 10240MB = virtual box vmdk file size XXX grab this value from vbox creation + # 10737418240 = 10GB + echo ">>> Setting DISKSECTIONALLOCATIONUNITS to 10737254400..." | tee -a ${LOGFILE} + file_search_replace DISKSECTIONALLOCATIONUNITS $OVA_DISKSECTIONALLOCATIONUNITS ${OVFPATH}/${PRODUCT_NAME}.ovf + echo ">>> Setting DISKSECTIONCAPACITY to 10737418240..." | tee -a ${LOGFILE} + file_search_replace DISKSECTIONCAPACITY $OVADISKSIZE ${OVFPATH}/${PRODUCT_NAME}.ovf + echo ">>> Repacking OVA with universal OVF file..." | tee -a ${LOGFILE} + mv ${OVFPATH}/${OVFVMDK} ${OVFPATH}/${PRODUCT_NAME}-disk1.vmdk + gtar -C ${OVFPATH} -cpf ${PRODUCT_NAME}.ova ${PRODUCT_NAME}.ovf ${PRODUCT_NAME}-disk1.vmdk + rm $OVFPATH/${PRODUCT_NAME}-disk1.vmdk + ls -lah ${OVFPATH}/${PRODUCT_NAME}*ov* +} + +# called from create_ova_image +ova_umount_mnt() { + # Unmount /dev/mdX + umount /mnt + sync ; sync +} + +# called from create_ova_image +ova_umount_mdconfig() { + MD=$1 + # Show gpart info + gpart show $MD + echo ">>> Unmounting ${MD}..." | tee -a ${LOGFILE} + mdconfig -d -u $MD + sync ; sync +} + +# called from create_ova_image +ova_mount_mnt() { + MD=$1 + echo ">>> Mounting image to /mnt..." | tee -a ${LOGFILE} + mount -o rw /dev/${MD}p2 /mnt/ +} + +# called from create_ova_image +ova_setup_ovf_file() { + if [ -f ${OVFFILE} ]; then + cp ${OVFFILE} ${OVFPATH}/${PRODUCT_NAME}.ovf + fi + + if [ ! -f ${OVFPATH}/${PRODUCT_NAME}.ovf ]; then + cp ${BUILDER_SCRIPTS}/conf/ovf/${PRODUCT_NAME}.ovf ${OVFPATH}/${PRODUCT_NAME}.ovf + file_search_replace PRODUCT_VERSION $PRODUCT_VERSION ${OVFPATH}/${PRODUCT_NAME}.ovf + file_search_replace PRODUCT_URL $PRODUCT_URL ${OVFPATH}/${PRODUCT_NAME}.ovf + fi +} + +# called from create_ova_image +ova_prereq_check() { + if [ ! -f /usr/local/bin/vmdktool ]; then + echo ">>> ERROR: vmdktool is not present please check port sysutils/vmdktool installation" | tee -a ${LOGFILE} + print_error_pfS + fi + sysctl kern.geom.debugflags=16 +} + +# called from create_ova_image +ova_calculate_mnt_size() { + /bin/echo -n ">>> Calculating size of /mnt..." | tee -a ${LOFGILE} + INSTALLSIZE=$(du -s /mnt/ | awk '{ print $1 }') + INSTALLSIZEH=$(du -d0 -h /mnt/ | awk '{ print $1 }') + echo $INSTALLSIZEH +} + +# called from create_ova_image +ova_create_raw_backed_file() { + DISKSIZE=$OVADISKSIZE + BLOCKSIZE=$OVABLOCKSIZE + COUNT=$((${DISKSIZE}/${BLOCKSIZE})) + DISKFILE=${OVFPATH}/${OVFVMDK}.raw + echo ">>> Creating raw backing file ${DISKFILE} (Disk Size: ${DISKSIZE}, Block Size: ${BLOCKSIZE}, Count: ${COUNT})..." | tee -a ${LOGFILE} + dd if=/dev/zero of=$DISKFILE bs=$BLOCKSIZE count=0 seek=$COUNT +} + +# called from create_ova_image +ova_remove_old_tmp_files() { + rm ${OVFPATH}/*.ovf.final 2>/dev/null + rm ${OVFPATH}/*.ova 2>/dev/null +} + +# called from create_ova_image +ova_create_vbox_image() { + # VirtualBox + echo ">>> Creating image using vmdktool..." | tee -a ${LOGFILE} + rm ${OVFPATH}/${OVFVMDK} 2>/dev/null + vmdktool -v ${OVFPATH}/${OVFVMDK} ${OVFPATH}/${OVFVMDK}.raw + rm -rf ${OVFPATH}/${OVFVMDK}.raw + echo ">>> ${OVFPATH}/${OVFVMDK} created." | tee -a ${LOGFILE} +} + +# called from create_ova_image +ova_cpdup_files() { + echo ">>> Populating vmdk staging area..." + cpdup -o ${FINAL_CHROOT_DIR}/COPYRIGHT /mnt/COPYRIGHT + cpdup -o ${FINAL_CHROOT_DIR}/boot /mnt/boot + cpdup -o ${FINAL_CHROOT_DIR}/bin /mnt/bin + cpdup -o ${FINAL_CHROOT_DIR}/cf/conf /mnt/cf/conf + cpdup -o ${FINAL_CHROOT_DIR}/conf.default /mnt/conf.default + cpdup -o ${FINAL_CHROOT_DIR}/dev /mnt/dev + cpdup -o ${FINAL_CHROOT_DIR}/etc /mnt/etc + cpdup -o ${FINAL_CHROOT_DIR}/home /mnt/home + cpdup -o ${FINAL_CHROOT_DIR}/pkgs /mnt/pkgs + cpdup -o ${FINAL_CHROOT_DIR}/libexec /mnt/libexec + cpdup -o ${FINAL_CHROOT_DIR}/lib /mnt/lib + cpdup -o ${FINAL_CHROOT_DIR}/root /mnt/root + cpdup -o ${FINAL_CHROOT_DIR}/sbin /mnt/sbin + cpdup -o ${FINAL_CHROOT_DIR}/usr /mnt/usr + cpdup -o ${FINAL_CHROOT_DIR}/var /mnt/var + sync ; sync ; sync ; sync +} + +ova_setup_platform_specific() { + echo ">>> Installing platform specific items..." | tee -a ${LOGFILE} + echo "/dev/label/${PRODUCT_NAME} / ufs rw 0 0" > /mnt/etc/fstab + echo "/dev/label/swap0 none swap sw 0 0" >> /mnt/etc/fstab + echo ${PRODUCT_NAME} > /mnt/etc/platform + rmdir /mnt/conf + mkdir -p /mnt/cf + mkdir -p /mnt/cf/conf + cp /mnt/conf.default/config.xml /mnt/cf/conf/ + chroot /mnt /bin/ln -s /cf/conf /conf + mkdir -p /mnt/tmp +} + +# called from create_ova_image +ova_partition_gpart() { + # XXX: Switch to mkimg tool!! + MD=$1 + echo ">>> Creating GPT..." | tee -a ${LOGFILE} + gpart create -s gpt $MD + echo ">>> Embedding GPT bootstrap into protective MBR..." | tee -a ${LOGFILE} + gpart bootcode -b /boot/pmbr $MD + echo ">>> Creating GPT boot partition..." | tee -a ${LOGFILE} + gpart add -b 34 -s 128 -t freebsd-boot $MD + gpart bootcode -p /boot/gptboot -i 1 $MD + echo ">>> Setting up disk slices: ${MD}p2 (Size: ${OVA_FIRST_PART_SIZE})..." | tee -a ${LOGFILE} + gpart add -s $OVA_FIRST_PART_SIZE -t freebsd-ufs -i 2 $MD + echo ">>> Setting up disk slices: ${MD}p3 (swap) (Size: ${OVA_SWAP_PART_SIZE})..." | tee -a ${LOGFILE} + gpart add -s $OVA_SWAP_PART_SIZE -t freebsd-swap -i 3 $MD + echo ">>> Running newfs..." | tee -a ${LOGFILE} + newfs -U /dev/${MD}p2 + sync ; sync ; sync ; sync + echo ">>> Labeling partitions: ${MD}p2..." | tee -a ${LOGFILE} + glabel label ${PRODUCT_NAME} ${MD}p2 + sync ; sync + echo ">>> Labeling partitions: ${MD}p3..." | tee -a ${LOGFILE} + glabel label swap0 ${MD}p3 + sync ; sync +} + +# called from create_ova_image +# This routine will replace a string in a file +file_search_replace() { + local SEARCH="$1" + local REPLACE="$2" + local FILENAME="$3" + + if [ -f "${FILENAME}" ]; then + sed -i '' -e "s/${SEARCH}/${REPLACE}/g" ${FILENAME} + fi +} + +# Cleans up previous builds +clean_obj_dir() { + # Clean out directories + echo ">>> Cleaning up previous build environment...Please wait!" + + staginareas_clean_each_run + + if [ -d "${STAGE_CHROOT_DIR}" ]; then + BASENAME=$(basename ${STAGE_CHROOT_DIR}) + echo -n ">>> Cleaning ${STAGE_CHROOT_DIR} ..." + chflags -R noschg ${STAGE_CHROOT_DIR} 2>&1 >/dev/null + rm -rf ${STAGE_CHROOT_DIR} 2>/dev/null + echo "Done." + fi + mkdir -p ${STAGE_CHROOT_DIR} + + if [ -z "${NO_CLEANFREEBSDOBJDIR}" -a -d "${SRCDIR}" ]; then + OBJTREE=$(env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} make -C ${SRCDIR} -V OBJTREE) + if [ -d "${OBJTREE}" ]; then + echo -n ">>> Cleaning FreeBSD objects dir staging..." + echo -n "." + chflags -R noschg ${OBJTREE} 2>&1 >/dev/null + echo -n "." + rm -rf ${OBJTREE}/* + echo "Done!" + fi + if [ -n "${KERNEL_DESTDIR}" -a -d "${KERNEL_BUILD_PATH}" ]; then + echo -n ">>> Cleaning previously built kernel stage area..." + rm -rf $KERNEL_BUILD_PATH/* + echo "Done!" + fi + fi + mkdir -p $KERNEL_BUILD_PATH + + echo -n ">>> Cleaning previously built images..." + rm -rf $MAKEOBJDIRPREFIXFINAL/* + echo "Done!" + + if [ -z "${NO_CLEANREPOS}" ]; then + if [ -d "${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}" ]; then + echo -n ">>> Cleaning ${PRODUCT_NAME} repo checkout..." + echo -n "." + rm -rf "${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}" + echo "Done!" + fi + + if [ -d "$SRCDIR" ]; then + echo -n ">>> Ensuring $SRCDIR is clean..." + rm -rf ${SRCDIR} + echo "Done!" + fi + fi + + echo -n ">>> Cleaning previous builder logs..." + if [ -d "$BUILDER_LOGS" ]; then + rm -rf ${BUILDER_LOGS} + fi + mkdir -p ${BUILDER_LOGS} + + echo "Done!" + + echo ">>> Cleaning of builder environment has finished." +} + +# This routine ensures that the $SRCDIR has sources +# and is ready for action / building. +ensure_source_directories_present() { + update_freebsd_sources + + update_product_repository +} + +clone_directory_contents() { + if [ ! -d "$1" -o ! -d "$2" ]; then + if [ -z "${LOGFILE}" ]; then + echo ">>> ERROR: Argument $1 supplied is not a directory!" + else + echo ">>> ERROR: Argument $1 supplied is not a directory!" | tee -a ${LOGFILE} + fi + print_error_pfS + fi + echo -n ">>> Using TAR to clone $1 to $2 ..." + tar -C ${1} -c -f - . | tar -C ${2} -x -p -f - + echo "Done!" +} + +clone_to_staging_area() { + # Clone everything to the final staging area + echo -n ">>> Cloning everything to ${STAGE_CHROOT_DIR} staging area..." + LOGFILE=${BUILDER_LOGS}/cloning.${TARGET}.log + + if [ ! -d ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} ]; then + echo "ERROR!" + echo ">>> ERROR: ${PRODUCT_NAME} repository does not seem to be there please fix...STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + + tar --exclude=\.git -C ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} -c -f - . | \ + tar -C ${STAGE_CHROOT_DIR} -x -p -f - + + if [ -f ${STAGE_CHROOT_DIR}/etc/master.passwd ]; then + chroot ${STAGE_CHROOT_DIR} cap_mkdb /etc/master.passwd + chroot ${STAGE_CHROOT_DIR} pwd_mkdb /etc/master.passwd + fi + mkdir -p ${STAGE_CHROOT_DIR}/etc/mtree + mtree -Pcp ${STAGE_CHROOT_DIR}/var > ${STAGE_CHROOT_DIR}/etc/mtree/var.dist + mtree -Pcp ${STAGE_CHROOT_DIR}/etc > ${STAGE_CHROOT_DIR}/etc/mtree/etc.dist + if [ -d ${STAGE_CHROOT_DIR}/usr/local/etc ]; then + mtree -Pcp ${STAGE_CHROOT_DIR}/usr/local/etc > ${STAGE_CHROOT_DIR}/etc/mtree/localetc.dist + fi + + ## Add buildtime and lastcommit information + # This is used for detecting updates. + echo "$BUILTDATESTRING" > $STAGE_CHROOT_DIR/etc/version.buildtime + # Record last commit info if it is available. + if [ -f $SCRATCHDIR/build_commit_info.txt ]; then + cp $SCRATCHDIR/build_commit_info.txt $STAGE_CHROOT_DIR/etc/version.lastcommit + fi + + local _exclude_files="${CORE_PKG_TMP}/base_exclude_files" + sed \ + -e "s,%%PRODUCT_NAME%%,${PRODUCT_NAME},g" \ + -e "s,%%FLAVOR%%,${_flavor},g" \ + -e "s,%%VERSION%%,${_version},g" \ + ${BUILDER_SCRIPTS}/templates/core_pkg/base/exclude_files \ + > ${_exclude_files} + + mkdir -p ${STAGE_CHROOT_DIR}/usr/local/share/${PRODUCT_NAME} >/dev/null 2>&1 + mtree \ + -c \ + -k uid,gid,mode,size,flags,sha256digest \ + -p ${STAGE_CHROOT_DIR} \ + -X ${_exclude_files} \ + > ${STAGE_CHROOT_DIR}/usr/local/share/${PRODUCT_NAME}/base.mtree + tar \ + -C ${STAGE_CHROOT_DIR} \ + -cJf ${STAGE_CHROOT_DIR}/usr/local/share/${PRODUCT_NAME}/base.txz \ + -X ${_exclude_files} \ + . + + core_pkg_create base "" ${CORE_PKG_VERSION} ${STAGE_CHROOT_DIR} + core_pkg_create default-config "" ${CORE_PKG_VERSION} ${STAGE_CHROOT_DIR} + + local DEFAULTCONF=${STAGE_CHROOT_DIR}/conf.default/config.xml + # Activate serial console in config.xml + # If it was there before, clear the setting to be sure we don't add it twice. + sed -i "" -e "/ /d" ${DEFAULTCONF} + # Enable serial in the config + sed -i "" -e "s/ <\/system>/ \\$(echo -e \\\n) <\/system>/" ${DEFAULTCONF} + + echo force > ${STAGE_CHROOT_DIR}/cf/conf/enableserial_force + + core_pkg_create default-config-serial "" ${CORE_PKG_VERSION} ${STAGE_CHROOT_DIR} + + rm -f ${STAGE_CHROOT_DIR}/cf/conf/enableserial_force + rm -f ${STAGE_CHROOT_DIR}/cf/conf/config.xml + + # Make sure pkg is present + pkg_bootstrap ${STAGE_CHROOT_DIR} + + pkg_chroot_add ${STAGE_CHROOT_DIR} base + + echo "Done!" +} + +create_final_staging_area() { + if [ -z "${FINAL_CHROOT_DIR}" ]; then + echo ">>> ERROR: FINAL_CHROOT_DIR is not set, cannot continue!" | tee -a ${LOGFILE} + print_error_pfS + fi + + if [ -d "${FINAL_CHROOT_DIR}" ]; then + echo -n ">>> Previous ${FINAL_CHROOT_DIR} detected cleaning up..." | tee -a ${LOGFILE} + chflags -R noschg ${FINAL_CHROOT_DIR} 2>&1 1>/dev/null + rm -rf ${FINAL_CHROOT_DIR}/* 2>&1 1>/dev/null + echo "Done." | tee -a ${LOGFILE} + fi + mkdir -p ${FINAL_CHROOT_DIR} + + echo ">>> Preparing Final image staging area: $(LC_ALL=C date)" 2>&1 | tee -a ${LOGFILE} + echo ">>> Cloning ${STAGE_CHROOT_DIR} to ${FINAL_CHROOT_DIR}" 2>&1 | tee -a ${LOGFILE} + clone_directory_contents ${STAGE_CHROOT_DIR} ${FINAL_CHROOT_DIR} + + if [ ! -f $FINAL_CHROOT_DIR/sbin/init ]; then + echo ">>> ERROR: Something went wrong during cloning -- Please verify!" 2>&1 | tee -a ${LOGFILE} + print_error_pfS + fi +} + +customize_stagearea_for_image() { + # Prepare final stage area + create_final_staging_area + + if [ "${1}" = "iso" -o \ + "${1}" = "memstick" -o \ + "${1}" = "memstickserial" -o \ + "${1}" = "memstickadi" ]; then + install_bsdinstaller + mkdir -p ${FINAL_CHROOT_DIR}/pkgs + cp ${CORE_PKG_PATH}/*default-config*.txz ${FINAL_CHROOT_DIR}/pkgs + fi + + if [ "${1}" = "nanobsd" -o \ + "${1}" = "memstickserial" -o \ + "${1}" = "memstickadi" ]; then + pkg_chroot_add ${FINAL_CHROOT_DIR} default-config-serial + else + pkg_chroot_add ${FINAL_CHROOT_DIR} default-config + fi +} + +create_distribution_tarball() { + mkdir -p ${FINAL_CHROOT_DIR}/install + + tar -C ${FINAL_CHROOT_DIR} --exclude ./install --exclude ./pkgs -cJf ${FINAL_CHROOT_DIR}/install/${PRODUCT_NAME}.txz . +} + +create_iso_image() { + LOGFILE=${BUILDER_LOGS}/isoimage.${TARGET} + echo ">>> Building bootable ISO image for ${TARGET}" | tee -a ${LOGFILE} + if [ -z "${DEFAULT_KERNEL}" ]; then + echo ">>> ERROR: Could not identify DEFAULT_KERNEL to install on image!" | tee -a ${LOGFILE} + print_error_pfS + fi + + customize_stagearea_for_image "iso" + install_default_kernel ${DEFAULT_KERNEL} + + echo cdrom > $FINAL_CHROOT_DIR/etc/platform + + FSLABEL=$(echo ${PRODUCT_NAME} | tr '[:lower:]' '[:upper:]') + echo "/dev/iso9660/${FSLABEL} / cd9660 ro 0 0" > ${FINAL_CHROOT_DIR}/etc/fstab + + # This check is for supporting create memstick/ova images + echo -n ">>> Running command: script -aq $LOGFILE makefs -t cd9660 -o bootimage=\"i386;${FINAL_CHROOT_DIR}/boot/cdboot \"-o no-emul-boot -o rockridge " | tee -a ${LOGFILE} + echo "-o label=${FSLABEL} -o publisher=\"${PRODUCT_NAME} project.\" $ISOPATH ${FINAL_CHROOT_DIR}" | tee -a ${LOGFILE} + + create_distribution_tarball + + # Remove /rescue from iso since cd9660 cannot deal with hardlinks + rm -rf ${FINAL_CHROOT_DIR}/rescue + + makefs -t cd9660 -o bootimage="i386;${FINAL_CHROOT_DIR}/boot/cdboot" -o no-emul-boot -o rockridge \ + -o label=${FSLABEL} -o publisher="${PRODUCT_NAME} project." $ISOPATH ${FINAL_CHROOT_DIR} 2>&1 >> ${LOGFILE} + if [ $? -ne 0 -o ! -f $ISOPATH ]; then + if [ -f ${ISOPATH} ]; then + rm -f $ISOPATH + fi + echo ">>> ERROR: Something wrong happened during ISO image creation. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + gzip -qf $ISOPATH & + + echo ">>> ISO created: $(LC_ALL=C date)" | tee -a ${LOGFILE} +} + +create_memstick_image() { + + LOGFILE=${BUILDER_LOGS}/memstick.${TARGET} + if [ "${MEMSTICKPATH}" = "" ]; then + echo ">>> MEMSTICKPATH is empty skipping generation of memstick image!" | tee -a ${LOGFILE} + return + fi + + if [ ! -d ${FINAL_CHROOT_DIR}/boot ]; then + customize_stagearea_for_image "memstick" + install_default_kernel ${DEFAULT_KERNEL} + fi + + echo cdrom > $FINAL_CHROOT_DIR/etc/platform + + echo ">>> Creating memstick to ${MEMSTICKPATH}." 2>&1 | tee -a ${LOGFILE} + echo "/dev/ufs/${PRODUCT_NAME} / ufs ro 0 0" > ${FINAL_CHROOT_DIR}/etc/fstab + echo "kern.cam.boot_delay=10000" >> ${FINAL_CHROOT_DIR}/boot/loader.conf.local + + create_distribution_tarball + + makefs -B little -o label=${PRODUCT_NAME} ${MEMSTICKPATH} ${FINAL_CHROOT_DIR} + if [ $? -ne 0 ]; then + if [ -f ${MEMSTICKPATH} ]; then + rm -f $MEMSTICKPATH + fi + echo ">>> ERROR: Something wrong happened during MEMSTICK image creation. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + MD=$(mdconfig -a -t vnode -f $MEMSTICKPATH) + # Just in case + trap "mdconfig -d -u ${MD}" 1 2 15 EXIT + gpart create -s BSD ${MD} 2>&1 >> ${LOGFILE}| + gpart bootcode -b ${FINAL_CHROOT_DIR}/boot/boot ${MD} 2>&1 >> ${LOGFILE} + gpart add -t freebsd-ufs ${MD} 2>&1 >> ${LOGFILE} + trap "-" 1 2 15 EXIT + mdconfig -d -u ${MD} 2>&1 | tee -a ${LOGFILE} + gzip -qf $MEMSTICKPATH & + + echo ">>> MEMSTICK created: $(LC_ALL=C date)" | tee -a ${LOGFILE} +} + +create_memstick_serial_image() { + LOGFILE=${BUILDER_LOGS}/memstickserial.${TARGET} + if [ "${MEMSTICKSERIALPATH}" = "" ]; then + echo ">>> MEMSTICKSERIALPATH is empty skipping generation of memstick image!" | tee -a ${LOGFILE} + return + fi + + if [ ! -d ${FINAL_CHROOT_DIR}/boot ]; then + customize_stagearea_for_image "memstickserial" + install_default_kernel ${DEFAULT_KERNEL} + fi + + echo cdrom > $FINAL_CHROOT_DIR/etc/platform + + echo "/dev/ufs/${PRODUCT_NAME} / ufs ro 0 0" > ${FINAL_CHROOT_DIR}/etc/fstab + echo "kern.cam.boot_delay=10000" >> ${FINAL_CHROOT_DIR}/boot/loader.conf.local + + echo ">>> Creating serial memstick to ${MEMSTICKSERIALPATH}." 2>&1 | tee -a ${LOGFILE} + + BOOTCONF=${FINAL_CHROOT_DIR}/boot.config + LOADERCONF=${FINAL_CHROOT_DIR}/boot/loader.conf + + echo ">>> Activating serial console..." 2>&1 | tee -a ${LOGFILE} + # Activate serial console in boot.config + if [ -f ${BOOTCONF} ]; then + sed -i "" '/-D$/d' ${BOOTCONF} + fi + echo "-S115200 -D" >> ${BOOTCONF} + + # Remove old console options if present. + [ -f "${LOADERCONF}" ] \ + && sed -i "" -Ee "/(console|boot_multicons|boot_serial)/d" ${LOADERCONF} + # Activate serial console+video console in loader.conf + echo 'boot_multicons="YES"' >> ${LOADERCONF} + echo 'boot_serial="YES"' >> ${LOADERCONF} + echo 'console="comconsole,vidconsole"' >> ${LOADERCONF} + echo 'comconsole_speed="115200"' >> ${LOADERCONF} + + create_distribution_tarball + + makefs -B little -o label=${PRODUCT_NAME} ${MEMSTICKSERIALPATH} ${FINAL_CHROOT_DIR} + if [ $? -ne 0 ]; then + if [ -f ${MEMSTICKSERIALPATH} ]; then + rm -f $MEMSTICKSERIALPATH + fi + echo ">>> ERROR: Something wrong happened during MEMSTICKSERIAL image creation. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + MD=$(mdconfig -a -t vnode -f $MEMSTICKSERIALPATH) + # Just in case + trap "mdconfig -d -u ${MD}" 1 2 15 EXIT + gpart create -s BSD ${MD} 2>&1 >> ${LOGFILE} + gpart bootcode -b ${FINAL_CHROOT_DIR}/boot/boot ${MD} 2>&1 >> ${LOGFILE} + gpart add -t freebsd-ufs ${MD} 2>&1 >> ${LOGFILE} + trap "-" 1 2 15 EXIT + mdconfig -d -u ${MD} 2>&1 >> ${LOGFILE} + gzip -qf $MEMSTICKSERIALPATH & + + echo ">>> MEMSTICKSERIAL created: $(LC_ALL=C date)" | tee -a ${LOGFILE} +} + +create_memstick_adi_image() { + LOGFILE=${BUILDER_LOGS}/memstickadi${TARGET} + if [ "${MEMSTICKADIPATH}" = "" ]; then + echo ">>> MEMSTICKADIPATH is empty skipping generation of memstick image!" | tee -a ${LOGFILE} + return + fi + + if [ ! -d ${FINAL_CHROOT_DIR}/boot ]; then + customize_stagearea_for_image "memstickadi" + install_default_kernel ${DEFAULT_KERNEL} + fi + + echo cdrom > $FINAL_CHROOT_DIR/etc/platform + + echo "/dev/ufs/${PRODUCT_NAME} / ufs ro 0 0" > ${FINAL_CHROOT_DIR}/etc/fstab + echo "kern.cam.boot_delay=10000" >> ${FINAL_CHROOT_DIR}/boot/loader.conf.local + + echo ">>> Creating serial memstick to ${MEMSTICKADIPATH}." 2>&1 | tee -a ${LOGFILE} + + BOOTCONF=${FINAL_CHROOT_DIR}/boot.config + LOADERCONF=${FINAL_CHROOT_DIR}/boot/loader.conf + + echo ">>> Activating serial console..." 2>&1 | tee -a ${LOGFILE} + # Activate serial console in boot.config + if [ -f ${BOOTCONF} ]; then + sed -i "" '/-[Dh]$/d' ${BOOTCONF} + fi + echo "-S115200 -h" >> ${BOOTCONF} + + # Remove old console options if present. + [ -f "${LOADERCONF}" ] \ + && sed -i "" -Ee "/(console|boot_multicons|boot_serial|hint.uart)/d" ${LOADERCONF} + # Activate serial console+video console in loader.conf + echo 'boot_serial="YES"' >> ${LOADERCONF} + echo 'console="comconsole"' >> ${LOADERCONF} + echo 'comconsole_speed="115200"' >> ${LOADERCONF} + echo 'comconsole_port="0x2F8"' >> ${LOADERCONF} + echo 'hint.uart.0.flags="0x00"' >> ${LOADERCONF} + echo 'hint.uart.1.flags="0x10"' >> ${LOADERCONF} + + create_distribution_tarball + + makefs -B little -o label=${PRODUCT_NAME} ${MEMSTICKADIPATH} ${FINAL_CHROOT_DIR} + if [ $? -ne 0 ]; then + if [ -f ${MEMSTICKADIPATH} ]; then + rm -f $MEMSTICKADIPATH + fi + echo ">>> ERROR: Something wrong happened during MEMSTICKADI image creation. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + MD=$(mdconfig -a -t vnode -f $MEMSTICKADIPATH) + # Just in case + trap "mdconfig -d -u ${MD}" 1 2 15 EXIT + gpart create -s BSD ${MD} 2>&1 >> ${LOGFILE} + gpart bootcode -b ${FINAL_CHROOT_DIR}/boot/boot ${MD} 2>&1 >> ${LOGFILE} + gpart add -t freebsd-ufs ${MD} 2>&1 >> ${LOGFILE} + trap "-" 1 2 15 EXIT + mdconfig -d -u ${MD} 2>&1 >> ${LOGFILE} + gzip -qf $MEMSTICKADIPATH & + + echo ">>> MEMSTICKADI created: $(LC_ALL=C date)" | tee -a ${LOGFILE} +} + +# Create pkg conf on desired place with desired arch/branch +setup_pkg_repo() { + if [ -z "${4}" ]; then + return + fi + + local _target="${1}" + local _arch="${2}" + local _target_arch="${3}" + local _branch="${4}" + + mkdir -p $(dirname ${_target}) >/dev/null 2>&1 + + sed \ + -e "s/%%ARCH%%/${_arch}_${_target_arch}/" \ + -e "s/%%GIT_REPO_BRANCH_OR_TAG%%/${_branch}/g" \ + -e "s,%%PKG_REPO_SERVER%%,${PKG_REPO_SERVER},g" \ + -e "s/%%PRODUCT_NAME%%/${PRODUCT_NAME}/g" \ + ${SRCDIR}/release/pkg_repos/${PRODUCT_NAME}.conf.template \ + > ${_target} +} + +# This routine ensures any ports / binaries that the builder +# system needs are on disk and ready for execution. +builder_setup() { + # If Product-builder is already installed, just leave + if pkg info -e -q ${PRODUCT_NAME}-builder; then + return + fi + + if [ ! -f /usr/local/etc/pkg/repos/${PRODUCT_NAME}.conf ]; then + [ -d /usr/local/etc/pkg/repos ] \ + || mkdir -p /usr/local/etc/pkg/repos + + local _arch=$(uname -m) + setup_pkg_repo /usr/local/etc/pkg/repos/${PRODUCT_NAME}.conf ${_arch} ${_arch} ${PKG_REPO_CONF_BRANCH} + fi + + pkg install ${PRODUCT_NAME}-builder +} + +# Updates FreeBSD sources +update_freebsd_sources() { + if [ "${1}" = "full" ]; then + local _full=1 + local _clone_params="" + else + local _full=0 + local _clone_params="--depth 1 --single-branch" + fi + + if [ ! -d "${SRCDIR}" ]; then + mkdir -p ${SRCDIR} + fi + + if [ -n "${NO_BUILDWORLD:-}" -a -n "${NO_BUILDKERNEL:-}" ]; then + echo ">>> NO_BUILDWORLD and NO_BUILDKERNEL set, skipping update of freebsd sources" | tee -a ${LOGFILE} + return + fi + + echo -n ">>> Obtaining FreeBSD sources ${FREEBSD_BRANCH}..." + local _FREEBSD_BRANCH=${FREEBSD_BRANCH:-"devel"} + local _CLONE=1 + + if [ -d "${SRCDIR}/.git" ]; then + CUR_BRANCH=$(cd ${SRCDIR} && git branch | grep '^\*' | cut -d' ' -f2) + if [ ${_full} -eq 0 -a "${CUR_BRANCH}" = "${_FREEBSD_BRANCH}" ]; then + _CLONE=0 + ( cd ${SRCDIR} && git clean -fxd; git fetch origin; git reset --hard origin/${_FREEBSD_BRANCH} ) 2>&1 | grep -C3 -i -E 'error|fatal' + else + rm -rf ${SRCDIR} + fi + fi + + if [ ${_CLONE} -eq 1 ]; then + ( git clone --branch ${_FREEBSD_BRANCH} ${_clone_params} ${FREEBSD_REPO_BASE} ${SRCDIR} ) 2>&1 | grep -C3 -i -E 'error|fatal' + fi + + if [ ! -d "${SRCDIR}/.git" ]; then + echo ">>> ERROR: It was not possible to clone FreeBSD src repo" + print_error_pfS + fi + + if [ -n "${GIT_FREEBSD_COSHA1}" ]; then + ( cd ${SRCDIR} && git checkout ${GIT_FREEBSD_COSHA1} ) 2>&1 | grep -C3 -i -E 'error|fatal' + fi + echo "Done!" +} + +pkg_chroot() { + local _root="${1}" + shift + + if [ $# -eq 0 ]; then + return -1 + fi + + if [ -z "${_root}" -o "${_root}" = "/" -o ! -d "${_root}" ]; then + return -1 + fi + + mkdir -p \ + ${SCRATCHDIR}/pkg_cache \ + ${_root}/var/cache/pkg \ + ${_root}/dev + + /sbin/mount -t nullfs ${SCRATCHDIR}/pkg_cache ${_root}/var/cache/pkg + /sbin/mount -t devfs devfs ${_root}/dev + cp -f /etc/resolv.conf ${_root}/etc/resolv.conf + touch ${BUILDER_LOGS}/install_pkg_install_ports.txt + script -aq ${BUILDER_LOGS}/install_pkg_install_ports.txt pkg -c ${_root} $@ >/dev/null 2>&1 + rm -f ${_root}/etc/resolv.conf + /sbin/umount -f ${_root}/dev + /sbin/umount -f ${_root}/var/cache/pkg +} + + +pkg_chroot_add() { + if [ -z "${1}" -o -z "${2}" ]; then + return 1 + fi + + local _target="${1}" + local _pkg="$(get_pkg_name ${2}).txz" + + if [ ! -d "${_target}" ]; then + echo ">>> ERROR: Target dir ${_target} not found" + print_error_pfS + fi + + if [ ! -f ${CORE_PKG_PATH}/${_pkg} ]; then + echo ">>> ERROR: Package ${_pkg} not found" + print_error_pfS + fi + + cp ${CORE_PKG_PATH}/${_pkg} ${_target} + pkg_chroot ${_target} add /${_pkg} + rm -f ${_target}/${_pkg} +} + +pkg_bootstrap() { + local _root=${1:-"${STAGE_CHROOT_DIR}"} + + setup_pkg_repo ${_root}/usr/local/etc/pkg/repos/${PRODUCT_NAME}.conf ${TARGET} ${TARGET_ARCH} ${PKG_REPO_CONF_BRANCH} + + pkg_chroot ${_root} bootstrap -f +} + +# This routine assists with installing various +# freebsd ports files into the pfsenese-fs staging +# area. +install_pkg_install_ports() { + local MAIN_PKG="${1}" + + if [ -z "${MAIN_PKG}" ]; then + MAIN_PKG=${PRODUCT_NAME} + fi + + echo ">>> Installing pkg repository in chroot (${STAGE_CHROOT_DIR})..." + + [ -d ${STAGE_CHROOT_DIR}/var/cache/pkg ] || \ + mkdir -p ${STAGE_CHROOT_DIR}/var/cache/pkg + + [ -d ${SCRATCHDIR}/pkg_cache ] || \ + mkdir -p ${SCRATCHDIR}/pkg_cache + + echo ">>> Installing built ports (packages) in chroot (${STAGE_CHROOT_DIR})... (starting)" + pkg_chroot ${STAGE_CHROOT_DIR} install ${MAIN_PKG} ${custom_package_list} + pkg_chroot ${STAGE_CHROOT_DIR} autoremove + echo ">>> Installing built ports (packages) in chroot (${STAGE_CHROOT_DIR})... (finshied)" +} + +install_bsdinstaller() { + echo ">>> Installing BSDInstaller in chroot (${FINAL_CHROOT_DIR})... (starting)" + pkg_chroot ${FINAL_CHROOT_DIR} install -f bsdinstaller ${MAIN_PKG} ${custom_package_list} + echo ">>> Installing BSDInstaller in chroot (${FINAL_CHROOT_DIR})... (finished)" +} + +staginareas_clean_each_run() { + echo -n ">>> Cleaning build directories: " + if [ -d "${FINAL_CHROOT_DIR}" ]; then + BASENAME=$(basename ${FINAL_CHROOT_DIR}) + echo -n "$BASENAME " + chflags -R noschg ${FINAL_CHROOT_DIR} 2>&1 >/dev/null + rm -rf ${FINAL_CHROOT_DIR} 2>/dev/null + fi + echo "Done!" +} + +# Imported from FreeSBIE +buildkernel() { + if [ -n "${NO_BUILDKERNEL:-}" ]; then + echo ">>> NO_BUILDKERNEL set, skipping build" | tee -a ${LOGFILE} + return + fi + + if [ -z "${KERNCONF:-}" ]; then + echo ">>> ERROR: No kernel configuration defined probably this is not what you want! STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + + if [ -n "${KERNELCONF:-}" ]; then + export KERNCONFDIR=$(dirname ${KERNELCONF}) + export KERNCONF=$(basename ${KERNELCONF}) + fi + + SRCCONFBASENAME=$(basename ${SRC_CONF}) + echo ">>> KERNCONFDIR: ${KERNCONFDIR}" + echo ">>> ARCH: ${TARGET}" + echo ">>> SRC_CONF: ${SRCCONFBASENAME}" + + makeargs="${MAKEJ_KERNEL:-} SRCCONF=${SRC_CONF} __MAKE_CONF=${MAKE_CONF} TARGET_ARCH=${TARGET_ARCH} TARGET=${TARGET}" + echo ">>> Builder is running the command: script -aq $LOGFILE make -DNO_KERNELCLEAN $makeargs buildkernel KERNCONF=${KERNCONF}" | tee -a $LOGFILE + (script -q $LOGFILE make -C ${SRCDIR} -DNO_KERNELCLEAN $makeargs buildkernel KERNCONF=${KERNCONF} || print_error_pfS;) | egrep '^>>>' +} + +# Imported from FreeSBIE +installkernel() { + if [ -z "${KERNCONF:-}" ]; then + echo ">>> ERROR: No kernel configuration defined probably this is not what you want! STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + + if [ -n "${KERNELCONF:-}" ]; then + export KERNCONFDIR=$(dirname ${KERNELCONF}) + export KERNCONF=$(basename ${KERNELCONF}) + fi + + mkdir -p ${STAGE_CHROOT_DIR}/boot + makeargs="${MAKEJ_KERNEL:-} SRCCONF=${SRC_CONF} __MAKE_CONF=${MAKE_CONF} TARGET_ARCH=${TARGET_ARCH} TARGET=${TARGET} DESTDIR=${KERNEL_DESTDIR}" + echo ">>> Builder is running the command: script -aq $LOGFILE make ${makeargs:-} installkernel KERNCONF=${KERNCONF}" | tee -a $LOGFILE + (script -aq $LOGFILE make -C ${SRCDIR} ${makeargs:-} installkernel KERNCONF=${KERNCONF} || print_error_pfS;) | egrep '^>>>' + gzip -f9 $KERNEL_DESTDIR/boot/kernel/kernel +} + +# Launch is ran first to setup a few variables that we need +# Imported from FreeSBIE +launch() { + + if [ ! -f $SCRATCHDIR/${PRODUCT_NAME}_builder_set_time ]; then + echo ">>> Updating system clock..." + ntpdate 0.freebsd.pool.ntp.org + touch $SCRATCHDIR/${PRODUCT_NAME}_builder_set_time + fi + + if [ "$(id -u)" != "0" ]; then + echo "Sorry, this must be done as root." + fi + + echo ">>> Operation $0 has started at $(date)" +} + +finish() { + echo ">>> Operation $0 has ended at $(date)" +} + +poudriere_create_patch() { + local _jail_patch="${SCRATCHDIR}/poudriere_jail.${GIT_REPO_BRANCH_OR_TAG}.patch" + + if [ -z "${FREEBSD_PARENT_BRANCH}" ]; then + echo ">>> ERROR: FREEBSD_PARENT_BRANCH is not set" + fi + + LOGFILE=${BUILDER_LOGS}/poudriere.log + + # Get FreeBSD source and apply patches + update_freebsd_sources full + + [ -f "${_jail_patch}" ] && \ + rm -f "${_jail_patch}" + + # Create a big patch with all our changes to use on jail + ( \ + cd ${SRCDIR} && \ + git diff $(git merge-base origin/${FREEBSD_PARENT_BRANCH} ${FREEBSD_BRANCH}) > ${_jail_patch} + ) >/dev/null 2>&1 + + # Check if patch was created + if [ ! -s "${_jail_patch}" ]; then + echo ">>> ERROR: Patch does not exist or is empty, aborting..." | tee -a ${LOGFILE} + print_error_pfS + fi +} + +poudriere_possible_archs() { + local _arch=$(uname -m) + local _archs="i386.i386" + + # IF host is amd64, we'll create both repos, and if possible armv6 + if [ "${_arch}" = "amd64" ]; then + _archs="amd64.amd64 ${_archs}" + + if [ -f /usr/local/bin/qemu-arm-static ]; then + # Make sure binmiscctl is ok + /usr/local/etc/rc.d/qemu_user_static forcestart >/dev/null 2>&1 + + if binmiscctl lookup armv6 >/dev/null 2>&1; then + _archs="${_archs} arm.armv6" + fi + fi + fi + + echo ${_archs} +} + +poudriere_jail_name() { + local _jail_arch="${1}" + + if [ -z "${_jail_arch}" ]; then + return 1 + fi + + # Poudriere doesn't like periods in jail names + _jail_arch=$(echo "${_jail_arch}" | tr '.' '_') + + echo "${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}_${_jail_arch}" +} + +poudriere_create_ports_tree() { + LOGFILE=${BUILDER_LOGS}/poudriere.log + + if ! poudriere ports -l | grep -q -E "^${POUDRIERE_PORTS_NAME}[[:blank:]]"; then + local _branch="" + if [ -z "${POUDRIERE_PORTS_GIT_URL}" ]; then + echo ">>> ERROR: POUDRIERE_PORTS_GIT_URL is not defined" + print_error_pfS + fi + if [ -n "${POUDRIERE_PORTS_GIT_BRANCH}" ]; then + _branch="-B ${POUDRIERE_PORTS_GIT_BRANCH}" + fi + echo -n ">>> Creating poudriere ports tree, it may take some time... " | tee -a ${LOGFILE} + if ! script -aq ${LOGFILE} poudriere ports -c -p "${POUDRIERE_PORTS_NAME}" -m git ${_branch} >/dev/null 2>&1; then + echo "" | tee -a ${LOGFILE} + echo ">>> ERROR: Error creating poudriere ports tree, aborting..." | tee -a ${LOGFILE} + print_error_pfS + fi + echo "Done!" | tee -a ${LOGFILE} + fi +} + +poudriere_init() { + local _error=0 + local _archs=$(poudriere_possible_archs) + local _jail_patch="${SCRATCHDIR}/poudriere_jail.${GIT_REPO_BRANCH_OR_TAG}.patch" + + LOGFILE=${BUILDER_LOGS}/poudriere.log + + # Sanity checks + if [ -z "${ZFS_TANK}" ]; then + echo ">>> ERROR: \$ZFS_TANK is empty" | tee -a ${LOGFILE} + error=1 + fi + + if [ -z "${ZFS_ROOT}" ]; then + echo ">>> ERROR: \$ZFS_ROOT is empty" | tee -a ${LOGFILE} + error=1 + fi + + if [ -z "${POUDRIERE_PORTS_NAME}" ]; then + echo ">>> ERROR: \$POUDRIERE_PORTS_NAME is empty" | tee -a ${LOGFILE} + error=1 + fi + + if [ ${_error} -eq 1 ]; then + print_error_pfS + fi + + # Check if zpool exists + if ! zpool list ${ZFS_TANK} >/dev/null 2>&1; then + echo ">>> ERROR: ZFS tank ${ZFS_TANK} not found, please create it and try again..." | tee -a ${LOGFILE} + print_error_pfS + fi + + # Check if zfs rootfs exists + if ! zfs list ${ZFS_TANK}${ZFS_ROOT} >/dev/null 2>&1; then + echo ">>> ERROR: ZFS filesystem ${ZFS_TANK}${ZFS_ROOT} not found, please create it and try again..." | tee -a ${LOGFILE} + print_error_pfS + fi + + # Make sure poudriere is installed + if ! pkg info --quiet poudriere; then + echo ">>> Installing poudriere..." | tee -a ${LOGFILE} + if ! pkg install poudriere >/dev/null 2>&1; then + echo ">>> ERROR: poudriere was not installed, aborting..." | tee -a ${LOGFILE} + print_error_pfS + fi + fi + + # Create poudriere.conf + if [ -z "${POUDRIERE_PORTS_GIT_URL}" ]; then + echo ">>> ERROR: POUDRIERE_PORTS_GIT_URL is not defined" + print_error_pfS + fi + echo ">>> Creating poudriere.conf" | tee -a ${LOGFILE} + cat </usr/local/etc/poudriere.conf +ZPOOL=${ZFS_TANK} +ZROOTFS=${ZFS_ROOT} +RESOLV_CONF=/etc/resolv.conf +BASEFS=/usr/local/poudriere +USE_PORTLINT=no +USE_TMPFS=yes +NOLINUX=yes +DISTFILES_CACHE=/usr/ports/distfiles +CHECK_CHANGED_OPTIONS=yes +CHECK_CHANGED_DEPS=yes +ATOMIC_PACKAGE_REPOSITORY=yes +COMMIT_PACKAGES_ON_FAILURE=no +GIT_URL="${POUDRIERE_PORTS_GIT_URL}" +EOF + + # Remove old jails + for jail_arch in ${_archs}; do + jail_name=$(poudriere_jail_name ${jail_arch}) + + if poudriere jail -i -j "${jail_name}" >/dev/null 2>&1; then + echo ">>> Poudriere jail ${jail_name} already exists, deleting it..." | tee -a ${LOGFILE} + poudriere jail -d -j "${jail_name}" >/dev/null 2>&1 + fi + done + + # Remove old ports tree + if poudriere ports -l | grep -q -E "^${POUDRIERE_PORTS_NAME}[[:blank:]]"; then + echo ">>> Poudriere ports tree ${POUDRIERE_PORTS_NAME} already exists, deleting it..." | tee -a ${LOGFILE} + poudriere ports -d -p "${POUDRIERE_PORTS_NAME}" + fi + + poudriere_create_patch + + local native_xtools="" + # Now we are ready to create jails + for jail_arch in ${_archs}; do + jail_name=$(poudriere_jail_name ${jail_arch}) + + if [ "${jail_arch}" = "arm.armv6" ]; then + native_xtools="-x" + else + native_xtools="" + fi + + echo -n ">>> Creating jail ${jail_name}, it may take some time... " | tee -a ${LOGFILE} + # XXX: Change -m to git when it's available in poudriere + if ! script -aq ${LOGFILE} poudriere jail -c -j "${jail_name}" -v ${FREEBSD_PARENT_BRANCH} \ + -a ${jail_arch} -m svn -P ${_jail_patch} ${native_xtools} >/dev/null 2>&1; then + echo "" | tee -a ${LOGFILE} + echo ">>> ERROR: Error creating jail ${jail_name}, aborting..." | tee -a ${LOGFILE} + print_error_pfS + fi + echo "Done!" | tee -a ${LOGFILE} + done + + poudriere_create_ports_tree + + echo ">>> Poudriere is now configured!" | tee -a ${LOGFILE} +} + +poudriere_update_jails() { + local _archs=$(poudriere_possible_archs) + local _jail_patch="${SCRATCHDIR}/poudriere_jail.${GIT_REPO_BRANCH_OR_TAG}.patch" + + LOGFILE=${BUILDER_LOGS}/poudriere.log + + poudriere_create_patch + + local native_xtools="" + for jail_arch in ${_archs}; do + local _run=0 + if [ -n "${ARCH_LIST}" ]; then + for _arch in ${ARCH_LIST}; do + if [ "${jail_arch##*.}" = "${_arch}" ]; then + _run=1 + fi + done + else + _run=1 + fi + + [ ${_run} -eq 0 ] \ + && continue + + jail_name=$(poudriere_jail_name ${jail_arch}) + + if ! poudriere jail -i -j "${jail_name}" >/dev/null 2>&1; then + echo ">>> Poudriere jail ${jail_name} not found, skipping..." | tee -a ${LOGFILE} + continue + fi + + if [ "${jail_arch}" = "arm.armv6" ]; then + native_xtools="-x" + else + native_xtools="" + fi + + echo -n ">>> Updating jail ${jail_name}, it may take some time... " | tee -a ${LOGFILE} + if ! script -aq ${LOGFILE} poudriere jail -u -j "${jail_name}" -P ${_jail_patch} ${native_xtools} >/dev/null 2>&1; then + echo "" | tee -a ${LOGFILE} + echo ">>> ERROR: Error updating jail ${jail_name}, aborting..." | tee -a ${LOGFILE} + print_error_pfS + fi + echo "Done!" | tee -a ${LOGFILE} + done +} + +poudriere_update_ports() { + LOGFILE=${BUILDER_LOGS}/poudriere.log + + # Create ports tree if necessary + if ! poudriere ports -l | grep -q -E "^${POUDRIERE_PORTS_NAME}[[:blank:]]"; then + poudriere_create_ports_tree + else + echo -n ">>> Updating ports tree ${POUDRIERE_PORTS_NAME}... " | tee -a ${LOGFILE} + script -aq ${LOGFILE} poudriere ports -u -p "${POUDRIERE_PORTS_NAME}" >/dev/null 2>&1 + echo "Done!" | tee -a ${LOGFILE} + fi +} + +poudriere_bulk() { + local _archs=$(poudriere_possible_archs) + + LOGFILE=${BUILDER_LOGS}/poudriere.log + + poudriere_create_ports_tree + + local CUR_BRANCH=$(cd ${BUILDER_TOOLS} && git branch | grep '^\*' | cut -d' ' -f2) + + [ -d /usr/local/etc/poudriere.d ] || \ + mkdir -p /usr/local/etc/poudriere.d + + if [ "${CUR_BRANCH}" != "${GIT_REPO_BRANCH_OR_TAG}" ] && git show "${GIT_REPO_BRANCH_OR_TAG}:builder_scripts/conf/pfPorts/make.conf" >/dev/null 2>&1; then + git show "origin/${GIT_REPO_BRANCH_OR_TAG}:builder_scripts/conf/pfPorts/make.conf" > /usr/local/etc/poudriere.d/${POUDRIERE_PORTS_NAME}-make.conf + elif [ -f "${BUILDER_SCRIPTS}/conf/pfPorts/make.conf" ]; then + cp -f "${BUILDER_SCRIPTS}/conf/pfPorts/make.conf" /usr/local/etc/poudriere.d/${POUDRIERE_PORTS_NAME}-make.conf + fi + + for jail_arch in ${_archs}; do + jail_name=$(poudriere_jail_name ${jail_arch}) + + if ! poudriere jail -i -j "${jail_name}" >/dev/null 2>&1; then + echo ">>> Poudriere jail ${jail_name} not found, skipping..." | tee -a ${LOGFILE} + continue + fi + + if [ -f "${POUDRIERE_BULK}.${jail_arch}" ]; then + _ref_bulk="${POUDRIERE_BULK}.${jail_arch}" + else + _ref_bulk="${POUDRIERE_BULK}" + fi + + _bulk=${SCRATCHDIR}/poudriere_bulk.${GIT_REPO_BRANCH_OR_TAG} + sed -e "s,%%PRODUCT_NAME%%,${PRODUCT_NAME},g" ${_ref_bulk} > ${_bulk} + + if ! poudriere bulk -f ${_bulk} -j ${jail_name} -p ${POUDRIERE_PORTS_NAME}; then + echo ">>> ERROR: Something went wrong..." + print_error_pfS + fi + + echo ">>> Cleaning up old packages from repo..." + if ! poudriere pkgclean -f ${_bulk} -j ${jail_name} -p ${POUDRIERE_PORTS_NAME} -y; then + echo ">>> ERROR: Something went wrong..." + print_error_pfS + fi + + # ./ is intentional, it's a rsync trick to make it chdir to directory before send it + REPO_PATH="/usr/local/poudriere/data/packages/./${jail_name}-${POUDRIERE_PORTS_NAME}" + if [ -z "${DO_NOT_UPLOAD}" -a -d "${REPO_PATH}" ]; then + echo -n ">>> Sending updated repository to ${PKG_RSYNC_HOSTNAME}... " | tee -a ${LOGFILE} + if script -aq ${LOGFILE} rsync -ave "ssh -p ${PKG_RSYNC_SSH_PORT}" \ + --timeout=60 --delete-delay ${REPO_PATH} \ + ${PKG_RSYNC_USERNAME}@${PKG_RSYNC_HOSTNAME}:${PKG_RSYNC_DESTDIR} >/dev/null 2>&1 + then + echo "Done!" | tee -a ${LOGFILE} + else + echo "Failed!" | tee -a ${LOGFILE} + echo ">>> ERROR: An error occurred sending repo to remote hostname" + print_error_pfS + fi + fi + done +} diff --git a/tools/builder_defaults.sh b/tools/builder_defaults.sh new file mode 100644 index 0000000..ffd139f --- /dev/null +++ b/tools/builder_defaults.sh @@ -0,0 +1,299 @@ +#!/bin/sh +# +# build.sh +# +# Copyright (c) 2004-2015 Electric Sheep Fencing, LLC. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# 3. All advertising materials mentioning features or use of this software +# must display the following acknowledgment: +# "This product includes software developed by the pfSense Project +# for use in the pfSense® software distribution. (http://www.pfsense.org/). +# +# 4. The names "pfSense" and "pfSense Project" must not be used to +# endorse or promote products derived from this software without +# prior written permission. For written permission, please contact +# coreteam@pfsense.org. +# +# 5. Products derived from this software may not be called "pfSense" +# nor may "pfSense" appear in their names without prior written +# permission of the Electric Sheep Fencing, LLC. +# +# 6. Redistributions of any form whatsoever must retain the following +# acknowledgment: +# +# "This product includes software developed by the pfSense Project +# for use in the pfSense software distribution (http://www.pfsense.org/). +# +# THIS SOFTWARE IS PROVIDED BY THE pfSense PROJECT ``AS IS'' AND ANY +# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE pfSense PROJECT OR +# ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +# OF THE POSSIBILITY OF SUCH DAMAGE. +# + +########################################### +# Product builder configuration file # +# Please don't modify this file, you # +# can put your settings and options # +# in build.conf, which is sourced at the # +# beginning of this file # +########################################### + +BUILD_CONF="$(realpath $(dirname ${0}))/build.conf" + +# Ensure file exists +if [ -f ${BUILD_CONF} ]; then + . ${BUILD_CONF} +else + echo + echo "You must first run ./set_version.sh !" + echo + exit 2 +fi + +# Make sure pkg will not be interactive +export ASSUME_ALWAYS_YES=true + +OIFS=$IFS +IFS=% + +# Architecture, supported ARCH values are: +# Tier 1: i386, AMD64, and PC98 +# Tier 2: ARM, PowerPC, ia64, Sparc64 and sun4v +# Tier 3: MIPS and S/390 +# Tier 4: None at the moment +# Source: http://www.freebsd.org/doc/en/articles/committers-guide/archs.html +export TARGET=${TARGET:-"`uname -m`"} +export TARGET_ARCH=${TARGET_ARCH:-${TARGET}} +# Set TARGET_ARCH_CONF_DIR +if [ "$TARGET_ARCH" = "" ]; then + export TARGET_ARCH=`uname -p` +fi +export BUILDER_HOST_TARGET=`uname -m` + +# This is used for using svn for retrieving src +export FREEBSD_REPO_BASE=${FREEBSD_REPO_BASE:-"git@git.pfmechanics.com:pfsense/freebsd-src.git"} +export FREEBSD_BRANCH=${FREEBSD_BRANCH:-"devel"} +export FREEBSD_PARENT_BRANCH=${FREEBSD_PARENT_BRANCH:-"stable/10"} + +# Product details +export PRODUCT_NAME=${PRODUCT_NAME:-pfSense} +export PRODUCT_VERSION=${PRODUCT_VERSION:-2.3-DEVELOPMENT} +export PRODUCT_URL=${PRODUCT_VERSION:-"https://www.pfsense.org/"} + +# Product repository tag to build +export GIT_REPO_BRANCH_OR_TAG=${GIT_REPO_BRANCH_OR_TAG:-master} + +# Directory to be used for writing temporary information +export SCRATCHDIR=${SCRATCHDIR:-/tmp} + +# Area that the final image will appear in +export MAKEOBJDIRPREFIXFINAL=${MAKEOBJDIRPREFIXFINAL:-${SCRATCHDIR}/${PRODUCT_NAME}/} + +export BUILDER_SCRIPTS=${BUILDER_SCRIPTS:-$(realpath $(dirname ${0}))} +export BUILDER_TOOLS=${BUILDER_TOOLS:-$(realpath ${BUILDER_SCRIPTS}/..)} +export TOOLS_DIR=${TOOLS_DIR:-$(basename ${BUILDER_TOOLS})} +export BASE_DIR=${BASE_DIR:-$(realpath ${BUILDER_TOOLS}/..)} + +export BUILDER_LOGS=${BUILDER_LOGS:-${BUILDER_TOOLS}/logs} +if [ ! -d ${BUILDER_LOGS} ]; then + mkdir -p ${BUILDER_LOGS} +fi + +# Poudriere +export ZFS_TANK=${ZFS_TANK:-"tank"} +export ZFS_ROOT=${ZFS_ROOT:-"/poudriere"} +export POUDRIERE_PORTS_NAME=${POUDRIERE_PORTS_NAME:-${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}} + +export POUDRIERE_BULK=${POUDRIERE_BULK:-${BUILDER_SCRIPTS}/conf/pfPorts/poudriere_bulk} +export POUDRIERE_PORTS_GIT_URL=${POUDRIERE_PORTS_GIT_URL:-"git@git.pfmechanics.com:pfsense/freebsd-ports.git"} +export POUDRIERE_PORTS_GIT_BRANCH=${POUDRIERE_PORTS_GIT_BRANCH:-"devel"} + +# This is where files will be staged +export STAGE_CHROOT_DIR=${STAGE_CHROOT_DIR:-/usr/local/stage-dir} + +export SRCDIR=${SRCDIR:-/usr/${PRODUCT_NAME}src/src.${GIT_REPO_BRANCH_OR_TAG}} + +# 400M is not enough for amd64 +export MEMORYDISK_SIZE=${MEMORYDISK_SIZE:-"768M"} + +# OVF/vmdk parms +export OVFPATH=${OVFPATH:-${MAKEOBJDIRPREFIXFINAL}} +# Name of ovf file included inside OVA archive +export OVFFILE=${OVFFILE:-${PRODUCT_NAME}.ovf} +# On disk name of VMDK file included in OVA +export OVFVMDK=${OVFVMDK:-${PRODUCT_NAME}.vmdk} +# optional +export OVFCERT=${OVFCERT:-""} +# 10 gigabyte on disk VMDK size +export OVADISKSIZE=${OVADISKSIZE:-"10737418240"} +# dd buffering size when creating raw backed VMDK +export OVABLOCKSIZE=${OVABLOCKSIZE:-"409600"} +# first partition size (freebsd-ufs) GPT +export OVA_FIRST_PART_SIZE=${OVA_FIRST_PART_SIZE:-"8G"} +# swap partition size (freebsd-swap) GPT - +# remaining space of 10G-8G - 128 block beginning/loader +export OVA_SWAP_PART_SIZE=${OVA_SWAP_PART_SIZE:-"4193725"} +# 10737254400 = 10240MB = virtual box vmdk file size XXX grab this value from vbox creation +export OVA_DISKSECTIONALLOCATIONUNITS=${OVA_DISKSECTIONALLOCATIONUNITS:-"10737254400"} +# end of OVF + +# Leave this alone. +export SRC_CONF=${SRC_CONF:-"${SRCDIR}/release/conf/${PRODUCT_NAME}_src.conf"} +export MAKE_CONF=${MAKE_CONF:-"${SRCDIR}/release/conf/${PRODUCT_NAME}_make.conf"} + +# Extra tools to be added to ITOOLS +export EXTRA_TOOLS=${EXTRA_TOOLS:-"uuencode uudecode ex"} + +# Path to kernel files being built +export KERNEL_BUILD_PATH=${KERNEL_BUILD_PATH:-"${SCRATCHDIR}/kernels"} + +# Controls how many concurrent make processes are run for each stage +if [ "${NO_MAKEJ}" = "" ]; then + CPUS=`sysctl -n kern.smp.cpus` + CPUS=`expr $CPUS '*' 2` + export MAKEJ_WORLD=${MAKEJ_WORLD:-"-j$CPUS"} + export MAKEJ_KERNEL=${MAKEJ_KERNEL:-"-j$CPUS"} +else + export MAKEJ_WORLD=${MAKEJ_WORLD:-""} + export MAKEJ_KERNEL=${MAKEJ_KERNEL:-""} +fi +if [ "${TARGET}" = "i386" ]; then + export MODULES_OVERRIDE=${MODULES_OVERRIDE:-"i2c ipmi ndis ipfw ipdivert dummynet fdescfs opensolaris zfs glxsb if_stf coretemp amdtemp hwpmc"} +else + export MODULES_OVERRIDE=${MODULES_OVERRIDE:-"i2c ipmi ndis ipfw ipdivert dummynet fdescfs opensolaris zfs glxsb if_stf coretemp amdtemp aesni sfxge hwpmc"} +fi + +# Number of code images on media (1 or 2) +export NANO_IMAGES=2 +# 0 -> Leave second image all zeroes so it compresses better. +# 1 -> Initialize second image with a copy of the first +export NANO_INIT_IMG2=1 +export NANO_WITH_VGA=${NANO_WITH_VGA:-""} +export NANO_NEWFS="-b 4096 -f 512 -i 8192 -O1" +export FLASH_SIZE=${FLASH_SIZE:-"1g"} +# Size of code file system in 512 bytes sectors +# If zero, size will be as large as possible. +export NANO_CODESIZE=0 +# Size of data file system in 512 bytes sectors +# If zero: no partition configured. +# If negative: max size possible +export NANO_DATASIZE=0 +# Size of Product /conf partition # 102400 = 50 megabytes. +export NANO_CONFSIZE=102400 +# packet is OK for 90% of embedded +export NANO_BOOT0CFG="-o packet -s 1 -m 3" + +# " - UNBREAK TEXTMATE FORMATTING - PLEASE LEAVE. + +# GIT repositories settings +export GIT_REPO_URL=${GIT_REPO_URL:-"git@git.pfmechanics.com:pfsense/pfsense.git"} +export GIT_REPO_DIR="${BASE_DIR}/${PRODUCT_NAME}_GIT_REPO" +export GIT_REPO_TOOLS=${GIT_REPO_TOOLS:-"git@git.pfmechanics.com:pfsense/pfsense-tools.git"} + +# Host to rsync pkg repos from poudriere +export PKG_RSYNC_HOSTNAME=${PKG_RSYNC_HOSTNAME:-"beta.pfsense.org"} +export PKG_RSYNC_USERNAME=${PKG_RSYNC_USERNAME:-"wwwsync"} +export PKG_RSYNC_SSH_PORT=${PKG_RSYNC_SSH_PORT:-"22"} +export PKG_RSYNC_DESTDIR=${PKG_RSYNC_DESTDIR:-"/usr/local/www/beta/packages"} +export PKG_REPO_SERVER=${PKG_REPO_SERVER:-"pkg+http://beta.pfsense.org/packages"} +export PKG_REPO_CONF_BRANCH=${PKG_REPO_CONF_BRANCH:-"${GIT_REPO_BRANCH_OR_TAG}"} + +# Package overlay. This gives people a chance to build product +# installable image that already contains certain extra packages. +# +# Needs to contain comma separated package names. Of course +# package names must be valid. Using non existent +# package name would yield an error. +# +#export custom_package_list="" + +# Directory that will clone to in order to create +# iso staging area. +export FINAL_CHROOT_DIR=${FINAL_CHROOT_DIR:-/usr/local/final-dir} + +# NOTE: Date string is used for creating file names of images +# The file is used for sharing the same value with build_snapshots.sh +export _BUILDER_EPOCH=$(date +"%s") +export DATESTRINGFILE=${DATESTRINGFILE:-$SCRATCHDIR/version.snapshots} +if [ "${DATESTRING}" = "" ]; then + if [ -f $DATESTRINGFILE ]; then + # If the file is more than 30 minutes old regenerate it + TMPDATESTRINGFILE=$(($_BUILDER_EPOCH - `stat -f %m $DATESTRINGFILE`)) + if [ -z "${_USE_OLD_DATESTRING}" -a $TMPDATESTRINGFILE -gt 1800 ]; then + export DATESTRING=`date "+%Y%m%d-%H%M"` + else + export DATESTRING=`cat $DATESTRINGFILE` + fi + unset TMPDATESTRINGFILE + else + export DATESTRING=`date "+%Y%m%d-%H%M"` + fi + echo "$DATESTRING" > $DATESTRINGFILE +fi + +# NOTE: Date string is placed on the final image etc folder to help detect new updates +# The file is used for sharing the same value with build_snapshots.sh +export BUILTDATESTRINGFILE=${BUILTDATESTRINGFILE:-$SCRATCHDIR/version.buildtime} +if [ "${BUILTDATESTRING}" = "" ]; then + if [ -f $BUILTDATESTRINGFILE ]; then + # If the file is more than 30 minutes old regenerate it + TMPBUILTDATESTRINGFILE=$(($_BUILDER_EPOCH - `stat -f %m $BUILTDATESTRINGFILE`)) + if [ $TMPBUILTDATESTRINGFILE -gt 1800 ]; then + export BUILTDATESTRING=`date "+%a %b %d %T %Z %Y"` + else + export BUILTDATESTRING=`cat $BUILTDATESTRINGFILE` + fi + unset TMPBUILTDATESTRINGFILE + else + export BUILTDATESTRING=`date "+%a %b %d %T %Z %Y"` + fi + echo "$BUILTDATESTRING" > $BUILTDATESTRINGFILE +fi + +# Define base package version, based on date for snaps +CORE_PKG_VERSION=${PRODUCT_VERSION%%-*} +if echo "${PRODUCT_VERSION}" | grep -qv -- '-RELEASE'; then + CORE_PKG_VERSION="${CORE_PKG_VERSION}.${DATESTRING}" +fi +export CORE_PKG_PATH=${CORE_PKG_PATH:-"${SCRATCHDIR}/core_pkg"} +export CORE_PKG_TMP=${CORE_PKG_TMP:-"${SCRATCHDIR}/core_pkg_tmp"} + +# General builder output filenames +export UPDATESDIR=${UPDATESDIR:-$MAKEOBJDIRPREFIXFINAL/updates} +export ISOPATH=${ISOPATH:-${MAKEOBJDIRPREFIXFINAL}/${PRODUCT_NAME}-LiveCD-${PRODUCT_VERSION}-${TARGET}-${DATESTRING}.iso} +export MEMSTICKPATH=${MEMSTICKPATH:-${MAKEOBJDIRPREFIXFINAL}/${PRODUCT_NAME}-memstick-${PRODUCT_VERSION}-${TARGET}-${DATESTRING}.img} +export MEMSTICKSERIALPATH=${MEMSTICKSERIALPATH:-${MAKEOBJDIRPREFIXFINAL}/${PRODUCT_NAME}-memstick-serial-${PRODUCT_VERSION}-${TARGET}-${DATESTRING}.img} +export MEMSTICKADIPATH=${MEMSTICKADIPATH:-${MAKEOBJDIRPREFIXFINAL}/${PRODUCT_NAME}-memstick-ADI-${PRODUCT_VERSION}-${TARGET}-${DATESTRING}.img} + +# set full-update update filename +export UPDATES_TARBALL_FILENAME=${UPDATES_TARBALL_FILENAME:-"${UPDATESDIR}/${PRODUCT_NAME}-Full-Update-${PRODUCT_VERSION}-${TARGET}-${DATESTRING}.tgz"} + +# " - UNBREAK TEXTMATE FORMATTING - PLEASE LEAVE. + +if [ "${TARGET}" = "i386" ]; then + export BUILD_KERNELS=${BUILD_KERNELS:-"${PRODUCT_NAME} ${PRODUCT_NAME}_WRAP ${PRODUCT_NAME}_WRAP_VGA"} +else + export BUILD_KERNELS=${BUILD_KERNELS:-"${PRODUCT_NAME}"} +fi + +# This needs to be at the very end of the file. +IFS=$OIFS diff --git a/tools/conf/ovf/pfSense.ovf b/tools/conf/ovf/pfSense.ovf new file mode 100755 index 0000000..aeb82d3 --- /dev/null +++ b/tools/conf/ovf/pfSense.ovf @@ -0,0 +1,136 @@ + + + + + + + List of the virtual disks used in the package + + + + The list of logical networks + + The VM Network network + + + + pfSense virtual machine + pfSense + + FreeBSD XX-Bit + FreeBSD XX-Bit (pfSense) + + + Virtual hardware requirements + + Virtual Hardware Family + 0 + pfSense + vmx-07 + + + hertz * 10^6 + Number of Virtual CPUs + 1 virtual CPU(s) + 1 + 3 + 1 + + + byte * 2^20 + Memory Size + 1024MB of memory + 2 + 4 + 1024 + + + 0 + SCSI Controller + scsi0 + 3 + lsilogic + 6 + + + 1 + IDE Controller + VirtualIDEController 1 + 4 + 5 + + + 0 + IDE Controller + VirtualIDEController 0 + 5 + 5 + + + 7 + true + VM Network + E1000 ethernet adapter on "VM Network" + ethernet0 + 6 + E1000 + 10 + + + 8 + true + VM Network + E1000 ethernet adapter on "VM Network" + ethernet0 + 7 + E1000 + 10 + + + 0 + scsi0:0 + ovf:/disk/vmdisk1 + 8 + 3 + 17 + + + + Describes product information for the service + pfSense + Electric Sheep Fencing, LLC + PRODUCT_VERSION + PRODUCT_URL + PRODUCT_URL + + + An annotation + pfSense is a free, open source customized distribution of FreeBSD tailored for use as a firewall and router. In addition to being a powerful, flexible firewalling and routing platform, it includes a long list of related features and a package system allowing further expandability without adding bloat and potential security vulnerabilities to the base distribution. pfSense is a popular project with more than 1 million downloads since its inception, and proven in countless installations ranging from small home networks protecting a PC and an Xbox to large corporations, universities and other organizations protecting thousands of network devices. + + + The End User License Agreement + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, +OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + + + diff --git a/tools/conf/pfPorts/make.conf b/tools/conf/pfPorts/make.conf new file mode 100644 index 0000000..bd716a2 --- /dev/null +++ b/tools/conf/pfPorts/make.conf @@ -0,0 +1,125 @@ +# Generic options +OPTIONS_UNSET_FORCE= X11 DOCS EXAMPLES MAN INFO MANPAGES + +# Individual ports options +net_apinger_SET_FORCE= IPV6 + +www_elinks_UNSET_FORCE= FASTMEM + +www_lighttpd_SET_FORCE= NODELAY + +net_miniupnpd_SET_FORCE= PF_ENABLE_FILTER_RULES + +security_openssl_SET_FORCE= PADLOCK + +security_openvpn_SET_FORCE= PW_SAVE ENGINEFIX2 +security_openvpn_UNSET_FORCE= EASYRSA + +net_relayd_SET_FORCE= LIBEVENT_STATIC + +databases_rrdtool12_UNSET_FORCE= PERL + +security_snortsam_UNSET_FORCE= IPFW + +emulators_qemu_UNSET_FORCE= SDL + +security_ca_root_nss_SET_FORCE= ETCSYMLINK + +security_strongswan_SET_FORCE= CURL EAPDYNAMIC EAPRADIUS EAPSIMFILE \ + IKEv1 UNBOUND XAUTH IPSECKEY SMP PKI UNITY +security_strongswan_UNSET_FORCE= EAPAKA3GPP2 + +dns_unbound_SET_FORCE= LIBEVENT + +.if ${.CURDIR:N*net/rsync*}=="" +CFLAGS+= -static +.endif + +.if ${.CURDIR:N*security/openiked*}=="" +CONFIGURE_ARGS= --without-openssl-header-check +.endif + +.if ${.CURDIR:N*security/strongswan*}=="" +ETCDIR= /var/etc/ipsec/strongswan +.endif + +# OS version specific options +OSRELDATE!= /sbin/sysctl -n kern.osreldate +.if ${OSRELDATE} >= 1000000 +DEFAULT_VERSIONS= php=55 +WITH_OPENSSL_BASE= yes +sysutils_syslog-ng_SET_FORCE= SYS_SSL +sysutils_syslog-ng_UNSET_FORCE= PORTS_SSL +lang_php55_SET_FORCE= FPM + +. if ${.CURDIR:N*sysutils/check_reload_status*}=="" +WITH_DEBUG= yes +. endif + +.endif + +# Options for packages +sysutils_apcupsd_SET_FORCE=APCDUMB_DRV APCSMART_DRV PCNET_DRV SNMP_DRV TCP_WRAPPERS USB +sysutils_apcupsd_UNSET_FORCE=CGI CLIENT_ONLY GAPCMON SNMP_DRV_OLD TEST_DRV + +devel_apr1_SET_FORCE=IPV6 SQLITE SSL THREADS +devel_apr1_UNSET_FORCE=BDB MYSQL NSS PGSQL + +net_asterisk_SET_FORCE=SRTP +net_asterisk_UNSET_FORCE=FREETDS PGSQL + +net_avahi_UNSET_FORCE=GTK + +security_barnyard2_SET_FORCE=BRO GRE IPV6 MPLS MYSQL PORT_PCAP +security_barnyard2_UNSET_FORCE=ODBC PGSQL PRELUDE + +dns_bind99_SET_FORCE=DLZ_FILESYSTEM FILTER_AAAA IPV6 LINKS RRL SIGCHASE SSL THREADS XML +dns_bind99_UNSET_FORCE=FIXED_RRSET GSSAPI IDN LARGE_FILE REPLACE_BASE + +www_c-icap_SET_FORCE=LARGE_FILES +www_c-icap_UNSET_FORCE=IPV6 + +graphics_cairo_UNSET_FORCE=X11 XCB OPENGL + +net_freeradius2_SET_FORCE=KERBEROS LDAP MYSQL PERL PGSQL PYTHON SSL_PORT + +lang_gcc_UNSET_FORCE=JAVA + +devel_git_SET_FORCE=CONTRIB CURL ETCSHELLS ICONV NLS PERL +devel_git_UNSET_FORCE=CVS GITWEB GUI HTMLDOCS P4 SVN + +graphics_graphviz_SET_FORCE=ICONV NLS +graphics_graphviz_UNSET_FORCE=DIGCOLA IPSEPCOLA PANGOCAIRO XPM + +net_haproxy_SET_FORCE=OPENSSL SPCRE LUA +net_haproxy_UNSET_FORCE=DPCRE + +net_haproxy-devel_SET_FORCE=OPENSSL SPCRE LUA +net_haproxy-devel_UNSET_FORCE=DPCRE + +graphics_gd_SET_FORCE=ICONV +graphics_gd_UNSET_FORCE=FONTCONFIG XPM + +net_libpcap_SET_FORCE=IPV6 +net_libpcap_UNSET_FORCE=DAG + +www_lightsquid_SET_FORCE=GD + +net_mtr_UNSET_FORCE=X11 + +security_snort_SET_FORCE=APPID BARNYARD GRE IPV6 NORMALIZER PERFPROFILE SOURCEFIRE +security_snort_UNSET_FORCE=FILEINSPECT HA PULLEDPORK + +www_squidguard_SET_FORCE=DNS_BL LDAP STRIP_NTDOMAIN +www_squidguard_UNSET_FORCE=QUOTE_STRING + +www_squid_SET_FORCE=ARP_ACL AUTH_LDAP AUTH_NIS AUTH_SASL CACHE_DIGESTS DELAY_POOLS FOLLOW_XFF FS_AUFS FS_DISKD GSSAPI_MIT HTCP ICAP ICMP IDENT IPV6 KQUEUE LARGEFILE LAX_HTTP SNMP SSL SSL_CRTD TP_PF WCCP WCCPV2 +www_squid_UNSET_FORCE=AUTH_SMB AUTH_SQL DEBUG DNS_HELPER ECAP ESI FS_ROCK GSSAPI_NONE GSSAPI_BASE GSSAPI_HEIMDAL STACKTRACES TP_IPF TP_IPFW VIA_DB + +security_suricata_SET_FORCE=GEOIP HTP_PORT IPFW JSON LUAJIT NSS PORTS_PCAP +security_suricata_UNSET_FORCE=LUA PRELUDE SC TESTS + +net-mgmt_zabbix22-agent_SET_FORCE=IPV6 SQLITE + +net-mgmt_zabbix22-proxy_SET_FORCE=IPV6 SQLITE +net-mgmt_zabbix22-proxy_UNSET_FORCE=GSSAPI JABBER MYSQL diff --git a/tools/conf/pfPorts/poudriere_bulk b/tools/conf/pfPorts/poudriere_bulk new file mode 100644 index 0000000..58fcb25 --- /dev/null +++ b/tools/conf/pfPorts/poudriere_bulk @@ -0,0 +1,47 @@ +benchmarks/iperf +benchmarks/iperf3 +security/%%PRODUCT_NAME%% +security/%%PRODUCT_NAME%%-vm +sysutils/%%PRODUCT_NAME%%-builder +sysutils/bsdinstaller +net/%%PRODUCT_NAME%%-pkg-pfBlockerNG +net/%%PRODUCT_NAME%%-pkg-haproxy +net/%%PRODUCT_NAME%%-pkg-haproxy-devel +net/%%PRODUCT_NAME%%-pkg-Avahi +sysutils/%%PRODUCT_NAME%%-pkg-Notes +sysutils/%%PRODUCT_NAME%%-pkg-Backup +sysutils/%%PRODUCT_NAME%%-pkg-Cron +security/%%PRODUCT_NAME%%-pkg-snort +sysutils/%%PRODUCT_NAME%%-pkg-Shellcmd +net/%%PRODUCT_NAME%%-pkg-routed +net/%%PRODUCT_NAME%%-pkg-siproxd +net/%%PRODUCT_NAME%%-pkg-OpenBGPD +www/%%PRODUCT_NAME%%-pkg-Lightsquid +emulators/%%PRODUCT_NAME%%-pkg-Open-VM-Tools +sysutils/%%PRODUCT_NAME%%-pkg-AutoConfigBackup +net/%%PRODUCT_NAME%%-pkg-arping +security/%%PRODUCT_NAME%%-pkg-nmap +net-mgmt/%%PRODUCT_NAME%%-pkg-darkstat +benchmarks/%%PRODUCT_NAME%%-pkg-iperf +net/%%PRODUCT_NAME%%-pkg-mtr-nox11 +www/%%PRODUCT_NAME%%-pkg-squidGuard +security/%%PRODUCT_NAME%%-pkg-openvpn-client-export +sysutils/%%PRODUCT_NAME%%-pkg-blinkled +sysutils/%%PRODUCT_NAME%%-pkg-gwled +mail/%%PRODUCT_NAME%%-pkg-mailreport +sysutils/%%PRODUCT_NAME%%-pkg-RRD_Summary +net/%%PRODUCT_NAME%%-pkg-Quagga_OSPF +net-mgmt/%%PRODUCT_NAME%%-pkg-iftop +sysutils/%%PRODUCT_NAME%%-pkg-System_Patches +net-mgmt/%%PRODUCT_NAME%%-pkg-softflowd +security/%%PRODUCT_NAME%%-pkg-sudo +sysutils/%%PRODUCT_NAME%%-pkg-Service_Watchdog +ftp/%%PRODUCT_NAME%%-pkg-FTP_Client_Proxy +security/%%PRODUCT_NAME%%-pkg-suricata +www/%%PRODUCT_NAME%%-pkg-squid +#net/%%PRODUCT_NAME%%-pkg-ntopng +net-mgmt/%%PRODUCT_NAME%%-pkg-zabbix-agent +net-mgmt/%%PRODUCT_NAME%%-pkg-zabbix-proxy +net/%%PRODUCT_NAME%%-pkg-LADVD +net/%%PRODUCT_NAME%%-pkg-freeradius2 +sysutils/%%PRODUCT_NAME%%-pkg-syslog-ng diff --git a/tools/conf/pfPorts/poudriere_bulk.arm.armv6 b/tools/conf/pfPorts/poudriere_bulk.arm.armv6 new file mode 100644 index 0000000..37cfafb --- /dev/null +++ b/tools/conf/pfPorts/poudriere_bulk.arm.armv6 @@ -0,0 +1 @@ +security/%%PRODUCT_NAME%% diff --git a/tools/scripts/generate-privdefs.php b/tools/scripts/generate-privdefs.php new file mode 100755 index 0000000..917a94d --- /dev/null +++ b/tools/scripts/generate-privdefs.php @@ -0,0 +1,216 @@ +#!/usr/local/bin/php -f +/usr/local/www + * directory and builds a privilege definition file + * based on the embedded metadata tags. For more info + * please see /etc/inc/meta.inc + */ + +if (count($argv) < 2) { + echo "usage: generate-privdefs \n"; + echo "\n"; + echo "This utility generates privilege definitions and writes them to\n"; + echo "'/etc/inc/priv.defs.inc'. The parameter should\n"; + echo "be specified as your base pfSense working directory.\n"; + echo "\n"; + echo "Examples:\n"; + echo "#generate-privdefs /\n"; + echo "#generate-privdefs /home/pfsense/RELENG_1/pfSense/\n"; + echo "\n"; + exit -1; +} + +$prefix = $argv[1]; +if (!file_exists($prefix)) { + echo "prefix {$prefix} is invalid"; + exit -1; +} + +$metainc = $prefix."etc/inc/meta.inc"; + +if (!file_exists($metainc)) { + echo "unable to locate {$metainc} file\n"; + exit -1; +} + +require_once($metainc); + +echo "--Locating www php files--\n"; + +$path = $prefix."/usr/local/www"; +list_phpfiles($path, $found); + +echo "--Gathering privilege metadata--\n"; + +$data; +foreach ($found as $fname) + read_file_metadata($path."/".$fname, $data, "PRIV"); + +echo "--Generating privilege definitions--\n"; +$privdef = $prefix."etc/inc/priv.defs.inc"; + +$fp = fopen($privdef,"w"); +if (!$fp) { + echo "unable to open {$privdef}\n"; + exit -2; +} + +$pdata; +$pdata = " $tags) { + + foreach ($tags as $tname => $vals) { + + $ident = ""; + $name = ""; + $descr = ""; + $match = array(); + + foreach ($vals as $vname => $vlist) { + + switch ($vname) { + case "IDENT": + $ident = $vlist[0]; + break; + case "NAME": + $name = $vlist[0]; + break; + case "DESCR": + $descr = $vlist[0]; + break; + case "MATCH": + $match = $vlist; + break; + } + } + + if (!$ident) { + echo "invalid IDENT in {$fname} privilege\n"; + continue; + } + + if (!count($match)) { + echo "invalid MATCH in {$fname} privilege\n"; + continue; + } + + $pdata .= "\$priv_list['{$ident}'] = array();\n"; + $pdata .= "\$priv_list['{$ident}']['name'] = \"WebCfg - {$name}\";\n"; + $pdata .= "\$priv_list['{$ident}']['descr'] = \"{$descr}\";\n"; + $pdata .= "\$priv_list['{$ident}']['match'] = array();\n"; + + foreach ($match as $url) + $pdata .= "\$priv_list['{$ident}']['match'][] = \"{$url}\";\n"; + + $pdata .= "\n"; + } +} + +$pdata .= "\n"; +$pdata .= "\$priv_rmvd = array();\n"; +$pdata .= "\n"; + +$pdata .= "?>\n"; +fwrite($fp, $pdata); + +fclose($fp); + +/* + * TODO : Build additional functionality + * + +echo "--Checking for pages without privilege definitions--\n"; + +foreach ($found as $fname) { + $match = false; + foreach ($pages_current as $pname => $pdesc) { + if (!strcmp($pname,$fname)) { + $match = true; + break; + } + } + if (!$match) + echo "missing: $fname\n"; +} + +echo "--Checking for stale privilege definitions--\n"; + +foreach ($pages_current as $pname => $pdesc) { + $match = false; + foreach ($found as $fname) { + if (!strncmp($fname,$pname,strlen($fname))) { + $match = true; + break; + } + } + if (!$match) + echo "stale: $pname\n"; +} + + */ + +?> diff --git a/tools/scripts/update_package_pfPorts.php b/tools/scripts/update_package_pfPorts.php new file mode 100755 index 0000000..648ebca --- /dev/null +++ b/tools/scripts/update_package_pfPorts.php @@ -0,0 +1,449 @@ +#!/usr/local/bin/php -q + 0) { + echo "!!! Ignoring {$pkg['name']}, maximum version is {$pkg['maximum_version']}\n"; + continue; + } + } + + if (isset($pkg['required_version']) && !empty($pkg['required_version'])) { + if (version_compare_numeric($pfs_version, $pkg['required_version']) < 0) { + echo "!!! Ignoring {$pkg['name']}, required version is {$pkg['required_version']}\n"; + continue; + } + } + + create_port($pkg); +} + +function fix_php_calls($file) { + if (!file_exists($file)) { + return; + } + + if (!preg_match('/\.(php|inc)$/', $file)) { + return; + } + + $content = file_get_contents($file); + $new_content = preg_replace('/\/usr\/local\/bin\/php/', '/usr/local/bin/php-cgi', $content); + file_put_contents($file, $new_content); + unset($content, $new_content); +} + +function create_port($pkg) { + global $ports_dir, $tools_dir, $builder_dir, $packages_dir, $product_name; + + if (isset($pkg['internal_name'])) { + $pkg_name = $pkg['internal_name']; + } else { + $pkg_name = $pkg['name']; + } + + if (empty($pkg_name)) { + echo "!!! Error: Package name cannot be empty\n"; + exit(1); + } + + if (!preg_match('/^[a-zA-Z0-9\.\-_]+$/', $pkg_name)) { + echo "!!! Error: Package name '{$pkg_name}' is invalid\n"; + exit(1); + } + + if (isset($pkg['port_category']) && !empty($pkg['port_category'])) + $category = $pkg['port_category']; + else + $category = 'sysutils'; + + $port_name_prefix = $product_name . '-pkg-'; + $port_name = $port_name_prefix . $pkg_name; + $port_path = $ports_dir . '/' . $category . '/' . $port_name; + + if (is_dir($port_path)) { + $_gb = exec("rm -rf {$port_path}"); + } + + mkdir($port_path . "/files", 0755, true); + + if (isset($pkg['descr']) && !empty($pkg['descr'])) { + $pkg_descr = $pkg['descr']; + } else { + /* provide a generic description when it's not available */ + $pkg_descr = "{$pkg_name} {$product_name} package"; + } + + if (isset($pkg['pkginfolink']) && !empty($pkg['pkginfolink'])) { + $pkg_descr .= "\n\nWWW: {$pkg['pkginfolink']}"; + } + $pkg_descr .= "\n"; + + file_put_contents($port_path . "/pkg-descr.tmp", $pkg_descr); + unset($pkg_descr); + + $_gb = exec("/usr/bin/fmt -w 80 {$port_path}/pkg-descr.tmp > {$port_path}/pkg-descr 2>/dev/null"); + @unlink("{$port_path}/pkg-descr.tmp"); + + if (isset($pkg['after_install_info']) && !empty($pkg['after_install_info'])) { + file_put_contents($port_path . "/pkg-message", $pkg['after_install_info'] . "\n"); + } + + $pkg_install = file_get_contents($builder_dir . "/templates/pkg-install.in"); + file_put_contents($port_path . "/files/pkg-install.in", $pkg_install); + unset($pkg_install); + + $pkg_deinstall = file_get_contents($builder_dir . "/templates/pkg-deinstall.in"); + file_put_contents($port_path . "/files/pkg-deinstall.in", $pkg_deinstall); + unset($pkg_deinstall); + + $config_file = preg_replace('/^https*:\/\/[^\/]+\/packages\//', '', $pkg['config_file']); + + if (!file_exists($packages_dir . '/' . $config_file)) { + echo "!!! Error, config file {$config_file} not found\n"; + exit(1); + } + + $pkg_config = parse_xml_config_pkg($packages_dir . '/' . $config_file, "packagegui"); + + if (empty($pkg_config)) { + echo "!!! Error, config file {$config_file} is invalid\n"; + exit(1); + } + + if (!is_dir($port_path . '/files/usr/local/pkg')) { + mkdir($port_path . '/files/usr/local/pkg', 0755, true); + } + copy($packages_dir . '/' . $config_file, $port_path . '/files/usr/local/pkg/' . basename($config_file)); + + $plist_files = array('pkg/' . basename($config_file)); + $plist_dirs = array(); + $mkdirs = array('${MKDIR} ${STAGEDIR}${PREFIX}/pkg'); + $install = array('${INSTALL_DATA} -m 0644 ${FILESDIR}${PREFIX}/pkg/' . basename($config_file) . " \\\n\t\t" . '${STAGEDIR}${PREFIX}/pkg'); + if (!empty($pkg_config['additional_files_needed'])) { + foreach ($pkg_config['additional_files_needed'] as $item) { + if (is_array($item['item'])) { + $item['item'] = $item['item'][0]; + } + if (isset($item['do_not_add_to_port'])) + continue; + $file_relpath = preg_replace('/^https*:\/\/[^\/]+\/packages\//', '', $item['item']); + if (!file_exists($packages_dir . '/' . $file_relpath)) { + echo "!!! Error: Additional file needed {$file_relpath} not found\n"; + exit(1); + } + + if (!is_dir($port_path . '/files' . $item['prefix'])) { + mkdir($port_path . '/files' . $item['prefix'], 0755, true); + } + + copy($packages_dir . '/' . $file_relpath, $port_path . '/files' . $item['prefix'] . '/' . basename($file_relpath)); + fix_php_calls($port_path . '/files' . $item['prefix'] . '/' . basename($file_relpath)); + /* Remove /usr/local/ from prefix */ + $plist_entry = preg_replace('/^\/usr\/local\//', '', $item['prefix']); + $plist_entry = preg_replace('/\/*$/', '', $plist_entry); + + if (substr($plist_entry, 0, 1) == '/' && + !in_array("@dir {$plist_entry}", $plist_dirs)) { + $plist_dirs[] = "@dir {$plist_entry}"; + } + + $plist_entry .= '/' . basename($item['item']); + if (!in_array($plist_entry, $plist_files)) { + $plist_files[] = $plist_entry; + } + unset($plist_entry); + + if (preg_match('/^\/usr\/local\//', $item['prefix'])) { + $mkdirs_entry = preg_replace('/^\/usr\/local\//', '${PREFIX}/', $item['prefix']); + } else { + $mkdirs_entry = $item['prefix']; + } + $mkdirs_entry = preg_replace('/\/*$/', '', $mkdirs_entry); + + $install_entry = '${INSTALL_DATA} '; + + if (isset($item['chmod']) && !empty($item['chmod'])) { + $install_entry .= "-m {$item['chmod']} "; + } + + $install_entry .= '${FILESDIR}' . $mkdirs_entry . '/' . basename($item['item']) . " \\\n\t\t"; + $install_entry .= '${STAGEDIR}' . $mkdirs_entry; + $mkdirs_entry = '${MKDIR} ${STAGEDIR}' . $mkdirs_entry; + + if (!in_array($mkdirs_entry, $mkdirs)) { + $mkdirs[] = $mkdirs_entry; + } + if (!in_array($install_entry, $install)) { + $install[] = $install_entry; + } + + unset($install_entry, $mkdirs_entry); + } + } + + if (!is_dir($port_path . '/files/usr/local/share/' . $port_name)) { + mkdir($port_path . '/files/usr/local/share/' . $port_name, 0755, true); + } + + $info['package'][] = $pkg; + $info_xml = dump_xml_config($info, 'pfsensepkgs'); + file_put_contents($port_path . '/files/usr/local/share/' . $port_name . '/info.xml', $info_xml); + unset($info, $info_xml); + $plist_files[] = '%%DATADIR%%/info.xml'; + $mkdirs[] = '${MKDIR} ${STAGEDIR}${DATADIR}'; + $install[] = '${INSTALL_DATA} ${FILESDIR}${DATADIR}/info.xml ' . "\\\n\t\t" . '${STAGEDIR}${DATADIR}'; + + $version = $pkg['version']; + + /* Detect PORTEPOCH */ + if (($pos = strpos($version, ',')) != FALSE) { + $epoch = substr($version, $pos+1); + $version = substr($version, 0, $pos); + } + + /* Detect PORTREVISION */ + if (($pos = strpos($version, '_')) != FALSE) { + $revision = substr($version, $pos+1); + $version = substr($version, 0, $pos); + } + + $makefile = array(); + $makefile[] = '# $FreeBSD$'; + $makefile[] = ''; + $makefile[] = "PORTNAME=\t{$port_name}"; + $makefile[] = "PORTVERSION=\t{$version}"; + if (isset($revision)) { + $makefile[] = "PORTREVISION=\t{$revision}"; + } + if (isset($epoch)) { + $makefile[] = "PORTEPOCH=\t{$epoch}"; + } + // XXX: use categories from xml */ + $makefile[] = "CATEGORIES=\t{$category}"; + $makefile[] = "MASTER_SITES=\t# empty"; + $makefile[] = "DISTFILES=\t# empty"; + $makefile[] = "EXTRACT_ONLY=\t# empty"; + $makefile[] = ""; + $makefile[] = "MAINTAINER=\tcoreteam@pfsense.org"; + // XXX: Provide comment on xml */ + $makefile[] = "COMMENT=\t{$product_name} package {$pkg_name}"; + if (isset($pkg['run_depends']) && !empty($pkg['run_depends'])) { + $run_depends = array(); + foreach (preg_split('/\s+/', trim($pkg['run_depends'])) as $depend) { + list($file_depend, $port_depend) = explode(':', $depend); + $file_depend = '${LOCALBASE}/' . $file_depend; + $port_depend = '${PORTSDIR}/' . $port_depend; + $run_depends[] = $file_depend . ':' . $port_depend; + } + if (!empty($run_depends)) { + $makefile[] = ""; + $first = true; + foreach ($run_depends as $run_depend) { + if ($first) { + $makefile_entry = "RUN_DEPENDS=\t" . $run_depend; + $first = false; + } else { + $makefile_entry .= " \\\n\t\t" . $run_depend; + } + } + $makefile[] = $makefile_entry; + unset($makefile_entry); + } + unset($run_depends); + } + if (isset($pkg['lib_depends']) && !empty($pkg['lib_depends'])) { + $lib_depends = array(); + foreach (preg_split('/\s+/', trim($pkg['lib_depends'])) as $depend) { + list($lib_depend, $port_depend) = explode(':', $depend); + $port_depend = '${PORTSDIR}/' . $port_depend; + $lib_depends[] = $lib_depend . ':' . $port_depend; + } + if (!empty($lib_depends)) { + $makefile[] = ""; + $first = true; + foreach ($lib_depends as $lib_depend) { + if ($first) { + $makefile_entry = "LIB_DEPENDS=\t" . $lib_depend; + $first = false; + } else { + $makefile_entry .= " \\\n\t\t" . $lib_depend; + } + } + $makefile[] = $makefile_entry; + unset($makefile_entry); + } + unset($run_depends); + } + if (isset($pkg['port_uses']) && !empty($pkg['port_uses'])) { + $makefile[] = ""; + foreach (preg_split('/\s+/', trim($pkg['port_uses'])) as $port_use) { + $port_use = preg_replace('/=/', "=\t", $port_use); + $makefile[] = $port_use; + } + } + if (isset($pkg['conflicts']) && !empty($pkg['conflicts'])) { + $makefile[] = ""; + $makefile[] = "CONFLICTS=\t" . $port_name_prefix . $pkg['conflicts'] . '-[0-9]*'; + } + $makefile[] = ""; + $makefile[] = "NO_BUILD=\tyes"; + $makefile[] = "NO_MTREE=\tyes"; + $makefile[] = ""; + $makefile[] = "SUB_FILES=\tpkg-install pkg-deinstall"; + $makefile[] = "SUB_LIST=\tPORTNAME=\${PORTNAME}"; + $makefile[] = ""; + $makefile[] = "do-extract:"; + $makefile[] = "\t\${MKDIR} \${WRKSRC}"; + $makefile[] = ""; + $makefile[] = "do-install:"; + foreach ($mkdirs as $item) { + $makefile[] = "\t" . $item; + } + foreach ($install as $item) { + $makefile[] = "\t" . $item; + } + $makefile[] = ""; + $makefile[] = ".include "; + + file_put_contents($port_path . '/Makefile', implode("\n", $makefile) . "\n"); + unset($makefile); + + file_put_contents($port_path . '/pkg-plist', implode("\n", $plist_files) . "\n"); + if (!empty($plist_dirs)) { + file_put_contents($port_path . '/pkg-plist', implode("\n", $plist_dirs) . "\n", FILE_APPEND); + } + unset($plist_files, $plist_dirs); +} + +function usage() { + global $argv; + echo "Usage: {$argv[0]} -x [-p ]\n"; + echo " Flags:\n"; + echo " -s Product version to pass to set_version.sh during chroot build\n"; + echo " -U Do NOT run build.sh --update-sources\n"; + echo " Examples:\n"; + echo " {$argv[0]} -x /home/packages/pkg_info.10.xml -p squid -s RELENG_2_2\n"; + exit; +} +?> diff --git a/tools/set_version.sh b/tools/set_version.sh new file mode 100755 index 0000000..7689dd7 --- /dev/null +++ b/tools/set_version.sh @@ -0,0 +1,127 @@ +#!/bin/sh +# +# build_snapshots.sh +# +# Copyright (c) 2007-2015 Electric Sheep Fencing, LLC +# All rights reserved +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE pfSense PROJECT ``AS IS'' AND ANY +# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE pfSense PROJECT OR +# ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +# OF THE POSSIBILITY OF SUCH DAMAGE. +# + +if [ $# -lt 1 ]; then + cat <&2 +Usage : $0 BRANCH [FREEBSD_REPO_BASE] +Example: $0 master git@git.pfmechanics.com:pfsense/freebsd-src.git + +BRANCH is required. +FREEBSD_REPO_BASE is required or the default of git@git.pfmechanics.com:pfsense/freebsd-src.git +will be used. + +END_OF_USAGE + exit 127 +fi + +export BUILDER_SCRIPTS=$(realpath $(dirname ${0})) + +# Ensure file exists +rm -f ${BUILDER_SCRIPTS}/build.conf +touch ${BUILDER_SCRIPTS}/build.conf + +# Source build.conf variables +. ${BUILDER_SCRIPTS}/builder_defaults.sh + +# Default FREEBSD_REPO_BASE +if [ "$2" != "" ]; then + FREEBSD_REPO_BASE="$2" +else + echo "WARNING: Setting FREEBSD repository to host git@git.pfmechanics.com:pfsense/freebsd-src.git" + echo + FREEBSD_REPO_BASE="git@git.pfmechanics.com:pfsense/freebsd-src.git" + sleep 2 +fi + +strip_build_conf() { + # Strip dynamic values + cat $BUILDER_SCRIPTS/build.conf | \ + grep -v FREEBSD_BRANCH | \ + grep -v FREEBSD_PARENT_BRANCH | \ + grep -v GIT_REPO_BRANCH_OR_TAG | \ + grep -v "set_version.sh" | \ + grep -v PRODUCT_VERSION > /tmp/build.conf + mv /tmp/build.conf $BUILDER_SCRIPTS/build.conf +} + +set_items() { + strip_build_conf + # Add our custom dynamic values + echo "# set_version.sh generated defaults" >> $BUILDER_SCRIPTS/build.conf + echo export PRODUCT_VERSION="${PRODUCT_VERSION}" >> $BUILDER_SCRIPTS/build.conf + echo export GIT_REPO_BRANCH_OR_TAG="${GIT_REPO_BRANCH_OR_TAG}" >> $BUILDER_SCRIPTS/build.conf + if [ -n "${FREEBSD_REPO_BASE}" ]; then + echo "export FREEBSD_REPO_BASE=${FREEBSD_REPO_BASE}" >> $BUILDER_SCRIPTS/build.conf + fi + echo export FREEBSD_BRANCH="${FREEBSD_BRANCH}" >> $BUILDER_SCRIPTS/build.conf + echo export FREEBSD_PARENT_BRANCH="${FREEBSD_PARENT_BRANCH}" >> $BUILDER_SCRIPTS/build.conf + if [ -n "$GIT_FREEBSD_COSHA1}" ]; then + echo "export GIT_FREEBSD_COSHA1=${GIT_FREEBSD_COSHA1}" >> $BUILDER_SCRIPTS/build.conf + fi + + # To speedup builds and reduce internet traffic + # Also recommended for snapshot builders + echo "#export NO_CLEANFREEBSDOBJDIR=YES " >> $BUILDER_SCRIPTS/build.conf + echo "#export NO_CLEANREPOS=YES " >> $BUILDER_SCRIPTS/build.conf + + # Output build.conf + echo + echo ">>> Custom build.conf contains:" + echo "---------------------------------------------------------------------------------------" + cat ${BUILDER_SCRIPTS}/build.conf + echo "---------------------------------------------------------------------------------------" + echo + echo " NOTE: build.conf values updated. These values override builder_defaults.sh !!" + echo +} + +echo + +case $1 in + HEAD|master) + export PRODUCT_VERSION=2.3-DEVELOPMENT + export GIT_REPO_BRANCH_OR_TAG=master + export FREEBSD_BRANCH=devel + export FREEBSD_PARENT_BRANCH=stable/10 + #export GIT_FREEBSD_COSHA1=30e366f556dde8950782845d6a3bdbc2c5a84b6f + set_items + ;; + *) + echo "Invalid version." + exit 1 +esac + +echo ">>> Setting builder environment to use ${GIT_REPO_BRANCH_OR_TAG} + ${FREEBSD_BRANCH} ..." + +(cd ${BUILDER_SCRIPTS} && ./build.sh --clean-builder) + +echo ">>> Please run './build.sh --setup' to get necessary packages installed" diff --git a/tools/templates/core_pkg/base/exclude_files b/tools/templates/core_pkg/base/exclude_files new file mode 100644 index 0000000..d21c093 --- /dev/null +++ b/tools/templates/core_pkg/base/exclude_files @@ -0,0 +1,26 @@ +./boot.config +./boot/loader.conf +./boot/loader.conf.local +./boot/modules/bwi_v3_ucode.ko +./cf/conf/config.xml +./cf/conf/enableserial_force +./cf/conf/trigger_initial_wizard +./conf.default/config.xml +./etc/bogons +./etc/bogonsv6 +./etc/fstab +./etc/group +./etc/master.passwd +./etc/master.passwd.db +./etc/passwd +./etc/platform +./etc/pwd.db +./etc/rc.conf +./etc/spwd.db +./media +./mnt +./proc +./sys +./tmp +./usr/local/share/%%PRODUCT_NAME%%/base.mtree +./usr/local/share/%%PRODUCT_NAME%%/base.txz diff --git a/tools/templates/core_pkg/base/metadir/+DESC b/tools/templates/core_pkg/base/metadir/+DESC new file mode 100644 index 0000000..34b9159 --- /dev/null +++ b/tools/templates/core_pkg/base/metadir/+DESC @@ -0,0 +1,3 @@ +%%PRODUCT_NAME%% core files + +WWW: %%PRODUCT_URL%% diff --git a/tools/templates/core_pkg/base/metadir/+MANIFEST b/tools/templates/core_pkg/base/metadir/+MANIFEST new file mode 100644 index 0000000..94b2057 --- /dev/null +++ b/tools/templates/core_pkg/base/metadir/+MANIFEST @@ -0,0 +1,15 @@ +name: "%%PRODUCT_NAME%%-base" +version: "%%VERSION%%" +origin: "security/%%PRODUCT_NAME%%-base" +comment: < Keeping a copy of current kernel in /boot/kernel.old" + +# Check if there is a current kernel to be moved +if [ ! -f /boot/kernel/kernel.gz ]; then + echo "ERROR: Current kernel not found" + exit 1 +fi + +[ -d /boot/kernel.old ] \ + rm -rf /boot/kernel.old + +cp -r /boot/kernel /boot/kernel.old +exit $? diff --git a/tools/templates/core_pkg/kernel/metadir/+DESC b/tools/templates/core_pkg/kernel/metadir/+DESC new file mode 100644 index 0000000..14c9552 --- /dev/null +++ b/tools/templates/core_pkg/kernel/metadir/+DESC @@ -0,0 +1,3 @@ +%%PRODUCT_NAME%% kernel + +WWW: %%PRODUCT_URL%% diff --git a/tools/templates/core_pkg/kernel/metadir/+MANIFEST b/tools/templates/core_pkg/kernel/metadir/+MANIFEST new file mode 100644 index 0000000..1770333 --- /dev/null +++ b/tools/templates/core_pkg/kernel/metadir/+MANIFEST @@ -0,0 +1,15 @@ +name: "%%PRODUCT_NAME%%-kernel-%%FLAVOR%%" +version: "%%VERSION%%" +origin: "security/%%PRODUCT_NAME%%-kernel" +comment: <