diff options
Diffstat (limited to 'tools/builder_common.sh')
-rw-r--r-- | tools/builder_common.sh | 2062 |
1 files changed, 2062 insertions, 0 deletions
diff --git a/tools/builder_common.sh b/tools/builder_common.sh new file mode 100644 index 0000000..8ff0b39 --- /dev/null +++ b/tools/builder_common.sh @@ -0,0 +1,2062 @@ +#!/bin/sh +# +# builder_common.sh +# +# Copyright (c) 2004-2015 Electric Sheep Fencing, LLC +# Copyright (C) 2014 Ermal Luçi +# All rights reserved. +# +# NanoBSD portions of the code +# Copyright (c) 2005 Poul-Henning Kamp. +# and copied from nanobsd.sh +# All rights reserved. +# +# FreeSBIE portions of the code +# Copyright (c) 2005 Dario Freni +# and copied from FreeSBIE project +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE pfSense PROJECT ``AS IS'' AND ANY +# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE pfSense PROJECT OR +# ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +# OF THE POSSIBILITY OF SUCH DAMAGE. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# + +if [ "$MAKEOBJDIRPREFIXFINAL" != "" ]; then + mkdir -p $MAKEOBJDIRPREFIXFINAL +else + echo "MAKEOBJDIRPREFIXFINAL is not defined" + print_error_pfS +fi + +lc() { + echo "${1}" | tr '[[:upper:]]' '[[:lower:]]' +} + +git_last_commit() { + if [ -d "${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}/.git" ]; then + CURRENT_COMMIT=$(cd ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} && git log -1 --format='%H') + CURRENT_AUTHOR=$(cd ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} && git log -1 --format='%an') + echo ">>> Last known commit $CURRENT_AUTHOR - $CURRENT_COMMIT" + echo "$CURRENT_COMMIT" > $SCRATCHDIR/build_commit_info.txt + else + echo ">>> WARNING: git repo is not cloned! Previous commit functions disabled." + fi +} + +# Create core pkg (base, kernel) +core_pkg_create() { + local _template="${1}" + local _flavor="${2}" + local _version="${3}" + local _root="${4}" + + [ -d "${CORE_PKG_TMP}" ] \ + && rm -rf ${CORE_PKG_TMP} + + local _templates_path=${BUILDER_SCRIPTS}/templates/core_pkg/${_template} + local _template_metadir=${_templates_path}/metadir + local _metadir=${CORE_PKG_TMP}/${_template}_metadir + + if [ ! -d ${_template_metadir} ]; then + echo "ERROR: Template dir not found for pkg ${_template}" + exit + fi + + mkdir -p ${CORE_PKG_TMP} + + cp -r ${_template_metadir} ${_metadir} + + local _manifest=${_metadir}/+MANIFEST + local _plist=${CORE_PKG_TMP}/${_template}_plist + local _exclude_plist=${CORE_PKG_TMP}/${_template}_exclude_plist + + if [ -f "${_templates_path}/pkg-plist" ]; then + cp ${_templates_path}/pkg-plist ${_plist} + else + (cd ${_root} && find . -type f -or -type l | sed 's,^.,,' | sort -u) > ${_plist} + fi + + if [ -f "${_templates_path}/exclude_plist" ]; then + cp ${_templates_path}/exclude_plist ${_exclude_plist} + else + touch ${_exclude_plist} + fi + + sed \ + -i '' \ + -e "s,%%PRODUCT_NAME%%,${PRODUCT_NAME},g" \ + -e "s,%%PRODUCT_URL%%,${PRODUCT_URL},g" \ + -e "s,%%FLAVOR%%,${_flavor},g" \ + -e "s,%%VERSION%%,${_version},g" \ + ${_metadir}/* \ + ${_plist} \ + ${exclude_plist} + + if [ -f "${_exclude_plist}" ]; then + sort -u ${_exclude_plist} > ${_plist}.exclude + mv ${_plist} ${_plist}.tmp + comm -23 ${_plist}.tmp ${_plist}.exclude > ${_plist} + rm -f ${_plist}.tmp ${plist}.exclude + fi + + mkdir -p ${CORE_PKG_PATH} + if ! pkg create -o ${CORE_PKG_PATH} -p ${_plist} -r ${_root} -m ${_metadir}; then + echo ">>> ERROR: Error building package ${_template} ${_flavor}" + print_error_pfS + fi +} + +# This routine will output that something went wrong +print_error_pfS() { + echo + echo "####################################" + echo "Something went wrong, check errors!" >&2 + echo "####################################" + echo + echo "NOTE: a lot of times you can run './build.sh --clean-builder' to resolve." + echo + if [ "$1" != "" ]; then + echo $1 + fi + [ -n "${LOGFILE:-}" ] && \ + echo "Log saved on ${LOGFILE}" && \ + tail -n20 ${LOGFILE} >&2 + echo + echo "Press enter to continue." + read ans + kill $$ + exit 1 +} + +prestage_on_ram_setup() { + _AVAIL_MEM=$(($(sysctl -n hw.usermem) / 1024 / 1024)) + if [ $_AVAIL_MEM -lt 2000 ]; then + echo ">>> Builder has less than 2GiB RAM skipping memory disks" + return + else + echo "######################################################################################" + echo + echo ">>> Builder has more than 1.4GiB RAM enabling memory disks" + echo ">>> WARNING: Remember to remove these memory disks by running $0 --disable-memorydisks" + echo + echo "######################################################################################" + fi + + if df /dev/ufs/prestagebacking >/dev/null 2>&1; then + echo ">>> Detected preexisting memory disk enabled for ${STAGE_CHROOT_DIR}." + else + mdconfig -a -t swap -u 10001 -s ${MEMORYDISK_SIZE} + mkdir -p ${STAGE_CHROOT_DIR} + newfs -L prestagebacking -U /dev/md10001 + mount /dev/ufs/prestagebacking ${STAGE_CHROOT_DIR} + fi + + if df /dev/ufs/finalstagebacking >/dev/null 2>&1; then + echo ">>> Detected preexisting memory disk enabled for ${FINAL_CHROOT_DIR}." + else + mdconfig -a -t swap -u 10002 -s ${MEMORYDISK_SIZE} + mkdir -p ${FINAL_CHROOT_DIR} + newfs -L finalstagebacking -U /dev/md10002 + mount /dev/ufs/finalstagebacking ${FINAL_CHROOT_DIR} + fi +} + +prestage_on_ram_cleanup() { + if [ -c /dev/md10001 ]; then + if [ -d ${STAGE_CHROOT_DIR} ]; then + umount ${STAGE_CHROOT_DIR} + fi + mdconfig -d -u 10001 + fi + if [ -c /dev/md10002 ]; then + if [ -d ${FINAL_CHROOT_DIR} ]; then + umount ${FINAL_CHROOT_DIR} + fi + mdconfig -d -u 10002 + fi +} + +# This routine will verify that the kernel has been +# installed OK to the staging area. +ensure_kernel_exists() { + if [ ! -f "$1/boot/kernel/kernel.gz" ]; then + echo ">>> ERROR: Could not locate $1/boot/kernel.gz" + print_error_pfS + fi + KERNEL_SIZE=$(stat -f "%z" $1/boot/kernel/kernel.gz) + if [ "$KERNEL_SIZE" -lt 3500 ]; then + echo ">>> ERROR: Kernel $1/boot/kernel.gz appears to be smaller than it should be: $KERNEL_SIZE" + print_error_pfS + fi +} + +get_pkg_name() { + echo "${PRODUCT_NAME}-${1}-${CORE_PKG_VERSION}" +} + +# This routine builds all related kernels +build_all_kernels() { + # Set KERNEL_BUILD_PATH if it has not been set + if [ -z "${KERNEL_BUILD_PATH}" ]; then + KERNEL_BUILD_PATH=$SCRATCHDIR/kernels + echo ">>> KERNEL_BUILD_PATH has not been set. Setting to ${KERNEL_BUILD_PATH}!" + fi + + [ -d "${KERNEL_BUILD_PATH}" ] \ + && rm -rf ${KERNEL_BUILD_PATH} + + # Build embedded kernel + for BUILD_KERNEL in $BUILD_KERNELS; do + unset KERNCONF + unset KERNEL_DESTDIR + unset KERNELCONF + unset KERNEL_NAME + export KERNCONF=$BUILD_KERNEL + export KERNEL_DESTDIR="$KERNEL_BUILD_PATH/$BUILD_KERNEL" + export KERNELCONF="$SRCDIR/sys/${TARGET}/conf/$BUILD_KERNEL" + export KERNEL_NAME=${BUILD_KERNEL} + + LOGFILE="${BUILDER_LOGS}/kernel.${KERNCONF}.${TARGET}.log" + echo ">>> Building $BUILD_KERNEL kernel." | tee -a ${LOGFILE} + + if [ ! -e "${SRCDIR}/sys/${TARGET}/conf/${BUILD_KERNEL}" ]; then + echo ">>> ERROR: Could not find $KERNELCONF" + print_error_pfS + fi + + if [ -n "${NO_BUILDKERNEL:-}" -a -f "${CORE_PKG_PATH}/$(get_pkg_name kernel-${KERNEL_NAME}).txz" ]; then + echo ">>> NO_BUILDKERNEL set, skipping build" | tee -a ${LOGFILE} + continue + fi + + export SRC_CONF=${SRC_CONF} + buildkernel + + echo ">>> Staging $BUILD_KERNEL kernel..." | tee -a ${LOGFILE} + installkernel + + ensure_kernel_exists $KERNEL_DESTDIR + + echo -n ">>> Creating pkg of $KERNEL_NAME kernel to staging area..." | tee -a ${LOGFILE} + core_pkg_create kernel ${KERNEL_NAME} ${CORE_PKG_VERSION} ${KERNEL_DESTDIR} + + rm -rf $KERNEL_DESTDIR 2>&1 1>/dev/null + + echo ".Done" | tee -a ${LOGFILE} + done +} + +install_default_kernel() { + if [ -z "${1}" ]; then + echo ">>> ERROR: install_default_kernel called without a kernel config name"| tee -a ${LOGFILE} + print_error_pfS + fi + + export KERNEL_NAME="${1}" + + echo -n ">>> Installing kernel to be used by image ${KERNEL_NAME}..." | tee -a ${LOGFILE} + + # Copy kernel package to chroot, otherwise pkg won't find it to install + if ! pkg_chroot_add ${FINAL_CHROOT_DIR} kernel-${KERNEL_NAME}; then + echo ">>> ERROR: Error installing kernel package $(get_pkg_name kernel-${KERNEL_NAME}).txz" | tee -a ${LOGFILE} + print_error_pfS + fi + + # Lock kernel to avoid user end up removing it for any reason + pkg_chroot ${FINAL_CHROOT_DIR} lock -q -y $(get_pkg_name kernel-${KERNEL_NAME}) + + if [ ! -f $FINAL_CHROOT_DIR/boot/kernel/kernel.gz ]; then + echo ">>> ERROR: No kernel installed on $FINAL_CHROOT_DIR and the resulting image will be unusable. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + mkdir -p $FINAL_CHROOT_DIR/pkgs + if [ -z "${2}" -o -n "${INSTALL_EXTRA_KERNELS}" ]; then + cp ${CORE_PKG_PATH}/$(get_pkg_name kernel-${KERNEL_NAME}).txz $FINAL_CHROOT_DIR/pkgs + if [ -n "${INSTALL_EXTRA_KERNELS}" ]; then + for _EXTRA_KERNEL in $INSTALL_EXTRA_KERNELS; do + _EXTRA_KERNEL_PATH=${CORE_PKG_PATH}/$(get_pkg_name kernel-${_EXTRA_KERNEL}).txz + if [ -f "${_EXTRA_KERNEL_PATH}" ]; then + echo -n ". adding ${_EXTRA_KERNEL_PATH} on image /pkgs folder" + cp ${_EXTRA_KERNEL_PATH} $FINAL_CHROOT_DIR/pkgs + else + echo ">>> ERROR: Requested kernel $(get_pkg_name kernel-${_EXTRA_KERNEL}).txz was not found to be put on image /pkgs folder!" + print_error_pfS + fi + done + fi + fi + echo "Done." | tee -a ${LOGFILE} + + unset KERNEL_NAME +} + +# Items that need to be fixed up that are +# specific to nanobsd builds +cust_fixup_nanobsd() { + local _NANO_WITH_VGA="${1}" + + echo ">>> Fixing up NanoBSD Specific items..." | tee -a ${LOGFILE} + + echo "nanobsd" > $FINAL_CHROOT_DIR/etc/platform + + local BOOTCONF=${FINAL_CHROOT_DIR}/boot.config + local LOADERCONF=${FINAL_CHROOT_DIR}/boot/loader.conf + + if [ "${_NANO_WITH_VGA}" = "nanobsd" ]; then + # Tell loader to use serial console early. + echo "-S115200 -h" >> ${BOOTCONF} + else + # Empty file to identify nanobsd_vga images + touch ${FINAL_CHROOT_DIR}/etc/nano_use_vga.txt + fi + + # Remove old console options if present. + [ -f "${LOADERCONF}" ] \ + && sed -i "" -Ee "/(console|boot_multicons|boot_serial|hint.uart)/d" ${LOADERCONF} + # Activate serial console+video console in loader.conf + echo 'autoboot_delay="5"' >> ${LOADERCONF} + echo 'loader_color="NO"' >> ${LOADERCONF} + echo 'beastie_disable="YES"' >> ${LOADERCONF} + echo 'boot_serial="YES"' >> ${LOADERCONF} + echo 'console="comconsole"' >> ${LOADERCONF} + echo 'comconsole_speed="115200"' >> ${LOADERCONF} +} + +# Creates a full update file +create_Full_update_tarball() { + mkdir -p $UPDATESDIR + + customize_stagearea_for_image "fullupdate" + install_default_kernel ${DEFAULT_KERNEL} + + rm -rf ${FINAL_CHROOT_DIR}/cf + rm -rf ${FINAL_CHROOT_DIR}/conf + rm -f ${FINAL_CHROOT_DIR}/etc/rc.conf + rm -f ${FINAL_CHROOT_DIR}/etc/pwd.db 2>/dev/null + rm -f ${FINAL_CHROOT_DIR}/etc/group 2>/dev/null + rm -f ${FINAL_CHROOT_DIR}/etc/spwd.db 2>/dev/null + rm -f ${FINAL_CHROOT_DIR}/etc/passwd 2>/dev/null + rm -f ${FINAL_CHROOT_DIR}/etc/master.passwd 2>/dev/null + rm -f ${FINAL_CHROOT_DIR}/etc/fstab 2>/dev/null + rm -f ${FINAL_CHROOT_DIR}/etc/bogons 2>/dev/null + # Remove loader.conf and friends. Ticket #560 + rm ${FINAL_CHROOT_DIR}/boot/loader.conf 2>/dev/null + rm ${FINAL_CHROOT_DIR}/boot/loader.conf.local 2>/dev/null + + echo ">>> Creating ${UPDATES_TARBALL_FILENAME} ..." | tee -a ${LOGFILE} + tar --exclude=./dev -czPf ${UPDATES_TARBALL_FILENAME} -C ${FINAL_CHROOT_DIR} . +} + +# Outputs various set variables aka env +print_flags() { + + echo + printf " Product version: %s\n" $PRODUCT_VERSION + printf " BASE_DIR: %s\n" $BASE_DIR + printf " Stage DIR: %s\n" $STAGE_CHROOT_DIR + printf " Updates dir: %s\n" $UPDATESDIR + printf " Image Preparation Stage DIR: %s\n" $FINAL_CHROOT_DIR + printf " Source DIR: %s\n" $SRCDIR + printf " FreeBSD repository: %s\n" $FREEBSD_REPO_BASE + printf " FreeBSD-src branch: %s\n" $FREEBSD_BRANCH + printf " FreeBSD original branch: %s\n" $FREEBSD_PARENT_BRANCH + printf " BUILD_KERNELS: %s\n" $BUILD_KERNELS + printf " Git Repository: %s\n" $GIT_REPO_URL + printf " Git Branch or Tag: %s\n" $GIT_REPO_BRANCH_OR_TAG + printf " MODULES_OVERRIDE: %s\n" $MODULES_OVERRIDE + printf " OVADISKSIZE: %s\n" $OVADISKSIZE + printf " OVABLOCKSIZE: %s\n" $OVABLOCKSIZE + printf " OVA_FIRST_PART_SIZE: %s\n" $OVA_FIRST_PART_SIZE + printf " OVA_SWAP_PART_SIZE: %s\n" $OVA_SWAP_PART_SIZE + printf " OVFFILE: %s\n" $OVFFILE + printf " OVFVMDK: %s\n" $OVFVMDK + printf " OVFCERT: %s\n" $OVFCERT + printf " SRC_CONF: %s\n" $SRC_CONF + printf " ISOPATH: %s\n" $ISOPATH + printf " MEMSTICKPATH: %s\n" $MEMSTICKPATH + printf " UPDATES_TARBALL_FILENAME: %s\n" $UPDATES_TARBALL_FILENAME +if [ -n "$SHOW_ENV" ]; then + for LINE in $(env | egrep -v '(terminal|PASS|NAME|USER|SSH|GROUP|HOST)'); do + echo "SHOW_ENV: $LINE" + done +fi + echo +} + +# This updates the product sources +update_product_repository() { + if [ ! -d "${GIT_REPO_DIR}" ]; then + echo ">>> Creating ${GIT_REPO_DIR}" + mkdir -p ${GIT_REPO_DIR} + fi + + echo ">>> Using GIT to checkout ${GIT_REPO_BRANCH_OR_TAG}" + + # There is already a cloned repo, test if branch is correct + if [ -d "${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}/.git" ]; then + if ! (cd ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} && git rev-parse --verify ${GIT_REPO_BRANCH_OR_TAG} >/dev/null 2>&1); then + echo -n ">>> ${PRODUCT_NAME} git repo is at wrong branch, removing it... " + rm -rf ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} + echo "Done!" + fi + fi + + if [ ! -d "${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}/.git" ]; then + echo -n ">>> Cloning ${GIT_REPO_URL} (${GIT_REPO_BRANCH_OR_TAG})... " + (git clone --depth 1 --single-branch --branch ${GIT_REPO_BRANCH_OR_TAG} ${GIT_REPO_URL} ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}) 2>&1 | \ + egrep -B3 -A3 -wi -E '(error)|fatal' + if [ ! -d "${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}/conf.default" ]; then + echo "Failed!" + echo + echo "!!!! An error occurred while checking out ${PRODUCT_NAME}" + echo " Could not locate ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}/conf.default" + echo + print_error_pfS + fi + echo "Done!" + else + # It is necessary for the case when a tag has moved + local TAG=$(cd ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} && git tag) + if [ "${TAG}" = "${GIT_REPO_BRANCH_OR_TAG}" ]; then + RESET_TARGET="${GIT_REPO_BRANCH_OR_TAG}" + else + RESET_TARGET="origin/${GIT_REPO_BRANCH_OR_TAG}" + fi + + # Fetch an update of the repository + if ! (cd ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} && \ + git fetch origin; \ + git reset --hard ${RESET_TARGET}; \ + git clean -fxd ) >/dev/null 2>&1 + then + echo ">>> ERROR: Something went wrong while checking out GIT." + print_error_pfS + fi + fi +} + +# This builds FreeBSD (make buildworld) +# Imported from FreeSBIE +make_world() { + + LOGFILE=${BUILDER_LOGS}/buildworld.${TARGET} + if [ -n "${NO_BUILDWORLD:-}" ]; then + echo ">>> NO_BUILDWORLD set, skipping build" | tee -a ${LOGFILE} + return + fi + + # Set SRC_CONF variable if it's not already set. + if [ -z "${SRC_CONF:-}" ]; then + echo ">>> SRC_CONF is unset make sure this is what you want!" | tee -a ${LOGFILE} + else + echo ">>> Setting SRC_CONF to $SRC_CONF" | tee -a ${LOGFILE} + fi + + # Set default parameters + export MAKE_ARGS="${MAKEJ_WORLD:-} __MAKE_CONF=${MAKE_CONF} SRCCONF=${SRC_CONF} TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH}" + + echo ">>> LOGFILE set to $LOGFILE." | tee -a ${LOGFILE} + makeargs="${MAKE_ARGS}" + echo ">>> Building world for ${TARGET} architecture... (Starting - $(LC_ALL=C date))" | tee -a ${LOGFILE} + echo ">>> Builder is running the command: env LOCAL_ITOOLS=\"${EXTRA_TOOLS}\" script -aq $LOGFILE make -C ${SRCDIR} -DNO_CLEAN ${makeargs:-} buildworld" | tee -a ${LOGFILE} + (env LOCAL_ITOOLS="${EXTRA_TOOLS}" script -aq $LOGFILE make -C ${SRCDIR} -DNO_CLEAN ${makeargs:-} buildworld || print_error_pfS;) | egrep '^>>>' | tee -a ${LOGFILE} + echo ">>> Building world for ${TARGET} architecture... (Finished - $(LC_ALL=C date))" | tee -a ${LOGFILE} + + LOGFILE=${BUILDER_LOGS}/installworld.${TARGET} + echo ">>> LOGFILE set to $LOGFILE." | tee -a ${LOGFILE} + # Create if cleaned up + mkdir -p ${STAGE_CHROOT_DIR} + makeargs="${MAKE_ARGS} DESTDIR=${STAGE_CHROOT_DIR} WITHOUT_TOOLCHAIN=1" + echo ">>> Installing world for ${TARGET} architecture... (Starting - $(LC_ALL=C date))" | tee -a ${LOGFILE} + echo ">>> Builder is running the command: env LOCAL_ITOOLS=\"${EXTRA_TOOLS}\" script -aq $LOGFILE make -C ${SRCDIR} ${makeargs:-} installworld" | tee -a ${LOGFILE} + (env LOCAL_ITOOLS="${EXTRA_TOOLS}" script -aq $LOGFILE make -C ${SRCDIR} ${makeargs:-} installworld || print_error_pfS;) | egrep '^>>>' | tee -a ${LOGFILE} + echo ">>> Installing world for ${TARGET} architecture... (Finished - $(LC_ALL=C date))" | tee -a ${LOGFILE} + + makeargs="${MAKE_ARGS} DESTDIR=${STAGE_CHROOT_DIR}" + echo ">>> Distribution world for ${TARGET} architecture... (Starting - $(LC_ALL=C date))" | tee -a ${LOGFILE} + echo ">>> Builder is running the command: script -aq $LOGFILE make -C ${SRCDIR} ${makeargs:-} distribution " | tee -a ${LOGFILE} + (script -aq $LOGFILE make -C ${SRCDIR} ${makeargs:-} distribution || print_error_pfS;) | egrep '^>>>' | tee -a ${LOGFILE} + echo ">>> Distribution world for ${TARGET} architecture... (Finished - $(LC_ALL=C date))" | tee -a ${LOGFILE} + + [ -d "${STAGE_CHROOT_DIR}/usr/local/bin" ] \ + || mkdir -p ${STAGE_CHROOT_DIR}/usr/local/bin + makeargs="${MAKE_ARGS} DESTDIR=${STAGE_CHROOT_DIR}" + echo ">>> Building and installing crypto tools and athstats for ${TARGET} architecture... (Starting - $(LC_ALL=C date))" | tee -a ${LOGFILE} + echo ">>> Builder is running the command: script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/crypto ${makeargs:-} clean all install " | tee -a ${LOGFILE} + (script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/crypto ${makeargs:-} clean all install || print_error_pfS;) | egrep '^>>>' | tee -a ${LOGFILE} + echo ">>> Builder is running the command: script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/ath/athstats ${makeargs:-} clean" | tee -a ${LOGFILE} + (script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/ath/athstats ${makeargs:-} clean || print_error_pfS;) | egrep '^>>>' | tee -a ${LOGFILE} + echo ">>> Builder is running the command: script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/ath/athstats ${makeargs:-} all" | tee -a ${LOGFILE} + (script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/ath/athstats ${makeargs:-} all || print_error_pfS;) | egrep '^>>>' | tee -a ${LOGFILE} + echo ">>> Builder is running the command: script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/ath/athstats ${makeargs:-} install" | tee -a ${LOGFILE} + (script -aq $LOGFILE make -C ${SRCDIR}/tools/tools/ath/athstats ${makeargs:-} install || print_error_pfS;) | egrep '^>>>' | tee -a ${LOGFILE} + echo ">>> Building and installing crypto tools and athstats for ${TARGET} architecture... (Finished - $(LC_ALL=C date))" | tee -a ${LOGFILE} + + unset makeargs +} + +# This routine originated in nanobsd.sh +nanobsd_set_flash_details () { + a1=$(echo $1 | tr '[:upper:]' '[:lower:]') + + # Source: + # SanDisk CompactFlash Memory Card + # Product Manual + # Version 10.9 + # Document No. 20-10-00038 + # April 2005 + # Table 2-7 + # NB: notice math error in SDCFJ-4096-388 line. + # + case "${a1}" in + 1024|1024m|1024mb|1g) + NANO_MEDIASIZE=$((997129216/512)) + ;; + 2048|2048m|2048mb|2g) + NANO_MEDIASIZE=$((1989999616/512)) + ;; + 4096|4096m|4096mb|4g) + NANO_MEDIASIZE=$((3989999616/512)) + ;; + 8192|8192m|8192mb|8g) + NANO_MEDIASIZE=$((7989999616/512)) + ;; + 16384|16384m|16384mb|16g) + NANO_MEDIASIZE=$((15989999616/512)) + ;; + *) + echo "Unknown Flash capacity" + exit 2 + ;; + esac + + NANO_HEADS=16 + NANO_SECTS=63 + + echo ">>> [nanoo] $1" + echo ">>> [nanoo] NANO_MEDIASIZE: $NANO_MEDIASIZE" + echo ">>> [nanoo] NANO_HEADS: $NANO_HEADS" + echo ">>> [nanoo] NANO_SECTS: $NANO_SECTS" + echo ">>> [nanoo] NANO_BOOT0CFG: $NANO_BOOT0CFG" +} + +# This routine originated in nanobsd.sh +create_nanobsd_diskimage () { + if [ -z "${1}" ]; then + echo ">>> ERROR: Type of image has not been specified" + print_error_pfS + fi + if [ -z "${2}" ]; then + echo ">>> ERROR: Size of image has not been specified" + print_error_pfS + fi + + if [ "${1}" = "nanobsd" ]; then + # It's serial + export NANO_BOOTLOADER="boot/boot0sio" + elif [ "${1}" = "nanobsd-vga" ]; then + # It's vga + export NANO_BOOTLOADER="boot/boot0" + else + echo ">>> ERROR: Type of image to create unknown" + print_error_pfS + fi + + if [ -z "${2}" ]; then + echo ">>> ERROR: Media size(s) not specified." + print_error_pfS + fi + + if [ -z "${2}" ]; then + echo ">>> ERROR: FLASH_SIZE is not set." + print_error_pfS + fi + + LOGFILE=${BUILDER_LOGS}/${1}.${TARGET} + # Prepare folder to be put in image + customize_stagearea_for_image "${1}" + install_default_kernel ${DEFAULT_KERNEL} "no" + + # Must be run after customize_stagearea_for_image + cust_fixup_nanobsd ${1} + + for _NANO_MEDIASIZE in ${2}; do + if [ -z "${_NANO_MEDIASIZE}" ]; then + continue; + fi + + echo ">>> building NanoBSD(${1}) disk image with size ${_NANO_MEDIASIZE} for platform (${TARGET})..." | tee -a ${LOGFILE} + echo "" > $BUILDER_LOGS/nanobsd_cmds.sh + + IMG="${MAKEOBJDIRPREFIXFINAL}/${PRODUCT_NAME}-${PRODUCT_VERSION}-${_NANO_MEDIASIZE}-${TARGET}-${1}-${DATESTRING}.img" + IMGUPDATE="${MAKEOBJDIRPREFIXFINAL}/${PRODUCT_NAME}-${PRODUCT_VERSION}-${_NANO_MEDIASIZE}-${TARGET}-${1}-upgrade-${DATESTRING}.img" + + nanobsd_set_flash_details ${_NANO_MEDIASIZE} + + # These are defined in FlashDevice and on builder_default.sh + echo $NANO_MEDIASIZE \ + $NANO_IMAGES \ + $NANO_SECTS \ + $NANO_HEADS \ + $NANO_CODESIZE \ + $NANO_CONFSIZE \ + $NANO_DATASIZE | +awk ' +{ + printf "# %s\n", $0 + + # size of cylinder in sectors + cs = $3 * $4 + + # number of full cylinders on media + cyl = int ($1 / cs) + + # output fdisk geometry spec, truncate cyls to 1023 + if (cyl <= 1023) + print "g c" cyl " h" $4 " s" $3 + else + print "g c" 1023 " h" $4 " s" $3 + + if ($7 > 0) { + # size of data partition in full cylinders + dsl = int (($7 + cs - 1) / cs) + } else { + dsl = 0; + } + + # size of config partition in full cylinders + csl = int (($6 + cs - 1) / cs) + + if ($5 == 0) { + # size of image partition(s) in full cylinders + isl = int ((cyl - dsl - csl) / $2) + } else { + isl = int (($5 + cs - 1) / cs) + } + + # First image partition start at second track + print "p 1 165 " $3, isl * cs - $3 + c = isl * cs; + + # Second image partition (if any) also starts offset one + # track to keep them identical. + if ($2 > 1) { + print "p 2 165 " $3 + c, isl * cs - $3 + c += isl * cs; + } + + # Config partition starts at cylinder boundary. + print "p 3 165 " c, csl * cs + c += csl * cs + + # Data partition (if any) starts at cylinder boundary. + if ($7 > 0) { + print "p 4 165 " c, dsl * cs + } else if ($7 < 0 && $1 > c) { + print "p 4 165 " c, $1 - c + } else if ($1 < c) { + print "Disk space overcommitted by", \ + c - $1, "sectors" > "/dev/stderr" + exit 2 + } + + # Force slice 1 to be marked active. This is necessary + # for booting the image from a USB device to work. + print "a 1" +} + ' > ${MAKEOBJDIRPREFIXFINAL}/_.fdisk + + MNT=${MAKEOBJDIRPREFIXFINAL}/_.mnt + mkdir -p ${MNT} + + dd if=/dev/zero of=${IMG} bs=${NANO_SECTS}b \ + count=0 seek=$((${NANO_MEDIASIZE}/${NANO_SECTS})) 2>&1 >> ${LOGFILE} + + MD=$(mdconfig -a -t vnode -f ${IMG} -x ${NANO_SECTS} -y ${NANO_HEADS}) + trap "mdconfig -d -u ${MD}; return" 1 2 15 EXIT + + fdisk -i -f ${MAKEOBJDIRPREFIXFINAL}/_.fdisk ${MD} 2>&1 >> ${LOGFILE} + fdisk ${MD} 2>&1 >> ${LOGFILE} + + boot0cfg -B -b ${FINAL_CHROOT_DIR}/${NANO_BOOTLOADER} ${NANO_BOOT0CFG} ${MD} 2>&1 >> ${LOGFILE} + + # Create first image + bsdlabel -m i386 -w -B -b ${FINAL_CHROOT_DIR}/boot/boot ${MD}s1 2>&1 >> ${LOGFILE} + bsdlabel -m i386 ${MD}s1 2>&1 >> ${LOGFILE} + local _label=$(lc ${PRODUCT_NAME}) + newfs -L ${_label}0 ${NANO_NEWFS} /dev/${MD}s1a 2>&1 >> ${LOGFILE} + mount /dev/ufs/${_label}0 ${MNT} + if [ $? -ne 0 ]; then + echo ">>> ERROR: Something wrong happened during mount of first slice image creation. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + # Consider the unmounting as well + trap "umount /dev/ufs/${_label}0; mdconfig -d -u ${MD}; return" 1 2 15 EXIT + + clone_directory_contents ${FINAL_CHROOT_DIR} ${MNT} + + # Set NanoBSD image size + echo "${_NANO_MEDIASIZE}" > ${MNT}/etc/nanosize.txt + rm -f $MNT/cf/conf/* 2>/dev/null + + echo "/dev/ufs/${_label}0 / ufs ro,sync,noatime 1 1" > ${MNT}/etc/fstab + if [ $NANO_CONFSIZE -gt 0 ] ; then + echo "/dev/ufs/cf /cf ufs ro,sync,noatime 1 1" >> ${MNT}/etc/fstab + fi + + umount ${MNT} + sync + # Restore the original trap + trap "mdconfig -d -u ${MD}; return" 1 2 15 EXIT + + # Setting NANO_IMAGES to 1 and NANO_INIT_IMG2 will tell + # NanoBSD to only create one partition. We default to 2 + # partitions in case anything happens to the first the + # operator can boot from the 2nd and should be OK. + + # Before just going to use dd for duplicate think! + # The images are created as sparse so lets take advantage + # of that by just exec some commands. + if [ $NANO_IMAGES -gt 1 -a $NANO_INIT_IMG2 -gt 0 ] ; then + # Duplicate to second image (if present) + echo ">>> Creating NanoBSD second slice by duplicating first slice." | tee -a ${LOGFILE} + # Create second image + dd if=/dev/${MD}s1 of=/dev/${MD}s2 conv=sparse bs=64k 2>&1 >> ${LOGFILE} + tunefs -L ${_label}1 /dev/${MD}s2a 2>&1 >> ${LOGFILE} + mount /dev/ufs/${_label}1 ${MNT} + if [ $? -ne 0 ]; then + echo ">>> ERROR: Something wrong happened during mount of second slice image creation. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + # Consider the unmounting as well + trap "umount /dev/ufs/${_label}1; mdconfig -d -u ${MD}; return" 1 2 15 EXIT + + echo "/dev/ufs/${_label}1 / ufs ro,sync,noatime 1 1" > ${MNT}/etc/fstab + if [ $NANO_CONFSIZE -gt 0 ] ; then + echo "/dev/ufs/cf /cf ufs ro,sync,noatime 1 1" >> ${MNT}/etc/fstab + fi + + umount ${MNT} + # Restore the trap back + trap "mdconfig -d -u ${MD}; return" 1 2 15 EXIT + fi + + # Create Data slice, if any. + # Note the changing of the variable to NANO_CONFSIZE + # from NANO_DATASIZE. We also added glabel support + # and populate the Product configuration from the /cf + # directory located in FINAL_CHROOT_DIR + if [ $NANO_CONFSIZE -gt 0 ] ; then + echo ">>> Creating /cf area to hold config.xml" + newfs -L cf ${NANO_NEWFS} /dev/${MD}s3 2>&1 >> ${LOGFILE} + # Mount data partition and copy contents of /cf + # Can be used later to create custom default config.xml while building + mount /dev/ufs/cf ${MNT} + if [ $? -ne 0 ]; then + echo ">>> ERROR: Something wrong happened during mount of cf slice image creation. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + # Consider the unmounting as well + trap "umount /dev/ufs/cf; mdconfig -d -u ${MD}; return" 1 2 15 EXIT + + clone_directory_contents ${FINAL_CHROOT_DIR}/cf ${MNT} + + umount ${MNT} + # Restore the trap back + trap "mdconfig -d -u ${MD}; return" 1 2 15 EXIT + else + ">>> [nanoo] NANO_CONFSIZE is not set. Not adding a /conf partition.. You sure about this??" | tee -a ${LOGFILE} + fi + + echo ">>> [nanoo] Creating NanoBSD upgrade file from first slice..." | tee -a ${LOGFILE} + dd if=/dev/${MD}s1 of=$IMGUPDATE conv=sparse bs=64k 2>&1 >> ${LOGFILE} + + mdconfig -d -u $MD + # Restore default action + trap "-" 1 2 15 EXIT + + # Check each image and ensure that they are over + # 3 megabytes. If either image is under 20 megabytes + # in size then error out. + IMGSIZE=$(stat -f "%z" ${IMG}) + IMGUPDATESIZE=$(stat -f "%z" ${IMGUPDATE}) + CHECKSIZE="20040710" + if [ "$IMGSIZE" -lt "$CHECKSIZE" ]; then + echo ">>> ERROR: Something went wrong when building NanoBSD. The image size is under 20 megabytes!" | tee -a ${LOGFILE} + print_error_pfS + fi + if [ "$IMGUPDATESIZE" -lt "$CHECKSIZE" ]; then + echo ">>> ERROR: Something went wrong when building NanoBSD upgrade image. The image size is under 20 megabytes!" | tee -a ${LOGFILE} + print_error_pfS + fi + + # Wrap up the show, Johnny + echo ">>> NanoBSD Image completed for size: $_NANO_MEDIASIZE." | tee -a ${LOGFILE} + + gzip -f $IMG & + gzip -f $IMGUPDATE & + done + + unset IMG + unset IMGUPDATE + unset IMGUPDATESIZE + unset IMGSIZE + + ls -lah $MAKEOBJDIRPREFIXFINAL +} + +# This routine creates a ova image that contains +# a ovf and vmdk file. These files can be imported +# right into vmware or virtual box. +# (and many other emulation platforms) +# http://www.vmware.com/pdf/ovf_whitepaper_specification.pdf +create_ova_image() { + # XXX create a .ovf php creator that you can pass: + # 1. populatedSize + # 2. license + # 3. product name + # 4. version + # 5. number of network interface cards + # 6. allocationUnits + # 7. capacity + # 8. capacityAllocationUnits + + LOGFILE=${BUILDER_LOGS}/ova.${TARGET}.log + + # Prepare folder to be put in image + customize_stagearea_for_image "ova" + install_default_kernel ${DEFAULT_KERNEL} "no" + + ova_prereq_check + ova_remove_old_tmp_files + ova_setup_ovf_file + ova_create_raw_backed_file + /bin/echo -n ">>> Creating mdconfig image ${OVFPATH}/${OVFVMDK}.raw... " | tee -a ${LOGFILE} + MD=$(mdconfig -a -t vnode -f ${OVFPATH}/${OVFVMDK}.raw) + # Just in case + trap "mdconfig -d -u ${MD}" 1 2 15 EXIT + + # comment out if using pc-sysinstall + ova_partition_gpart $MD + ova_mount_mnt $MD + ova_cpdup_files + ova_setup_platform_specific # after cpdup + ova_calculate_mnt_size + ova_umount_mnt $MD + # Restore default action + trap "-" 1 2 15 EXIT + ova_umount_mdconfig $MD + # We use vbox because it compresses the vmdk on export + ova_create_vbox_image + # We repack the file with a more universal xml file that + # works in both virtual box and esx server + ova_repack_vbox_image +} + +ova_repack_vbox_image() { + BUILDPLATFORM=$(uname -p) + POPULATEDSIZE=$(du -d0 -m $FINAL_CHROOT_DIR | awk '{ print \$1 }') + POPULATEDSIZEBYTES=$(echo "${POPULATEDSIZE}*1024^2" | bc) + REFERENCESSIZE=$(stat -f "%z" ${OVFPATH}/${OVFVMDK}) + echo ">>> Setting REFERENCESSIZE to ${REFERENCESSIZE}..." | tee -a ${LOGFILE} + file_search_replace REFERENCESSIZE ${REFERENCESSIZE} ${OVFPATH}/${PRODUCT_NAME}.ovf + echo ">>> Setting POPULATEDSIZEBYTES to ${POPULATEDSIZEBYTES}..." | tee -a ${LOGFILE} + # OperatingSystemSection (${PRODUCT_NAME}.ovf) + # 42 FreeBSD 32-Bit + # 78 FreeBSD 64-Bit + if [ "$BUILDPLATFORM" = "i386" ]; then + file_search_replace '"101"' '"42"' ${OVFPATH}/${PRODUCT_NAME}.ovf + file_search_replace 'FreeBSD XX-Bit' 'FreeBSD' ${OVFPATH}/${PRODUCT_NAME}.ovf + fi + if [ "$BUILDPLATFORM" = "amd64" ]; then + file_search_replace '"101"' '"78"' ${OVFPATH}/${PRODUCT_NAME}.ovf + file_search_replace 'FreeBSD XX-Bit' 'FreeBSD 64-Bit' ${OVFPATH}/${PRODUCT_NAME}.ovf + fi + file_search_replace DISKSECTIONPOPULATEDSIZE $POPULATEDSIZEBYTES ${OVFPATH}/${PRODUCT_NAME}.ovf + # 10737254400 = 10240MB = virtual box vmdk file size XXX grab this value from vbox creation + # 10737418240 = 10GB + echo ">>> Setting DISKSECTIONALLOCATIONUNITS to 10737254400..." | tee -a ${LOGFILE} + file_search_replace DISKSECTIONALLOCATIONUNITS $OVA_DISKSECTIONALLOCATIONUNITS ${OVFPATH}/${PRODUCT_NAME}.ovf + echo ">>> Setting DISKSECTIONCAPACITY to 10737418240..." | tee -a ${LOGFILE} + file_search_replace DISKSECTIONCAPACITY $OVADISKSIZE ${OVFPATH}/${PRODUCT_NAME}.ovf + echo ">>> Repacking OVA with universal OVF file..." | tee -a ${LOGFILE} + mv ${OVFPATH}/${OVFVMDK} ${OVFPATH}/${PRODUCT_NAME}-disk1.vmdk + gtar -C ${OVFPATH} -cpf ${PRODUCT_NAME}.ova ${PRODUCT_NAME}.ovf ${PRODUCT_NAME}-disk1.vmdk + rm $OVFPATH/${PRODUCT_NAME}-disk1.vmdk + ls -lah ${OVFPATH}/${PRODUCT_NAME}*ov* +} + +# called from create_ova_image +ova_umount_mnt() { + # Unmount /dev/mdX + umount /mnt + sync ; sync +} + +# called from create_ova_image +ova_umount_mdconfig() { + MD=$1 + # Show gpart info + gpart show $MD + echo ">>> Unmounting ${MD}..." | tee -a ${LOGFILE} + mdconfig -d -u $MD + sync ; sync +} + +# called from create_ova_image +ova_mount_mnt() { + MD=$1 + echo ">>> Mounting image to /mnt..." | tee -a ${LOGFILE} + mount -o rw /dev/${MD}p2 /mnt/ +} + +# called from create_ova_image +ova_setup_ovf_file() { + if [ -f ${OVFFILE} ]; then + cp ${OVFFILE} ${OVFPATH}/${PRODUCT_NAME}.ovf + fi + + if [ ! -f ${OVFPATH}/${PRODUCT_NAME}.ovf ]; then + cp ${BUILDER_SCRIPTS}/conf/ovf/${PRODUCT_NAME}.ovf ${OVFPATH}/${PRODUCT_NAME}.ovf + file_search_replace PRODUCT_VERSION $PRODUCT_VERSION ${OVFPATH}/${PRODUCT_NAME}.ovf + file_search_replace PRODUCT_URL $PRODUCT_URL ${OVFPATH}/${PRODUCT_NAME}.ovf + fi +} + +# called from create_ova_image +ova_prereq_check() { + if [ ! -f /usr/local/bin/vmdktool ]; then + echo ">>> ERROR: vmdktool is not present please check port sysutils/vmdktool installation" | tee -a ${LOGFILE} + print_error_pfS + fi + sysctl kern.geom.debugflags=16 +} + +# called from create_ova_image +ova_calculate_mnt_size() { + /bin/echo -n ">>> Calculating size of /mnt..." | tee -a ${LOFGILE} + INSTALLSIZE=$(du -s /mnt/ | awk '{ print $1 }') + INSTALLSIZEH=$(du -d0 -h /mnt/ | awk '{ print $1 }') + echo $INSTALLSIZEH +} + +# called from create_ova_image +ova_create_raw_backed_file() { + DISKSIZE=$OVADISKSIZE + BLOCKSIZE=$OVABLOCKSIZE + COUNT=$((${DISKSIZE}/${BLOCKSIZE})) + DISKFILE=${OVFPATH}/${OVFVMDK}.raw + echo ">>> Creating raw backing file ${DISKFILE} (Disk Size: ${DISKSIZE}, Block Size: ${BLOCKSIZE}, Count: ${COUNT})..." | tee -a ${LOGFILE} + dd if=/dev/zero of=$DISKFILE bs=$BLOCKSIZE count=0 seek=$COUNT +} + +# called from create_ova_image +ova_remove_old_tmp_files() { + rm ${OVFPATH}/*.ovf.final 2>/dev/null + rm ${OVFPATH}/*.ova 2>/dev/null +} + +# called from create_ova_image +ova_create_vbox_image() { + # VirtualBox + echo ">>> Creating image using vmdktool..." | tee -a ${LOGFILE} + rm ${OVFPATH}/${OVFVMDK} 2>/dev/null + vmdktool -v ${OVFPATH}/${OVFVMDK} ${OVFPATH}/${OVFVMDK}.raw + rm -rf ${OVFPATH}/${OVFVMDK}.raw + echo ">>> ${OVFPATH}/${OVFVMDK} created." | tee -a ${LOGFILE} +} + +# called from create_ova_image +ova_cpdup_files() { + echo ">>> Populating vmdk staging area..." + cpdup -o ${FINAL_CHROOT_DIR}/COPYRIGHT /mnt/COPYRIGHT + cpdup -o ${FINAL_CHROOT_DIR}/boot /mnt/boot + cpdup -o ${FINAL_CHROOT_DIR}/bin /mnt/bin + cpdup -o ${FINAL_CHROOT_DIR}/cf/conf /mnt/cf/conf + cpdup -o ${FINAL_CHROOT_DIR}/conf.default /mnt/conf.default + cpdup -o ${FINAL_CHROOT_DIR}/dev /mnt/dev + cpdup -o ${FINAL_CHROOT_DIR}/etc /mnt/etc + cpdup -o ${FINAL_CHROOT_DIR}/home /mnt/home + cpdup -o ${FINAL_CHROOT_DIR}/pkgs /mnt/pkgs + cpdup -o ${FINAL_CHROOT_DIR}/libexec /mnt/libexec + cpdup -o ${FINAL_CHROOT_DIR}/lib /mnt/lib + cpdup -o ${FINAL_CHROOT_DIR}/root /mnt/root + cpdup -o ${FINAL_CHROOT_DIR}/sbin /mnt/sbin + cpdup -o ${FINAL_CHROOT_DIR}/usr /mnt/usr + cpdup -o ${FINAL_CHROOT_DIR}/var /mnt/var + sync ; sync ; sync ; sync +} + +ova_setup_platform_specific() { + echo ">>> Installing platform specific items..." | tee -a ${LOGFILE} + echo "/dev/label/${PRODUCT_NAME} / ufs rw 0 0" > /mnt/etc/fstab + echo "/dev/label/swap0 none swap sw 0 0" >> /mnt/etc/fstab + echo ${PRODUCT_NAME} > /mnt/etc/platform + rmdir /mnt/conf + mkdir -p /mnt/cf + mkdir -p /mnt/cf/conf + cp /mnt/conf.default/config.xml /mnt/cf/conf/ + chroot /mnt /bin/ln -s /cf/conf /conf + mkdir -p /mnt/tmp +} + +# called from create_ova_image +ova_partition_gpart() { + # XXX: Switch to mkimg tool!! + MD=$1 + echo ">>> Creating GPT..." | tee -a ${LOGFILE} + gpart create -s gpt $MD + echo ">>> Embedding GPT bootstrap into protective MBR..." | tee -a ${LOGFILE} + gpart bootcode -b /boot/pmbr $MD + echo ">>> Creating GPT boot partition..." | tee -a ${LOGFILE} + gpart add -b 34 -s 128 -t freebsd-boot $MD + gpart bootcode -p /boot/gptboot -i 1 $MD + echo ">>> Setting up disk slices: ${MD}p2 (Size: ${OVA_FIRST_PART_SIZE})..." | tee -a ${LOGFILE} + gpart add -s $OVA_FIRST_PART_SIZE -t freebsd-ufs -i 2 $MD + echo ">>> Setting up disk slices: ${MD}p3 (swap) (Size: ${OVA_SWAP_PART_SIZE})..." | tee -a ${LOGFILE} + gpart add -s $OVA_SWAP_PART_SIZE -t freebsd-swap -i 3 $MD + echo ">>> Running newfs..." | tee -a ${LOGFILE} + newfs -U /dev/${MD}p2 + sync ; sync ; sync ; sync + echo ">>> Labeling partitions: ${MD}p2..." | tee -a ${LOGFILE} + glabel label ${PRODUCT_NAME} ${MD}p2 + sync ; sync + echo ">>> Labeling partitions: ${MD}p3..." | tee -a ${LOGFILE} + glabel label swap0 ${MD}p3 + sync ; sync +} + +# called from create_ova_image +# This routine will replace a string in a file +file_search_replace() { + local SEARCH="$1" + local REPLACE="$2" + local FILENAME="$3" + + if [ -f "${FILENAME}" ]; then + sed -i '' -e "s/${SEARCH}/${REPLACE}/g" ${FILENAME} + fi +} + +# Cleans up previous builds +clean_obj_dir() { + # Clean out directories + echo ">>> Cleaning up previous build environment...Please wait!" + + staginareas_clean_each_run + + if [ -d "${STAGE_CHROOT_DIR}" ]; then + BASENAME=$(basename ${STAGE_CHROOT_DIR}) + echo -n ">>> Cleaning ${STAGE_CHROOT_DIR} ..." + chflags -R noschg ${STAGE_CHROOT_DIR} 2>&1 >/dev/null + rm -rf ${STAGE_CHROOT_DIR} 2>/dev/null + echo "Done." + fi + mkdir -p ${STAGE_CHROOT_DIR} + + if [ -z "${NO_CLEANFREEBSDOBJDIR}" -a -d "${SRCDIR}" ]; then + OBJTREE=$(env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} make -C ${SRCDIR} -V OBJTREE) + if [ -d "${OBJTREE}" ]; then + echo -n ">>> Cleaning FreeBSD objects dir staging..." + echo -n "." + chflags -R noschg ${OBJTREE} 2>&1 >/dev/null + echo -n "." + rm -rf ${OBJTREE}/* + echo "Done!" + fi + if [ -n "${KERNEL_DESTDIR}" -a -d "${KERNEL_BUILD_PATH}" ]; then + echo -n ">>> Cleaning previously built kernel stage area..." + rm -rf $KERNEL_BUILD_PATH/* + echo "Done!" + fi + fi + mkdir -p $KERNEL_BUILD_PATH + + echo -n ">>> Cleaning previously built images..." + rm -rf $MAKEOBJDIRPREFIXFINAL/* + echo "Done!" + + if [ -z "${NO_CLEANREPOS}" ]; then + if [ -d "${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}" ]; then + echo -n ">>> Cleaning ${PRODUCT_NAME} repo checkout..." + echo -n "." + rm -rf "${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG}" + echo "Done!" + fi + + if [ -d "$SRCDIR" ]; then + echo -n ">>> Ensuring $SRCDIR is clean..." + rm -rf ${SRCDIR} + echo "Done!" + fi + fi + + echo -n ">>> Cleaning previous builder logs..." + if [ -d "$BUILDER_LOGS" ]; then + rm -rf ${BUILDER_LOGS} + fi + mkdir -p ${BUILDER_LOGS} + + echo "Done!" + + echo ">>> Cleaning of builder environment has finished." +} + +# This routine ensures that the $SRCDIR has sources +# and is ready for action / building. +ensure_source_directories_present() { + update_freebsd_sources + + update_product_repository +} + +clone_directory_contents() { + if [ ! -d "$1" -o ! -d "$2" ]; then + if [ -z "${LOGFILE}" ]; then + echo ">>> ERROR: Argument $1 supplied is not a directory!" + else + echo ">>> ERROR: Argument $1 supplied is not a directory!" | tee -a ${LOGFILE} + fi + print_error_pfS + fi + echo -n ">>> Using TAR to clone $1 to $2 ..." + tar -C ${1} -c -f - . | tar -C ${2} -x -p -f - + echo "Done!" +} + +clone_to_staging_area() { + # Clone everything to the final staging area + echo -n ">>> Cloning everything to ${STAGE_CHROOT_DIR} staging area..." + LOGFILE=${BUILDER_LOGS}/cloning.${TARGET}.log + + if [ ! -d ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} ]; then + echo "ERROR!" + echo ">>> ERROR: ${PRODUCT_NAME} repository does not seem to be there please fix...STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + + tar --exclude=\.git -C ${GIT_REPO_DIR}/${GIT_REPO_BRANCH_OR_TAG} -c -f - . | \ + tar -C ${STAGE_CHROOT_DIR} -x -p -f - + + if [ -f ${STAGE_CHROOT_DIR}/etc/master.passwd ]; then + chroot ${STAGE_CHROOT_DIR} cap_mkdb /etc/master.passwd + chroot ${STAGE_CHROOT_DIR} pwd_mkdb /etc/master.passwd + fi + mkdir -p ${STAGE_CHROOT_DIR}/etc/mtree + mtree -Pcp ${STAGE_CHROOT_DIR}/var > ${STAGE_CHROOT_DIR}/etc/mtree/var.dist + mtree -Pcp ${STAGE_CHROOT_DIR}/etc > ${STAGE_CHROOT_DIR}/etc/mtree/etc.dist + if [ -d ${STAGE_CHROOT_DIR}/usr/local/etc ]; then + mtree -Pcp ${STAGE_CHROOT_DIR}/usr/local/etc > ${STAGE_CHROOT_DIR}/etc/mtree/localetc.dist + fi + + ## Add buildtime and lastcommit information + # This is used for detecting updates. + echo "$BUILTDATESTRING" > $STAGE_CHROOT_DIR/etc/version.buildtime + # Record last commit info if it is available. + if [ -f $SCRATCHDIR/build_commit_info.txt ]; then + cp $SCRATCHDIR/build_commit_info.txt $STAGE_CHROOT_DIR/etc/version.lastcommit + fi + + local _exclude_files="${CORE_PKG_TMP}/base_exclude_files" + sed \ + -e "s,%%PRODUCT_NAME%%,${PRODUCT_NAME},g" \ + -e "s,%%FLAVOR%%,${_flavor},g" \ + -e "s,%%VERSION%%,${_version},g" \ + ${BUILDER_SCRIPTS}/templates/core_pkg/base/exclude_files \ + > ${_exclude_files} + + mkdir -p ${STAGE_CHROOT_DIR}/usr/local/share/${PRODUCT_NAME} >/dev/null 2>&1 + mtree \ + -c \ + -k uid,gid,mode,size,flags,sha256digest \ + -p ${STAGE_CHROOT_DIR} \ + -X ${_exclude_files} \ + > ${STAGE_CHROOT_DIR}/usr/local/share/${PRODUCT_NAME}/base.mtree + tar \ + -C ${STAGE_CHROOT_DIR} \ + -cJf ${STAGE_CHROOT_DIR}/usr/local/share/${PRODUCT_NAME}/base.txz \ + -X ${_exclude_files} \ + . + + core_pkg_create base "" ${CORE_PKG_VERSION} ${STAGE_CHROOT_DIR} + core_pkg_create default-config "" ${CORE_PKG_VERSION} ${STAGE_CHROOT_DIR} + + local DEFAULTCONF=${STAGE_CHROOT_DIR}/conf.default/config.xml + # Activate serial console in config.xml + # If it was there before, clear the setting to be sure we don't add it twice. + sed -i "" -e "/ <enableserial\/>/d" ${DEFAULTCONF} + # Enable serial in the config + sed -i "" -e "s/ <\/system>/ <enableserial\/>\\$(echo -e \\\n) <\/system>/" ${DEFAULTCONF} + + echo force > ${STAGE_CHROOT_DIR}/cf/conf/enableserial_force + + core_pkg_create default-config-serial "" ${CORE_PKG_VERSION} ${STAGE_CHROOT_DIR} + + rm -f ${STAGE_CHROOT_DIR}/cf/conf/enableserial_force + rm -f ${STAGE_CHROOT_DIR}/cf/conf/config.xml + + # Make sure pkg is present + pkg_bootstrap ${STAGE_CHROOT_DIR} + + pkg_chroot_add ${STAGE_CHROOT_DIR} base + + echo "Done!" +} + +create_final_staging_area() { + if [ -z "${FINAL_CHROOT_DIR}" ]; then + echo ">>> ERROR: FINAL_CHROOT_DIR is not set, cannot continue!" | tee -a ${LOGFILE} + print_error_pfS + fi + + if [ -d "${FINAL_CHROOT_DIR}" ]; then + echo -n ">>> Previous ${FINAL_CHROOT_DIR} detected cleaning up..." | tee -a ${LOGFILE} + chflags -R noschg ${FINAL_CHROOT_DIR} 2>&1 1>/dev/null + rm -rf ${FINAL_CHROOT_DIR}/* 2>&1 1>/dev/null + echo "Done." | tee -a ${LOGFILE} + fi + mkdir -p ${FINAL_CHROOT_DIR} + + echo ">>> Preparing Final image staging area: $(LC_ALL=C date)" 2>&1 | tee -a ${LOGFILE} + echo ">>> Cloning ${STAGE_CHROOT_DIR} to ${FINAL_CHROOT_DIR}" 2>&1 | tee -a ${LOGFILE} + clone_directory_contents ${STAGE_CHROOT_DIR} ${FINAL_CHROOT_DIR} + + if [ ! -f $FINAL_CHROOT_DIR/sbin/init ]; then + echo ">>> ERROR: Something went wrong during cloning -- Please verify!" 2>&1 | tee -a ${LOGFILE} + print_error_pfS + fi +} + +customize_stagearea_for_image() { + # Prepare final stage area + create_final_staging_area + + if [ "${1}" = "iso" -o \ + "${1}" = "memstick" -o \ + "${1}" = "memstickserial" -o \ + "${1}" = "memstickadi" ]; then + install_bsdinstaller + mkdir -p ${FINAL_CHROOT_DIR}/pkgs + cp ${CORE_PKG_PATH}/*default-config*.txz ${FINAL_CHROOT_DIR}/pkgs + fi + + if [ "${1}" = "nanobsd" -o \ + "${1}" = "memstickserial" -o \ + "${1}" = "memstickadi" ]; then + pkg_chroot_add ${FINAL_CHROOT_DIR} default-config-serial + else + pkg_chroot_add ${FINAL_CHROOT_DIR} default-config + fi +} + +create_distribution_tarball() { + mkdir -p ${FINAL_CHROOT_DIR}/install + + tar -C ${FINAL_CHROOT_DIR} --exclude ./install --exclude ./pkgs -cJf ${FINAL_CHROOT_DIR}/install/${PRODUCT_NAME}.txz . +} + +create_iso_image() { + LOGFILE=${BUILDER_LOGS}/isoimage.${TARGET} + echo ">>> Building bootable ISO image for ${TARGET}" | tee -a ${LOGFILE} + if [ -z "${DEFAULT_KERNEL}" ]; then + echo ">>> ERROR: Could not identify DEFAULT_KERNEL to install on image!" | tee -a ${LOGFILE} + print_error_pfS + fi + + customize_stagearea_for_image "iso" + install_default_kernel ${DEFAULT_KERNEL} + + echo cdrom > $FINAL_CHROOT_DIR/etc/platform + + FSLABEL=$(echo ${PRODUCT_NAME} | tr '[:lower:]' '[:upper:]') + echo "/dev/iso9660/${FSLABEL} / cd9660 ro 0 0" > ${FINAL_CHROOT_DIR}/etc/fstab + + # This check is for supporting create memstick/ova images + echo -n ">>> Running command: script -aq $LOGFILE makefs -t cd9660 -o bootimage=\"i386;${FINAL_CHROOT_DIR}/boot/cdboot \"-o no-emul-boot -o rockridge " | tee -a ${LOGFILE} + echo "-o label=${FSLABEL} -o publisher=\"${PRODUCT_NAME} project.\" $ISOPATH ${FINAL_CHROOT_DIR}" | tee -a ${LOGFILE} + + create_distribution_tarball + + # Remove /rescue from iso since cd9660 cannot deal with hardlinks + rm -rf ${FINAL_CHROOT_DIR}/rescue + + makefs -t cd9660 -o bootimage="i386;${FINAL_CHROOT_DIR}/boot/cdboot" -o no-emul-boot -o rockridge \ + -o label=${FSLABEL} -o publisher="${PRODUCT_NAME} project." $ISOPATH ${FINAL_CHROOT_DIR} 2>&1 >> ${LOGFILE} + if [ $? -ne 0 -o ! -f $ISOPATH ]; then + if [ -f ${ISOPATH} ]; then + rm -f $ISOPATH + fi + echo ">>> ERROR: Something wrong happened during ISO image creation. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + gzip -qf $ISOPATH & + + echo ">>> ISO created: $(LC_ALL=C date)" | tee -a ${LOGFILE} +} + +create_memstick_image() { + + LOGFILE=${BUILDER_LOGS}/memstick.${TARGET} + if [ "${MEMSTICKPATH}" = "" ]; then + echo ">>> MEMSTICKPATH is empty skipping generation of memstick image!" | tee -a ${LOGFILE} + return + fi + + if [ ! -d ${FINAL_CHROOT_DIR}/boot ]; then + customize_stagearea_for_image "memstick" + install_default_kernel ${DEFAULT_KERNEL} + fi + + echo cdrom > $FINAL_CHROOT_DIR/etc/platform + + echo ">>> Creating memstick to ${MEMSTICKPATH}." 2>&1 | tee -a ${LOGFILE} + echo "/dev/ufs/${PRODUCT_NAME} / ufs ro 0 0" > ${FINAL_CHROOT_DIR}/etc/fstab + echo "kern.cam.boot_delay=10000" >> ${FINAL_CHROOT_DIR}/boot/loader.conf.local + + create_distribution_tarball + + makefs -B little -o label=${PRODUCT_NAME} ${MEMSTICKPATH} ${FINAL_CHROOT_DIR} + if [ $? -ne 0 ]; then + if [ -f ${MEMSTICKPATH} ]; then + rm -f $MEMSTICKPATH + fi + echo ">>> ERROR: Something wrong happened during MEMSTICK image creation. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + MD=$(mdconfig -a -t vnode -f $MEMSTICKPATH) + # Just in case + trap "mdconfig -d -u ${MD}" 1 2 15 EXIT + gpart create -s BSD ${MD} 2>&1 >> ${LOGFILE}| + gpart bootcode -b ${FINAL_CHROOT_DIR}/boot/boot ${MD} 2>&1 >> ${LOGFILE} + gpart add -t freebsd-ufs ${MD} 2>&1 >> ${LOGFILE} + trap "-" 1 2 15 EXIT + mdconfig -d -u ${MD} 2>&1 | tee -a ${LOGFILE} + gzip -qf $MEMSTICKPATH & + + echo ">>> MEMSTICK created: $(LC_ALL=C date)" | tee -a ${LOGFILE} +} + +create_memstick_serial_image() { + LOGFILE=${BUILDER_LOGS}/memstickserial.${TARGET} + if [ "${MEMSTICKSERIALPATH}" = "" ]; then + echo ">>> MEMSTICKSERIALPATH is empty skipping generation of memstick image!" | tee -a ${LOGFILE} + return + fi + + if [ ! -d ${FINAL_CHROOT_DIR}/boot ]; then + customize_stagearea_for_image "memstickserial" + install_default_kernel ${DEFAULT_KERNEL} + fi + + echo cdrom > $FINAL_CHROOT_DIR/etc/platform + + echo "/dev/ufs/${PRODUCT_NAME} / ufs ro 0 0" > ${FINAL_CHROOT_DIR}/etc/fstab + echo "kern.cam.boot_delay=10000" >> ${FINAL_CHROOT_DIR}/boot/loader.conf.local + + echo ">>> Creating serial memstick to ${MEMSTICKSERIALPATH}." 2>&1 | tee -a ${LOGFILE} + + BOOTCONF=${FINAL_CHROOT_DIR}/boot.config + LOADERCONF=${FINAL_CHROOT_DIR}/boot/loader.conf + + echo ">>> Activating serial console..." 2>&1 | tee -a ${LOGFILE} + # Activate serial console in boot.config + if [ -f ${BOOTCONF} ]; then + sed -i "" '/-D$/d' ${BOOTCONF} + fi + echo "-S115200 -D" >> ${BOOTCONF} + + # Remove old console options if present. + [ -f "${LOADERCONF}" ] \ + && sed -i "" -Ee "/(console|boot_multicons|boot_serial)/d" ${LOADERCONF} + # Activate serial console+video console in loader.conf + echo 'boot_multicons="YES"' >> ${LOADERCONF} + echo 'boot_serial="YES"' >> ${LOADERCONF} + echo 'console="comconsole,vidconsole"' >> ${LOADERCONF} + echo 'comconsole_speed="115200"' >> ${LOADERCONF} + + create_distribution_tarball + + makefs -B little -o label=${PRODUCT_NAME} ${MEMSTICKSERIALPATH} ${FINAL_CHROOT_DIR} + if [ $? -ne 0 ]; then + if [ -f ${MEMSTICKSERIALPATH} ]; then + rm -f $MEMSTICKSERIALPATH + fi + echo ">>> ERROR: Something wrong happened during MEMSTICKSERIAL image creation. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + MD=$(mdconfig -a -t vnode -f $MEMSTICKSERIALPATH) + # Just in case + trap "mdconfig -d -u ${MD}" 1 2 15 EXIT + gpart create -s BSD ${MD} 2>&1 >> ${LOGFILE} + gpart bootcode -b ${FINAL_CHROOT_DIR}/boot/boot ${MD} 2>&1 >> ${LOGFILE} + gpart add -t freebsd-ufs ${MD} 2>&1 >> ${LOGFILE} + trap "-" 1 2 15 EXIT + mdconfig -d -u ${MD} 2>&1 >> ${LOGFILE} + gzip -qf $MEMSTICKSERIALPATH & + + echo ">>> MEMSTICKSERIAL created: $(LC_ALL=C date)" | tee -a ${LOGFILE} +} + +create_memstick_adi_image() { + LOGFILE=${BUILDER_LOGS}/memstickadi${TARGET} + if [ "${MEMSTICKADIPATH}" = "" ]; then + echo ">>> MEMSTICKADIPATH is empty skipping generation of memstick image!" | tee -a ${LOGFILE} + return + fi + + if [ ! -d ${FINAL_CHROOT_DIR}/boot ]; then + customize_stagearea_for_image "memstickadi" + install_default_kernel ${DEFAULT_KERNEL} + fi + + echo cdrom > $FINAL_CHROOT_DIR/etc/platform + + echo "/dev/ufs/${PRODUCT_NAME} / ufs ro 0 0" > ${FINAL_CHROOT_DIR}/etc/fstab + echo "kern.cam.boot_delay=10000" >> ${FINAL_CHROOT_DIR}/boot/loader.conf.local + + echo ">>> Creating serial memstick to ${MEMSTICKADIPATH}." 2>&1 | tee -a ${LOGFILE} + + BOOTCONF=${FINAL_CHROOT_DIR}/boot.config + LOADERCONF=${FINAL_CHROOT_DIR}/boot/loader.conf + + echo ">>> Activating serial console..." 2>&1 | tee -a ${LOGFILE} + # Activate serial console in boot.config + if [ -f ${BOOTCONF} ]; then + sed -i "" '/-[Dh]$/d' ${BOOTCONF} + fi + echo "-S115200 -h" >> ${BOOTCONF} + + # Remove old console options if present. + [ -f "${LOADERCONF}" ] \ + && sed -i "" -Ee "/(console|boot_multicons|boot_serial|hint.uart)/d" ${LOADERCONF} + # Activate serial console+video console in loader.conf + echo 'boot_serial="YES"' >> ${LOADERCONF} + echo 'console="comconsole"' >> ${LOADERCONF} + echo 'comconsole_speed="115200"' >> ${LOADERCONF} + echo 'comconsole_port="0x2F8"' >> ${LOADERCONF} + echo 'hint.uart.0.flags="0x00"' >> ${LOADERCONF} + echo 'hint.uart.1.flags="0x10"' >> ${LOADERCONF} + + create_distribution_tarball + + makefs -B little -o label=${PRODUCT_NAME} ${MEMSTICKADIPATH} ${FINAL_CHROOT_DIR} + if [ $? -ne 0 ]; then + if [ -f ${MEMSTICKADIPATH} ]; then + rm -f $MEMSTICKADIPATH + fi + echo ">>> ERROR: Something wrong happened during MEMSTICKADI image creation. STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + MD=$(mdconfig -a -t vnode -f $MEMSTICKADIPATH) + # Just in case + trap "mdconfig -d -u ${MD}" 1 2 15 EXIT + gpart create -s BSD ${MD} 2>&1 >> ${LOGFILE} + gpart bootcode -b ${FINAL_CHROOT_DIR}/boot/boot ${MD} 2>&1 >> ${LOGFILE} + gpart add -t freebsd-ufs ${MD} 2>&1 >> ${LOGFILE} + trap "-" 1 2 15 EXIT + mdconfig -d -u ${MD} 2>&1 >> ${LOGFILE} + gzip -qf $MEMSTICKADIPATH & + + echo ">>> MEMSTICKADI created: $(LC_ALL=C date)" | tee -a ${LOGFILE} +} + +# Create pkg conf on desired place with desired arch/branch +setup_pkg_repo() { + if [ -z "${4}" ]; then + return + fi + + local _target="${1}" + local _arch="${2}" + local _target_arch="${3}" + local _branch="${4}" + + mkdir -p $(dirname ${_target}) >/dev/null 2>&1 + + sed \ + -e "s/%%ARCH%%/${_arch}_${_target_arch}/" \ + -e "s/%%GIT_REPO_BRANCH_OR_TAG%%/${_branch}/g" \ + -e "s,%%PKG_REPO_SERVER%%,${PKG_REPO_SERVER},g" \ + -e "s/%%PRODUCT_NAME%%/${PRODUCT_NAME}/g" \ + ${SRCDIR}/release/pkg_repos/${PRODUCT_NAME}.conf.template \ + > ${_target} +} + +# This routine ensures any ports / binaries that the builder +# system needs are on disk and ready for execution. +builder_setup() { + # If Product-builder is already installed, just leave + if pkg info -e -q ${PRODUCT_NAME}-builder; then + return + fi + + if [ ! -f /usr/local/etc/pkg/repos/${PRODUCT_NAME}.conf ]; then + [ -d /usr/local/etc/pkg/repos ] \ + || mkdir -p /usr/local/etc/pkg/repos + + local _arch=$(uname -m) + setup_pkg_repo /usr/local/etc/pkg/repos/${PRODUCT_NAME}.conf ${_arch} ${_arch} ${PKG_REPO_CONF_BRANCH} + fi + + pkg install ${PRODUCT_NAME}-builder +} + +# Updates FreeBSD sources +update_freebsd_sources() { + if [ "${1}" = "full" ]; then + local _full=1 + local _clone_params="" + else + local _full=0 + local _clone_params="--depth 1 --single-branch" + fi + + if [ ! -d "${SRCDIR}" ]; then + mkdir -p ${SRCDIR} + fi + + if [ -n "${NO_BUILDWORLD:-}" -a -n "${NO_BUILDKERNEL:-}" ]; then + echo ">>> NO_BUILDWORLD and NO_BUILDKERNEL set, skipping update of freebsd sources" | tee -a ${LOGFILE} + return + fi + + echo -n ">>> Obtaining FreeBSD sources ${FREEBSD_BRANCH}..." + local _FREEBSD_BRANCH=${FREEBSD_BRANCH:-"devel"} + local _CLONE=1 + + if [ -d "${SRCDIR}/.git" ]; then + CUR_BRANCH=$(cd ${SRCDIR} && git branch | grep '^\*' | cut -d' ' -f2) + if [ ${_full} -eq 0 -a "${CUR_BRANCH}" = "${_FREEBSD_BRANCH}" ]; then + _CLONE=0 + ( cd ${SRCDIR} && git clean -fxd; git fetch origin; git reset --hard origin/${_FREEBSD_BRANCH} ) 2>&1 | grep -C3 -i -E 'error|fatal' + else + rm -rf ${SRCDIR} + fi + fi + + if [ ${_CLONE} -eq 1 ]; then + ( git clone --branch ${_FREEBSD_BRANCH} ${_clone_params} ${FREEBSD_REPO_BASE} ${SRCDIR} ) 2>&1 | grep -C3 -i -E 'error|fatal' + fi + + if [ ! -d "${SRCDIR}/.git" ]; then + echo ">>> ERROR: It was not possible to clone FreeBSD src repo" + print_error_pfS + fi + + if [ -n "${GIT_FREEBSD_COSHA1}" ]; then + ( cd ${SRCDIR} && git checkout ${GIT_FREEBSD_COSHA1} ) 2>&1 | grep -C3 -i -E 'error|fatal' + fi + echo "Done!" +} + +pkg_chroot() { + local _root="${1}" + shift + + if [ $# -eq 0 ]; then + return -1 + fi + + if [ -z "${_root}" -o "${_root}" = "/" -o ! -d "${_root}" ]; then + return -1 + fi + + mkdir -p \ + ${SCRATCHDIR}/pkg_cache \ + ${_root}/var/cache/pkg \ + ${_root}/dev + + /sbin/mount -t nullfs ${SCRATCHDIR}/pkg_cache ${_root}/var/cache/pkg + /sbin/mount -t devfs devfs ${_root}/dev + cp -f /etc/resolv.conf ${_root}/etc/resolv.conf + touch ${BUILDER_LOGS}/install_pkg_install_ports.txt + script -aq ${BUILDER_LOGS}/install_pkg_install_ports.txt pkg -c ${_root} $@ >/dev/null 2>&1 + rm -f ${_root}/etc/resolv.conf + /sbin/umount -f ${_root}/dev + /sbin/umount -f ${_root}/var/cache/pkg +} + + +pkg_chroot_add() { + if [ -z "${1}" -o -z "${2}" ]; then + return 1 + fi + + local _target="${1}" + local _pkg="$(get_pkg_name ${2}).txz" + + if [ ! -d "${_target}" ]; then + echo ">>> ERROR: Target dir ${_target} not found" + print_error_pfS + fi + + if [ ! -f ${CORE_PKG_PATH}/${_pkg} ]; then + echo ">>> ERROR: Package ${_pkg} not found" + print_error_pfS + fi + + cp ${CORE_PKG_PATH}/${_pkg} ${_target} + pkg_chroot ${_target} add /${_pkg} + rm -f ${_target}/${_pkg} +} + +pkg_bootstrap() { + local _root=${1:-"${STAGE_CHROOT_DIR}"} + + setup_pkg_repo ${_root}/usr/local/etc/pkg/repos/${PRODUCT_NAME}.conf ${TARGET} ${TARGET_ARCH} ${PKG_REPO_CONF_BRANCH} + + pkg_chroot ${_root} bootstrap -f +} + +# This routine assists with installing various +# freebsd ports files into the pfsenese-fs staging +# area. +install_pkg_install_ports() { + local MAIN_PKG="${1}" + + if [ -z "${MAIN_PKG}" ]; then + MAIN_PKG=${PRODUCT_NAME} + fi + + echo ">>> Installing pkg repository in chroot (${STAGE_CHROOT_DIR})..." + + [ -d ${STAGE_CHROOT_DIR}/var/cache/pkg ] || \ + mkdir -p ${STAGE_CHROOT_DIR}/var/cache/pkg + + [ -d ${SCRATCHDIR}/pkg_cache ] || \ + mkdir -p ${SCRATCHDIR}/pkg_cache + + echo ">>> Installing built ports (packages) in chroot (${STAGE_CHROOT_DIR})... (starting)" + pkg_chroot ${STAGE_CHROOT_DIR} install ${MAIN_PKG} ${custom_package_list} + pkg_chroot ${STAGE_CHROOT_DIR} autoremove + echo ">>> Installing built ports (packages) in chroot (${STAGE_CHROOT_DIR})... (finshied)" +} + +install_bsdinstaller() { + echo ">>> Installing BSDInstaller in chroot (${FINAL_CHROOT_DIR})... (starting)" + pkg_chroot ${FINAL_CHROOT_DIR} install -f bsdinstaller ${MAIN_PKG} ${custom_package_list} + echo ">>> Installing BSDInstaller in chroot (${FINAL_CHROOT_DIR})... (finished)" +} + +staginareas_clean_each_run() { + echo -n ">>> Cleaning build directories: " + if [ -d "${FINAL_CHROOT_DIR}" ]; then + BASENAME=$(basename ${FINAL_CHROOT_DIR}) + echo -n "$BASENAME " + chflags -R noschg ${FINAL_CHROOT_DIR} 2>&1 >/dev/null + rm -rf ${FINAL_CHROOT_DIR} 2>/dev/null + fi + echo "Done!" +} + +# Imported from FreeSBIE +buildkernel() { + if [ -n "${NO_BUILDKERNEL:-}" ]; then + echo ">>> NO_BUILDKERNEL set, skipping build" | tee -a ${LOGFILE} + return + fi + + if [ -z "${KERNCONF:-}" ]; then + echo ">>> ERROR: No kernel configuration defined probably this is not what you want! STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + + if [ -n "${KERNELCONF:-}" ]; then + export KERNCONFDIR=$(dirname ${KERNELCONF}) + export KERNCONF=$(basename ${KERNELCONF}) + fi + + SRCCONFBASENAME=$(basename ${SRC_CONF}) + echo ">>> KERNCONFDIR: ${KERNCONFDIR}" + echo ">>> ARCH: ${TARGET}" + echo ">>> SRC_CONF: ${SRCCONFBASENAME}" + + makeargs="${MAKEJ_KERNEL:-} SRCCONF=${SRC_CONF} __MAKE_CONF=${MAKE_CONF} TARGET_ARCH=${TARGET_ARCH} TARGET=${TARGET}" + echo ">>> Builder is running the command: script -aq $LOGFILE make -DNO_KERNELCLEAN $makeargs buildkernel KERNCONF=${KERNCONF}" | tee -a $LOGFILE + (script -q $LOGFILE make -C ${SRCDIR} -DNO_KERNELCLEAN $makeargs buildkernel KERNCONF=${KERNCONF} || print_error_pfS;) | egrep '^>>>' +} + +# Imported from FreeSBIE +installkernel() { + if [ -z "${KERNCONF:-}" ]; then + echo ">>> ERROR: No kernel configuration defined probably this is not what you want! STOPPING!" | tee -a ${LOGFILE} + print_error_pfS + fi + + if [ -n "${KERNELCONF:-}" ]; then + export KERNCONFDIR=$(dirname ${KERNELCONF}) + export KERNCONF=$(basename ${KERNELCONF}) + fi + + mkdir -p ${STAGE_CHROOT_DIR}/boot + makeargs="${MAKEJ_KERNEL:-} SRCCONF=${SRC_CONF} __MAKE_CONF=${MAKE_CONF} TARGET_ARCH=${TARGET_ARCH} TARGET=${TARGET} DESTDIR=${KERNEL_DESTDIR}" + echo ">>> Builder is running the command: script -aq $LOGFILE make ${makeargs:-} installkernel KERNCONF=${KERNCONF}" | tee -a $LOGFILE + (script -aq $LOGFILE make -C ${SRCDIR} ${makeargs:-} installkernel KERNCONF=${KERNCONF} || print_error_pfS;) | egrep '^>>>' + gzip -f9 $KERNEL_DESTDIR/boot/kernel/kernel +} + +# Launch is ran first to setup a few variables that we need +# Imported from FreeSBIE +launch() { + + if [ ! -f $SCRATCHDIR/${PRODUCT_NAME}_builder_set_time ]; then + echo ">>> Updating system clock..." + ntpdate 0.freebsd.pool.ntp.org + touch $SCRATCHDIR/${PRODUCT_NAME}_builder_set_time + fi + + if [ "$(id -u)" != "0" ]; then + echo "Sorry, this must be done as root." + fi + + echo ">>> Operation $0 has started at $(date)" +} + +finish() { + echo ">>> Operation $0 has ended at $(date)" +} + +poudriere_create_patch() { + local _jail_patch="${SCRATCHDIR}/poudriere_jail.${GIT_REPO_BRANCH_OR_TAG}.patch" + + if [ -z "${FREEBSD_PARENT_BRANCH}" ]; then + echo ">>> ERROR: FREEBSD_PARENT_BRANCH is not set" + fi + + LOGFILE=${BUILDER_LOGS}/poudriere.log + + # Get FreeBSD source and apply patches + update_freebsd_sources full + + [ -f "${_jail_patch}" ] && \ + rm -f "${_jail_patch}" + + # Create a big patch with all our changes to use on jail + ( \ + cd ${SRCDIR} && \ + git diff $(git merge-base origin/${FREEBSD_PARENT_BRANCH} ${FREEBSD_BRANCH}) > ${_jail_patch} + ) >/dev/null 2>&1 + + # Check if patch was created + if [ ! -s "${_jail_patch}" ]; then + echo ">>> ERROR: Patch does not exist or is empty, aborting..." | tee -a ${LOGFILE} + print_error_pfS + fi +} + +poudriere_possible_archs() { + local _arch=$(uname -m) + local _archs="i386.i386" + + # IF host is amd64, we'll create both repos, and if possible armv6 + if [ "${_arch}" = "amd64" ]; then + _archs="amd64.amd64 ${_archs}" + + if [ -f /usr/local/bin/qemu-arm-static ]; then + # Make sure binmiscctl is ok + /usr/local/etc/rc.d/qemu_user_static forcestart >/dev/null 2>&1 + + if binmiscctl lookup armv6 >/dev/null 2>&1; then + _archs="${_archs} arm.armv6" + fi + fi + fi + + echo ${_archs} +} + +poudriere_jail_name() { + local _jail_arch="${1}" + + if [ -z "${_jail_arch}" ]; then + return 1 + fi + + # Poudriere doesn't like periods in jail names + _jail_arch=$(echo "${_jail_arch}" | tr '.' '_') + + echo "${PRODUCT_NAME}_${GIT_REPO_BRANCH_OR_TAG}_${_jail_arch}" +} + +poudriere_create_ports_tree() { + LOGFILE=${BUILDER_LOGS}/poudriere.log + + if ! poudriere ports -l | grep -q -E "^${POUDRIERE_PORTS_NAME}[[:blank:]]"; then + local _branch="" + if [ -z "${POUDRIERE_PORTS_GIT_URL}" ]; then + echo ">>> ERROR: POUDRIERE_PORTS_GIT_URL is not defined" + print_error_pfS + fi + if [ -n "${POUDRIERE_PORTS_GIT_BRANCH}" ]; then + _branch="-B ${POUDRIERE_PORTS_GIT_BRANCH}" + fi + echo -n ">>> Creating poudriere ports tree, it may take some time... " | tee -a ${LOGFILE} + if ! script -aq ${LOGFILE} poudriere ports -c -p "${POUDRIERE_PORTS_NAME}" -m git ${_branch} >/dev/null 2>&1; then + echo "" | tee -a ${LOGFILE} + echo ">>> ERROR: Error creating poudriere ports tree, aborting..." | tee -a ${LOGFILE} + print_error_pfS + fi + echo "Done!" | tee -a ${LOGFILE} + fi +} + +poudriere_init() { + local _error=0 + local _archs=$(poudriere_possible_archs) + local _jail_patch="${SCRATCHDIR}/poudriere_jail.${GIT_REPO_BRANCH_OR_TAG}.patch" + + LOGFILE=${BUILDER_LOGS}/poudriere.log + + # Sanity checks + if [ -z "${ZFS_TANK}" ]; then + echo ">>> ERROR: \$ZFS_TANK is empty" | tee -a ${LOGFILE} + error=1 + fi + + if [ -z "${ZFS_ROOT}" ]; then + echo ">>> ERROR: \$ZFS_ROOT is empty" | tee -a ${LOGFILE} + error=1 + fi + + if [ -z "${POUDRIERE_PORTS_NAME}" ]; then + echo ">>> ERROR: \$POUDRIERE_PORTS_NAME is empty" | tee -a ${LOGFILE} + error=1 + fi + + if [ ${_error} -eq 1 ]; then + print_error_pfS + fi + + # Check if zpool exists + if ! zpool list ${ZFS_TANK} >/dev/null 2>&1; then + echo ">>> ERROR: ZFS tank ${ZFS_TANK} not found, please create it and try again..." | tee -a ${LOGFILE} + print_error_pfS + fi + + # Check if zfs rootfs exists + if ! zfs list ${ZFS_TANK}${ZFS_ROOT} >/dev/null 2>&1; then + echo ">>> ERROR: ZFS filesystem ${ZFS_TANK}${ZFS_ROOT} not found, please create it and try again..." | tee -a ${LOGFILE} + print_error_pfS + fi + + # Make sure poudriere is installed + if ! pkg info --quiet poudriere; then + echo ">>> Installing poudriere..." | tee -a ${LOGFILE} + if ! pkg install poudriere >/dev/null 2>&1; then + echo ">>> ERROR: poudriere was not installed, aborting..." | tee -a ${LOGFILE} + print_error_pfS + fi + fi + + # Create poudriere.conf + if [ -z "${POUDRIERE_PORTS_GIT_URL}" ]; then + echo ">>> ERROR: POUDRIERE_PORTS_GIT_URL is not defined" + print_error_pfS + fi + echo ">>> Creating poudriere.conf" | tee -a ${LOGFILE} + cat <<EOF >/usr/local/etc/poudriere.conf +ZPOOL=${ZFS_TANK} +ZROOTFS=${ZFS_ROOT} +RESOLV_CONF=/etc/resolv.conf +BASEFS=/usr/local/poudriere +USE_PORTLINT=no +USE_TMPFS=yes +NOLINUX=yes +DISTFILES_CACHE=/usr/ports/distfiles +CHECK_CHANGED_OPTIONS=yes +CHECK_CHANGED_DEPS=yes +ATOMIC_PACKAGE_REPOSITORY=yes +COMMIT_PACKAGES_ON_FAILURE=no +GIT_URL="${POUDRIERE_PORTS_GIT_URL}" +EOF + + # Remove old jails + for jail_arch in ${_archs}; do + jail_name=$(poudriere_jail_name ${jail_arch}) + + if poudriere jail -i -j "${jail_name}" >/dev/null 2>&1; then + echo ">>> Poudriere jail ${jail_name} already exists, deleting it..." | tee -a ${LOGFILE} + poudriere jail -d -j "${jail_name}" >/dev/null 2>&1 + fi + done + + # Remove old ports tree + if poudriere ports -l | grep -q -E "^${POUDRIERE_PORTS_NAME}[[:blank:]]"; then + echo ">>> Poudriere ports tree ${POUDRIERE_PORTS_NAME} already exists, deleting it..." | tee -a ${LOGFILE} + poudriere ports -d -p "${POUDRIERE_PORTS_NAME}" + fi + + poudriere_create_patch + + local native_xtools="" + # Now we are ready to create jails + for jail_arch in ${_archs}; do + jail_name=$(poudriere_jail_name ${jail_arch}) + + if [ "${jail_arch}" = "arm.armv6" ]; then + native_xtools="-x" + else + native_xtools="" + fi + + echo -n ">>> Creating jail ${jail_name}, it may take some time... " | tee -a ${LOGFILE} + # XXX: Change -m to git when it's available in poudriere + if ! script -aq ${LOGFILE} poudriere jail -c -j "${jail_name}" -v ${FREEBSD_PARENT_BRANCH} \ + -a ${jail_arch} -m svn -P ${_jail_patch} ${native_xtools} >/dev/null 2>&1; then + echo "" | tee -a ${LOGFILE} + echo ">>> ERROR: Error creating jail ${jail_name}, aborting..." | tee -a ${LOGFILE} + print_error_pfS + fi + echo "Done!" | tee -a ${LOGFILE} + done + + poudriere_create_ports_tree + + echo ">>> Poudriere is now configured!" | tee -a ${LOGFILE} +} + +poudriere_update_jails() { + local _archs=$(poudriere_possible_archs) + local _jail_patch="${SCRATCHDIR}/poudriere_jail.${GIT_REPO_BRANCH_OR_TAG}.patch" + + LOGFILE=${BUILDER_LOGS}/poudriere.log + + poudriere_create_patch + + local native_xtools="" + for jail_arch in ${_archs}; do + local _run=0 + if [ -n "${ARCH_LIST}" ]; then + for _arch in ${ARCH_LIST}; do + if [ "${jail_arch##*.}" = "${_arch}" ]; then + _run=1 + fi + done + else + _run=1 + fi + + [ ${_run} -eq 0 ] \ + && continue + + jail_name=$(poudriere_jail_name ${jail_arch}) + + if ! poudriere jail -i -j "${jail_name}" >/dev/null 2>&1; then + echo ">>> Poudriere jail ${jail_name} not found, skipping..." | tee -a ${LOGFILE} + continue + fi + + if [ "${jail_arch}" = "arm.armv6" ]; then + native_xtools="-x" + else + native_xtools="" + fi + + echo -n ">>> Updating jail ${jail_name}, it may take some time... " | tee -a ${LOGFILE} + if ! script -aq ${LOGFILE} poudriere jail -u -j "${jail_name}" -P ${_jail_patch} ${native_xtools} >/dev/null 2>&1; then + echo "" | tee -a ${LOGFILE} + echo ">>> ERROR: Error updating jail ${jail_name}, aborting..." | tee -a ${LOGFILE} + print_error_pfS + fi + echo "Done!" | tee -a ${LOGFILE} + done +} + +poudriere_update_ports() { + LOGFILE=${BUILDER_LOGS}/poudriere.log + + # Create ports tree if necessary + if ! poudriere ports -l | grep -q -E "^${POUDRIERE_PORTS_NAME}[[:blank:]]"; then + poudriere_create_ports_tree + else + echo -n ">>> Updating ports tree ${POUDRIERE_PORTS_NAME}... " | tee -a ${LOGFILE} + script -aq ${LOGFILE} poudriere ports -u -p "${POUDRIERE_PORTS_NAME}" >/dev/null 2>&1 + echo "Done!" | tee -a ${LOGFILE} + fi +} + +poudriere_bulk() { + local _archs=$(poudriere_possible_archs) + + LOGFILE=${BUILDER_LOGS}/poudriere.log + + poudriere_create_ports_tree + + local CUR_BRANCH=$(cd ${BUILDER_TOOLS} && git branch | grep '^\*' | cut -d' ' -f2) + + [ -d /usr/local/etc/poudriere.d ] || \ + mkdir -p /usr/local/etc/poudriere.d + + if [ "${CUR_BRANCH}" != "${GIT_REPO_BRANCH_OR_TAG}" ] && git show "${GIT_REPO_BRANCH_OR_TAG}:builder_scripts/conf/pfPorts/make.conf" >/dev/null 2>&1; then + git show "origin/${GIT_REPO_BRANCH_OR_TAG}:builder_scripts/conf/pfPorts/make.conf" > /usr/local/etc/poudriere.d/${POUDRIERE_PORTS_NAME}-make.conf + elif [ -f "${BUILDER_SCRIPTS}/conf/pfPorts/make.conf" ]; then + cp -f "${BUILDER_SCRIPTS}/conf/pfPorts/make.conf" /usr/local/etc/poudriere.d/${POUDRIERE_PORTS_NAME}-make.conf + fi + + for jail_arch in ${_archs}; do + jail_name=$(poudriere_jail_name ${jail_arch}) + + if ! poudriere jail -i -j "${jail_name}" >/dev/null 2>&1; then + echo ">>> Poudriere jail ${jail_name} not found, skipping..." | tee -a ${LOGFILE} + continue + fi + + if [ -f "${POUDRIERE_BULK}.${jail_arch}" ]; then + _ref_bulk="${POUDRIERE_BULK}.${jail_arch}" + else + _ref_bulk="${POUDRIERE_BULK}" + fi + + _bulk=${SCRATCHDIR}/poudriere_bulk.${GIT_REPO_BRANCH_OR_TAG} + sed -e "s,%%PRODUCT_NAME%%,${PRODUCT_NAME},g" ${_ref_bulk} > ${_bulk} + + if ! poudriere bulk -f ${_bulk} -j ${jail_name} -p ${POUDRIERE_PORTS_NAME}; then + echo ">>> ERROR: Something went wrong..." + print_error_pfS + fi + + echo ">>> Cleaning up old packages from repo..." + if ! poudriere pkgclean -f ${_bulk} -j ${jail_name} -p ${POUDRIERE_PORTS_NAME} -y; then + echo ">>> ERROR: Something went wrong..." + print_error_pfS + fi + + # ./ is intentional, it's a rsync trick to make it chdir to directory before send it + REPO_PATH="/usr/local/poudriere/data/packages/./${jail_name}-${POUDRIERE_PORTS_NAME}" + if [ -z "${DO_NOT_UPLOAD}" -a -d "${REPO_PATH}" ]; then + echo -n ">>> Sending updated repository to ${PKG_RSYNC_HOSTNAME}... " | tee -a ${LOGFILE} + if script -aq ${LOGFILE} rsync -ave "ssh -p ${PKG_RSYNC_SSH_PORT}" \ + --timeout=60 --delete-delay ${REPO_PATH} \ + ${PKG_RSYNC_USERNAME}@${PKG_RSYNC_HOSTNAME}:${PKG_RSYNC_DESTDIR} >/dev/null 2>&1 + then + echo "Done!" | tee -a ${LOGFILE} + else + echo "Failed!" | tee -a ${LOGFILE} + echo ">>> ERROR: An error occurred sending repo to remote hostname" + print_error_pfS + fi + fi + done +} |