diff options
4 files changed, 92 insertions, 93 deletions
diff --git a/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/bitbake/lib/toaster/bldcontrol/bbcontroller.py index 102606e..dbfb2f3 100644 --- a/bitbake/lib/toaster/bldcontrol/bbcontroller.py +++ b/bitbake/lib/toaster/bldcontrol/bbcontroller.py @@ -81,12 +81,17 @@ def getBuildEnvironmentController(**kwargs): raise Exception("FIXME: Implement BEC for type %s" % str(be.betype)) -def _getgitcheckoutdirectoryname(url): +def _get_git_clonedirectory(url, branch): """ Utility that returns the last component of a git path as directory """ import re components = re.split(r'[:\.\/]', url) - return components[-2] if components[-1] == "git" else components[-1] + base = components[-2] if components[-1] == "git" else components[-1] + + if branch != "HEAD": + return "_%s_%s.toaster_cloned" % (base, branch) + + return base class BuildEnvironmentController(object): @@ -166,12 +171,12 @@ class BuildEnvironmentController(object): raise Exception("Must override setLayers") - def getBBController(self, brbe): + def getBBController(self): """ returns a BitbakeController to an already started server; this is the point where the server starts if needed; or reconnects to the server if we can """ if not self.connection: - self.startBBServer(brbe) + self.startBBServer() self.be.lock = BuildEnvironment.LOCK_RUNNING self.be.save() diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py index 22d31e3..980751f 100644 --- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py @@ -30,11 +30,12 @@ import subprocess from toastermain import settings -from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname +from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _get_git_clonedirectory import logging logger = logging.getLogger("toaster") +from pprint import pprint, pformat class LocalhostBEController(BuildEnvironmentController): """ Implementation of the BuildEnvironmentController for the localhost; @@ -55,15 +56,16 @@ class LocalhostBEController(BuildEnvironmentController): p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (out,err) = p.communicate() + p.wait() if p.returncode: if len(err) == 0: err = "command: %s \n%s" % (command, out) else: err = "command: %s \n%s" % (command, err) - logger.debug("localhostbecontroller: shellcmd error %s" % err) + #logger.debug("localhostbecontroller: shellcmd error %s" % err) raise ShellCmdException(err) else: - logger.debug("localhostbecontroller: shellcmd success") + #logger.debug("localhostbecontroller: shellcmd success") return out def _createdirpath(self, path): @@ -80,10 +82,19 @@ class LocalhostBEController(BuildEnvironmentController): self._createdirpath(self.be.builddir) self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir)) - def startBBServer(self, brbe): + def startBBServer(self): assert self.pokydirname and os.path.exists(self.pokydirname) assert self.islayerset + # find our own toasterui listener/bitbake + from toaster.bldcontrol.management.commands.loadconf import _reduce_canon_path + + own_bitbake = _reduce_canon_path(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../bin/bitbake")) + + assert os.path.exists(own_bitbake) and os.path.isfile(own_bitbake) + + logger.debug("localhostbecontroller: running the listener at %s" % own_bitbake) + try: os.remove(os.path.join(self.be.builddir, "toaster_ui.log")) except OSError as e: @@ -91,7 +102,10 @@ class LocalhostBEController(BuildEnvironmentController): if e.errno != errno.ENOENT: raise - cmd = "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe) + + cmd = "bash -c \"source %s/oe-init-build-env %s && bitbake --read conf/toaster-pre.conf --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 && DATABASE_URL=%s BBSERVER=0.0.0.0:-1 daemon -d -i -D %s -o toaster_ui.log -- %s --observe-only -u toasterui &\"" % (self.pokydirname, self.be.builddir, + self.dburl, self.be.builddir, own_bitbake) + logger.debug("fullcommand |%s| " % cmd) port = "-1" for i in self._shellcmd(cmd).split("\n"): if i.startswith("Bitbake server address"): @@ -113,6 +127,16 @@ class LocalhostBEController(BuildEnvironmentController): time.sleep(0.5) logger.debug("localhostbecontroller: Started bitbake server") + + while port == "-1": + # the port specification is "autodetect"; read the bitbake.lock file + with open("%s/bitbake.lock" % self.be.builddir, "r") as f: + for line in f.readlines(): + if ":" in line: + port = line.split(":")[1].strip() + logger.debug("localhostbecontroller: Autodetected bitbake port %s", port) + break + assert self.be.sourcedir and os.path.exists(self.be.builddir) self.be.bbaddress = "localhost" self.be.bbport = port @@ -135,42 +159,62 @@ class LocalhostBEController(BuildEnvironmentController): assert len(bitbakes) == 1 # set layers in the layersource - # 1. get a list of repos, and map dirpaths for each layer + # 1. get a list of repos with branches, and map dirpaths for each layer gitrepos = {} - gitrepos[bitbakes[0].giturl] = [] - gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) + + gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = [] + gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append( ("bitbake", bitbakes[0].dirpath) ) for layer in layers: # we don't process local URLs if layer.giturl.startswith("file://"): continue - if not layer.giturl in gitrepos: - gitrepos[layer.giturl] = [] - gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit)) - for giturl in gitrepos.keys(): - commitid = gitrepos[giturl][0][2] - for e in gitrepos[giturl]: - if commitid != e[2]: - import pprint - raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos)) - - - logger.debug("localhostbecontroller, our git repos are %s" % gitrepos) + if not (layer.giturl, layer.commit) in gitrepos: + gitrepos[(layer.giturl, layer.commit)] = [] + gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) ) + + + logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) + + + # 2. find checked-out git repos in the sourcedir directory that may help faster cloning + + cached_layers = {} + for ldir in os.listdir(self.be.sourcedir): + fldir = os.path.join(self.be.sourcedir, ldir) + if os.path.isdir(fldir): + try: + for line in self._shellcmd("git remote -v", fldir).split("\n"): + try: + remote = line.split("\t")[1].split(" ")[0] + if remote not in cached_layers: + cached_layers[remote] = fldir + except IndexError: + pass + except ShellCmdException: + # ignore any errors in collecting git remotes + pass + layerlist = [] - # 2. checkout the repositories - for giturl in gitrepos.keys(): - localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl)) - logger.debug("localhostbecontroller: giturl %s checking out in current directory %s" % (giturl, localdirname)) + # 3. checkout the repositories + for giturl, commit in gitrepos.keys(): + localdirname = os.path.join(self.be.sourcedir, _get_git_clonedirectory(giturl, commit)) + logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) # make sure our directory is a git repository if os.path.exists(localdirname): if not giturl in self._shellcmd("git remote -v", localdirname): raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) else: - self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname)) - # checkout the needed commit - commit = gitrepos[giturl][0][2] + if giturl in cached_layers: + logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) + self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname)) + self._shellcmd("git remote remove origin", localdirname) + self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname) + else: + logger.debug("localhostbecontroller: cloning %s:%s in %s" % (giturl, commit, localdirname)) + self._shellcmd("git clone \"%s\" --single-branch --branch \"%s\" \"%s\"" % (giturl, commit, localdirname)) # branch magic name "HEAD" will inhibit checkout if commit != "HEAD": @@ -188,21 +232,22 @@ class LocalhostBEController(BuildEnvironmentController): self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) # verify our repositories - for name, dirpath, commit in gitrepos[giturl]: + for name, dirpath in gitrepos[(giturl, commit)]: localdirpath = os.path.join(localdirname, dirpath) + logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath) if not os.path.exists(localdirpath): raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) if name != "bitbake": layerlist.append(localdirpath.rstrip("/")) - logger.debug("localhostbecontroller: current layer list %s " % layerlist) + logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) - # 3. configure the build environment, so we have a conf/bblayers.conf + # 4. configure the build environment, so we have a conf/bblayers.conf assert self.pokydirname is not None self._setupBE() - # 4. update the bblayers.conf + # 5. update the bblayers.conf bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") if not os.path.exists(bblayerconf): raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py index bdce6ee..23ee855 100644 --- a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py +++ b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -2,7 +2,7 @@ from django.core.management.base import NoArgsCommand, CommandError from django.db import transaction from orm.models import Build from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException -from bldcontrol.models import BuildRequest, BuildEnvironment, BRError +from bldcontrol.models import BuildRequest, BuildEnvironment, BRError, BRVariable import os import logging @@ -47,6 +47,9 @@ class Command(NoArgsCommand): return logger.debug("runbuilds: starting build %s, environment %s" % (br, bec.be)) + + # write the build identification variable + BRVariable.objects.create(req = br, name="TOASTER_BRBE", value="%d:%d" % (br.pk, bec.be.pk)) # let the build request know where it is being executed br.environment = bec.be br.save() @@ -56,7 +59,7 @@ class Command(NoArgsCommand): bec.writePreConfFile(br.brvariable_set.all()) # get the bb server running with the build req id and build env id - bbctrl = bec.getBBController("%d:%d" % (br.pk, bec.be.pk)) + bbctrl = bec.getBBController() # trigger the build command task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, br.brtarget_set.all())) diff --git a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py index 45e1539..be797c9 100644 --- a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py +++ b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py @@ -29,7 +29,7 @@ import subprocess from toastermain import settings -from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname +from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _get_git_clonedirectory def DN(path): return "/".join(path.split("/")[0:-1]) @@ -124,62 +124,8 @@ class SSHBEController(BuildEnvironmentController): assert len(bitbakes) == 1 # set layers in the layersource - # 1. get a list of repos, and map dirpaths for each layer - gitrepos = {} - gitrepos[bitbakes[0].giturl] = [] - gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) - - for layer in layers: - # we don't process local URLs - if layer.giturl.startswith("file://"): - continue - if not layer.giturl in gitrepos: - gitrepos[layer.giturl] = [] - gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit)) - for giturl in gitrepos.keys(): - commitid = gitrepos[giturl][0][2] - for e in gitrepos[giturl]: - if commitid != e[2]: - raise BuildSetupException("More than one commit per git url, unsupported configuration") - - layerlist = [] - - # 2. checkout the repositories - for giturl in gitrepos.keys(): - import os - localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl)) - print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname - - # make sure our directory is a git repository - if self._pathexists(localdirname): - if not giturl in self._shellcmd("git remote -v", localdirname): - raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) - else: - self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname)) - # checkout the needed commit - commit = gitrepos[giturl][0][2] - - # branch magic name "HEAD" will inhibit checkout - if commit != "HEAD": - print "DEBUG: checking out commit ", commit, "to", localdirname - self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname) - - # take the localdirname as poky dir if we can find the oe-init-build-env - if self.pokydirname is None and self._pathexists(os.path.join(localdirname, "oe-init-build-env")): - print "DEBUG: selected poky dir name", localdirname - self.pokydirname = localdirname - - # verify our repositories - for name, dirpath, commit in gitrepos[giturl]: - localdirpath = os.path.join(localdirname, dirpath) - if not self._pathexists(localdirpath): - raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) - - if name != "bitbake": - layerlist.append(localdirpath) - - print "DEBUG: current layer list ", layerlist + raise Exception("Not implemented: SSH setLayers") # 3. configure the build environment, so we have a conf/bblayers.conf assert self.pokydirname is not None self._setupBE() |
