diff options
Diffstat (limited to 'bitbake/lib/bb/cooker.py')
| -rw-r--r-- | bitbake/lib/bb/cooker.py | 761 |
1 files changed, 482 insertions, 279 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 14ccfb5..8036d7e 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py @@ -7,7 +7,7 @@ # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer # Copyright (C) 2005 Holger Hans Peter Freyther # Copyright (C) 2005 ROAD GmbH -# Copyright (C) 2006 Richard Purdie +# Copyright (C) 2006 - 2007 Richard Purdie # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -25,9 +25,35 @@ import sys, os, getopt, glob, copy, os.path, re, time import bb from bb import utils, data, parse, event, cache, providers, taskdata, runqueue +from bb import command +import bb.server.xmlrpc import itertools, sre_constants -parsespin = itertools.cycle( r'|/-\\' ) +class MultipleMatches(Exception): + """ + Exception raised when multiple file matches are found + """ + +class ParsingErrorsFound(Exception): + """ + Exception raised when parsing errors are found + """ + +class NothingToBuild(Exception): + """ + Exception raised when there is nothing to build + """ + + +# Different states cooker can be in +cookerClean = 1 +cookerParsing = 2 +cookerParsed = 3 + +# Different action states the cooker can be in +cookerRun = 1 # Cooker is running normally +cookerShutdown = 2 # Active tasks should be brought to a controlled stop +cookerStop = 3 # Stop, now! #============================================================================# # BBCooker @@ -37,12 +63,14 @@ class BBCooker: Manages one bitbake build run """ - def __init__(self, configuration): + def __init__(self, configuration, server): self.status = None self.cache = None self.bb_cache = None + self.server = server.BitBakeServer(self) + self.configuration = configuration if self.configuration.verbose: @@ -58,17 +86,15 @@ class BBCooker: self.configuration.data = bb.data.init() - def parseConfiguration(self): - bb.data.inheritFromOS(self.configuration.data) - # Add conf/bitbake.conf to the list of configuration files to read - self.configuration.file.append( os.path.join( "conf", "bitbake.conf" ) ) + for f in self.configuration.file: + self.parseConfigurationFile( f ) - self.parseConfigurationFile(self.configuration.file) + self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) ) if not self.configuration.cmd: - self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data) or "build" + self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build" bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True) if bbpkgs and len(self.configuration.pkgs_to_build) == 0: @@ -80,9 +106,7 @@ class BBCooker: self.configuration.event_data = bb.data.createCopy(self.configuration.data) bb.data.update_data(self.configuration.event_data) - # # TOSTOP must not be set or our children will hang when they output - # fd = sys.stdout.fileno() if os.isatty(fd): import termios @@ -92,40 +116,91 @@ class BBCooker: tcattr[3] = tcattr[3] & ~termios.TOSTOP termios.tcsetattr(fd, termios.TCSANOW, tcattr) + self.command = bb.command.Command(self) + self.cookerState = cookerClean + self.cookerAction = cookerRun + + def parseConfiguration(self): + + # Change nice level if we're asked to nice = bb.data.getVar("BB_NICE_LEVEL", self.configuration.data, True) if nice: curnice = os.nice(0) nice = int(nice) - curnice bb.msg.note(2, bb.msg.domain.Build, "Renice to %s " % os.nice(nice)) - + + def parseCommandLine(self): + # Parse any commandline into actions + if self.configuration.show_environment: + self.commandlineAction = None + + if 'world' in self.configuration.pkgs_to_build: + bb.error("'world' is not a valid target for --environment.") + elif len(self.configuration.pkgs_to_build) > 1: + bb.error("Only one target can be used with the --environment option.") + elif self.configuration.buildfile and len(self.configuration.pkgs_to_build) > 0: + bb.error("No target should be used with the --environment and --buildfile options.") + elif len(self.configuration.pkgs_to_build) > 0: + self.commandlineAction = ["showEnvironmentTarget", self.configuration.pkgs_to_build] + else: + self.commandlineAction = ["showEnvironment", self.configuration.buildfile] + elif self.configuration.buildfile is not None: + self.commandlineAction = ["buildFile", self.configuration.buildfile, self.configuration.cmd] + elif self.configuration.revisions_changed: + self.commandlineAction = ["compareRevisions"] + elif self.configuration.show_versions: + self.commandlineAction = ["showVersions"] + elif self.configuration.parse_only: + self.commandlineAction = ["parseFiles"] + # FIXME - implement + #elif self.configuration.interactive: + # self.interactiveMode() + elif self.configuration.dot_graph: + if self.configuration.pkgs_to_build: + self.commandlineAction = ["generateDotGraph", self.configuration.pkgs_to_build, self.configuration.cmd] + else: + self.commandlineAction = None + bb.error("Please specify a package name for dependency graph generation.") + else: + if self.configuration.pkgs_to_build: + self.commandlineAction = ["buildTargets", self.configuration.pkgs_to_build, self.configuration.cmd] + else: + self.commandlineAction = None + bb.error("Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.") + + def runCommands(self, server, data, abort): + """ + Run any queued asynchronous command + This is done by the idle handler so it runs in true context rather than + tied to any UI. + """ + + return self.command.runAsyncCommand() def tryBuildPackage(self, fn, item, task, the_data): """ Build one task of a package, optionally build following task depends """ - bb.event.fire(bb.event.PkgStarted(item, the_data)) try: if not self.configuration.dry_run: bb.build.exec_task('do_%s' % task, the_data) - bb.event.fire(bb.event.PkgSucceeded(item, the_data)) return True except bb.build.FuncFailed: bb.msg.error(bb.msg.domain.Build, "task stack execution failed") - bb.event.fire(bb.event.PkgFailed(item, the_data)) raise except bb.build.EventException, e: event = e.args[1] bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event)) - bb.event.fire(bb.event.PkgFailed(item, the_data)) raise - def tryBuild(self, fn): + def tryBuild(self, fn, task): """ Build a provider and its dependencies. build_depends is a list of previous build dependencies (not runtime) If build_depends is empty, we're dealing with a runtime depends """ + the_data = self.bb_cache.loadDataFull(fn, self.configuration.data) item = self.status.pkg_fn[fn] @@ -133,9 +208,13 @@ class BBCooker: #if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data): # return True - return self.tryBuildPackage(fn, item, self.configuration.cmd, the_data) + return self.tryBuildPackage(fn, item, task, the_data) def showVersions(self): + + # Need files parsed + self.updateCache() + pkg_pn = self.status.pkg_pn preferred_versions = {} latest_versions = {} @@ -149,43 +228,36 @@ class BBCooker: pkg_list = pkg_pn.keys() pkg_list.sort() + bb.msg.plain("%-35s %25s %25s" % ("Package Name", "Latest Version", "Preferred Version")) + bb.msg.plain("%-35s %25s %25s\n" % ("============", "==============", "=================")) + for p in pkg_list: pref = preferred_versions[p] latest = latest_versions[p] - if pref != latest: - prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2] - else: + prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2] + lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2] + + if pref == latest: prefstr = "" - print "%-30s %20s %20s" % (p, latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2], - prefstr) + bb.msg.plain("%-35s %25s %25s" % (p, lateststr, prefstr)) + def compareRevisions(self): + ret = bb.fetch.fetcher_compare_revisons(self.configuration.data) + bb.event.fire(bb.command.CookerCommandSetExitCode(ret), self.configuration.event_data) - def showEnvironment(self , buildfile = None, pkgs_to_build = []): + def showEnvironment(self, buildfile = None, pkgs_to_build = []): """ Show the outer or per-package environment """ fn = None envdata = None - if 'world' in pkgs_to_build: - print "'world' is not a valid target for --environment." - sys.exit(1) - - if len(pkgs_to_build) > 1: - print "Only one target can be used with the --environment option." - sys.exit(1) - if buildfile: - if len(pkgs_to_build) > 0: - print "No target should be used with the --environment and --buildfile options." - sys.exit(1) self.cb = None self.bb_cache = bb.cache.init(self) fn = self.matchFile(buildfile) - if not fn: - sys.exit(1) elif len(pkgs_to_build) == 1: self.updateCache() @@ -193,13 +265,9 @@ class BBCooker: bb.data.update_data(localdata) bb.data.expandKeys(localdata) - taskdata = bb.taskdata.TaskData(self.configuration.abort, self.configuration.tryaltconfigs) - - try: - taskdata.add_provider(localdata, self.status, pkgs_to_build[0]) - taskdata.add_unresolved(localdata, self.status) - except bb.providers.NoProvider: - sys.exit(1) + taskdata = bb.taskdata.TaskData(self.configuration.abort) + taskdata.add_provider(localdata, self.status, pkgs_to_build[0]) + taskdata.add_unresolved(localdata, self.status) targetid = taskdata.getbuild_id(pkgs_to_build[0]) fnid = taskdata.build_targets[targetid][0] @@ -211,55 +279,69 @@ class BBCooker: try: envdata = self.bb_cache.loadDataFull(fn, self.configuration.data) except IOError, e: - bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e)) + bb.msg.error(bb.msg.domain.Parsing, "Unable to read %s: %s" % (fn, e)) + raise except Exception, e: - bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e) + bb.msg.error(bb.msg.domain.Parsing, "%s" % e) + raise + + class dummywrite: + def __init__(self): + self.writebuf = "" + def write(self, output): + self.writebuf = self.writebuf + output # emit variables and shell functions try: - data.update_data( envdata ) - data.emit_env(sys.__stdout__, envdata, True) + data.update_data(envdata) + wb = dummywrite() + data.emit_env(wb, envdata, True) + bb.msg.plain(wb.writebuf) except Exception, e: bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e) # emit the metadata which isnt valid shell - data.expandKeys( envdata ) + data.expandKeys(envdata) for e in envdata.keys(): if data.getVarFlag( e, 'python', envdata ): - sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, envdata, 1))) + bb.msg.plain("\npython %s () {\n%s}\n" % (e, data.getVar(e, envdata, 1))) - def generateDotGraph( self, pkgs_to_build, ignore_deps ): + def generateDepTreeData(self, pkgs_to_build, task): """ - Generate a task dependency graph. - - pkgs_to_build A list of packages that needs to be built - ignore_deps A list of names where processing of dependencies - should be stopped. e.g. dependencies that get + Create a dependency tree of pkgs_to_build, returning the data. """ - for dep in ignore_deps: - self.status.ignored_dependencies.add(dep) + # Need files parsed + self.updateCache() + + # If we are told to do the None task then query the default task + if (task == None): + task = self.configuration.cmd + + pkgs_to_build = self.checkPackages(pkgs_to_build) localdata = data.createCopy(self.configuration.data) bb.data.update_data(localdata) bb.data.expandKeys(localdata) - taskdata = bb.taskdata.TaskData(self.configuration.abort, self.configuration.tryaltconfigs) + taskdata = bb.taskdata.TaskData(self.configuration.abort) runlist = [] - try: - for k in pkgs_to_build: - taskdata.add_provider(localdata, self.status, k) - runlist.append([k, "do_%s" % self.configuration.cmd]) - taskdata.add_unresolved(localdata, self.status) - except bb.providers.NoProvider: - sys.exit(1) + for k in pkgs_to_build: + taskdata.add_provider(localdata, self.status, k) + runlist.append([k, "do_%s" % task]) + taskdata.add_unresolved(localdata, self.status) + rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist) rq.prepare_runqueue() seen_fnids = [] - depends_file = file('depends.dot', 'w' ) - tdepends_file = file('task-depends.dot', 'w' ) - print >> depends_file, "digraph depends {" - print >> tdepends_file, "digraph depends {" + depend_tree = {} + depend_tree["depends"] = {} + depend_tree["tdepends"] = {} + depend_tree["pn"] = {} + depend_tree["rdepends-pn"] = {} + depend_tree["packages"] = {} + depend_tree["rdepends-pkg"] = {} + depend_tree["rrecs-pkg"] = {} for task in range(len(rq.runq_fnid)): taskname = rq.runq_task[task] @@ -267,43 +349,118 @@ class BBCooker: fn = taskdata.fn_index[fnid] pn = self.status.pkg_fn[fn] version = "%s:%s-%s" % self.status.pkg_pepvpr[fn] - print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn) + if pn not in depend_tree["pn"]: + depend_tree["pn"][pn] = {} + depend_tree["pn"][pn]["filename"] = fn + depend_tree["pn"][pn]["version"] = version for dep in rq.runq_depends[task]: depfn = taskdata.fn_index[rq.runq_fnid[dep]] deppn = self.status.pkg_fn[depfn] - print >> tdepends_file, '"%s.%s" -> "%s.%s"' % (pn, rq.runq_task[task], deppn, rq.runq_task[dep]) + dotname = "%s.%s" % (pn, rq.runq_task[task]) + if not dotname in depend_tree["tdepends"]: + depend_tree["tdepends"][dotname] = [] + depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, rq.runq_task[dep])) if fnid not in seen_fnids: seen_fnids.append(fnid) packages = [] - print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn) - for depend in self.status.deps[fn]: - print >> depends_file, '"%s" -> "%s"' % (pn, depend) + + depend_tree["depends"][pn] = [] + for dep in taskdata.depids[fnid]: + depend_tree["depends"][pn].append(taskdata.build_names_index[dep]) + + depend_tree["rdepends-pn"][pn] = [] + for rdep in taskdata.rdepids[fnid]: + depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep]) + rdepends = self.status.rundeps[fn] for package in rdepends: - for rdepend in re.findall("([\w.-]+)(\ \(.+\))?", rdepends[package]): - print >> depends_file, '"%s" -> "%s%s" [style=dashed]' % (package, rdepend[0], rdepend[1]) + depend_tree["rdepends-pkg"][package] = [] + for rdepend in rdepends[package]: + depend_tree["rdepends-pkg"][package].append(rdepend) packages.append(package) + rrecs = self.status.runrecs[fn] for package in rrecs: - for rdepend in re.findall("([\w.-]+)(\ \(.+\))?", rrecs[package]): - print >> depends_file, '"%s" -> "%s%s" [style=dashed]' % (package, rdepend[0], rdepend[1]) + depend_tree["rrecs-pkg"][package] = [] + for rdepend in rrecs[package]: + depend_tree["rrecs-pkg"][package].append(rdepend) if not package in packages: packages.append(package) + for package in packages: - if package != pn: - print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn) - for depend in self.status.deps[fn]: - print >> depends_file, '"%s" -> "%s"' % (package, depend) - # Prints a flattened form of the above where subpackages of a package are merged into the main pn - #print >> depends_file, '"%s" [label="%s %s\\n%s\\n%s"]' % (pn, pn, taskname, version, fn) - #for rdep in taskdata.rdepids[fnid]: - # print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, taskdata.run_names_index[rdep]) - #for dep in taskdata.depids[fnid]: - # print >> depends_file, '"%s" -> "%s"' % (pn, taskdata.build_names_index[dep]) + if package not in depend_tree["packages"]: + depend_tree["packages"][package] = {} + depend_tree["packages"][package]["pn"] = pn + depend_tree["packages"][package]["filename"] = fn + depend_tree["packages"][package]["version"] = version + + return depend_tree + + + def generateDepTreeEvent(self, pkgs_to_build, task): + """ + Create a task dependency graph of pkgs_to_build. + Generate an event with the result + """ + depgraph = self.generateDepTreeData(pkgs_to_build, task) + bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.configuration.data) + + def generateDotGraphFiles(self, pkgs_to_build, task): + """ + Create a task dependency graph of pkgs_to_build. + Save the result to a set of .dot files. + """ + + depgraph = self.generateDepTreeData(pkgs_to_build, task) + + # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn + depends_file = file('pn-depends.dot', 'w' ) + print >> depends_file, "digraph depends {" + for pn in depgraph["pn"]: + fn = depgraph["pn"][pn]["filename"] + version = depgraph["pn"][pn]["version"] + print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn) + for pn in depgraph["depends"]: + for depend in depgraph["depends"][pn]: + print >> depends_file, '"%s" -> "%s"' % (pn, depend) + for pn in depgraph["rdepends-pn"]: + for rdepend in depgraph["rdepends-pn"][pn]: + print >> depends_file, '"%s" -> "%s" [style=dashed]' % (pn, rdepend) + print >> depends_file, "}" + bb.msg.plain("PN dependencies saved to 'pn-depends.dot'") + + depends_file = file('package-depends.dot', 'w' ) + print >> depends_file, "digraph depends {" + for package in depgraph["packages"]: + pn = depgraph["packages"][package]["pn"] + fn = depgraph["packages"][package]["filename"] + version = depgraph["packages"][package]["version"] + if package == pn: + print >> depends_file, '"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn) + else: + print >> depends_file, '"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn) + for depend in depgraph["depends"][pn]: + print >> depends_file, '"%s" -> "%s"' % (package, depend) + for package in depgraph["rdepends-pkg"]: + for rdepend in depgraph["rdepends-pkg"][package]: + print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend) + for package in depgraph["rrecs-pkg"]: + for rdepend in depgraph["rrecs-pkg"][package]: + print >> depends_file, '"%s" -> "%s" [style=dashed]' % (package, rdepend) print >> depends_file, "}" + bb.msg.plain("Package dependencies saved to 'package-depends.dot'") + + tdepends_file = file('task-depends.dot', 'w' ) + print >> tdepends_file, "digraph depends {" + for task in depgraph["tdepends"]: + (pn, taskname) = task.rsplit(".", 1) + fn = depgraph["pn"][pn]["filename"] + version = depgraph["pn"][pn]["version"] + print >> tdepends_file, '"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn) + for dep in depgraph["tdepends"][task]: + print >> tdepends_file, '"%s" -> "%s"' % (task, dep) print >> tdepends_file, "}" - bb.msg.note(1, bb.msg.domain.Collection, "Dependencies saved to 'depends.dot'") - bb.msg.note(1, bb.msg.domain.Collection, "Task dependencies saved to 'task-depends.dot'") + bb.msg.plain("Task dependencies saved to 'task-depends.dot'") def buildDepgraph( self ): all_depends = self.status.all_depends @@ -324,7 +481,7 @@ class BBCooker: try: (providee, provider) = p.split(':') except: - bb.msg.error(bb.msg.domain.Provider, "Malformed option in PREFERRED_PROVIDERS variable: %s" % p) + bb.msg.fatal(bb.msg.domain.Provider, "Malformed option in PREFERRED_PROVIDERS variable: %s" % p) continue if providee in self.status.preferred and self.status.preferred[providee] != provider: bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee])) @@ -362,19 +519,6 @@ class BBCooker: self.status.possible_world = None self.status.all_depends = None - def myProgressCallback( self, x, y, f, from_cache ): - """Update any tty with the progress change""" - if os.isatty(sys.stdout.fileno()): - sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) ) - sys.stdout.flush() - else: - if x == 1: - sys.stdout.write("Parsing .bb files, please wait...") - sys.stdout.flush() - if x == y: - sys.stdout.write("done.") - sys.stdout.flush() - def interactiveMode( self ): """Drop off into a shell""" try: @@ -383,12 +527,10 @@ class BBCooker: bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details ) else: shell.start( self ) - sys.exit( 0 ) - def parseConfigurationFile( self, afiles ): + def parseConfigurationFile( self, afile ): try: - for afile in afiles: - self.configuration.data = bb.parse.handle( afile, self.configuration.data ) + self.configuration.data = bb.parse.handle( afile, self.configuration.data ) # Handle any INHERITs and inherit the base class inherits = ["base"] + (bb.data.getVar('INHERIT', self.configuration.data, True ) or "").split() @@ -402,10 +544,10 @@ class BBCooker: bb.fetch.fetcher_init(self.configuration.data) - bb.event.fire(bb.event.ConfigParsed(self.configuration.data)) + bb.event.fire(bb.event.ConfigParsed(), self.configuration.data) except IOError, e: - bb.msg.fatal(bb.msg.domain.Parsing, "IO Error: %s" % str(e) ) + bb.msg.fatal(bb.msg.domain.Parsing, "Error when parsing %s: %s" % (afile, str(e))) except bb.parse.ParseError, details: bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) ) @@ -439,17 +581,17 @@ class BBCooker: """ if not bb.data.getVar("BUILDNAME", self.configuration.data): bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data) - bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data) + bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()), self.configuration.data) - def matchFile(self, buildfile): + def matchFiles(self, buildfile): """ - Convert the fragment buildfile into a real file - Error if there are too many matches + Find the .bb files which match the expression in 'buildfile'. """ + bf = os.path.abspath(buildfile) try: os.stat(bf) - return bf + return [bf] except OSError: (filelist, masked) = self.collect_bbfiles() regexp = re.compile(buildfile) @@ -458,27 +600,41 @@ class BBCooker: if regexp.search(f) and os.path.isfile(f): bf = f matches.append(f) - if len(matches) != 1: - bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches))) - for f in matches: - bb.msg.error(bb.msg.domain.Parsing, " %s" % f) - return False - return matches[0] + return matches - def buildFile(self, buildfile): + def matchFile(self, buildfile): + """ + Find the .bb file which matches the expression in 'buildfile'. + Raise an error if multiple files + """ + matches = self.matchFiles(buildfile) + if len(matches) != 1: + bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (buildfile, len(matches))) + for f in matches: + bb.msg.error(bb.msg.domain.Parsing, " %s" % f) + raise MultipleMatches + return matches[0] + + def buildFile(self, buildfile, task): """ Build the file matching regexp buildfile """ - # Make sure our target is a fully qualified filename + # Parse the configuration here. We need to do it explicitly here since + # buildFile() doesn't use the cache + self.parseConfiguration() + + # If we are told to do the None task then query the default task + if (task == None): + task = self.configuration.cmd + fn = self.matchFile(buildfile) - if not fn: - return False + self.buildSetVars() # Load data into the cache for fn and parse the loaded cache data self.bb_cache = bb.cache.init(self) self.status = bb.cache.CacheData() - self.bb_cache.loadData(fn, self.configuration.data, self.status) + self.bb_cache.loadData(fn, self.configuration.data, self.status) # Tweak some variables item = self.bb_cache.getVar('PN', fn, True) @@ -493,159 +649,157 @@ class BBCooker: # Remove stamp for target if force mode active if self.configuration.force: - bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, fn)) - bb.build.del_stamp('do_%s' % self.configuration.cmd, self.configuration.data) + bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (task, fn)) + bb.build.del_stamp('do_%s' % task, self.status, fn) # Setup taskdata structure - taskdata = bb.taskdata.TaskData(self.configuration.abort, self.configuration.tryaltconfigs) + taskdata = bb.taskdata.TaskData(self.configuration.abort) taskdata.add_provider(self.configuration.data, self.status, item) buildname = bb.data.getVar("BUILDNAME", self.configuration.data) - bb.event.fire(bb.event.BuildStarted(buildname, [item], self.configuration.event_data)) + bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.configuration.event_data) # Execute the runqueue - runlist = [[item, "do_%s" % self.configuration.cmd]] + runlist = [[item, "do_%s" % task]] + rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist) - rq.prepare_runqueue() - try: - failures = rq.execute_runqueue() - except runqueue.TaskFailure, fnids: + + def buildFileIdle(server, rq, abort): + + if abort or self.cookerAction == cookerStop: + rq.finish_runqueue(True) + elif self.cookerAction == cookerShutdown: + rq.finish_runqueue(False) failures = 0 - for fnid in fnids: - bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid]) - failures = failures + 1 - bb.event.fire(bb.event.BuildCompleted(buildname, [item], self.configuration.event_data, failures)) - return False - bb.event.fire(bb.event.BuildCompleted(buildname, [item], self.configuration.event_data, failures)) - return True + try: + retval = rq.execute_runqueue() + except runqueue.TaskFailure, fnids: + for fnid in fnids: + bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid]) + failures = failures + 1 + retval = False + if not retval: + self.command.finishAsyncCommand() + bb.event.fire(bb.event.BuildCompleted(buildname, item, failures), self.configuration.event_data) + return False + return 0.5 + + self.server.register_idle_function(buildFileIdle, rq) - def buildTargets(self, targets): + def buildTargets(self, targets, task): """ Attempt to build the targets specified """ - buildname = bb.data.getVar("BUILDNAME", self.configuration.data) - bb.event.fire(bb.event.BuildStarted(buildname, targets, self.configuration.event_data)) + # Need files parsed + self.updateCache() - localdata = data.createCopy(self.configuration.data) - bb.data.update_data(localdata) - bb.data.expandKeys(localdata) + # If we are told to do the NULL task then query the default task + if (task == None): + task = self.configuration.cmd - taskdata = bb.taskdata.TaskData(self.configuration.abort, self.configuration.tryaltconfigs) + targets = self.checkPackages(targets) - runlist = [] - try: - for k in targets: - taskdata.add_provider(localdata, self.status, k) - runlist.append([k, "do_%s" % self.configuration.cmd]) - taskdata.add_unresolved(localdata, self.status) - except bb.providers.NoProvider: - sys.exit(1) + def buildTargetsIdle(server, rq, abort): - rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist) - rq.prepare_runqueue() - try: - failures = rq.execute_runqueue() - except runqueue.TaskFailure, fnids: + if abort or self.cookerAction == cookerStop: + rq.finish_runqueue(True) + elif self.cookerAction == cookerShutdown: + rq.finish_runqueue(False) failures = 0 - for fnid in fnids: - bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid]) - failures = failures + 1 - bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures)) - sys.exit(1) - bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures)) + try: + retval = rq.execute_runqueue() + except runqueue.TaskFailure, fnids: + for fnid in fnids: + bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid]) + failures = failures + 1 + retval = False + if not retval: + self.command.finishAsyncCommand() + bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.event_data) + return None + return 0.5 - sys.exit(0) + self.buildSetVars() - def updateCache(self): - # Import Psyco if available and not disabled - import platform - if platform.machine() in ['i386', 'i486', 'i586', 'i686']: - if not self.configuration.disable_psyco: - try: - import psyco - except ImportError: - bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.") - else: - psyco.bind( self.parse_bbfiles ) - else: - bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.") + buildname = bb.data.getVar("BUILDNAME", self.configuration.data) + bb.event.fire(bb.event.BuildStarted(buildname, targets), self.configuration.event_data) - self.status = bb.cache.CacheData() + localdata = data.createCopy(self.configuration.data) + bb.data.update_data(localdata) + bb.data.expandKeys(localdata) - ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or "" - self.status.ignored_dependencies = set( ignore.split() ) + taskdata = bb.taskdata.TaskData(self.configuration.abort) - self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) ) + runlist = [] + for k in targets: + taskdata.add_provider(localdata, self.status, k) + runlist.append([k, "do_%s" % task]) + taskdata.add_unresolved(localdata, self.status) - bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files") - (filelist, masked) = self.collect_bbfiles() - bb.data.renameVar("__depends", "__base_depends", self.configuration.data) - self.parse_bbfiles(filelist, masked, self.myProgressCallback) - bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete") + rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist) - self.buildDepgraph() + self.server.register_idle_function(buildTargetsIdle, rq) - def cook(self): - """ - We are building stuff here. We do the building - from here. By default we try to execute task - build. - """ + def updateCache(self): - # Wipe the OS environment - bb.utils.empty_environment() + if self.cookerState == cookerParsed: + return - if self.configuration.show_environment: - self.showEnvironment(self.configuration.buildfile, self.configuration.pkgs_to_build) - sys.exit( 0 ) + if self.cookerState != cookerParsing: - self.buildSetVars() + self.parseConfiguration () - if self.configuration.interactive: - self.interactiveMode() + # Import Psyco if available and not disabled + import platform + if platform.machine() in ['i386', 'i486', 'i586', 'i686']: + if not self.configuration.disable_psyco: + try: + import psyco + except ImportError: + bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.") + else: + psyco.bind( CookerParser.parse_next ) + else: + bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.") - if self.configuration.buildfile is not None: - if not self.buildFile(self.configuration.buildfile): - sys.exit(1) - sys.exit(0) + self.status = bb.cache.CacheData() - # initialise the parsing status now we know we will need deps - self.updateCache() + ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or "" + self.status.ignored_dependencies = set(ignore.split()) + + for dep in self.configuration.extra_assume_provided: + self.status.ignored_dependencies.add(dep) + + self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) ) - if self.configuration.revisions_changed: - sys.exit(bb.fetch.fetcher_compare_revisons(self.configuration.data)) + bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files") + (filelist, masked) = self.collect_bbfiles() + bb.data.renameVar("__depends", "__base_depends", self.configuration.data) - if self.configuration.parse_only: - bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only. Exiting.") - return 0 + self.parser = CookerParser(self, filelist, masked) + self.cookerState = cookerParsing - pkgs_to_build = self.configuration.pkgs_to_build + if not self.parser.parse_next(): + bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete") + self.buildDepgraph() + self.cookerState = cookerParsed + return None - if len(pkgs_to_build) == 0 and not self.configuration.show_versions: - print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'" - print "for usage information." - sys.exit(0) + return True - try: - if self.configuration.show_versions: - self.showVersions() - sys.exit( 0 ) - if 'world' in pkgs_to_build: - self.buildWorldTargetList() - pkgs_to_build.remove('world') - for t in self.status.world_target: - pkgs_to_build.append(t) + def checkPackages(self, pkgs_to_build): - if self.configuration.dot_graph: - self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps ) - sys.exit( 0 ) + if len(pkgs_to_build) == 0: + raise NothingToBuild - return self.buildTargets(pkgs_to_build) + if 'world' in pkgs_to_build: + self.buildWorldTargetList() + pkgs_to_build.remove('world') + for t in self.status.world_target: + pkgs_to_build.append(t) - except KeyboardInterrupt: - bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.") - sys.exit(1) + return pkgs_to_build def get_bbfiles( self, path = os.getcwd() ): """Get list of default .bb files by reading out the current directory""" @@ -717,59 +871,108 @@ class BBCooker: return (finalfiles, masked) - def parse_bbfiles(self, filelist, masked, progressCallback = None): - parsed, cached, skipped, error = 0, 0, 0, 0 - for i in xrange( len( filelist ) ): - f = filelist[i] + def serve(self): - #bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f) + # Empty the environment. The environment will be populated as + # necessary from the data store. + bb.utils.empty_environment() - # read a file's metadata + if self.configuration.profile: try: - fromCache, skip = self.bb_cache.loadData(f, self.configuration.data, self.status) - if skip: - skipped += 1 - bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f) - self.bb_cache.skip(f) - continue - elif fromCache: cached += 1 - else: parsed += 1 - - # Disabled by RP as was no longer functional - # allow metadata files to add items to BBFILES - #data.update_data(self.pkgdata[f]) - #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None - #if addbbfiles: - # for aof in addbbfiles.split(): - # if not files.count(aof): - # if not os.path.isabs(aof): - # aof = os.path.join(os.path.dirname(f),aof) - # files.append(aof) - - # now inform the caller - if progressCallback is not None: - progressCallback( i + 1, len( filelist ), f, fromCache ) + import cProfile as profile + except: + import profile + + profile.runctx("self.server.serve_forever()", globals(), locals(), "profile.log") + + # Redirect stdout to capture profile information + pout = open('profile.log.processed', 'w') + so = sys.stdout.fileno() + os.dup2(pout.fileno(), so) + + import pstats + p = pstats.Stats('profile.log') + p.sort_stats('time') + p.print_stats() + p.print_callers() + p.sort_stats('cumulative') + p.print_stats() + + os.dup2(so, pout.fileno()) + pout.flush() + pout.close() + else: + self.server.serve_forever() + + bb.event.fire(CookerExit(), self.configuration.event_data) + +class CookerExit(bb.event.Event): + """ + Notify clients of the Cooker shutdown + """ + + def __init__(self): + bb.event.Event.__init__(self) + +class CookerParser: + def __init__(self, cooker, filelist, masked): + # Internal data + self.filelist = filelist + self.cooker = cooker + + # Accounting statistics + self.parsed = 0 + self.cached = 0 + self.error = 0 + self.masked = masked + self.total = len(filelist) + + self.skipped = 0 + self.virtuals = 0 + + # Pointer to the next file to parse + self.pointer = 0 + + def parse_next(self): + if self.pointer < len(self.filelist): + f = self.filelist[self.pointer] + cooker = self.cooker + + try: + fromCache, skipped, virtuals = cooker.bb_cache.loadData(f, cooker.configuration.data, cooker.status) + if fromCache: + self.cached += 1 + else: + self.parsed += 1 + + self.skipped += skipped + self.virtuals += virtuals except IOError, e: - self.bb_cache.remove(f) + self.error += 1 + cooker.bb_cache.remove(f) bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e)) pass except KeyboardInterrupt: - self.bb_cache.sync() + cooker.bb_cache.remove(f) + cooker.bb_cache.sync() raise except Exception, e: - error += 1 - self.bb_cache.remove(f) + self.error += 1 + cooker.bb_cache.remove(f) bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f)) except: - self.bb_cache.remove(f) + cooker.bb_cache.remove(f) raise + finally: + bb.event.fire(bb.event.ParseProgress(self.cached, self.parsed, self.skipped, self.masked, self.virtuals, self.error, self.total), cooker.configuration.event_data) - if progressCallback is not None: - print "\r" # need newline after Handling Bitbake files message - bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked )) + self.pointer += 1 - self.bb_cache.sync() + if self.pointer >= self.total: + cooker.bb_cache.sync() + if self.error > 0: + raise ParsingErrorsFound + return False + return True - if error > 0: - bb.msg.fatal(bb.msg.domain.Collection, "Parsing errors found, exiting...") |
