From 306b7c7a9757ead077363074e7bbac2e5c03e7c5 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Thu, 16 Nov 2006 15:02:15 +0000 Subject: bitbake: Upgrade from 1.4 -> 1.7.4ish git-svn-id: https://svn.o-hand.com/repos/poky/trunk@863 311d38ba-8fff-0310-9ca6-ca027cbcb966 --- bitbake/lib/bb/taskdata.py | 558 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 558 insertions(+) create mode 100644 bitbake/lib/bb/taskdata.py (limited to 'bitbake/lib/bb/taskdata.py') diff --git a/bitbake/lib/bb/taskdata.py b/bitbake/lib/bb/taskdata.py new file mode 100644 index 0000000..181bb5e --- /dev/null +++ b/bitbake/lib/bb/taskdata.py @@ -0,0 +1,558 @@ +#!/usr/bin/env python +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'TaskData' implementation + +Task data collection and handling + +Copyright (C) 2006 Richard Purdie + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License version 2 as published by the Free +Software Foundation + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +""" + +from bb import data, fetch, event, mkdirhier, utils +import bb, os + +class TaskData: + """ + BitBake Task Data implementation + """ + def __init__(self, abort = True): + self.build_names_index = [] + self.run_names_index = [] + self.fn_index = [] + + self.build_targets = {} + self.run_targets = {} + + self.external_targets = [] + + self.tasks_fnid = [] + self.tasks_name = [] + self.tasks_tdepends = [] + # Cache to speed up task ID lookups + self.tasks_lookup = {} + + self.depids = {} + self.rdepids = {} + + self.consider_msgs_cache = [] + + self.failed_deps = [] + self.failed_rdeps = [] + self.failed_fnids = [] + + self.abort = abort + + def getbuild_id(self, name): + """ + Return an ID number for the build target name. + If it doesn't exist, create one. + """ + if not name in self.build_names_index: + self.build_names_index.append(name) + return len(self.build_names_index) - 1 + + return self.build_names_index.index(name) + + def getrun_id(self, name): + """ + Return an ID number for the run target name. + If it doesn't exist, create one. + """ + if not name in self.run_names_index: + self.run_names_index.append(name) + return len(self.run_names_index) - 1 + + return self.run_names_index.index(name) + + def getfn_id(self, name): + """ + Return an ID number for the filename. + If it doesn't exist, create one. + """ + if not name in self.fn_index: + self.fn_index.append(name) + return len(self.fn_index) - 1 + + return self.fn_index.index(name) + + def gettask_id(self, fn, task): + """ + Return an ID number for the task matching fn and task. + If it doesn't exist, create one. + """ + fnid = self.getfn_id(fn) + + if fnid in self.tasks_lookup: + if task in self.tasks_lookup[fnid]: + return self.tasks_lookup[fnid][task] + + self.tasks_name.append(task) + self.tasks_fnid.append(fnid) + self.tasks_tdepends.append([]) + + listid = len(self.tasks_name) - 1 + + if fnid not in self.tasks_lookup: + self.tasks_lookup[fnid] = {} + self.tasks_lookup[fnid][task] = listid + + return listid + + def add_tasks(self, fn, dataCache): + """ + Add tasks for a given fn to the database + """ + + task_graph = dataCache.task_queues[fn] + task_deps = dataCache.task_deps[fn] + + fnid = self.getfn_id(fn) + + if fnid in self.failed_fnids: + bb.msg.fatal(bb.msg.domain.TaskData, "Trying to re-add a failed file? Something is broken...") + + # Check if we've already seen this fn + if fnid in self.tasks_fnid: + return + + # Work out task dependencies + for task in task_graph.allnodes(): + parentids = [] + for dep in task_graph.getparents(task): + parentid = self.gettask_id(fn, dep) + parentids.append(parentid) + taskid = self.gettask_id(fn, task) + self.tasks_tdepends[taskid].extend(parentids) + + # Work out build dependencies + if not fnid in self.depids: + dependids = {} + for depend in dataCache.deps[fn]: + bb.msg.debug(2, bb.msg.domain.TaskData, "Added dependency %s for %s" % (depend, fn)) + dependids[self.getbuild_id(depend)] = None + self.depids[fnid] = dependids.keys() + + # Work out runtime dependencies + if not fnid in self.rdepids: + rdependids = {} + rdepends = dataCache.rundeps[fn] + rrecs = dataCache.runrecs[fn] + for package in rdepends: + for rdepend in rdepends[package]: + bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime dependency %s for %s" % (rdepend, fn)) + rdependids[self.getrun_id(rdepend)] = None + for package in rrecs: + for rdepend in rrecs[package]: + bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime recommendation %s for %s" % (rdepend, fn)) + rdependids[self.getrun_id(rdepend)] = None + self.rdepids[fnid] = rdependids.keys() + + for dep in self.depids[fnid]: + if dep in self.failed_deps: + self.fail_fnid(fnid) + return + for dep in self.rdepids[fnid]: + if dep in self.failed_rdeps: + self.fail_fnid(fnid) + return + + def have_build_target(self, target): + """ + Have we a build target matching this name? + """ + targetid = self.getbuild_id(target) + + if targetid in self.build_targets: + return True + return False + + def have_runtime_target(self, target): + """ + Have we a runtime target matching this name? + """ + targetid = self.getrun_id(target) + + if targetid in self.run_targets: + return True + return False + + def add_build_target(self, fn, item): + """ + Add a build target. + If already present, append the provider fn to the list + """ + targetid = self.getbuild_id(item) + fnid = self.getfn_id(fn) + + if targetid in self.build_targets: + if fnid in self.build_targets[targetid]: + return + self.build_targets[targetid].append(fnid) + return + self.build_targets[targetid] = [fnid] + + def add_runtime_target(self, fn, item): + """ + Add a runtime target. + If already present, append the provider fn to the list + """ + targetid = self.getrun_id(item) + fnid = self.getfn_id(fn) + + if targetid in self.run_targets: + if fnid in self.run_targets[targetid]: + return + self.run_targets[targetid].append(fnid) + return + self.run_targets[targetid] = [fnid] + + def mark_external_target(self, item): + """ + Mark a build target as being externally requested + """ + targetid = self.getbuild_id(item) + + if targetid not in self.external_targets: + self.external_targets.append(targetid) + + def get_unresolved_build_targets(self, dataCache): + """ + Return a list of build targets who's providers + are unknown. + """ + unresolved = [] + for target in self.build_names_index: + if target in dataCache.ignored_dependencies: + continue + if self.build_names_index.index(target) in self.failed_deps: + continue + if not self.have_build_target(target): + unresolved.append(target) + return unresolved + + def get_unresolved_run_targets(self, dataCache): + """ + Return a list of runtime targets who's providers + are unknown. + """ + unresolved = [] + for target in self.run_names_index: + if target in dataCache.ignored_dependencies: + continue + if self.run_names_index.index(target) in self.failed_rdeps: + continue + if not self.have_runtime_target(target): + unresolved.append(target) + return unresolved + + def get_provider(self, item): + """ + Return a list of providers of item + """ + targetid = self.getbuild_id(item) + + return self.build_targets[targetid] + + def get_dependees(self, itemid): + """ + Return a list of targets which depend on item + """ + dependees = [] + for fnid in self.depids: + if itemid in self.depids[fnid]: + dependees.append(fnid) + return dependees + + def get_dependees_str(self, item): + """ + Return a list of targets which depend on item as a user readable string + """ + itemid = self.getbuild_id(item) + dependees = [] + for fnid in self.depids: + if itemid in self.depids[fnid]: + dependees.append(self.fn_index[fnid]) + return dependees + + def get_rdependees(self, itemid): + """ + Return a list of targets which depend on runtime item + """ + dependees = [] + for fnid in self.rdepids: + if itemid in self.rdepids[fnid]: + dependees.append(fnid) + return dependees + + def get_rdependees_str(self, item): + """ + Return a list of targets which depend on runtime item as a user readable string + """ + itemid = self.getrun_id(item) + dependees = [] + for fnid in self.rdepids: + if itemid in self.rdepids[fnid]: + dependees.append(self.fn_index[fnid]) + return dependees + + def add_provider(self, cfgData, dataCache, item): + try: + self.add_provider_internal(cfgData, dataCache, item) + except bb.providers.NoProvider: + if self.abort: + bb.msg.error(bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item))) + raise + targetid = self.getbuild_id(item) + self.remove_buildtarget(targetid) + + self.mark_external_target(item) + + def add_provider_internal(self, cfgData, dataCache, item): + """ + Add the providers of item to the task data + Mark entries were specifically added externally as against dependencies + added internally during dependency resolution + """ + + if item in dataCache.ignored_dependencies: + return + + if not item in dataCache.providers: + bb.msg.debug(1, bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item))) + bb.event.fire(bb.event.NoProvider(item, cfgData)) + raise bb.providers.NoProvider(item) + + if self.have_build_target(item): + return + + all_p = dataCache.providers[item] + + eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache) + + for p in eligible: + fnid = self.getfn_id(p) + if fnid in self.failed_fnids: + eligible.remove(p) + + if not eligible: + bb.msg.debug(1, bb.msg.domain.Provider, "No providers of build target %s after filtering (for %s)" % (item, self.get_dependees_str(item))) + bb.event.fire(bb.event.NoProvider(item, cfgData)) + raise bb.providers.NoProvider(item) + + prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, cfgData, 1) + if prefervar: + dataCache.preferred[item] = prefervar + + discriminated = False + if item in dataCache.preferred: + for p in eligible: + pn = dataCache.pkg_fn[p] + if dataCache.preferred[item] == pn: + bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item)) + eligible.remove(p) + eligible = [p] + eligible + discriminated = True + break + + if len(eligible) > 1 and discriminated == False: + if item not in self.consider_msgs_cache: + providers_list = [] + for fn in eligible: + providers_list.append(dataCache.pkg_fn[fn]) + bb.msg.note(1, bb.msg.domain.Provider, "multiple providers are available for %s (%s);" % (item, ", ".join(providers_list))) + bb.msg.note(1, bb.msg.domain.Provider, "consider defining PREFERRED_PROVIDER_%s" % item) + bb.event.fire(bb.event.MultipleProviders(item,providers_list,cfgData)) + self.consider_msgs_cache.append(item) + + for fn in eligible: + fnid = self.getfn_id(fn) + if fnid in self.failed_fnids: + continue + bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy %s" % (fn, item)) + self.add_build_target(fn, item) + self.add_tasks(fn, dataCache) + + + #item = dataCache.pkg_fn[fn] + + def add_rprovider(self, cfgData, dataCache, item): + """ + Add the runtime providers of item to the task data + (takes item names from RDEPENDS/PACKAGES namespace) + """ + + if item in dataCache.ignored_dependencies: + return + + if self.have_runtime_target(item): + return + + all_p = bb.providers.getRuntimeProviders(dataCache, item) + + if not all_p: + bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s (for %s)" % (item, self.get_rdependees_str(item))) + bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True)) + raise bb.providers.NoRProvider(item) + + eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache) + + for p in eligible: + fnid = self.getfn_id(p) + if fnid in self.failed_fnids: + eligible.remove(p) + + if not eligible: + bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s after filtering (for %s)" % (item, self.get_rdependees_str(item))) + bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True)) + raise bb.providers.NoRProvider(item) + + # Should use dataCache.preferred here? + preferred = [] + for p in eligible: + pn = dataCache.pkg_fn[p] + provides = dataCache.pn_provides[pn] + for provide in provides: + prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, cfgData, 1) + if prefervar == pn: + bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item)) + eligible.remove(p) + eligible = [p] + eligible + preferred.append(p) + + if len(eligible) > 1 and len(preferred) == 0: + if item not in self.consider_msgs_cache: + providers_list = [] + for fn in eligible: + providers_list.append(dataCache.pkg_fn[fn]) + bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available for runtime %s (%s);" % (item, ", ".join(providers_list))) + bb.msg.note(2, bb.msg.domain.Provider, "consider defining a PREFERRED_PROVIDER entry to match runtime %s" % item) + bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True)) + self.consider_msgs_cache.append(item) + + if len(preferred) > 1: + if item not in self.consider_msgs_cache: + providers_list = [] + for fn in preferred: + providers_list.append(dataCache.pkg_fn[fn]) + bb.msg.note(2, bb.msg.domain.Provider, "multiple preferred providers are available for runtime %s (%s);" % (item, ", ".join(providers_list))) + bb.msg.note(2, bb.msg.domain.Provider, "consider defining only one PREFERRED_PROVIDER entry to match runtime %s" % item) + bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True)) + self.consider_msgs_cache.append(item) + + # run through the list until we find one that we can build + for fn in eligible: + fnid = self.getfn_id(fn) + if fnid in self.failed_fnids: + continue + bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy runtime %s" % (fn, item)) + self.add_runtime_target(fn, item) + self.add_tasks(fn, dataCache) + + def fail_fnid(self, fnid): + """ + Mark a file as failed (unbuildable) + Remove any references from build and runtime provider lists + """ + if fnid in self.failed_fnids: + return + bb.msg.debug(1, bb.msg.domain.Provider, "Removing failed file %s" % self.fn_index[fnid]) + self.failed_fnids.append(fnid) + for target in self.build_targets: + if fnid in self.build_targets[target]: + self.build_targets[target].remove(fnid) + if len(self.build_targets[target]) == 0: + self.remove_buildtarget(target) + for target in self.run_targets: + if fnid in self.run_targets[target]: + self.run_targets[target].remove(fnid) + if len(self.run_targets[target]) == 0: + self.remove_runtarget(target) + + def remove_buildtarget(self, targetid): + """ + Mark a build target as failed (unbuildable) + Trigger removal of any files that have this as a dependency + """ + bb.msg.debug(1, bb.msg.domain.Provider, "Removing failed build target %s" % self.build_names_index[targetid]) + self.failed_deps.append(targetid) + dependees = self.get_dependees(targetid) + for fnid in dependees: + self.fail_fnid(fnid) + if self.abort and targetid in self.external_targets: + bb.msg.error(bb.msg.domain.Provider, "No buildable providers available for required build target %s" % self.build_names_index[targetid]) + raise bb.providers.NoProvider + + def remove_runtarget(self, targetid): + """ + Mark a run target as failed (unbuildable) + Trigger removal of any files that have this as a dependency + """ + bb.msg.note(1, bb.msg.domain.Provider, "Removing failed runtime build target %s" % self.run_names_index[targetid]) + self.failed_rdeps.append(targetid) + dependees = self.get_rdependees(targetid) + for fnid in dependees: + self.fail_fnid(fnid) + + def add_unresolved(self, cfgData, dataCache): + """ + Resolve all unresolved build and runtime targets + """ + bb.msg.note(1, bb.msg.domain.TaskData, "Resolving missing task queue dependencies") + while 1: + added = 0 + for target in self.get_unresolved_build_targets(dataCache): + try: + self.add_provider_internal(cfgData, dataCache, target) + added = added + 1 + except bb.providers.NoProvider: + targetid = self.getbuild_id(target) + if self.abort and targetid in self.external_targets: + raise + self.remove_buildtarget(targetid) + for target in self.get_unresolved_run_targets(dataCache): + try: + self.add_rprovider(cfgData, dataCache, target) + added = added + 1 + except bb.providers.NoRProvider: + self.remove_runtarget(self.getrun_id(target)) + bb.msg.debug(1, bb.msg.domain.TaskData, "Resolved " + str(added) + " extra dependecies") + if added == 0: + break + + def dump_data(self): + """ + Dump some debug information on the internal data structures + """ + bb.msg.debug(3, bb.msg.domain.TaskData, "build_names:") + bb.msg.debug(3, bb.msg.domain.TaskData, ", ".join(self.build_names_index)) + bb.msg.debug(3, bb.msg.domain.TaskData, "run_names:") + bb.msg.debug(3, bb.msg.domain.TaskData, ", ".join(self.run_names_index)) + bb.msg.debug(3, bb.msg.domain.TaskData, "build_targets:") + for target in self.build_targets.keys(): + bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.build_names_index[target], self.build_targets[target])) + bb.msg.debug(3, bb.msg.domain.TaskData, "run_targets:") + for target in self.run_targets.keys(): + bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.run_names_index[target], self.run_targets[target])) + bb.msg.debug(3, bb.msg.domain.TaskData, "tasks:") + for task in range(len(self.tasks_name)): + bb.msg.debug(3, bb.msg.domain.TaskData, " (%s)%s - %s: %s" % ( + task, + self.fn_index[self.tasks_fnid[task]], + self.tasks_name[task], + self.tasks_tdepends[task])) + bb.msg.debug(3, bb.msg.domain.TaskData, "runtime ids (per fn):") + for fnid in self.rdepids: + bb.msg.debug(3, bb.msg.domain.TaskData, " %s %s: %s" % (fnid, self.fn_index[fnid], self.rdepids[fnid])) + + -- cgit v1.1