diff options
Diffstat (limited to 'src/collagen/tinderbox/__init__.py')
| -rw-r--r-- | src/collagen/tinderbox/__init__.py | 550 |
1 files changed, 550 insertions, 0 deletions
diff --git a/src/collagen/tinderbox/__init__.py b/src/collagen/tinderbox/__init__.py new file mode 100644 index 0000000..c5d235d --- /dev/null +++ b/src/collagen/tinderbox/__init__.py @@ -0,0 +1,550 @@ +import sys + +import socket +try: + import cPickle as pickle +except ImportError: + import pickle +from time import sleep +import os +import errno +import subprocess +from subprocess import CalledProcessError +import re +import string +from string import atoi +from traceback import print_exc, format_exc +from logger import log, init_logging + +import portage +try: + import portage._sets as psets +except ImportError: + import portage.sets as psets + +import collagen.protocol as protocol +from collagen.util import WritableObject, flatten_deps +from collagen.common.exceptions import ChrootPreparationException +import tinderbox_config as config + + + + + +class Tinderbox(object): + """ + Class for basic worker object called Tinderbox. Tinderbox connects to Matchbox + and asks for package(s) to compile. Tinderbox tries to compile given packages + and responds to Matchbox either with package contents or error messages + """ + + NOMERGE_PKGS=['sys-apps/portage'] + """These packages will never be (re)merged""" + + def __init__(self): + self.hostname = config.MATCHBOX_HOST + self.port = config.MATCHBOX_PORT + self.sock = None + + self.settings = portage.config(clone=portage.settings) + self.trees = portage.create_trees() + self.settings["PORTAGE_VERBOSE"]="1" + self.settings.backup_changes("PORTAGE_VERBOSE") + self.setconf = psets.SetConfig([], self.settings, self.trees) + init_logging('/tmp') + + + def start_tinderbox(self): + """ + This function starts tinderbox process (connects to Matchbox + server) and then starts compiling packages requested by + Matchbox. + """ + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.connect((self.hostname,self.port)) + + while 1: + msg = protocol.GetNextPackage() + msg_pickled = pickle.dumps(msg) + self.sock.sendall(msg_pickled) + reply = self.sock.recv(1024) + reply_unpickled = pickle.loads(reply) + + if type(reply_unpickled) is protocol.GetNextPackageReply: + gnp = reply_unpickled + print "going to compile: %s\nuse flags: %s" %\ + (gnp.package_name,gnp.use_flags) + package = Package(gnp.package_name, gnp.version, gnp.use_flags) + sleep(5) + try: + self.emerge_package(package) + except Exception, e: + log.error("Fatal error when emerging package %s, see backtrace" % package.name) + log.error(format_exc()) + else: + print "Unknown reply: %s" % reply_unpickled + + + + def emerge_package(self, package): + """ + Top level emerge function for compiling packages + + @param package: package to be compiled, optionally with + version/useflag information + @type package: tinderbox Package + @return: None + """ + log.debug("emerge_package starting for %s" % package.name) + settings = self.settings + + porttree = self.trees[portage.root]['porttree'] + portdb = porttree.dbapi + if not package.version: + # we are compiling ALL versions of package + allversions = portdb.xmatch('match-all', package.name) + else: + # we were told exact version to compile + allversions = ["%s-%s" % (package.name, package.version)] + + for pkg in allversions: + archs = portdb.aux_get(pkg, ["KEYWORDS"])[0] + archs = archs.split() + if settings["ARCH"] not in archs and "~%s" % settings["ARCH"] not in archs: + log.warning("Was asked to compile %s but it doesn't have our arch in KEYWORDS" % pkg) + continue + deps = portdb.aux_get(pkg,["DEPEND"]) + + deps = portage.dep.paren_reduce(deps[0]) + + settings.setcpv(pkg, mydb=portdb) + use_enabled = set(settings["PORTAGE_USE"].split()) + iuse = set(settings["IUSE"].split()) + + # only count deps enabled by USE flags + use_deps = portage.dep.use_reduce(deps, list(use_enabled & iuse)) + use_deps = self._normalize_dependencies(use_deps) + + use_deps = flatten_deps(use_deps) + + log.debug("calling create_dep_groups for pkg %s use_deps %s" % (pkg, use_deps)) + dep_groups = self.create_dep_groups(use_deps) + + # prepare chroot & fork & do work + try: + subprocess.check_call([config.MK_CHROOT_SCRIPT,"-s",config.STAGE_TARBALL, + config.BASE_CHROOT, config.WORK_CHROOT]) + except CalledProcessError, cpe: + raise ChrootPreparationException("Chroot preparation for %s failed with error code: %d" + % (pkg, cpe.returncode)) + except OSError, ose: + raise ChrootPreparationException("Chroot preparation for %s failed with error(%d): %s\n\ + Check your settings" + % (pkg, ose.errno, ose.strerror)) + + self.child_pid = os.fork() + childpid = self.child_pid + if 0 == childpid: + # we are the child! + try: + # setup logging! + os.chroot(config.WORK_CHROOT) + os.chdir("/") + init_logging(config.CHROOT_LOGS) + pkgname, pkgver, pkgrev = portage.pkgsplit(pkg) + + if pkgrev is "r0": + package.version = pkgver + else: + package.version = "%s-%s" % (pkgver, pkgrev) + package.reinit() + self._emerge_package_subprocess(pkg, dep_groups, package) + + sys.exit(0) + except Exception, e: + print_exc() + print "Something went really bad and we need logging, stat!" + log.error("Unrecoverable error in tinderbox slave, see backtrace for possible solutions:") + log.error(format_exc()) + sys.exit(1) + + + (retpid, status) = os.waitpid(childpid, 0) + if 0 != status: + # something went really wrong, grab all the info we can + print "Something went really bad and we need logging, stat!" + log.error("Emerge of package %s failed with error code: %d" % (pkg, status)) + + try: + package_infos = self._load_info('package_infos') + except IOError, e: + log.error("Loading package_infos file failed, something has gone wrong in subprocess apparently") + package.attachments["error"] = "Loading package_infos failed, package was not compiled with any dependency combination" + package.version = 'all' + package_infos = [package.get_info()] + + msg = protocol.AddPackageInfo(package_infos) + self.sock.sendall(pickle.dumps(msg)) + + + def _emerge_package_subprocess(self, pkg, dep_groups, package): + """ + This functions is running inside chrooted environment. It's + purpose is to try and emerge packages group-by-group and then + package specified. All information is stored inside + package_infos to be retrieved later by calling function + + @param pkg: cpv string (category/package-version) + @type pkg: string + @param dep_groups: dependency groups as returned by create_dep_groups function + @type dep_groups: list of lists of dependency cpvs + @param package: package class for filling out information + @type package: tinderbox.Package + + @return None + """ + # We are chrooted inside WORK_CHROOT remember! + porttree = self.trees[portage.root]['porttree'] + portdb = porttree.dbapi + vartree = self.trees[portage.root]["vartree"] + ebuild = portdb.findname(pkg) + + package_infos = [] + + settings = self.settings + for group in dep_groups: + dep_failed = None + deps_processed = [] + for dep in group: + + try: + # this will need to change since it's only a quick hack so that + # we don't have to do dep resolution ourselves + import _emerge as emerge + dep_emergepid = os.fork() + # we need to run emerge_main() in child process since a lot of stuff in there + # likes to call sys.exit() and we don't want that do we? + if 0 == dep_emergepid: + try: + extra_use = dep[0] + if extra_use: + os.environ["USE"]=" ".join(extra_use) + os.environ["FEATURES"] = "-strict" + sys.argv = ["emerge","--verbose", "--usepkg", "--buildpkg" ,"=%s" % dep[1]] + exit_code = emerge.emerge_main() + sys.exit(exit_code) + except Exception, e: + print_exc() + log.error(format_exc()) + sys.exit(1) + ret = os.waitpid(dep_emergepid, 0) + if 0 != ret[1]: + raise Exception("emerge_main() failed with error code %d" % ret[1]) + except Exception, e: + log.error(format_exc()) + log.error("Unable to merge dependency %s for package %s" % (dep, pkg)) + dep_failed = dep[1] + + deps_processed.append(dep) + + settings.setcpv(dep[1], mydb=portdb) + dep_use_enabled = set(settings["PORTAGE_USE"].split()) + dep_iuse = set(settings["IUSE"].split()) + dep_name, dep_ver, dep_rev = portage.pkgsplit(dep[1]) + + real_use_enabled = list(dep_use_enabled & dep_iuse) + if dep[0]: + for useflag in dep[0]: + if useflag.startswith('-'): + if 0 is not l.count(useflag[1:]): + real_use_enabled.remove(useflag[1:]) + elif 0 == real_use_enabled.count(useflag): + real_use_enabled.append(useflag) + + if dep_rev is'r0': + dep_ver_full = dep_ver + else: + dep_ver_full = "%s-%s" % (dep_ver, dep_rev) + dep_pkg = Package(dep_name, dep_ver_full, real_use_enabled) + if dep_failed == dep[1]: + build_dir = self.get_build_dir(dep_failed) + self._add_attachment(dep_pkg, "%s/temp/build.log" % build_dir) + self._add_attachment(dep_pkg, "%s/temp/environment" % build_dir) + package.attachments['emerge_info']=self.get_emerge_info() + package_infos.append(dep_pkg.get_info()) + + + if dep_failed: + log.error("Unable to emerge package %s with deps %s" % (pkg, group)) + # TODO unmerge succeeded deps + self._add_attachment(package, "/var/log/emerge.log") + self._add_attachment(package, "%s/tinderbox.log" % config.CHROOT_LOGS) + package.attachments['emerge_info']=self.get_emerge_info() + package.depends = [x[1] for x in group] + package_infos.append(package.get_info()) + continue + settings.setcpv(pkg, mydb=portdb) + + ret = portage.doebuild(ebuild, "merge", portage.root, settings, debug = False, tree="porttree") + if 0 != ret: + # error installing, grab logs + self._add_attachment(package, "%s/build.log" % settings["T"]) + self._add_attachment(package, "%s/environment" % settings["T"]) + self._add_attachment(package, "%s/tinderbox.log" % config.CHROOT_LOGS) + self._add_attachment(package, "/var/log/emerge.log") + package.attachments['emerge_info']=self.get_emerge_info() + + + package.depends = [x[1] for x in deps_processed] + package_infos.append(package.get_info()) + + for dep in group: + if dep[1] in self.setconf.getSetAtoms("system"): + pass + dep_cat, dep_pv = portage.catsplit(dep[1]) + + ret = portage.unmerge(dep_cat, dep_pv, portage.root, settings, True, vartree=vartree) + if 0 != ret: + log.error("Unable to unmerge dep %s" % dep) + + pkg_cat, pkg_pv = portage.catsplit(pkg) + ret = portage.unmerge(pkg_cat, pkg_pv, portage.root, settings, True, vartree=vartree) + if ret != 0: + log.error("Unable to unmerge package %s" % dep) + + self._save_info('package_infos', package_infos) + + def _add_attachment(self, pkg, path): + """ + Adds file content with given path to package pkg as attachment + + @param pkg: Package that will get embedded data + @type pkg: tinderbox.Package + @param path: Path to attachment file + @type path: string + + @return None + """ + try: + attfile = open(path,"r") + except IOError, (errno, strerror): + print "Unable to read file %s: %d: %s" % (path, errno, strerror) + log.warning("Unable to read file (%s) %d: %s" % (path, errno, strerror)) + pkg.attachments[os.path.basename(path)] = "Unable to read file %s" % path + else: + pkg.attachments[os.path.basename(path)] = attfile.read() + attfile.close() + del attfile + + def _save_info(self, key, data): + """ + Save data inside CHROOT_LOGS directory, under name 'key'. This + function is called from within _emerge_package_subprocess to + save data for parent process + + + @param key: key used to identify data inside CHROOT_LOGS directory + @type key: string + @param data: data to be saved + @rtype: None + """ + outfile = open("%s/%s" % (config.CHROOT_LOGS, key), "w") + pickle.dump(data, outfile) + outfile.close() + + def _load_info(self, key): + """ + Load data from CHROOT_LOGS directory (under WORK_CHROOT) with filename 'key' + + + @param key: key used to identify data inside directory + @type key: string + @rtype: depends on key + @returns: data loaded from filename, usually blob + """ + infile = open("%s/%s/%s" % (config.WORK_CHROOT, config.CHROOT_LOGS, + key),"r") + return pickle.load(infile) + + + + def get_emerge_info(self): + """ + @rtype: string + @returns: emerge --info output + + """ + infoout = WritableObject() + sys.stdout = infoout + + cfg, trees, mtimedb = emerge.load_emerge_config() + emerge.action_info(cfg, trees, [], []) + # REDO withouth emerge.action_info + ret = sys.stdout.content + sys.stdout = sys.__stdout__ + + return string.join(infoout.content,sep='') + + def _normalize_dependencies(self, deps): + """ + Normalizes dependencies by replacing '||' by with alternative + + @param deps: depedencies (list of package atoms) + @type deps: List + @rtype: List + @returns: Normalized list of package atoms with replaced || groups + + Example: + >>> from tinderbox import Tinderbox as tb + >>> t = tb() + >>> t._normalize_dependencies(['||', ['=virtual/jdk-1.5*', '=virtual/jdk-1.4*'], 'app-arch/unzip']) + ['=virtual/jdk-1.5*', 'app-arch/unzip'] + >>> + """ + log.debug("normalize_dependencies called with deps: %s" % deps) + new_deps = [] + for i in range(len(deps)): + if '||' == deps[i]: + next = deps[i+1] + if type(next) is list and len(next) > i: + deps[i+1] = next[0] + elif type(deps[i]) == list: + new_deps.extend(self._normalize_dependencies(deps[i])) + else: + new_deps.append(deps[i]) + return new_deps + + def create_dep_groups(self, deps): + """ + Create dependency groups from package dependencies + + Every valid version of dep (according to input spec) has to be at least in + one output dependency group + + @param deps: dependencies of package (atoms) + @type deps: List + @rtype: List of Lists + @returns: List of dependency groups (list of dependency versions) together with special use flags needed + + Example: + Input: ['=dev-libs/glib-2*', '>=net-fs/samba-3.0.0', 'x11-libs/libX11','dev-util/subversion[-dso]'] + Output: + [[(None,'net-fs/samba-3.2.11'),(None, 'dev-libs/glib-2.18.4-r1'),(None, 'x11-libs/libX11-1.1.2-r1'), (['-dso'],'dev-util/subversion-1.5.5'], + [(None,'net-fs/samba-3.0.32'),(None, 'dev-libs/glib-2.18.4-r1'),(None, 'x11-libs/libX11-1.1.3-r1'), (['-dso'],'dev-util/subversion-1.5.5'], + [(None,'net-fs/samba-3.0.32'),(None, 'dev-libs/glib-2.20.3'),(None, 'x11-libs/libX11-1.1.3'), (['-dso'],'dev-util/subversion-1.5.5'], + """ + log.debug("create_dep_groups called with deps %s" % deps) + result = None + porttree = self.trees[portage.root]['porttree'] + portdb = porttree.dbapi + deps_expanded = [] + max_dep_versions = 0 + for dep in deps: + if dep[0].startswith('!') or dep in self.NOMERGE_PKGS: + continue + dep_useflag = list(portage.dep.dep_getusedeps(dep)) + if 0 == len(dep_useflag): + dep_useflag = None + if not portage.dep.isvalidatom(dep): + log.error("%s is not valid atom in %s" % (dep, str(deps))) + continue + depversions = portdb.xmatch('match-all',dep) + depversions = [(dep_useflag,x) for x in depversions] + deps_expanded.append(depversions) + if len(depversions) > max_dep_versions: + max_dep_versions = len(depversions) + + for dep in deps_expanded: + while len(dep) != 0 and len(dep) < max_dep_versions: + dep.append(dep[0]) + + result = [] + for i in range(max_dep_versions): + group = [] + for dep in deps_expanded: + if 0 == len(dep): + continue + group.append(dep.pop()) + result.append(group) + return result + + def get_build_dir(self, cpv): + settings = self.settings + bldprefix = settings["PORTAGE_TMPDIR"]+"/portage" + return os.path.join(bldprefix, cpv) + + + + +class Package(object): + + def __init__(self, name, version, use_flags): + """ + @param name: category/name of given package (excluding version/release) + @type name: string + @param version: version part of CPV e.g. 1.2.0-r4 + @type version: string + @param use_flags: list of enabled use flags for given package + @type use_flags: list + + """ + self.name = name + self.version = version + self.use_flags = use_flags + self.reinit() + + def reinit(self): + self.content = None + self.attachments = {} + self.depends = None + + + def get_info(self): + """Returns protocol.PackageInfo with information about this package + + @returns: PackageInfo for sending across network + @rtype: protocol.PackageInfo + """ + pi = protocol.PackageInfo() + pi.name = self.name + pi.version = self.version + pi.use_flags = self.use_flags + if self.content is None: + self.content = self.get_package_contents() + pi.content = self.content + pi.attachments = self.attachments + if 0 < len(self.attachments): + pi.error = 1 + pi.depends = self.depends + pi.profile = self._get_current_profile() + return pi + + def _get_current_profile(self): + ret = None + try: + profpath = os.readlink("%s/etc/make.profile" % portage.root) + ret = profpath[profpath.rindex("profiles")+len("profiles")+1:] + except Exception, e: + print e + return ret + + + def get_package_contents(self): + """Returns package contents as dict with paths as keys + + data values are tuples of information (such as hash and size) + + @returns: package contents (or {} if it's not installed) + @rtype: dict + """ + vartree = portage.db[portage.root]["vartree"] + cpv = "%s-%s" % (self.name, self.version) + if not vartree.dbapi.cpv_exists(cpv): + # maybe raise exception? instead + return {} + + cat, pkg = portage.catsplit(cpv) + dblink = portage.dblink(cat, pkg, portage.root, vartree.settings, + treetype="vartree", vartree=vartree) + + return dblink.getcontents() |