#!/usr/bin/env python3 # pylint: disable=line-too-long,missing-docstring,invalid-name # vim:noet sts=4 ts=4 # Copyright 2015 Doug Freed (dwfreed) # # - Acquire local copy of repo/gentoo (.git directory not required) # - Ensure mtime of files matches last time they were committed to, including # merge commits (for optimal behavior, this should be the last time they were # actually modified, but the last time a commit affected them in any way is # alright too) # - Run egencache against the checkout (this step is NOT optional; manifest generation is torturously slow without md5-cache) # - Run thicken-manifests.py # # Takes an optional --jobs parameter, defaults to the number of CPUs on the host; also takes an optional directory to run against, defaults to the current working directory # For GPG signing, add --sign parameter. # It behaves exactly like repoman manifest, so you can override GPG behavior by setting PORTAGE_GPG_SIGNING_COMMAND, PORTAGE_GPG_KEY, and PORTAGE_GPG_DIR in make.conf or the environment. # You do NOT need to modify layout.conf for this script to work. Stick with GPG 2.0 for now, because 2.1 moves key operations into the agent, which makes them not parallel. import os import multiprocessing import subprocess import logging import argparse import errno import math import collections import portage import portage.exception import portage.manifest import portage.util portage_settings = None portage_portdbapi = None logger = None args = None dir_contents = collections.defaultdict(list) def gpg_sign(filename): gpgcmd = portage_settings.get("PORTAGE_GPG_SIGNING_COMMAND") if gpgcmd in [None, '']: raise portage.exception.MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset! Is make.globals missing?") if "${PORTAGE_GPG_KEY}" in gpgcmd and "PORTAGE_GPG_KEY" not in portage_settings: raise portage.exception.MissingParameter("PORTAGE_GPG_KEY is unset!") if "${PORTAGE_GPG_DIR}" in gpgcmd: if "PORTAGE_GPG_DIR" not in portage_settings: portage_settings["PORTAGE_GPG_DIR"] = os.path.expanduser("~/.gnupg") else: portage_settings["PORTAGE_GPG_DIR"] = os.path.expanduser(portage_settings["PORTAGE_GPG_DIR"]) if not os.access(portage_settings["PORTAGE_GPG_DIR"], os.X_OK): raise portage.exception.InvalidLocation("Unable to access directory: PORTAGE_GPG_DIR='%s'" % portage_settings["PORTAGE_GPG_DIR"]) gpgvars = {"FILE": filename} for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"): v = portage_settings.get(k) if v is not None: gpgvars[k] = v gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars) gpgcmd = portage.util.shlex_split(gpgcmd) gpgcmd = [portage._unicode_encode(arg, encoding=portage._encodings['fs'], errors='strict') for arg in gpgcmd] # pylint: disable=protected-access return_code = subprocess.call(gpgcmd) if return_code == os.EX_OK: os.rename(filename + ".asc", filename) else: raise portage.exception.PortageException("!!! gpg exited with '" + str(return_code) + "' status") def worker_init(): global portage_settings, portage_portdbapi # pylint: disable=global-statement portage_settings = portage.config(clone=portage.settings) # pylint: disable=no-member portage_portdbapi = portage.portdbapi(portage_settings) # pylint: disable=no-member def maybe_thicken_manifest(pkg_dir): try: manifest_mtime = math.floor(os.stat(os.path.join(pkg_dir, 'Manifest')).st_mtime) except OSError as e: if e.errno != errno.ENOENT: logger.exception("%s OSError thrown trying to stat Manifest", pkg_dir) manifest_mtime = 0 newest_mtime = manifest_mtime manifest_entries = [] for root in pkg_dir, os.path.join(pkg_dir, 'files'): for filename in dir_contents[root]: if filename == 'Manifest' and root == pkg_dir: continue file_mtime = math.floor(os.stat(os.path.join(root, filename)).st_mtime) if file_mtime > newest_mtime: newest_mtime = file_mtime if root != pkg_dir: manifest_entries.append('AUX ' + filename) elif filename.endswith('.ebuild'): manifest_entries.append('EBUILD ' + filename) else: manifest_entries.append('MISC ' + filename) if newest_mtime == manifest_mtime: try: with open(os.path.join(pkg_dir, 'Manifest'), 'r') as f: for line in f: if not line.startswith(('AUX', 'EBUILD', 'MISC')): continue manifest_entry = ' '.join(line.split(' ')[0:2]) if manifest_entry in manifest_entries: manifest_entries.remove(manifest_entry) else: newest_mtime += 1 break except: pass fetchlistdict = portage.FetchlistDict(pkg_dir, portage_settings, portage_portdbapi) # We use an empty dir so that we do not introduce any changes to # existing DIST entries, or inject DIST entries that were otherwise # missing. manifest = portage.manifest.Manifest(pkg_dir, fetchlist_dict=fetchlistdict, distdir='/var/empty') try: manifest.create(assumeDistHashesAlways=True) except portage.exception.FileNotFound: logger.exception("%s is missing DIST entries!", pkg_dir) # If any exception was triggered on the Manifest, it is NOT safe to write. return # If it changed, write it out if manifest.write(): if newest_mtime == manifest_mtime: # portage says the manifest changed, but mtime is supposed to stay the same # bump the mtime by 1 to be safe newest_mtime += 1 if args.sign: try: gpg_sign(manifest.getFullname()) except: logger.exception("%s Exception thrown during GPG signing", pkg_dir) # Always reset mtime on manifest os.utime(manifest.getFullname(), (newest_mtime, newest_mtime)) def main(): global logger, args, dir_contents # pylint: disable=global-statement parser = argparse.ArgumentParser(description='Thicken ebuild manifests as needed') parser.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int, help='Number of parallel jobs to run; default: number of CPUs') parser.add_argument('-s', '--sign', action='store_true', help='Sign manifests with GPG') parser.add_argument('location', nargs='?', default='.', help='The location to thicken manifests in; default: .') args = parser.parse_args() args.location = os.path.realpath(args.location) logger = multiprocessing.log_to_stderr() logger.setLevel(logging.WARN) os.environ['PORTAGE_REPOSITORIES'] = ''' [DEFAULT] main-repo = gentoo [gentoo] location = %s ''' % args.location os.chdir(args.location) pkg_dirs = [] for root, subdirectories, files in os.walk('.'): if root == '.': for dirname in ('eclass', 'licenses', 'metadata', 'profiles', 'scripts', '.git'): if dirname in subdirectories: subdirectories.remove(dirname) if (not subdirectories or subdirectories == ['files']) and 'metadata.xml' in files: if os.path.sep not in os.path.split(root)[0]: # EDGE CASE: category with single package called "files" continue pkg_dirs.append(root[2:]) dir_contents[root[2:]] = files if 'files' in subdirectories: subdirectories.remove('files') filesdir = os.path.join(root, 'files') dir_contents[filesdir[2:]] = [] os.chdir(filesdir) for root2, subdirectories2, files2 in os.walk('.'): for filename in files2: dir_contents[filesdir[2:]].append(os.path.join(root2[2:], filename)) os.chdir(args.location) pkg_dirs.sort() pool = multiprocessing.Pool(args.jobs, worker_init) pool.map(maybe_thicken_manifest, pkg_dirs) pool.close() if __name__ == '__main__': main()