Initial commit.

Copied from git+ssh://git@git.gentoo.org/infra/cfengine.git repo as:
timestamp 2015-11-29T21:57:00Z
commit 3b63da8fbbb848d5a1f7e7cd7c6989638ed0d817

No passwords, passphrases or key material is contained herein, but it
may reference the pathes to such.

Reviewed-by: Robin H. Johnson <robbat2@gentoo.org>
Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>

1 files changed, 150 insertions, 0 deletions

diff --git a/thicken-manifests.py b/thicken-manifests.py
new file mode 100644
index 0000000..3b04fa4
--- /dev/null
+++ b/thicken-manifests.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+# Copyright 2015 Doug Freed (dwfreed)
+#
+# - Acquire local copy of repo/gentoo (.git directory not required)
+# - Ensure mtime of files matches last time they were committed to, including
+#   merge commits (for optimal behavior, this should be the last time they were
+#   actually modified, but the last time a commit affected them in any way is
+#   alright too)
+# - Run egencache against the checkout (this step is NOT optional; manifest generation is torturously slow without md5-cache)
+# - Run thicken-manifests.py
+# 
+# Takes an optional --jobs parameter, defaults to the number of CPUs on the host; also takes an optional directory to run against, defaults to the current working directory.  
+# For GPG signing, add --sign parameter.  
+# It behaves exactly like repoman manifest, so you can override GPG behavior by setting PORTAGE_GPG_SIGNING_COMMAND, PORTAGE_GPG_KEY, and PORTAGE_GPG_DIR in make.conf or the environment.  
+# You do NOT need to modify layout.conf for this script to work. Stick with GPG 2.0 for now, because 2.1 moves key operations into the agent, which makes them not parallel.
+
+import os
+import multiprocessing
+import subprocess
+import logging
+import argparse
+import errno
+import math
+
+import portage
+import portage.exception
+import portage.manifest
+import portage.util
+
+portage_settings = None
+portage_portdbapi = None
+logger = None
+args = None
+
+
+def gpg_sign(filename):
+	gpgcmd = portage_settings.get("PORTAGE_GPG_SIGNING_COMMAND")
+	if gpgcmd in [None, '']:
+		raise portage.exception.MissingParameter(
+			"PORTAGE_GPG_SIGNING_COMMAND is unset! Is make.globals missing?")
+	if "${PORTAGE_GPG_KEY}" in gpgcmd and "PORTAGE_GPG_KEY" not in portage_settings:
+		raise portage.exception.MissingParameter("PORTAGE_GPG_KEY is unset!")
+	if "${PORTAGE_GPG_DIR}" in gpgcmd:
+		if "PORTAGE_GPG_DIR" not in portage_settings:
+			portage_settings["PORTAGE_GPG_DIR"] = os.path.expanduser("~/.gnupg")
+		else:
+			portage_settings["PORTAGE_GPG_DIR"] = os.path.expanduser(portage_settings["PORTAGE_GPG_DIR"])
+		if not os.access(portage_settings["PORTAGE_GPG_DIR"], os.X_OK):
+			raise portage.exception.InvalidLocation(
+				"Unable to access directory: PORTAGE_GPG_DIR='%s'" %
+				portage_settings["PORTAGE_GPG_DIR"])
+	gpgvars = {"FILE": filename}
+	for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"):
+		v = portage_settings.get(k)
+		if v is not None:
+			gpgvars[k] = v
+	gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars)
+	gpgcmd = portage.util.shlex_split(gpgcmd)
+	gpgcmd = [portage._unicode_encode(arg, encoding=portage._encodings['fs'], errors='strict') for arg in gpgcmd]
+
+	return_code = subprocess.call(gpgcmd)
+	if return_code == os.EX_OK:
+		os.rename(filename + ".asc", filename)
+	else:
+		raise portage.exception.PortageException("!!! gpg exited with '" + str(return_code) + "' status")
+
+
+def worker_init():
+	global portage_settings, portage_portdbapi
+	portage_settings = portage.config(clone=portage.settings)
+	portage_portdbapi = portage.portdbapi(portage_settings)
+
+
+def maybe_thicken_manifest(pkg_dir):
+	try:
+		manifest_mtime = math.floor(os.stat(os.path.join(pkg_dir, 'Manifest')).st_mtime)
+	except OSError as e:
+		if e.errno != errno.ENOENT:
+			logger.exception("%s OSError thrown trying to stat Manifest", pkg_dir)
+		manifest_mtime = 0
+	newest_mtime = manifest_mtime
+	for root, subdirectories, files in os.walk(pkg_dir):
+		for file in files:
+			if file == 'Manifest':
+				continue
+
+			file_mtime = math.floor(os.stat(os.path.join(root, file)).st_mtime)
+			if file_mtime > newest_mtime:
+				newest_mtime = file_mtime
+
+	if newest_mtime == manifest_mtime:
+		try:
+			with open(os.path.join(pkg_dir, 'Manifest'), 'r') as f:
+				if f.readline().startswith('-'):
+					return
+		except:
+			pass
+
+	fetchlistdict = portage.FetchlistDict(pkg_dir, portage_settings, portage_portdbapi)
+	manifest = portage.manifest.Manifest(pkg_dir, fetchlist_dict=fetchlistdict, distdir=portage_settings['DISTDIR'])
+	try:
+		manifest.create(assumeDistHashesAlways=True)
+	except portage.exception.FileNotFound:
+		logger.exception("%s is missing DIST entries!", pkg_dir)
+	if manifest.write():
+		if args.sign:
+			try:
+				gpg_sign(manifest.getFullname())
+			except:
+				logger.exception("%s Exception thrown during GPG signing", pkg_dir)
+		os.utime(manifest.getFullname(), (newest_mtime, newest_mtime))
+
+
+def main():
+	global logger, args
+	parser = argparse.ArgumentParser(description='Thicken ebuild manifests as needed')
+	parser.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int,
+				help='Number of parallel jobs to run; default: number of CPUs')
+	parser.add_argument('-s', '--sign', action='store_true')
+	parser.add_argument('location', nargs='?', default='.', help='The location to thicken manifests in; default: .')
+	args = parser.parse_args()
+
+	logger = multiprocessing.log_to_stderr()
+	logger.setLevel(logging.WARN)
+
+	os.environ['PORTAGE_REPOSITORIES'] = '''
+[DEFAULT]
+main-repo = gentoo
+
+[gentoo]
+location = %s
+''' % os.path.realpath(args.location)
+
+	os.chdir(args.location)
+	pkg_dirs = []
+	for root, subdirectories, files in os.walk('.'):
+		for dir in ('eclass', 'metadata', 'profiles', '.git', 'files'):
+			if dir in subdirectories:
+				subdirectories.remove(dir)
+
+		if not subdirectories and 'metadata.xml' in files:
+			pkg_dirs.append(root)
+
+	pool = multiprocessing.Pool(args.jobs, worker_init)
+	pool.map(maybe_thicken_manifest, pkg_dirs)
+	pool.close()
+
+
+if __name__ == '__main__':
+	main()