diff options
| -rw-r--r-- | Makefile.gpyutils | 71 | ||||
| -rwxr-xr-x | cgi-bin/get-git-file.sh | 35 | ||||
| -rwxr-xr-x | cgi-bin/trigger-pull.sh | 36 | ||||
| -rwxr-xr-x | check_eclasses_eapis.sh | 17 | ||||
| -rwxr-xr-x | create-dev-keyrings.bash | 12 | ||||
| m--------- | dev-timeline | 0 | ||||
| -rwxr-xr-x | eapi-usage.sh | 86 | ||||
| -rwxr-xr-x | eapi_usage.sh | 25 | ||||
| -rwxr-xr-x | eclass-eapi-matrix.py | 6 | ||||
| -rwxr-xr-x | find-binary-files.sh | 22 | ||||
| -rwxr-xr-x | gen-dev-wot.sh | 2 | ||||
| -rwxr-xr-x | genrdeps-index.py | 131 | ||||
| -rwxr-xr-x | genrdeps/genrdeps.py | 46 | ||||
| -rwxr-xr-x | genrdeps/run-genrdeps-index.sh | 40 | ||||
| -rwxr-xr-x | gpy-impl-list | 17 | ||||
| -rw-r--r-- | htdocs/index.html | 217 | ||||
| -rw-r--r-- | htdocs/robots.txt | 2 | ||||
| -rwxr-xr-x | keyrings-export.bash | 71 | ||||
| -rwxr-xr-x | keyrings-recv-keys.gentoo.org.bash | 16 | ||||
| -rwxr-xr-x | keyrings-recv-sks.bash | 5 | ||||
| -rwxr-xr-x | keyrings-send-keys.gentoo.org.bash | 2 | ||||
| -rw-r--r-- | keyrings.inc.bash | 49 | ||||
| -rw-r--r-- | kuroneko.exclude | 23 | ||||
| -rwxr-xr-x | leaf-packages.py | 118 | ||||
| -rwxr-xr-x | pkg-newest-commit.py | 67 | ||||
| m--------- | pkgcheck2html | 0 |
26 files changed, 821 insertions, 295 deletions
diff --git a/Makefile.gpyutils b/Makefile.gpyutils index 525057e..75fee4e 100644 --- a/Makefile.gpyutils +++ b/Makefile.gpyutils @@ -5,78 +5,47 @@ timestamp = $(PORTDIR)/metadata/timestamp.commit # pass via make! outdir = /dev/null +# Adding new implementations: +# 1. Create rules below & add to upgr_all +# 2. Later, when dot doesn't take too long for it, move to upgr_txt +# 3. Once the impl is stable, move to upgr_base +# +# Please prioritize edges like python-any-r1 consumers to speed up getting +# from 1->2. Especially e.g. git, ninja, meson, doxygen. + # upgrades for stable impls -upgr_base = $(outdir)/34-to-35.txt $(outdir)/35-to-36.txt $(outdir)/36-to-37.txt +upgr_base = $(outdir)/310-to-311.txt $(outdir)/311-to-312.txt # related stabilizations upgr_streq = $(patsubst %.txt,%-stablereq.txt,$(upgr_base)) # all upgrade stuff -upgr_txt = $(upgr_base) $(upgr_streq) $(outdir)/pypy3-to-35.txt +upgr_txt = $(upgr_base) $(upgr_streq) upgr_dot = $(patsubst %.txt,%.dot,$(upgr_txt)) upgr_svg = $(patsubst %.dot,%.svg,$(upgr_dot)) +# add new impls here if not stable yet, to avoid insanely huge generation times upgr_all = $(upgr_txt) $(upgr_dot) $(upgr_svg) -all = $(upgr_all) $(outdir)/cands.txt $(outdir)/missing-meta.txt +all = $(upgr_all) all: $(all) -$(outdir)/cands.txt: $(timestamp) - mkdir -p $(outdir) - gpy-cands > $@.new - mv $@.new $@ - -$(outdir)/depcands.txt: $(timestamp) - mkdir -p $(outdir) - gpy-depcands > $@.new - mv $@.new $@ - -$(outdir)/depcands-all.txt: $(timestamp) - mkdir -p $(outdir) - gpy-depcands --all > $@.new - mv $@.new $@ - -$(outdir)/depcheck.txt: $(timestamp) - mkdir -p $(outdir) - gpy-depcheck > $@.new - mv $@.new $@ - -$(outdir)/34-to-35.txt: $(timestamp) - mkdir -p $(outdir) - gpy-upgrade-impl python3_4 python3_5 > $@.new - mv $@.new $@ - -$(outdir)/34-to-35-stablereq.txt: $(timestamp) - mkdir -p $(outdir) - gpy-upgrade-impl -s python3_4 python3_5 > $@.new - mv $@.new $@ - -$(outdir)/35-to-36.txt: $(timestamp) - mkdir -p $(outdir) - gpy-upgrade-impl python3_5 python3_6 > $@.new - mv $@.new $@ - -$(outdir)/35-to-36-stablereq.txt: $(timestamp) - mkdir -p $(outdir) - gpy-upgrade-impl -s python3_5 python3_6 > $@.new - mv $@.new $@ - -$(outdir)/36-to-37.txt: $(timestamp) +$(outdir)/311-to-312.txt: $(timestamp) mkdir -p $(outdir) - gpy-upgrade-impl python3_6 python3_7 > $@.new + gpy-upgrade-impl -m python3_11 python3_12 > $@.new mv $@.new $@ -$(outdir)/36-to-37-stablereq.txt: $(timestamp) +$(outdir)/311-to-312-stablereq.txt: $(timestamp) mkdir -p $(outdir) - gpy-upgrade-impl -s python3_6 python3_7 > $@.new + gpy-upgrade-impl -m -s python3_11 python3_12 > $@.new mv $@.new $@ -$(outdir)/pypy3-to-35.txt: $(timestamp) +$(outdir)/310-to-311.txt: $(timestamp) mkdir -p $(outdir) - gpy-upgrade-impl pypy3 python3_5 > $@.new + gpy-upgrade-impl -m python3_10 python3_11 > $@.new mv $@.new $@ -$(outdir)/missing-meta.txt: $(timestamp) +$(outdir)/310-to-311-stablereq.txt: $(timestamp) mkdir -p $(outdir) - gpy-find-missing-meta > $@.new + gpy-upgrade-impl -m -s python3_10 python3_11 > $@.new mv $@.new $@ %.dot: %.txt diff --git a/cgi-bin/get-git-file.sh b/cgi-bin/get-git-file.sh index 5e9fd0d..9721f71 100755 --- a/cgi-bin/get-git-file.sh +++ b/cgi-bin/get-git-file.sh @@ -12,6 +12,25 @@ main() { local commit=${qs%%;*} qs=${qs#*;} local file=${qs%%;*} + [[ ${qs} == *\;* ]] && qs=${qs#*;} || qs= + + local filter_maint= projects= + while [[ -n ${qs} ]]; do + local q=${qs%%;*} + case ${q} in + maintainer=*) + filter_maint="--maintainer ${q#maintainer=}" + ;; + include-projects) + projects=--projects + ;; + pkg=*) + pkgs=${q#pkg=} + filter_pkg="--pkg ${pkgs//://}" + ;; + esac + [[ ${qs} == *\;* ]] && qs=${qs#*;} || qs= + done if [[ ${repo} == */* ]]; then echo "DANGER! SOMEONE TRIES TO ABUSE ME!" >&2 @@ -28,7 +47,12 @@ main() { fi # generate HTML from XML + local verbose= local lfile=${file} + if [[ ${file} == *.verbose.html ]]; then + file=${file%.verbose.html}.html + verbose=--verbose + fi [[ ${file} == *.html ]] && lfile=${file%.html}.xml local tree=( $(git ls-tree "${commit}" "${lfile}" 2>/dev/null) ) @@ -37,13 +61,17 @@ main() { lfile=${file} tree=( $(git ls-tree "${commit}" "${lfile}" 2>/dev/null) ) if [[ ! ${tree[*]} ]]; then - echo "Status: 404 Not Found" + echo "Status: 503 Service Unavailable" + echo "Retry-After: 30" echo echo "404 Not Found (if the report was just published, you may need to wait a minute or two for sync)" exit 0 fi fi + local revarg= + [[ ${commit} != HEAD ]] && revarg="--revision ${commit}" + local ct case "${file}" in *.css) ct=text/css;; @@ -58,7 +86,10 @@ main() { local ts=$(TZ=UTC git log --format='%cd' --date=iso-local -1 | cut -d' ' -f1-2) git cat-file -p "${tree[2]}" \ - | PYTHONIOENCODING=utf8 python "${topdir}"/pkgcheck2html/pkgcheck2html.py -t "${ts}" - + | PYTHONIOENCODING=utf8 python \ + "${topdir}"/pkgcheck2html/pkgcheck2html.py ${verbose} \ + -x "${topdir}"/pkgcheck2html/excludes.json \ + ${filter_maint} ${projects} ${filter_pkg} -t "${ts}" - else git cat-file -p "${tree[2]}" fi diff --git a/cgi-bin/trigger-pull.sh b/cgi-bin/trigger-pull.sh new file mode 100755 index 0000000..89aff7a --- /dev/null +++ b/cgi-bin/trigger-pull.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +if [[ ! ${QUERY_STRING} ]]; then + echo "Script must be run through CGI" >&2 + exit 1 +fi + +main() { + local repo=${QUERY_STRING} + if [[ ${repo} == */* ]]; then + echo "DANGER! DANGER! DON'T TALK TO STRANGERS!" >&2 + exit 1 + fi + + local topdir=$(dirname "${0}")/.. + + if ! cd "${topdir}/htdocs/output/${repo}" 2>/dev/null; then + echo "Status: 404 Not Found" + echo + echo "404 Not Found" + exit 0 + fi + + local output=$(git pull -q 2>&1) + if [ $? -eq 0 ]; then + echo "Status: 200 OK" + echo + echo "Done." + else + echo "Status: 500 Failed" + echo + echo "${output}" + fi +} + +main diff --git a/check_eclasses_eapis.sh b/check_eclasses_eapis.sh index f3e1ff8..899b1c3 100755 --- a/check_eclasses_eapis.sh +++ b/check_eclasses_eapis.sh @@ -10,16 +10,17 @@ else fi #[[ $(type pquery 2> /dev/null) ]] || exit 1 +[[ $(type gawk 2> /dev/null) ]] || exit 1 -TMPEAPIS="/tmp/$(basename $0).global.$$.tmp" -TMPECLASS="/tmp/$(basename $0).eclass.$$.tmp" -REPO_PATH=$(portageq get_repo_path / gentoo) +TMPEAPIS="$(mktemp -t $(basename $0).global.tmp.XXXXXX)" +TMPECLASS="$(mktemp -t $(basename $0).eclass.tmp.XXXXXX)" +REPO_PATH=$(portageq get_repo_path ${EROOT:-/} gentoo) pushd "${REPO_PATH}/eclass" > /dev/null ECLASSES=$(echo *.eclass) popd > /dev/null #pquery --attr eapi --attr inherited --raw --all --repo portdir > "${TMPEAPIS}" -find "${REPO_PATH}/metadata/md5-cache" -type f -exec awk -F= ' +find "${REPO_PATH}/metadata/md5-cache" -type f -name '*-[0-9]*' -exec gawk -F= ' BEGINFILE { n = split(FILENAME, f, "/") file = f[n-1] "/" f[n] @@ -38,7 +39,7 @@ find "${REPO_PATH}/metadata/md5-cache" -type f -exec awk -F= ' } ' '{}' \+ > "${TMPEAPIS}" -KNOWN_EAPIS=$(awk ' +KNOWN_EAPIS=$(gawk ' { e = gensub(/eapi="(.*)"/, "\\1", "", $2); eapi[e] = e } END { PROCINFO["sorted_in"]="@ind_num_asc"; for (e in eapi) print e } ' "${TMPEAPIS}") @@ -49,11 +50,11 @@ for x in ${ECLASSES}; do echo "Processing eclass \"${x}\"" rm -rf "${x}" mkdir "${x}" - awk -F'=' '$3 ~ /[ "]'"${x%.eclass}"'[ "]/ {print $1" "$2}' "${TMPEAPIS}" > "${TMPECLASS}" + gawk -F'=' '$3 ~ /[ "]'"${x%.eclass}"'[ "]/ {print $1" "$2}' "${TMPEAPIS}" > "${TMPECLASS}" pushd "${x}" > /dev/null - echo "Overall statistic for eclass \"${x}\":" > "STATS.txt" + echo "Overall statistics for eclass \"${x}\":" > "STATS.txt" for y in ${KNOWN_EAPIS}; do - awk -F ' ' '$3 ~ /"'"${y}"'"/ {print $1}' "${TMPECLASS}" > "${y}.txt" + gawk -F ' ' '$3 ~ /"'"${y}"'"/ {print $1}' "${TMPECLASS}" > "${y}.txt" tmpval=$(wc -l "${y}.txt" |cut -d' ' -f1) echo "EAPI=${y} count: ${tmpval}" >> "STATS.txt" done diff --git a/create-dev-keyrings.bash b/create-dev-keyrings.bash index f2772d8..3ab1b58 100755 --- a/create-dev-keyrings.bash +++ b/create-dev-keyrings.bash @@ -11,13 +11,17 @@ source "${BASEDIR}"/keyrings.inc.bash set -e export_ldap_data_to_env -export KEYSERVERS=( "${KS_SKS}" "${KS_GENTOO}" ) +export KEYSERVERS=( "${KS_SKS}" "${KS_OPENPGP}" "${KS_GENTOO}" ) export KEYSERVER_TIMEOUT=20m grab_keys "${SYSTEM_KEYS[@]}" export_keys "${OUTPUT_DIR}"/service-keys.gpg \ "${SYSTEM_KEYS[@]}" +grab_keys "${INFRA_SYSTEM_KEYS[@]}" +export_keys "${OUTPUT_DIR}"/infra-service-keys.gpg \ + "${INFRA_SYSTEM_KEYS[@]}" + grab_keys "${COMMITTING_DEVS[@]}" export_keys "${OUTPUT_DIR}"/committing-devs.gpg \ "${COMMITTING_DEVS[@]}" @@ -27,6 +31,10 @@ export_keys "${OUTPUT_DIR}"/active-devs.gpg \ "${COMMITTING_DEVS[@]}" \ "${NONCOMMITTING_DEVS[@]}" +grab_keys "${INFRA_DEVS[@]}" +export_keys "${OUTPUT_DIR}"/infra-devs.gpg \ + "${INFRA_DEVS[@]}" + # -- not all are on keyservers # -- and are unlikely to turn up now # -- this needs to fetch from some archive instead @@ -39,6 +47,7 @@ export_keys "${OUTPUT_DIR}"/all-devs.gpg \ "${SYSTEM_KEYS[@]}" \ "${COMMITTING_DEVS[@]}" \ "${NONCOMMITTING_DEVS[@]}" \ + "${INFRA_DEVS[@]}" \ "${RETIRED_DEVS[@]}" # Populate keys.gentoo.org with the keys we have, since they might have come from SKS @@ -47,4 +56,5 @@ export KEYSERVER_TIMEOUT=20m push_keys "${SYSTEM_KEYS[@]}" push_keys "${COMMITTING_DEVS[@]}" push_keys "${NONCOMMITTING_DEVS[@]}" +push_keys "${INFRA_DEVS[@]}" push_keys "${RETIRED_DEVS[@]}" diff --git a/dev-timeline b/dev-timeline -Subproject a3ebb35413700113bffd4ee38cd9b6d77fc3612 +Subproject 6d256c01a1631c7985a4339ad02bca93390b27f diff --git a/eapi-usage.sh b/eapi-usage.sh new file mode 100755 index 0000000..b7a1112 --- /dev/null +++ b/eapi-usage.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# This script respects EINFO_QUIET=1 etc to silence stdout +# Always puts a summary in $1/README.html. +# Arguments: +# $1: output directory. Defaults to eapi-usage. +# $2: file to place stats in within $1. +. /lib/gentoo/functions.sh + +dir=${1} + +if [[ -n ${1} && -e ${dir} && ! -d ${dir} ]] ; then + eerror "Output directory given (${dir}) is not a directory! Exiting." + exit 1 +elif [[ -z ${dir} ]] ; then + ewarn "No output directory argument given! Defaulting to 'eapi-usage'." + dir=eapi-usage +fi + +stats=${2:-$dir/STATS.txt} + +mkdir -p ${dir} || exit 1 + +REPO_PATH=$(portageq get_repo_path ${EROOT:-/} gentoo || exit 1) +TMPDIR="$(mktemp -d || exit 1)" + +shopt -s nullglob + +einfo "Working in TMPDIR=${TMPDIR}" +pushd "${TMPDIR}" &>/dev/null || exit 1 +mkdir -p eapi-usage || exit 1 +cd eapi-usage || exit 1 + +ebegin "Finding ebuilds" +( + while IFS= read -r ebuild ; do + cpf_eapi="${ebuild#${REPO_PATH}/metadata/md5-cache/}" + cpf="${cpf_eapi%%:*}" + eapi="${cpf_eapi##*:EAPI=}" + echo "${cpf}" >> ${eapi}.txt + done < <(find "${REPO_PATH}/metadata/md5-cache" -mindepth 2 -maxdepth 2 -type f -name '*-[0-9]*' -exec grep '^EAPI=' {} +) +) || { eend $? || exit 1; } +eend ${?} + +ebegin "Sorting EAPI files" +for eapi in *.txt ; do + sort -u ${eapi} > ${eapi}.sorted + mv ${eapi}.sorted ${eapi} +done || { eend $? || exit 1; } +eend $? + +popd &>/dev/null || exit 1 +# No exit here because it's fine if we removed nothing +rm ${dir}/*.txt +mv ${TMPDIR}/eapi-usage/*.txt ${dir}/ || exit 1 + +rm -r "${TMPDIR}" || exit 1 + +# Now generate the numbers/summary (copied in from previous eapi_usage.sh script) +# Someday it would be nice to graph this output, or maybe keep some running history? +# TODO: We used pinspect in the past which is cleaner but it's slower than the +# awk method used below (we also stopped using it because at the time, pkgcore +# lacked newer EAPI support). +#[[ $(type pinspect 2> /dev/null) ]] || exit 1 +# +#pinspect eapi_usage /usr/portage + +find "${REPO_PATH}"/metadata/md5-cache -type f ! -name '*.gz' \ + -exec grep -h '^EAPI=' '{}' + \ + | awk ' + { sub("EAPI=",""); eapi[$1]++ } + END { + PROCINFO["sorted_in"]="@val_num_desc" + for (i in eapi) { + s=""; for (j=1; j<eapi[i]*50./NR+0.5; j++) s=s"#" + printf "EAPI %s: %7d ebuilds (%5.02f%%) %s\n", + i, eapi[i], eapi[i]*100.0/NR, s + } + printf "total: %7d ebuilds\n", NR + }' > ${stats} + +echo >> ${stats} +echo "Date generated: $(date -u '+%Y-%m-%d %H:%M:%S %Z')" >> ${stats} + +echo "<pre>" > ${dir}/README.html +cat ${stats} >> ${dir}/README.html +echo "</pre>" >> ${dir}/README.html diff --git a/eapi_usage.sh b/eapi_usage.sh deleted file mode 100755 index 29c3831..0000000 --- a/eapi_usage.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Boring 'script' that just uses pkgcore's pinspect command. Someday it would be -# nice to graph this output, or maybe keep some running history? - -#[[ $(type pinspect 2> /dev/null) ]] || exit 1 -# -#pinspect eapi_usage /usr/portage - -find /usr/portage/metadata/md5-cache -type f ! -name '*.gz' \ - -exec grep -h '^EAPI=' '{}' + \ - | awk ' - { sub("EAPI=",""); eapi[$1]++ } - END { - PROCINFO["sorted_in"]="@val_num_desc" - for (i in eapi) { - s=""; for (j=1; j<eapi[i]*50./NR+0.5; j++) s=s"#" - printf "EAPI %s: %7d ebuilds (%5.02f%%) %s\n", - i, eapi[i], eapi[i]*100.0/NR, s - } - printf "total: %7d ebuilds\n", NR - }' - -echo -echo "Date generated: $(date)" diff --git a/eclass-eapi-matrix.py b/eclass-eapi-matrix.py index 8117d63..0f465e1 100755 --- a/eclass-eapi-matrix.py +++ b/eclass-eapi-matrix.py @@ -83,8 +83,10 @@ def format_table(data): supp_eapis, eapis = ecl_data for eapi in all_eapis: if supp_eapis is not None and eapi not in supp_eapis: - assert eapis.get(eapi, 0) == 0 - line['eapi_'+eapi] = 'xx' + if eapis.get(eapi, 0) > 0: + line['eapi_'+eapi] = '%d?!' % eapis[eapi] + else: + line['eapi_'+eapi] = 'xx' else: line['eapi_'+eapi] = eapis.get(eapi, 0) ret += format_str.format(**line) diff --git a/find-binary-files.sh b/find-binary-files.sh index 6714422..b053d2f 100755 --- a/find-binary-files.sh +++ b/find-binary-files.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2014-2018 Gentoo Foundation +# Copyright 2014-2024 Ulrich Müller # Distributed under the terms of the GNU GPL version 2 or later # Author: Ulrich Müller <ulm@gentoo.org> @@ -8,10 +8,8 @@ shopt -s extglob portdir=$(portageq get_repo_path / gentoo) cd "${portdir}" || exit 1 -find . \( -path ./distfiles -o -path ./local -o -path ./metadata \ - -o -path ./packages \) -prune \ - -o ! -type d ! \( -type f -name 'Manifest*.gz' \) -exec file -ih '{}' + \ -| while read line; do +count=0 +while read line; do path=${line%:*} type=${line##*:*( )} case ${type} in @@ -32,13 +30,23 @@ find . \( -path ./distfiles -o -path ./local -o -path ./metadata \ text/*) ;; # text file application/*"; charset=us-ascii") ;; application/*"; charset=utf-8") ;; + "image/svg; charset=us-ascii") ;; # SVG image "image/svg+xml; charset=us-ascii") ;; # SVG image "image/x-xpmi; charset=us-ascii") ;; # XPM image + "image/x-xpixmap; charset=us-ascii") ;; # XPM image "message/rfc822; charset=us-ascii") ;; + "message/rfc822; charset=utf-8") ;; *) size=$(stat -c "%s" "${path}") echo "${path#./}: ${type} (size=${size})" + (( count++ )) ;; esac -done \ -| sort +done < <(find \( -path ./distfiles -o -path ./local -o -path ./metadata \ + -o -path ./packages \) -prune -o ! -type d ! -name 'Manifest*.gz' \ + -exec file -ih '{}' + | sort) + +[[ ${count} -gt 0 ]] || echo "No binary files found. :-)" + +# Output the file version for debugging of false positives/negatives +echo; file --version | head -n1 diff --git a/gen-dev-wot.sh b/gen-dev-wot.sh index d15f461..8f01995 100755 --- a/gen-dev-wot.sh +++ b/gen-dev-wot.sh @@ -17,6 +17,6 @@ gpg -q --keyid-format=long --keyring "${1}/active-devs.gpg" --list-sigs | \ /usr/bin/sig2dot -q -t "Gentoo Dev WoT" -s wot-stats.html \ > keys.dot -dot -Gcharset=latin1 -Tsvg keys.dot > "${1}/wot-graph.svg" +dot -Gcharset=UTF-8 -Tsvg keys.dot > "${1}/wot-graph.svg" mv wot-stats.html "${1}" diff --git a/genrdeps-index.py b/genrdeps-index.py new file mode 100755 index 0000000..9b1cd85 --- /dev/null +++ b/genrdeps-index.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# Rewrite of genrdeps-index to stop using horrible Portage API. +# (c) 2020 Michał Górny +# 2-clause BSD license + +import argparse +import collections +import errno +import os +import os.path +import shutil +import subprocess +import sys +import tempfile + +import pkgcore.config +from pkgcore.ebuild.atom import atom +from pkgcore.restrictions.boolean import AndRestriction, OrRestriction +from pkgcore.restrictions.packages import Conditional + + +DepTuple = collections.namedtuple('DepTuple', ('cpv', 'blocks', 'use')) + + +GROUPS = ( + ('bdepend', 'bindex'), + ('depend', 'dindex'), + ('idepend', 'iindex'), + ('pdepend', 'pindex'), + ('rdepend', 'rindex'), +) + + +def process_deps(deps, useflags=frozenset()): + for d in deps: + if isinstance(d, atom): + yield DepTuple(d.key, d.blocks, useflags) + elif isinstance(d, OrRestriction) or isinstance(d, AndRestriction): + # || deps and nested () blocks + for sd in process_deps(d, useflags): + yield sd + elif isinstance(d, Conditional): + # foo? deps + assert d.attr == 'use' + assert len(d.restriction.vals) == 1 + r = next(iter(d.restriction.vals)) + if d.restriction.negate: + r = '!' + r + for sd in process_deps(d, useflags | frozenset((r,))): + yield sd + else: + raise AssertionError("Unknown dep type: " + d.__class__) + + +def rmtree_ignore_enoent(func, path, exc_info): + if not isinstance(exc_info[1], FileNotFoundError): + raise + + +def main(): + argp = argparse.ArgumentParser() + argp.add_argument('outputdir', + help='Directory to create rdep index in') + args = argp.parse_args() + + c = pkgcore.config.load_config() + repo = c.objects.repo['gentoo'] + + rindex = {} + for g, gi in GROUPS: + rindex[g] = collections.defaultdict(set) + + for p in repo: + for g, gi in GROUPS: + deps = frozenset(process_deps(getattr(p, g))) + for dep, blocks, flags in deps: + rindex[g][dep].add(DepTuple(p.cpvstr, blocks, flags)) + + for g, gi in GROUPS: + outdir = os.path.join(args.outputdir, '.' + gi + '.new') + shutil.rmtree(outdir, onerror=rmtree_ignore_enoent) + + for p, revdeps in rindex[g].items(): + outpath = os.path.join(outdir, p) + os.makedirs(os.path.dirname(outpath), exist_ok=True) + with open(outpath, 'w') as f: + for dep, blocks, flags in sorted(revdeps): + if blocks: + dep = '[B]' + dep + if flags: + dep += ':' + '+'.join(sorted(flags)) + f.write(dep + '\n') + + for g, gi in GROUPS: + outdir = os.path.join(args.outputdir, gi) + olddir = os.path.join(args.outputdir, '.' + gi + '.old') + newdir = os.path.join(args.outputdir, '.' + gi + '.new') + + shutil.rmtree(olddir, onerror=rmtree_ignore_enoent) + try: + os.rename(outdir, olddir) + except FileNotFoundError as e: + pass + os.rename(newdir, outdir) + shutil.rmtree(olddir, onerror=rmtree_ignore_enoent) + + with tempfile.NamedTemporaryFile(prefix='.tmp.rdeps-', suffix='.tar', dir=args.outputdir, delete=False) as tmpf: + try: + subprocess.check_call( + ['tar', '-cf', tmpf.name] + [gi for g, gi in GROUPS], + cwd=args.outputdir) + subprocess.check_call( + ['xz', '-9', tmpf.name], + cwd=args.outputdir) + os.rename(tmpf.name + '.xz', os.path.join(args.outputdir, 'rdeps.tar.xz')) + os.chmod(os.path.join(args.outputdir, 'rdeps.tar.xz'), 0o644) + except Exception as e: + raise e + finally: + # Cleanup: + for f in [tmpf.name, (tmpf.name + '.xz')]: + try: + os.unlink(f) + except FileNotFoundError as e: + pass + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/genrdeps/genrdeps.py b/genrdeps/genrdeps.py deleted file mode 100755 index d7eeb76..0000000 --- a/genrdeps/genrdeps.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/python2.7 -# Pepper and solar.. - -import os, sys, types - -os.environ["PORTDIR_OVERLAY"]="" -import portage, portage.dep - -def processDeps(deps,use=""): - returnMe=[] - for (index,x) in enumerate(deps): - if type(x)==types.ListType: - continue - elif x=="||": - returnMe.extend(processDeps(deps[index+1],use)) - elif x[-1]=="?": - returnMe.extend(processDeps(deps[index+1],"+".join(x for x in (use,x[:-1]) if x))) - elif x[0]=="!": - returnMe.append((portage.dep_getkey(x),"[B]",use)) - else: - returnMe.append((portage.dep_getkey(x),"",use)) - return returnMe - -revdeps = {} -for cpv in portage.portdb.cpv_all(): - try: - deps = processDeps(portage.dep.paren_reduce(portage.portdb.aux_get(cpv, [sys.argv[1]])[0])) - except: - continue - - for dep in deps: - if dep[0] not in revdeps: - revdeps[dep[0]] = [] - revdeps[dep[0]].append((cpv, dep[1], dep[2] and ":"+dep[2])) - -dirs = [] -for cp in revdeps: - c = cp.split("/")[0] - if c not in dirs: - os.makedirs(c) - dirs.append(c) - - revdeps[cp].sort() - f = open(cp, "w") - f.write("\n".join([ b+cpv+use for (cpv,b,use) in revdeps[cp] ])+"\n") - f.close() diff --git a/genrdeps/run-genrdeps-index.sh b/genrdeps/run-genrdeps-index.sh deleted file mode 100755 index fd89301..0000000 --- a/genrdeps/run-genrdeps-index.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh -OUTPUTDIR=/var/www/qa-reports.gentoo.org/htdocs/output/genrdeps -[ -d "${OUTPUTDIR}" ] || mkdir -p ${OUTPUTDIR} -cd ${OUTPUTDIR} - -mkdir .rindex_new -pushd .rindex_new > /dev/null -/var/www/qa-reports.gentoo.org/qa-scripts/genrdeps/genrdeps.py RDEPEND -find | cut -c 3- > .rindex -popd > /dev/null -mv rindex rindex_old -mv .rindex_new rindex -rm -rf rindex_old - -mkdir .dindex_new -pushd .dindex_new > /dev/null -/var/www/qa-reports.gentoo.org/qa-scripts/genrdeps/genrdeps.py DEPEND -find | cut -c 3- > .dindex -popd > /dev/null -mv dindex dindex_old -mv .dindex_new dindex -rm -rf dindex_old - -mkdir .pindex_new -pushd .pindex_new > /dev/null -/var/www/qa-reports.gentoo.org/qa-scripts/genrdeps/genrdeps.py PDEPEND -find | cut -c 3- > .pindex -popd > /dev/null -mv pindex pindex_old -mv .pindex_new pindex -rm -rf pindex_old - -mkdir .bindex_new -pushd .bindex_new > /dev/null -/var/www/qa-reports.gentoo.org/qa-scripts/genrdeps/genrdeps.py BDEPEND -find | cut -c 3- > .bindex -popd > /dev/null -mv bindex bindex_old -mv .bindex_new bindex -rm -rf bindex_old diff --git a/gpy-impl-list b/gpy-impl-list new file mode 100755 index 0000000..be70554 --- /dev/null +++ b/gpy-impl-list @@ -0,0 +1,17 @@ +#!/bin/bash + +OUTDIR=$1 + +if [[ -z ${OUTDIR} ]]; then + echo "Usage: $0 <out-dir>" >&2 + exit 1 +fi + +IRC='ircs://irc.libera.chat/#gentoo-python' + +irk "${IRC}" - <<<$'\0' +gpy-list-pkg-impls > "${OUTDIR}"/python-pkg-impls.txt.new || exit 1 +while read -r x; do + irk "${IRC}" "${x}" +done < <(git diff --no-index -s --word-diff -U0 "${OUTDIR}"/python-pkg-impls.txt{,.new} | tail -n +5 | grep -v '^@@') +mv "${OUTDIR}"/python-pkg-impls.txt{.new,} diff --git a/htdocs/index.html b/htdocs/index.html index 44ad5d6..60955d2 100644 --- a/htdocs/index.html +++ b/htdocs/index.html @@ -5,65 +5,69 @@ <meta charset="utf-8"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="theme-color" content="#54487a"> - <link href="https://1b9a50f4f9de4348cd9f-e703bc50ba0aa66772a874f8c7698be7.ssl.cf5.rackcdn.com/bootstrap.min.css" rel="stylesheet" media="screen"> - <link href="https://1b9a50f4f9de4348cd9f-e703bc50ba0aa66772a874f8c7698be7.ssl.cf5.rackcdn.com/tyrian.min.css" rel="stylesheet" media="screen"> + <link href="https://assets.gentoo.org/tyrian/v2/tyrian.min.css" rel="stylesheet" media="screen"> + <link href="https://assets.gentoo.org/tyrian/v2/components/additional-listgroup-link.min.css" rel="stylesheet" media="screen"> <link rel="icon" href="https://www.gentoo.org/favicon.ico" type="image/x-icon"> </head> <body> <header> <div class="site-title"> <div class="container"> - <div class="row"> + <div class="row justify-content-between"> + <div class="logo"> + <a href="/" title="Back to the homepage" class="site-logo"> + <img src="https://assets.gentoo.org/tyrian/v2/site-logo.png" alt="Gentoo" srcset="https://assets.gentoo.org/tyrian/v2/site-logo.svg"> + </a> + <span class="site-label">QA Reports</span> + </div> <div class="site-title-buttons"> <div class="btn-group btn-group-sm"> - <a href="https://get.gentoo.org/" role="button" class="btn get-gentoo"><span class="fa fa-fw fa-download"></span> <strong>Get Gentoo!</strong></a> + <a href="https://get.gentoo.org/" role="button" class="btn get-gentoo"><span class="fa fa-fw fa-download"></span> + <strong>Get Gentoo!</strong></a> <div class="btn-group btn-group-sm"> <a class="btn gentoo-org-sites dropdown-toggle" data-toggle="dropdown" data-target="#" href="#"> - <span class="fa fa-fw fa-map-o"></span> <span class="hidden-xs">gentoo.org sites</span> <span class="caret"></span> + <span class="fa fa-fw fa-map-o"></span> <span class="d-none d-sm-inline">gentoo.org sites</span> + <span class="caret"></span> </a> - <ul class="dropdown-menu dropdown-menu-right"> - <li><a href="https://www.gentoo.org/" title="Main Gentoo website"><span class="fa fa-home fa-fw"></span> gentoo.org</a></li> - <li><a href="https://wiki.gentoo.org/" title="Find and contribute documentation"><span class="fa fa-file-text-o fa-fw"></span> Wiki</a></li> - <li><a href="https://bugs.gentoo.org/" title="Report issues and find common issues"><span class="fa fa-bug fa-fw"></span> Bugs</a></li> - <li><a href="https://forums.gentoo.org/" title="Discuss with the community"><span class="fa fa-comments-o fa-fw"></span> Forums</a></li> - <li><a href="https://packages.gentoo.org/" title="Find software for your Gentoo"><span class="fa fa-hdd-o fa-fw"></span> Packages</a></li> - <li class="divider"></li> - <li><a href="https://planet.gentoo.org/" title="Find out what's going on in the developer community"><span class="fa fa-rss fa-fw"></span> Planet</a></li> - <li><a href="https://archives.gentoo.org/" title="Read up on past discussions"><span class="fa fa-archive fa-fw"></span> Archives</a></li> - <li><a href="https://sources.gentoo.org/" title="Browse our source code"><span class="fa fa-code fa-fw"></span> Sources</a></li> - <li class="divider"></li> - <li><a href="https://infra-status.gentoo.org/" title="Get updates on the services provided by Gentoo"><span class="fa fa-server fa-fw"></span> Infra Status</a></li> - </ul> + <div class="dropdown-menu dropdown-menu-right"> + <a class="dropdown-item" href="https://www.gentoo.org/" title="Main Gentoo website"><span class="fa fa-home fa-fw"></span> + gentoo.org</a> + <a class="dropdown-item" href="https://wiki.gentoo.org/" title="Find and contribute documentation"><span class="fa fa-file-text-o fa-fw"></span> + Wiki</a> + <a class="dropdown-item" href="https://bugs.gentoo.org/" title="Report issues and find common issues"><span class="fa fa-bug fa-fw"></span> + Bugs</a> + <a class="dropdown-item" href="https://forums.gentoo.org/" title="Discuss with the community"><span class="fa fa-comments-o fa-fw"></span> + Forums</a> + <a class="dropdown-item" href="https://packages.gentoo.org/" title="Find software for your Gentoo"><span class="fa fa-hdd-o fa-fw"></span> + Packages</a> + <div class="dropdown-divider"></div> + <a class="dropdown-item" href="https://planet.gentoo.org/" title="Find out what's going on in the developer community"><span class="fa fa-rss fa-fw"></span> + Planet</a> + <a class="dropdown-item" href="https://archives.gentoo.org/" title="Read up on past discussions"><span class="fa fa-archive fa-fw"></span> + Archives</a> + <a class="dropdown-item" href="https://sources.gentoo.org/" title="Browse our source code"><span class="fa fa-code fa-fw"></span> + Sources</a> + <div class="dropdown-divider"></div> + <a class="dropdown-item" href="https://infra-status.gentoo.org/" title="Get updates on the services provided by Gentoo"><span class="fa fa-server fa-fw"></span> + Infra Status</a> + </div> </div> </div> </div> - <div class="logo"> - <a href="/" title="Back to the homepage" class="site-logo"> - <object data="https://1b9a50f4f9de4348cd9f-e703bc50ba0aa66772a874f8c7698be7.ssl.cf5.rackcdn.com/site-logo.svg" type="image/svg+xml"> - <img src="https://1b9a50f4f9de4348cd9f-e703bc50ba0aa66772a874f8c7698be7.ssl.cf5.rackcdn.com/site-logo.png" alt="Gentoo Linux Logo"> - </object> - </a> - <span class="site-label">QA Reports</span> - </div> </div> </div> </div> - <nav class="tyrian-navbar" role="navigation"> + <nav class="tyrian-navbar navbar navbar-dark navbar-expand-lg bg-primary" role="navigation"> <div class="container"> - <div class="row"> - <div class="navbar-header"> - <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-main-collapse"> - <span class="sr-only">Toggle navigation</span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - </div> - <div class="collapse navbar-collapse navbar-main-collapse"> - <ul class="nav navbar-nav"> - <li class="active"><a href="/">Home</a></li> - </ul> - </div> + <div class="navbar-header"> + <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbar-main-collapse" aria-controls="navbar-main-collapse" aria-expanded="false" aria-label="Toggle navigation"> + <span class="navbar-toggler-icon"></span> + </button> + </div> + <div class="collapse navbar-collapse navbar-main-collapse" id="navbar-main-collapse"> + <ul class="navbar-nav mr-auto"> + <li class="nav-item active"><a class="nav-link" href="/">Home</a></li> + </ul> </div> </div> </nav> @@ -72,7 +76,7 @@ <div class="container"> <div class="row"> <div class="col-lg-12"> - <h1 class="first-header">Gentoo QA Reports</h1> + <h1 class="first-header mb-4">Gentoo QA Reports</h1> <div class="row"> <div class="col-lg-4"> @@ -84,57 +88,99 @@ --> <h3>Misc</h3> - <div class="list-group"> - <a href="/output/wot-graph.svg" class="list-group-item">Gentoo Dev Web of Trust (WoT)</a> - <a href="/output/wot-stats.html" class="list-group-item">WoT Node Stats</a> - <a href="/output/maintainer-needed.html" class="list-group-item">"Maintainer Needed" Packages</a> - <a href="/output/dev-timeline.html" class="list-group-item">Developer commit timeline</a> - <a href="/output/active-devs.html" class="list-group-item">Active developer commit timeline</a> + <div class="list-group mb-3"> + <a href="/output/wot-graph.svg" class="list-group-item list-group-item-action">Gentoo Dev Web of Trust (WoT)</a> + <a href="/output/wot-stats.html" class="list-group-item list-group-item-action">WoT Node Stats</a> + <a href="/output/maintainer-needed.html" class="list-group-item list-group-item-action">"Maintainer Needed" (orphaned) Packages</a> + <a href="/output/dev-timeline.html" class="list-group-item list-group-item-action">Developer commit timeline</a> + <a href="/output/active-devs.html" class="list-group-item list-group-item-action">Active developer commit timeline</a> + <a href="/output/repos/" class="list-group-item list-group-item-action">Repository status reports</a> </div> <h3>Python reports</h3> - <div class="list-group"> - <a href="/output/gpyutils/35-to-36.txt" class="list-group-item">Py3.5 → 3.6 testing</a> - <a href="/output/gpyutils/35-to-36-stablereq.txt" class="list-group-item">Py3.5 → 3.6 stabilizations</a> - <a href="/output/gpyutils/36-to-37.txt" class="list-group-item">Py3.6 → 3.7 testing</a> - <a href="/output/gpyutils/pypy3-to-35.txt" class="list-group-item">PyPy3 → 3.5 testing</a> - <a href="/output/gpyutils/cands.txt" class="list-group-item">Packages missing python*-r1 eclass</a> - <a href="/output/gpyutils/missing-meta.txt" class="list-group-item">python*-r1 eclass usage issues</a> + <div class="list-group mb-3"> + <div class="list-group-item list-group-item-action p-0"> + <a href="/output/gpyutils/311-to-312.txt" class="list-group-item-base-link">Py3.11 → 3.12 testing</a> + <a href="/output/gpyutils/311-to-312.svg" class="list-group-item-additional-link p-1"> + <span class="badge badge-primary">svg</span> + </a> + </div> + <div class="list-group-item list-group-item-action p-0"> + <a href="/output/gpyutils/311-to-312-stablereq.txt" class="list-group-item-base-link">Py3.11 → 3.12 stabilizations</a> + <a href="/output/gpyutils/311-to-312-stablereq.svg" class="list-group-item-additional-link p-1"> + <span class="badge badge-primary">svg</span> + </a> + </div> + <div class="list-group-item list-group-item-action p-0"> + <a href="/output/gpyutils/310-to-311.txt" class="list-group-item-base-link">Py3.10 → 3.11 testing</a> + <a href="/output/gpyutils/310-to-311.svg" class="list-group-item-additional-link p-1"> + <span class="badge badge-primary">svg</span> + </a> + </div> + <div class="list-group-item list-group-item-action p-0"> + <a href="/output/gpyutils/310-to-311-stablereq.txt" class="list-group-item-base-link">Py3.10 → 3.11 stabilizations</a> + <a href="/output/gpyutils/310-to-311-stablereq.svg" class="list-group-item-additional-link p-1"> + <span class="badge badge-primary">svg</span> + </a> + </div> + </div> + <h3>Generated artifacts</h3> + <div class="list-group mb-3"> + <a href="/output/service-keys.gpg" class="list-group-item list-group-item-action">GPG keyring: service keys</a> + <a href="/output/active-devs.gpg" class="list-group-item list-group-item-action">GPG keyring: all active devs</a> + <a href="/output/committing-devs.gpg" class="list-group-item list-group-item-action">GPG keyring: active gentoo.git devs</a> + <a href="/output/retired-devs.gpg" class="list-group-item list-group-item-action">GPG keyring: retired devs</a> + <a href="/output/infra-devs.gpg" class="list-group-item list-group-item-action">GPG keyring: infra devs</a> + <a href="/output/all-devs.gpg" class="list-group-item list-group-item-action">GPG keyring: all devs</a> </div> </div> <div class="col-lg-4"> <h3>Tree-wide Scanning</h3> - <div class="list-group"> - <a href="/output/genrdeps/rindex/" class="list-group-item">Reverse runtime dependencies (RDEPEND)</a> - <a href="/output/genrdeps/dindex/" class="list-group-item">Reverse buildtime dependencies (DEPEND)</a> - <a href="/output/genrdeps/pindex/" class="list-group-item">Reverse post dependencies (PDEPEND)</a> - <a href="/output/genrdeps/bindex/" class="list-group-item">Reverse buildtime dependencies (BDEPEND)</a> - <a href="/output/eapi_usage.txt" class="list-group-item">EAPI Usage</a> - <a href="/output/eapi-per-eclass" class="list-group-item">EAPI Usage per eclass</a> - <a href="/output/eapi-per-eclass/matrix.txt" class="list-group-item">Eclass-EAPI usage matrix</a> - <a href="/output/invalid-mask.txt" class="list-group-item">Invalid package.mask entries</a> - <a href="/output/filesdir-usage.txt" class="list-group-item">Top 100 largest FILESDIRs</a> - <a href="/output/find-binary-files.txt" class="list-group-item">Binary files in gentoo-x86</a> - <a href="/output/eclass-usage/" class="list-group-item">Listing of packages that inherit an eclass</a> - <a href="/output/gentoo-ci/output.html" class="list-group-item">Gentoo CI – QA check results</a> - <a href="/output/repos/" class="list-group-item">Repository status reports</a> + <div class="list-group mb-3"> + <a href="/output/eapi-usage/" class="list-group-item list-group-item-action">EAPI Usage</a> + <a href="/output/eapi-per-eclass" class="list-group-item list-group-item-action">EAPI Usage per eclass</a> + <a href="/output/eapi-per-eclass/matrix.txt" class="list-group-item list-group-item-action">Eclass-EAPI usage matrix</a> + <a href="/output/invalid-mask.txt" class="list-group-item list-group-item-action">Invalid package.mask entries</a> + <a href="/output/filesdir-usage.txt" class="list-group-item list-group-item-action">Top 100 largest FILESDIRs</a> + <a href="/output/find-binary-files.txt" class="list-group-item list-group-item-action">Binary files in ::gentoo</a> + <a href="/output/eclass-usage/" class="list-group-item list-group-item-action">Listing of packages that inherit an eclass</a> + <a href="/output/leaf-packages.txt" class="list-group-item list-group-item-action">Leaf Packages</a> + <div class="list-group-item list-group-item-action p-0"> + <a href="/output/gentoo-ci/output.html" class="list-group-item-base-link">Gentoo CI – QA check results</a> + <a href="/output/gentoo-ci/output.verbose.html" class="list-group-item-additional-link p-1"> + <span class="badge badge-primary">-v</span> + </a> + </div> + </div> + + <h3>Reverse dependencies</h3> + <div class="list-group mb-3"> + <a href="/output/genrdeps/rindex/" class="list-group-item list-group-item-action">Reverse RDEPEND</a> + <a href="/output/genrdeps/dindex/" class="list-group-item list-group-item-action">Reverse DEPEND</a> + <a href="/output/genrdeps/iindex/" class="list-group-item list-group-item-action">Reverse IDEPEND</a> + <a href="/output/genrdeps/pindex/" class="list-group-item list-group-item-action">Reverse PDEPEND</a> + <a href="/output/genrdeps/bindex/" class="list-group-item list-group-item-action">Reverse BDEPEND</a> </div> </div> <div class="col-lg-4"> <h3>Bugs last touched by year</h3> - <div class="list-group"> - <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2007-01-01&v2=2006-01-01" class="list-group-item">2006</a> - <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2008-01-01&v2=2007-01-01" class="list-group-item">2007</a> - <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2009-01-01&v2=2008-01-01" class="list-group-item">2008</a> - <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2010-01-01&v2=2009-01-01" class="list-group-item">2009</a> - <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2011-01-01&v2=2010-01-01" class="list-group-item">2010</a> - <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2012-01-01&v2=2011-01-01" class="list-group-item">2011</a> - <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2013-01-01&v2=2012-01-01" class="list-group-item">2012</a> - <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2014-01-01&v2=2013-01-01" class="list-group-item">2013</a> - <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2015-01-01&v2=2014-01-01" class="list-group-item">2014</a> - <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2016-01-01&v2=2015-01-01" class="list-group-item">2015</a> - <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2017-01-01&v2=2016-01-01" class="list-group-item">2016</a> - <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2018-01-01&v2=2017-01-01" class="list-group-item">2017</a> + <div class="list-group mb-3"> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2008-01-01&v2=2007-01-01" class="list-group-item list-group-item-action">2007</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2009-01-01&v2=2008-01-01" class="list-group-item list-group-item-action">2008</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2010-01-01&v2=2009-01-01" class="list-group-item list-group-item-action">2009</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2011-01-01&v2=2010-01-01" class="list-group-item list-group-item-action">2010</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2012-01-01&v2=2011-01-01" class="list-group-item list-group-item-action">2011</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2013-01-01&v2=2012-01-01" class="list-group-item list-group-item-action">2012</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2014-01-01&v2=2013-01-01" class="list-group-item list-group-item-action">2013</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2015-01-01&v2=2014-01-01" class="list-group-item list-group-item-action">2014</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2016-01-01&v2=2015-01-01" class="list-group-item list-group-item-action">2015</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2017-01-01&v2=2016-01-01" class="list-group-item list-group-item-action">2016</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2018-01-01&v2=2017-01-01" class="list-group-item list-group-item-action">2017</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2019-01-01&v2=2018-01-01" class="list-group-item list-group-item-action">2018</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2020-01-01&v2=2019-01-01" class="list-group-item list-group-item-action">2019</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2021-01-01&v2=2020-01-01" class="list-group-item list-group-item-action">2020</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2022-01-01&v2=2021-01-01" class="list-group-item list-group-item-action">2021</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2023-01-01&v2=2022-01-01" class="list-group-item list-group-item-action">2022</a> </div> </div> </div> @@ -162,7 +208,7 @@ </ul> </div> <div class="col-xs-10 col-sm-9 col-md-10"> - <strong>© 2001–2019 Gentoo Foundation, Inc.</strong><br> + <strong>© 2001–2023 Gentoo Authors</strong><br> <small> Gentoo is a trademark of the Gentoo Foundation, Inc. The contents of this document, unless otherwise expressly stated, are licensed under the @@ -174,7 +220,8 @@ </div> </footer> - <script src="https://1b9a50f4f9de4348cd9f-e703bc50ba0aa66772a874f8c7698be7.ssl.cf5.rackcdn.com/jquery.min.js"></script> - <script src="https://1b9a50f4f9de4348cd9f-e703bc50ba0aa66772a874f8c7698be7.ssl.cf5.rackcdn.com/bootstrap.min.js"></script> + <script src="https://assets.gentoo.org/tyrian/v2/jquery-3.3.slim.js"></script> + <script src="https://assets.gentoo.org/tyrian/v2/popper.min.js"></script> + <script src="https://assets.gentoo.org/tyrian/v2/bootstrap.min.js"></script> </body> </html> diff --git a/htdocs/robots.txt b/htdocs/robots.txt new file mode 100644 index 0000000..1f53798 --- /dev/null +++ b/htdocs/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/keyrings-export.bash b/keyrings-export.bash index 143cda2..56e4e51 100755 --- a/keyrings-export.bash +++ b/keyrings-export.bash @@ -6,6 +6,8 @@ # - requires keeping state to detect changes in keys, there is no usable mtime data in a key itself OUTPUT_DIR=${1:-.} +# Ensure output is absolute +OUTPUT_DIR=$(readlink -f "${OUTPUT_DIR}") BASEDIR="$(dirname "$0")" # shellcheck source=./keyrings.inc.bash source "${BASEDIR}"/keyrings.inc.bash @@ -16,23 +18,72 @@ export -a COMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${COMMIT_ export -a NONCOMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${NONCOMMIT_RULE}") ) export -a RETIRED_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${RETIRED_RULE}") ) export -a SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${NONCOMMIT_RULE}") ) +export -a INFRA_SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${INFRA_SYSTEM_RULE}") ) +export -a KEYRINGS=( ) -export_keys "${OUTPUT_DIR}"/service-keys.gpg \ - "${SYSTEM_KEYS[@]}" +export_keys "${OUTPUT_DIR}"/keys/service-keys.gpg \ + "${SYSTEM_KEYS[@]}" \ +&& KEYRINGS+=( service-keys ) + +export_keys "${OUTPUT_DIR}"/keys/infra-service-keys.gpg \ + "${INFRA_SYSTEM_KEYS[@]}" \ +&& KEYRINGS+=( infra-service-keys ) -export_keys "${OUTPUT_DIR}"/committing-devs.gpg \ - "${COMMITTING_DEVS[@]}" +export_keys "${OUTPUT_DIR}"/keys/committing-devs.gpg \ + "${COMMITTING_DEVS[@]}" \ +&& KEYRINGS+=( committing-devs ) -export_keys "${OUTPUT_DIR}"/active-devs.gpg \ +export_keys "${OUTPUT_DIR}"/keys/active-devs.gpg \ "${COMMITTING_DEVS[@]}" \ - "${NONCOMMITTING_DEVS[@]}" + "${NONCOMMITTING_DEVS[@]}" \ +&& KEYRINGS+=( active-devs ) -export_keys "${OUTPUT_DIR}"/retired-devs.gpg \ - "${RETIRED_DEVS[@]}" +export_keys "${OUTPUT_DIR}"/keys/infra-devs.gpg \ + "${INFRA_DEVS[@]}" \ +&& KEYRINGS+=( infra-devs ) + +export_keys "${OUTPUT_DIR}"/keys/retired-devs.gpg \ + "${RETIRED_DEVS[@]}" \ +&& KEYRINGS+=( retired-devs ) # Everybody together now -export_keys "${OUTPUT_DIR}"/all-devs.gpg \ +export_keys "${OUTPUT_DIR}"/keys/all-devs.gpg \ "${SYSTEM_KEYS[@]}" \ + "${INFRA_SYSTEM_KEYS[@]}" \ "${COMMITTING_DEVS[@]}" \ "${NONCOMMITTING_DEVS[@]}" \ - "${RETIRED_DEVS[@]}" + "${INFRA_DEVS[@]}" \ + "${RETIRED_DEVS[@]}" \ +&& KEYRINGS+=( all-devs ) + +for key in "${KEYRINGS[@]}" ; do + if [[ ! -L "${OUTPUT_DIR}"/${key}.gpg ]] ; then + # Compatibility symlink + ln -sf "${OUTPUT_DIR}"/keys/${key}.gpg "${OUTPUT_DIR}"/${key}.gpg + fi + + if [[ $(date -u +%A) == Monday ]] ; then + # We don't want to run on Mondays to avoid last/next week confusion + break + fi + + timestamp=$(date -u +%Y%m%d-%A -d "last monday") + + if [[ ${timestamp} != *-Monday ]] ; then + break + fi + + timestamp=${timestamp/-Monday/} + + # Don't clobber existing timestamped keys for this period (weekly) + # if we're running several times a day. + if [[ -f "${OUTPUT_DIR}"/keys/${key}-${timestamp}.gpg ]] ; then + continue + fi + + mkdir -p "${OUTPUT_DIR}"/keys + + cp "${OUTPUT_DIR}"/${key}.gpg "${OUTPUT_DIR}"/keys/${key}-${timestamp}.gpg +done + +clean_tmp diff --git a/keyrings-recv-keys.gentoo.org.bash b/keyrings-recv-keys.gentoo.org.bash index 916fe54..7cc1b62 100755 --- a/keyrings-recv-keys.gentoo.org.bash +++ b/keyrings-recv-keys.gentoo.org.bash @@ -1,16 +1,17 @@ #!/bin/bash # Import key updates from Keyservers: keys.gentoo.org -# -# TODO: -# - Turn off export in this script BASEDIR="$(dirname "$0")" +DEBUG=${DEBUG:=0} # shellcheck source=./keyrings.inc.bash source "${BASEDIR}"/keyrings.inc.bash set -e -export_ldap_data_to_env + +# export_ldap_data_to_env +# TODO: for unclear reason this does not populate correctly inside a function export -a COMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${COMMIT_RULE}") ) +export -a INFRA_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${INFRA_RULE}") ) export -a NONCOMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${NONCOMMIT_RULE}") ) export -a RETIRED_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${RETIRED_RULE}") ) export -a SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${NONCOMMIT_RULE}") ) @@ -18,10 +19,17 @@ export -a SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${NONCOMMI export KEYSERVERS=( "${KS_GENTOO}" ) export KEYSERVER_TIMEOUT=5m +[[ $DEBUG -ne 0 ]] && echo SYSTEM_KEYS grab_keys "${SYSTEM_KEYS[@]}" +[[ $DEBUG -ne 0 ]] && echo COMITTING_DEVS grab_keys "${COMMITTING_DEVS[@]}" +[[ $DEBUG -ne 0 ]] && echo NONCOMITTING_DEVS grab_keys "${NONCOMMITTING_DEVS[@]}" +[[ $DEBUG -ne 0 ]] && echo INFRA_DEVS +grab_keys "${INFRA_DEVS[@]}" # -- not all are on keyservers # -- and are unlikely to turn up now # -- this needs to fetch from some archive instead #grab_keys "${RETIRED_DEVS[@]}" + +clean_tmp diff --git a/keyrings-recv-sks.bash b/keyrings-recv-sks.bash index 4e613b8..81ba0a2 100755 --- a/keyrings-recv-sks.bash +++ b/keyrings-recv-sks.bash @@ -15,13 +15,16 @@ export -a NONCOMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${NONC export -a RETIRED_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${RETIRED_RULE}") ) export -a SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${NONCOMMIT_RULE}") ) -export KEYSERVERS=( "${KS_SKS}" ) +export KEYSERVERS=( "${KS_SKS}" "${KS_OPENPGP}" ) export KEYSERVER_TIMEOUT=20m grab_keys "${SYSTEM_KEYS[@]}" grab_keys "${COMMITTING_DEVS[@]}" grab_keys "${NONCOMMITTING_DEVS[@]}" +grab_keys "${INFRA_DEVS[@]}" # -- not all are on keyservers # -- and are unlikely to turn up now # -- this needs to fetch from some archive instead #grab_keys "${RETIRED_DEVS[@]}" + +clean_tmp diff --git a/keyrings-send-keys.gentoo.org.bash b/keyrings-send-keys.gentoo.org.bash index ae67acf..1691867 100755 --- a/keyrings-send-keys.gentoo.org.bash +++ b/keyrings-send-keys.gentoo.org.bash @@ -20,3 +20,5 @@ push_keys "${SYSTEM_KEYS[@]}" push_keys "${COMMITTING_DEVS[@]}" push_keys "${NONCOMMITTING_DEVS[@]}" push_keys "${RETIRED_DEVS[@]}" + +clean_tmp diff --git a/keyrings.inc.bash b/keyrings.inc.bash index a3879d5..d2668e0 100644 --- a/keyrings.inc.bash +++ b/keyrings.inc.bash @@ -6,18 +6,26 @@ SYSTEM_BASE='ou=system,dc=gentoo,dc=org' COMMIT_RULE='(&(gentooAccess=git.gentoo.org/repo/gentoo.git)(gentooStatus=active))' NONCOMMIT_RULE='(&(!(gentooAccess=git.gentoo.org/repo/gentoo.git))(gentooStatus=active))' RETIRED_RULE='(!(gentooStatus=active))' +INFRA_RULE='(&(gentooAccess=infra.group)(gentooStatus=active))' +INFRA_SYSTEM_RULE='(&(gentooAccess=infra-system.group)(gentooStatus=active))' export KS_GENTOO=hkps://keys.gentoo.org/ -export KS_SKS=hkps://hkps.pool.sks-keyservers.net/ +# Use local keyserver for speedup +# KS_GENTOO_LOCAL=${HOSTNAME%.gentoo.org}.keys.gentoo.org +# dig $KS_GENTOO_LOCAL +short |grep -sq . && export KS_GENTOO=hkps://${KS_GENTOO_LOCAL} + +#export KS_SKS=hkps://hkps.pool.sks-keyservers.net/ # Disabled pending security announcement +export KS_OPENPGP=hkps://keys.openpgp.org/ # runs Hagrid export KEYSERVERS=( ) # empty by default export COMMITTING_DEVS=( ) export NONCOMMITTING_DEVS=( ) export RETIRED_DEVS=( ) +export INFRA_DEVS=( ) export SYSTEM_KEYS=( ) # grab_ldap_fingerprints <ldap-rule> grab_ldap_fingerprints() { - ldapsearch "${@}" -Z gpgfingerprint -LLL | + ldapsearch "${@}" -x -Z gpgfingerprint -LLL | sed -n -e '/: undefined/d' -e '/^gpgfingerprint: /{s/^.*://;s/ //g;p}' | sort -u } @@ -28,10 +36,14 @@ grab_keys() { local missing=() local remaining=( "${@}" ) - KEYSERVER_TIMEOUT=${KEYSERVER_TIMEOUT:=1m} + KEYSERVER_TIMEOUT=${KEYSERVER_TIMEOUT:=2m} + # quickly handle empty keyservers set + [ "${#KEYSERVERS[@]}" -eq 0 ] && return while :; do for ks in "${KEYSERVERS[@]}" ; do - timeout ${KEYSERVER_TIMEOUT} gpg --keyserver "$ks" -q --recv-keys "${remaining[@]}" || : + timeout ${KEYSERVER_TIMEOUT} gpg \ + --keyserver-options no-import-clean,no-self-sigs-only \ + --keyserver "$ks" -q --recv-keys "${remaining[@]}" || : done missing=() for key in "${remaining[@]}"; do @@ -56,6 +68,8 @@ grab_keys() { # push_keys <fingerprint>... push_keys() { + # quickly handle empty keyservers set + [ "${#KEYSERVERS[@]}" -eq 0 ] && return # Only send keys that we have local remaining=( $(gpg --with-colon --list-public "${@}" | sed -n '/^pub/{n; /fpr/p }' |cut -d: -f10) ) KEYSERVER_TIMEOUT=${KEYSERVER_TIMEOUT:=1m} @@ -66,6 +80,8 @@ push_keys() { export GPG_TMPDIR='' clean_tmp() { + # Ensure any agent is closed down + gpgconf --kill all [ -n "$GPG_TMPDIR" ] && [ -d "$GPG_TMPDIR" ] && rm -rf "$GPG_TMPDIR" } setup_tmp() { @@ -80,7 +96,8 @@ export_keys() { DST="$1" shift setup_tmp - TMP="${GPG_TMPDIR}"/$(basename "${DST}") + BASENAME=$(basename "${DST}") + TMP="${GPG_TMPDIR}/${BASENAME}" # Must not exist, otherwise GPG will give error [[ -f "${TMP}" ]] && rm -f "${TMP}" # 'gpg --export' returns zero if there was no error with the command itself @@ -100,15 +117,21 @@ export_keys() { echo "Unable to export keys to $DST: GPG failed to list packets" exit 1 fi + + # Ensure we have a checksum to verify the file. + rhash --bsd --sha256 --sha512 --blake2b "${TMP}" |sed "s,${TMP},${BASENAME},g" >"${TMP}.DIGESTS" + # Check if the textual format has changed at all, and emit the new version # if there are ANY changes at all. - if ! cmp -s "${DST}.packets.txt" "${TMP}.packets.txt"; then - chmod a+r "${TMP}" - mv -f "${TMP}" "${DST}" - mv -f "${TMP}.packets.txt" "${DST}.packets.txt" - fi - # Cleanup anyway - rm -f "${TMP}.packets.txt" "${TMP}" + cmp -s "${DST}.packets.txt" "${TMP}.packets.txt" + cmp_rc=$? + chmod a+r "${TMP}" "${TMP}.packets.txt" "${TMP}.DIGESTS" + for suffix in '' '.packets.txt' '.DIGESTS'; do + # If these file do not exist, or the cmp was different, move them. + [ ! -f "${DST}${suffix}" -o $cmp_rc -ne 0 ] && mv -f "${TMP}${suffix}" "${DST}${suffix}" + # Cleanup anyway + rm -f "${TMP}${suffix}" + done } # populate common variables @@ -117,5 +140,7 @@ export_ldap_data_to_env() { export -a COMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${COMMIT_RULE}") ) export -a NONCOMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${NONCOMMIT_RULE}") ) export -a RETIRED_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${RETIRED_RULE}") ) + export -a INFRA_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${INFRA_RULE}") ) export -a SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${NONCOMMIT_RULE}") ) + export -a INFRA_SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${INFRA_SYSTEM_RULE}") ) } diff --git a/kuroneko.exclude b/kuroneko.exclude new file mode 100644 index 0000000..3cc605f --- /dev/null +++ b/kuroneko.exclude @@ -0,0 +1,23 @@ +# upstream restarted versioning +404095 +421031 +429522 +433389 +438750 +442478 +452432 +461714 +466822 +499464 +501472 +508270 +524560 +537940 +559532 +565842 +567850 +572716 +578300 +581028 +609562 +619458 diff --git a/leaf-packages.py b/leaf-packages.py new file mode 100755 index 0000000..1454eb8 --- /dev/null +++ b/leaf-packages.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# From Leo3418's GSoC 2021 work +# https://leo3418.github.io/2021/07/18/find-leaf-packages.html + +import concurrent.futures +import os +import re +import subprocess +import sys + +METHOD="pkgcore" +VERBOSE=int(os.getenv('VERBOSE') or os.getenv('V') or 0) # V=1..N to use + +def main() -> None: + if len(sys.argv) > 1: + repo = sys.argv[1] + else: + repo = 'gentoo' + zero_in_degree = create_ebuild_dict(repo) + with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()) \ + as executor: + for ebuild in zero_in_degree: + # Let the executor run function call + # update_for_deps_of(ebuild, zero_in_degree) + if METHOD == "pkgcore": + executor.submit(update_for, ebuild, zero_in_degree, repo) + else: + executor.submit(update_for_deps_of, ebuild, zero_in_degree) + + # Print leaf ebuilds to standard output + for ebuild in zero_in_degree: + if zero_in_degree[ebuild]: + print(ebuild) + + +def update_for(ebuild: str, zero_in_degree: dict, repo: str) -> None: + """ + Update the boolean value for the specified ebuild in the given dictionary. + Reverse dependencies of the ebuild will be searched in the specified + repository only. + """ + # This spams infra cronjobs, only enable if there is a problem + if VERBOSE >= 1: + print(f"Processing {ebuild} ...", file=sys.stderr) + proc = subprocess.run(f'pquery --first --restrict-revdep ={ebuild} ' + f'--repo {repo} --raw --unfiltered', + capture_output=True, text=True, shell=True) + zero_in_degree[ebuild] = len(proc.stdout) == 0 + +def create_ebuild_dict(repo: str) -> dict: + """ + Create a dictionary with all ebuilds in the specified repository as keys + that maps each key to a boolean value indicating whether it is a leaf + ebuild with zero in-degree. + """ + zero_in_degree = {} + proc = subprocess.run(f'pquery --repo {repo} --raw --unfiltered', + capture_output=True, text=True, + shell=True, check=True) + ebuilds = proc.stdout.splitlines() + for ebuild in ebuilds: + zero_in_degree[ebuild] = True + return zero_in_degree + + +def update_for_deps_of(ebuild: str, zero_in_degree: dict) -> None: + """ + For ebuilds that can be pulled as the specified ebuild's dependencies, + update the boolean value for them in the given dictionary accordingly. + """ + + def get_dep_atoms() -> list: + """ + Return a list of all dependency specification atoms. + """ + dep_atoms = [] + equery_dep_atom_pattern = re.compile(r'\(.+/.+\)') + proc = subprocess.run(f'equery -CN depgraph -MUl {ebuild}', + capture_output=True, text=True, shell=True) + out_lines = proc.stdout.splitlines() + for line in out_lines: + dep_atom_match = equery_dep_atom_pattern.findall(line) + dep_atom = [dep.strip('()') for dep in dep_atom_match] + dep_atoms.extend(dep_atom) + return dep_atoms + + def find_matching_ebuilds(atom: str) -> list: + """ + Return a list of ebuilds that satisfy an atom. + """ + proc = subprocess.run(f"equery list -op -F '$cpv' '{atom}'", + capture_output=True, text=True, shell=True) + return proc.stdout.splitlines() + + if VERBOSE >= 2: + print(f"Processing {ebuild} ...", file=sys.stderr) + + # Get dependency specifications in the ebuild; + # equivalent to dep_graph[ebuild] in the examples above + dep_atoms = get_dep_atoms() + + # Convert list of atoms to list of ebuilds that satisfy them + dep_ebuilds = [] + for dep_atom in dep_atoms: + dep_ebuilds.extend(find_matching_ebuilds(dep_atom)) + + # Register dependency ebuilds as non-leaves + for dep_ebuild in dep_ebuilds: + # An ebuild in an overlay might depend on ebuilds from ::gentoo and/or + # other repositories, but we only care about ebuilds in the dictionary + # passed to this function + if dep_ebuild in zero_in_degree: + zero_in_degree[dep_ebuild] = False + + +if __name__ == '__main__': + main() + diff --git a/pkg-newest-commit.py b/pkg-newest-commit.py new file mode 100755 index 0000000..ef0af89 --- /dev/null +++ b/pkg-newest-commit.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +import argparse +import glob +import os +import os.path +import subprocess +import sys + + +def main(argv): + argp = argparse.ArgumentParser(prog=argv[0]) + argp.add_argument('--git-dir', default='.', + help='Path to repo git clone (used for git log)') + argp.add_argument('--work-dir', default='.', + help='Path to repo working directory (used to grab ' + 'package list when no packages specified)') + argp.add_argument('package', nargs='*', + help='List of packages to find (defaults to all)') + args = argp.parse_args(argv[1:]) + + packages = set(args.package) + if not packages: + with open(os.path.join(args.work_dir, 'profiles/categories')) as f: + for cat in f: + packages.update('/'.join(x.rsplit('/', 3)[-3:-1]) + for x in glob.glob(os.path.join( + args.work_dir, cat.strip(), '*/'))) + if not packages: + print('No packages specified or found', file=sys.stderr) + return 1 + pkg_max_len = max(len(x) for x in packages) + pkg_format = '{{:.<{}}} {{}} {{}}'.format(pkg_max_len+1) + + excludes = frozenset([ + # specify EAPI=0 explicitly + '4a409a1ecd75d064e8b471f6131bb1feb83c37a8', + # drop $id + '61b861acd7b49083dab687e133f30f3331cb7480', + # initial git commit + '56bd759df1d0c750a065b8c845e93d5dfa6b549d', + ]) + + os.environ['TZ'] = 'UTC' + s = subprocess.Popen(['git', 'log', '--date=iso-local', '--name-only', + '--diff-filter=AM', '--no-renames', + '--pretty=COMMIT|%H|%cd', '**.ebuild'], + cwd=args.git_dir, + stdout=subprocess.PIPE) + for l in s.stdout: + l = l.decode('utf8').strip() + if l.startswith('COMMIT|'): + commit_data = l[7:] + elif l: + pkg = '/'.join(l.split('/', 2)[:2]) + if pkg in packages: + commit, date = commit_data.split('|') + if commit in excludes: + continue + print(pkg_format.format(pkg + ' ', date, commit)) + packages.remove(pkg) + if not packages: + break + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/pkgcheck2html b/pkgcheck2html -Subproject 1bfec8e37a2a770e47ed0971c4c3684292073ac +Subproject bb9807de3be23ef879e492c0142e770d54ff450 |