From 50bf86830113bdf0278dad413d3b236c0ba53e5f Mon Sep 17 00:00:00 2001 From: Mykyta Holubakha Date: Wed, 28 Jun 2017 05:21:31 +0300 Subject: Major updates and refactorings added and updated parameter descriptions to most function docstrings cleaned up imports over the whole project converted Package to use file path mappings internally fixed errors in manifest generation fixed handling slots and adding paths to index in merge process converted package removal (by name) to use repository metadata truly integrated the local ebuild package source module separated package-related utilities into a separate pkg module separated portage repo-related utilities into a separate portage module excluded the tests package from installation --- pomu/package.py | 41 ++++++++++++++++------------- pomu/repo/init.py | 33 ++++++++++++++++++++---- pomu/repo/repo.py | 47 +++++++++++++++++++--------------- pomu/source/__init__.py | 1 + pomu/source/file.py | 18 ++++++------- pomu/source/portage.py | 68 +++---------------------------------------------- pomu/util/fs.py | 7 ++++- pomu/util/pkg.py | 41 +++++++++++++++++++++++++++++ pomu/util/portage.py | 47 ++++++++++++++++++++++++++++++++++ pomu/util/query.py | 7 ++--- setup.py | 2 +- 11 files changed, 188 insertions(+), 124 deletions(-) create mode 100644 pomu/util/pkg.py create mode 100644 pomu/util/portage.py diff --git a/pomu/package.py b/pomu/package.py index 4a07cbe..d59bd4f 100644 --- a/pomu/package.py +++ b/pomu/package.py @@ -14,7 +14,7 @@ from pomu.util.fs import strip_prefix from pomu.util.result import Result class Package(): - def __init__(self, backend, name, root, category=None, version=None, slot='0', d_path=None, files=None): + def __init__(self, backend, name, root, category=None, version=None, slot='0', d_path=None, files=None, filemap=None): """ Parameters: backend - specific source module object/class @@ -23,6 +23,7 @@ class Package(): d_path - a subdirectory of the root path, which would be sourced recursively. could be a relative or an absolute path files - a set of files to build a package from + filemap - a mapping from destination files to files in the filesystem category, version, slot - self-descriptive """ self.backend = backend @@ -31,21 +32,24 @@ class Package(): self.category = category self.version = version self.slot = slot - self.files = [] - if d_path is None and files is None: + self.filemap = {} + if d_path is None and files is None and filemap is None: self.d_path = None self.read_path(self.root) - elif files is None: + elif d_path: self.d_path = self.strip_root(d_path) self.read_path(path.join(self.root, self.d_path)) - elif d_path is None: + elif files: for f in files: - self.files.append(path.split(self.strip_root(f))) + dst = self.strip_root(f) + self.filemap[dst] = path.join(self.root, dst) + elif filemap: + self.filemap = filemap else: - raise ValueError('You should specify either d_path, or files') + raise ValueError('You should specify either d_path, files or filemap') def strip_root(self, d_path): - """Strip the root component of a path""" + """Strip the root component of d_path""" # the path should be either relative, or a child of root if d_path.startswith('/'): if path.commonprefix(d_path, self.root) != self.root: @@ -54,36 +58,37 @@ class Package(): return d_path def read_path(self, d_path): - """Recursively add files from a subtree""" + """Recursively add files from a subtree (specified by d_path)""" for wd, dirs, files in walk(d_path): wd = self.strip_root(wd) - self.files.extend([(wd, f) for f in files]) + self.filemap.update({path.join(wd, f): path.join(self.root, wd, f) for f in files}) def merge_into(self, dst): - """Merges contents of the package into a specified directory""" - for wd, f in self.files: + """Merges contents of the package into a specified directory (dst)""" + for trg, src in self.filemap.items(): + wd, _ = path.split(trg) dest = path.join(dst, wd) try: - makedirs(dest, exist_ok=True) - copy2(path.join(self.root, wd, f), dest) + makedirs(wd, exists_ok=True) + copy2(src, dest) except PermissionError: return Result.Err('You do not have enough permissions') return Result.Ok() def gen_manifests(self, dst): """ - Generate manifests for the installed package. + Generate manifests for the installed package (in the dst directory). TODO: use portage APIs instead of calling repoman. """ - dirs = [x for wd, f in self.files if y.endswith('.ebuild')] + dirs = [wd for wd, f in self.files if f.endswith('.ebuild')] dirs = list(set(dirs)) res = [] for d_ in dirs: - d = path.join(dst, d) + d = path.join(dst, d_) ret = subprocess.run(['repoman', 'manifest'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=d) - if r != 0: + if ret.returncode != 0: return Result.Err('Failed to generate manifest at', d) if path.exists(path.join(d, 'Manifest')): res.append(path.join(d, 'Manifest')) diff --git a/pomu/repo/init.py b/pomu/repo/init.py index c8ecd2f..ce3678a 100644 --- a/pomu/repo/init.py +++ b/pomu/repo/init.py @@ -7,8 +7,14 @@ import portage from pomu.util.result import Result, ResultException -def init_plain_repo(create, repo_path, name=''): #name might be extraneous - """Initialize a plain repository""" +def init_plain_repo(create, repo_path): + """ + Initialize a plain repository + Parameters: + create - if true, create a new git repo, + else, reuse an existing one + repo_path - a path for the repository to reside in + """ if not repo_path: return Result.Err('repository path required') if create: @@ -30,7 +36,14 @@ def init_plain_repo(create, repo_path, name=''): #name might be extraneous return init_pomu(repo_path) def init_portage_repo(create, repo, repo_dir): - """Initialize a portage repository""" + """ + Initialize a portage repository + Parameters: + create - if true, create a new portage repo with git, + else, reuse an existing one + repo - name of the repository + repo_dir - location of the newly created repository, if applicable + """ if not repo: return Result.Err('repository name required') rsets = portage.db[portage.root]['vartree'].settings.repositories @@ -61,7 +74,12 @@ def init_portage_repo(create, repo, repo_dir): return init_pomu(rsets.prepos[repo], repo) def init_new(repo_path, name=''): - """Initialize a newly created repository (metadata/layout.conf and pomu)""" + """ + Initialize a newly created repository (metadata/layout.conf and pomu) + Parameters: + repo_path - path to the repository + name - name of the repository + """ cnf = path.join(repo_path, 'metadata', 'layout.conf') if not path.isfile(cnf): try: @@ -73,7 +91,12 @@ def init_new(repo_path, name=''): return init_pomu(repo_path, name) def init_pomu(repo_path, name=''): - """Initialise pomu for a repository""" + """ + Initialise pomu for a repository + Parameters: + repo_path - path to the repository + name - name of the repository + """ pomu_path = path.join(repo_path, 'metadata', 'pomu') if not path.isdir(path.join(repo_path, '.git')): return Result.Err('target repository should be a git repo') diff --git a/pomu/repo/repo.py b/pomu/repo/repo.py index 4318d3e..df891e1 100644 --- a/pomu/repo/repo.py +++ b/pomu/repo/repo.py @@ -1,16 +1,22 @@ """Subroutines with repositories""" -from os import path, makedirs, rmdir -from shutil import copy2 +from os import path, rmdir from git import Repo import portage +from pomu.package import Package +from pomu.source import dispatcher from pomu.util.cache import cached -from pomu.util.fs import remove_file +from pomu.util.fs import remove_file, strip_prefix from pomu.util.result import Result class Repository(): def __init__(self, root, name=None): + """ + Parameters: + root - root of the repository + name - name of the repository + """ if not pomu_status(root): raise ValueError('This path is not a valid pomu repository') self.root = root @@ -25,14 +31,14 @@ class Repository(): return path.join(self.root, 'metadata/pomu') def merge(self, package): - """Merge a package into the repository""" + """Merge a package (a pomu.package.Package package) into the repository""" r = self.repo pkgdir = path.join(self.pomu_dir, package.category, package.name) - if slot != 0: - pkgdir = path.join(pkgdir, slot) + if package.slot != 0: + pkgdir = path.join(pkgdir, package.slot) package.merge_into(self.root).expect('Failed to merge package') for wd, f in package.files: - r.index.add(path.join(dst, f)) + r.index.add(path.join(self.root, wd, f)) manifests = package.gen_manifests(self.root).expect() for m in manifests: r.index.add(m) @@ -46,6 +52,13 @@ class Repository(): return Result.Ok('Merged package ' + package.name + ' successfully') def write_meta(self, pkgdir, package, manifests): + """ + Write metadata for a Package object + Parameters: + pkgdir - destination directory + package - the package object + manifests - list of generated manifest files + """ with open(path.join(pkgdir, 'FILES'), 'w') as f: for w, f in package.files: f.write('{}/{}\n'.format(w, f)) @@ -75,19 +88,11 @@ class Repository(): def remove_package(self, name): """Remove a package (by name) from the repository""" - r = self.repo - pf = path.join(self.pomu_dir, name, 'FILES') - if not path.isfile(pf): - return Result.Err('Package not found') - with open(pf, 'w') as f: - for insf in f: - remove_file(path.join(self.root, insf)) - remove_file(path.join(self.pomu_dir, name)) - r.commit('Removed package ' + name + ' successfully') - return Result.Ok('Removed package ' + name + ' successfully') + pkg = self.get_package(name).expect() + return self.unmerge(pkg) def _get_package(self, category, name, slot='0'): - """Get an existing package""" + """Get an existing package (by category, name and slot), reading the manifest""" if slot == '0': pkgdir = path.join(self.pomu_dir, category, name) else: @@ -102,7 +107,7 @@ class Repository(): return Package(backend, name, self.root, category=category, version=version, slot=slot, files=files) def get_package(self, name, category=None, slot=None): - """Get a package by name""" + """Get a package by name, category and slot""" with open(path.join(self.pomu_dir, 'world'), 'r') as f: for spec in f: cat, _, nam = spec.partition('/') @@ -121,7 +126,7 @@ def portage_repos(): yield repo def portage_repo_path(repo): - """Get the path of a given portage repository""" + """Get the path of a given portage repository (repo)""" rsets = portage.db[portage.root]['vartree'].settings.repositories if repo in rsets.prepos: @@ -129,7 +134,7 @@ def portage_repo_path(repo): return None def pomu_status(repo_path): - """Check if pomu is enabled for a repository at a given path""" + """Check if pomu is enabled for a repository at a given path (repo_path)""" return path.isdir(path.join(repo_path, 'metadata', 'pomu')) def pomu_active_portage_repo(): diff --git a/pomu/source/__init__.py b/pomu/source/__init__.py index 027227e..3e78150 100644 --- a/pomu/source/__init__.py +++ b/pomu/source/__init__.py @@ -3,3 +3,4 @@ from pomu.source.manager import PackageDispatcher dispatcher = PackageDispatcher() import pomu.source.portage +import pomu.source.file diff --git a/pomu/source/file.py b/pomu/source/file.py index 8b1a650..d3fc7ed 100644 --- a/pomu/source/file.py +++ b/pomu/source/file.py @@ -2,15 +2,11 @@ A package source module to import packages from filesystem locations (ebuilds) """ -import os - from os import path -from shutil import copy2 -from tempfile import mkdtemp from pomu.package import Package from pomu.source import dispatcher -from pomu.source.portage import cpv_split, ver_str +from pomu.util.pkg import cpv_split, ver_str from pomu.util.query import query from pomu.util.result import Result @@ -26,11 +22,13 @@ class LocalEbuild(): self.path = path def fetch(self): - root = mkdtemp() - pkgpath = path.join(root, self.category, self.name) - os.makedirs(pkgpath) - copy2(self.path, pkgpath) - return Package(self, self.name, root, self.category, self.version) + return Package(self, self.name, '/', self.category, self.version, + filemap = { + path.join( + self.category, + self.name, + '{}/{}-{}.ebuild'.format(self.category, self.name, self.version) + ) : self.path}) @staticmethod def from_data_file(path): diff --git a/pomu/source/portage.py b/pomu/source/portage.py index 40ad964..e71f539 100644 --- a/pomu/source/portage.py +++ b/pomu/source/portage.py @@ -1,19 +1,17 @@ """ A package source module to import packages from configured portage repositories """ -import os -import re - from functools import cmp_to_key from os import path -from portage.versions import best, suffix_value, vercmp +from portage.versions import vercmp from pomu.package import Package from pomu.repo.repo import portage_repos, portage_repo_path from pomu.source import dispatcher +from pomu.util.pkg import cpv_split, ver_str +from pomu.util.portage import repo_pkgs from pomu.util.result import Result -from pomu.util.str import pivot class PortagePackage(): """A class to represent a portage package""" @@ -58,8 +56,6 @@ class PortagePackage(): return '{}/{}-{}{}::{}'.format(self.category, self.name, self.version, '' if self.slot == '0' else ':' + self.slot, self.repo) -suffixes = [x[0] for x in sorted(suffix_value.items(), key=lambda x:x[1])] -misc_dirs = ['profiles', 'licenses', 'eclass', 'metadata', 'distfiles', 'packages', 'scripts', '.git'] @dispatcher.source class PortageSource(): @@ -154,62 +150,4 @@ def sanity_check(repo, category, name, vernum, suff, rev, slot, ver=None): pkg = sorted(pkgs, key=cmp_to_key(lambda x,y:vercmp(x[3],y[3])), reverse=True)[0] return PortagePackage(*pkg) - -def ver_str(vernum, suff, rev): - """Gets the string representation of the version""" - return vernum + (suff if suff else '') + (rev if rev else '') - -def best_ver(repo, category, name, ver=None): - """Gets the best (newest) version of a package in the repo""" - ebuilds = [category + '/' + name + x[len(name):-7] for x in - os.listdir(path.join(portage_repo_path(repo), category, name)) - if x.endswith('.ebuild')] - cat, name, vernum, suff, rev = cpv_split(best(ebuilds)) - return ver_str(vernum, suff, rev) - -def repo_pkgs(repo, category, name, ver=None, slot=None): - """List of package occurences in the repo""" - if not repo: - res = [] - for r in portage_repos(): - res.extend(repo_pkgs(r, category, name, ver, slot)) - return res - if category: - if path.exists(path.join(portage_repo_path(repo), category, name)): - return [(repo, category, name, best_ver(repo, category, name))] - return [] - rpath = portage_repo_path(repo) - dirs = set(os.listdir(rpath)) - set(misc_dirs) - res = [] - for d in dirs: - if path.isdir(path.join(rpath, d, name)): - res.append((repo, d, name, best_ver(repo, d, name))) - return res - -#NOTE: consider moving cpv_split and ver_str into util -def cpv_split(pkg): - # dev-libs/openssl-0.9.8z_p8-r100 - category, _, pkg = pkg.rpartition('/') # category may be omitted - # openssl-0.9.8z_p8-r100 - m = re.search(r'-r\d+$', pkg) # revision is optional - if m: - pkg, rev = pivot(pkg, m.start(0), False) - else: - rev = None - # openssl-0.9.8z_p8 - m = re.search(r'_({})(\d*)$'.format('|'.join(suffixes)), pkg) - if m: - pkg, suff = pivot(pkg, m.start(0), False) - else: - suff = None - # openssl-0.9.8z - m = re.search(r'-(\d+(\.\d+)*)([a-z])?$', pkg) - if m: - pkg, vernum = pivot(pkg, m.start(0), False) - else: - vernum = None - # openssl - name = pkg - return category, name, vernum, suff, rev - __all__ = [PortagePackage, PortageSource] diff --git a/pomu/util/fs.py b/pomu/util/fs.py index 5d25ec5..7016717 100644 --- a/pomu/util/fs.py +++ b/pomu/util/fs.py @@ -11,6 +11,11 @@ def strip_prefix(string, prefix): return string def remove_file(repo, dst): - """Removes a file from a repository""" + """ + Removes a file from a repository (adding changes to the index) + Parameters: + repo - the repo + dst - the file + """ repo.index.remove(dst) os.remove(dst) diff --git a/pomu/util/pkg.py b/pomu/util/pkg.py new file mode 100644 index 0000000..26fdb39 --- /dev/null +++ b/pomu/util/pkg.py @@ -0,0 +1,41 @@ +""" +A set of utility function to manipulate package descriptions +""" + +import re + +from pomu.util.portage import suffixes +from pomu.util.str import pivot + +def ver_str(vernum, suff, rev): + """Gets the string representation of the version (specified by number, suffix and rev)""" + return vernum + (suff if suff else '') + (rev if rev else '') + +def cpv_split(pkg): + """ + Extracts category, name, version number, suffix, revision from a package descriptor + e.g. dev-libs/openssl-0.9.8z_p8-r100 -> dev-libs, openssl, 0.9.8z, p8, r100 + """ + # dev-libs/openssl-0.9.8z_p8-r100 + category, _, pkg = pkg.rpartition('/') # category may be omitted + # openssl-0.9.8z_p8-r100 + m = re.search(r'-r\d+$', pkg) # revision is optional + if m: + pkg, rev = pivot(pkg, m.start(0), False) + else: + rev = None + # openssl-0.9.8z_p8 + m = re.search(r'_({})(\d*)$'.format('|'.join(suffixes)), pkg) + if m: + pkg, suff = pivot(pkg, m.start(0), False) + else: + suff = None + # openssl-0.9.8z + m = re.search(r'-(\d+(\.\d+)*)([a-z])?$', pkg) + if m: + pkg, vernum = pivot(pkg, m.start(0), False) + else: + vernum = None + # openssl + name = pkg + return category, name, vernum, suff, rev diff --git a/pomu/util/portage.py b/pomu/util/portage.py new file mode 100644 index 0000000..32d8e5d --- /dev/null +++ b/pomu/util/portage.py @@ -0,0 +1,47 @@ +""" +A set of utility functions to query portage repos +""" + +import os + +from os import path + +from portage.versions import best, suffix_value + +from pomu.repo.repo import portage_repos, portage_repo_path +from pomu.util.pkg import cpv_split, ver_str + +suffixes = [x[0] for x in sorted(suffix_value.items(), key=lambda x:x[1])] +misc_dirs = ['profiles', 'licenses', 'eclass', 'metadata', 'distfiles', 'packages', 'scripts', '.git'] + +def best_ver(repo, category, name, ver=None): + """Gets the best (newest) version of a package in the repo""" + if ver: + return ver if path.exists(path.join(portage_repo_path(repo), + category, name, '{}-{}.ebuild'.format(name, ver))) else None + ebuilds = [category + '/' + name + x[len(name):-7] for x in + os.listdir(path.join(portage_repo_path(repo), category, name)) + if x.endswith('.ebuild')] + cat, name, vernum, suff, rev = cpv_split(best(ebuilds)) + return ver_str(vernum, suff, rev) + +def repo_pkgs(repo, category, name, ver=None, slot=None): + """List of package occurences in the repo""" + if not repo: + res = [] + for r in portage_repos(): + res.extend(repo_pkgs(r, category, name, ver, slot)) + return res + if category: + if path.exists(path.join(portage_repo_path(repo), category, name)): + res = best_ver(repo, category, name, ver) + return [(repo, category, name, res)] if res else [] + return [] + rpath = portage_repo_path(repo) + dirs = set(os.listdir(rpath)) - set(misc_dirs) + res = [] + for d in dirs: + if path.isdir(path.join(rpath, d, name)): + res.append((repo, d, name, best_ver(repo, d, name, ver))) + return res + diff --git a/pomu/util/query.py b/pomu/util/query.py index 035fe1b..d9dc09e 100644 --- a/pomu/util/query.py +++ b/pomu/util/query.py @@ -1,14 +1,15 @@ """ A module to (non)interactively query the user for impure values """ - -import sys - from pomu.util.result import Result def query(name, prompt=None, default=None): """ Queries the impure world for name + Parameters: + name - the name + prompt - prompt text + default - default value used for errors, forced non-interactive etc. TODO: non-interactive """ if not prompt: diff --git a/setup.py b/setup.py index aa4e82b..c247110 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( author='Mykyta Holubakha', author_email='hilobakho@gmail.com', license='GNU GPLv2', - packages=find_packages(), + packages=find_packages(exclude=['tests']), install_requires=['Click', 'GitPython'], entry_points={ 'console_scripts':['pomu = pomu.cli:main_'] -- cgit v1.2.3-18-g5258