diff options
38 files changed, 1722 insertions, 207 deletions
@@ -1,6 +1,6 @@ # Pomu -##Prerequisites +## Prerequisites pomu requires the following packages to be installed: @@ -8,58 +8,84 @@ pomu requires the following packages to be installed: - app-portage/repoman : Manifest generation - dev-python/click : CLI implementation - dev-python/git-python : Git repository management and initialization +- dev-python/pbraw : A library to fetch plaintexts from pastebins +- dev-python/curtsies : Curse-like terminal wrapper, used for TUIs (both fullscreen and inline) In addition to the necessary dependencies, pomu requires a git ebuild repository (optionally set up with portage), though it can create a new one as well (with or without setting it up in portage). ## Usage +``` Usage: pomu [OPTIONS] COMMAND [ARGS]... - A utility to manage portage overlays + A utility to import and manage ebuilds portage overlays Options: --no-portage Do not setup the portage repo --repo-path TEXT Path to the repo directory (used with --no-portage) - --help Show this message and exit. + -h, --help Show this message and exit. Commands: - fetch Fetch a package into a directory - init Initialise pomu for a repository - install - show - status - uninstall + commit Commit user changes + fetch Fetch a package into a directory (or display... + import Import a package into a repository + init Initialise a pomu repository + patch Patch an existing package + search Search gpo.zugaina.org + show Display installed package info + status Display pomu status + uninstall Uninstall a package +``` + +## Done work (for GSOC) + +During the summer coding period the following projects were implemented (from scratch): + +- The `pomu` utility (in this repository) +- The `pbraw` library (and utility) to fetch plaintexts from arbitrary pastebin services (see https://github.com/Hummer12007/pbraw) ## Planned functionality -1) create/use a git repository/directory, in a configurable location (default may be /usr/local/portage), +- [x] create/use a git repository/directory, in a configurable location (default may be /usr/local/portage), with an option to set it up as a repo in /etc/portage/repos.conf; -2) import ebuilds and related files (FILESDIR, eclasses, dependent ebuilds (if required) etc.) in atomic +- [ ] import ebuilds and related files (FILESDIR, eclasses, dependent ebuilds (if required) etc.) in atomic commits with meaningful descriptions; - -3) import packages from: -* the portage tree -* overlays (either in fs or in remote repositories, including, but not limited to, overlays in the layman + - [x] ebuilds + - [ ] FILESDIR (imports all the files, relevant import is WIP) + - [ ] dependent ebuilds + - [x] atomic commits + - [x] commit message generation + +- [ ] import packages from: + - [x] the portage tree (repositories set up in portage) + - [x] overlays (either in fs or in remote repositories, including, but not limited to, overlays in the layman catalog, arbitrary git repos (with proper layout) etc.) -* local/remote text files -* bugzilla tickets (from ebuild/patch attachments, and (optionally) from links in comments (at least (may + - [x] local/remote text files + - [x] (possibly) some sane pastebin services (like github gists, paste.pound-python.org etc.); + - [x] bugzilla tickets (from ebuild/patch attachments, and (optionally) from links in comments (at least (may be extended) serving text/plain content)) -* github pull requests (may be extended to support repos other than gentoo/gentoo) -* (possibly) some sane pastebin services (like github gists, paste.pound-python.org etc.); + - [ ] github pull requests (may be extended to support repos other than gentoo/gentoo) (backend is practically done, only thing left's to hook it up to GH, so 70% done) 4) when importing packages: -* (from overlays) detect dependencies/eclasses unique to the overlay (unavailable in parent overlays/main -tree) and (prompt the user to) import them -* (from overlays/portage) detect user changes and merge remote changes hunk by hunk (in a manner -similar to dispatch-conf) -* allow the user to specify package category/name (when importing from files/pastebins/patches, and new + - [ ] (from overlays) detect dependencies/eclasses unique to the overlay (unavailable in parent overlays/main +tree) and (prompt the user to) import them (WIP) + - [x] (from overlays/portage) detect user changes and merge remote changes hunk by hunk (in a manner +similar to dispatch-conf) (not tested enough) + - [x] allow the user to specify package category/name (when importing from files/pastebins/patches, and new package version -* generate required files (Manifests etc.) + - [x] generate required files (Manifests etc.) -4) search for packages, interfacing with gpo.zugaina.org; +- [x] search for packages, interfacing with gpo.zugaina.org; -5) update ebuilds pulled in from portage/overlays, merging upstream and user changes; +- [ ] update ebuilds pulled in from portage/overlays, merging upstream and user changes; (work on patch sequences and reordering complete, 30% done) -6) remove packages from the repository (if possible, by simply reverting the relevant commits, unless some +- [x] remove packages from the repository (if possible, by simply reverting the relevant commits, unless some non-leaf packages/eclasses pulled in by the package are required for another one, ergo we should track dependencies). + +### TODO (future plans) + +- migrate from curtsies to a custom TUI library (with widgets and stuff) +- generify handlers (?) +- expand and extend global options +- properly sanitize user input diff --git a/pomu/__init__.py b/pomu/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pomu/__init__.py diff --git a/pomu/cli.py b/pomu/cli.py index 72cec9b..bb3fe58 100644 --- a/pomu/cli.py +++ b/pomu/cli.py @@ -3,9 +3,13 @@ import click from os import path +from pomu.data.zugaina import ZugainaDataSource +from pomu.patch.patch import process_changes from pomu.repo.init import init_plain_repo, init_portage_repo from pomu.repo.repo import portage_repo_path, portage_repos, pomu_active_repo +from pomu.search import PSPrompt from pomu.source import dispatcher +from pomu.util.pkg import cpv_split from pomu.util.result import ResultException #TODO: global --repo option, (env var?) @@ -22,20 +26,21 @@ class needs_repo(): def __init__(self, func): self.func = func self.__name__ = func.__name__ + self.__doc__ = func.__doc__ - def __call__(self, *args): + def __call__(self, *args, **kwargs): pomu_active_repo(g_params.no_portage, g_params.repo_path) - self.func(*args) + self.func(*args, **kwargs) pass_globals = click.make_pass_decorator(GlobalVars, ensure=True) -@click.group() +@click.group(context_settings=dict(help_option_names=['-h', '--help'])) @click.option('--no-portage', is_flag=True, help='Do not setup the portage repo') @click.option('--repo-path', help='Path to the repo directory (used with --no-portage)') def main(no_portage, repo_path): - """A utility to manage portage overlays""" + """A utility to import and manage ebuilds portage overlays""" g_params.no_portage = no_portage g_params.repo_path = repo_path @@ -47,8 +52,8 @@ def main(no_portage, repo_path): @click.option('--repo-dir', envvar='POMU_REPO_DIR', default='/var/lib/pomu', help='Path for creating new repos') @click.argument('repo', required=False) -def init(g_params, list_repos, create, repo_dir, repo): - """Initialise pomu for a repository""" +def init(list_repos, create, repo_dir, repo): + """Initialise a pomu repository""" if list_repos: print('Available repos:') for prepo in portage_repos(): @@ -69,15 +74,34 @@ def status(): else: print('pomu is initialized at', repo.root) -@main.command() +@main.command(name='import') @click.argument('package', required=True) +@click.option('--patch', nargs=1, multiple=True) @needs_repo -def install(package): - """Install a package""" - res = dispatcher.install_package(pomu_active_repo(), package).expect() +def import_cmd(package, patch): + """Import a package into a repository""" + pkg = dispatcher.get_package(package).expect() + pkg.patch(patch) + res = pomu_active_repo().merge(pkg).expect() print(res) @main.command() +@click.argument('package', required=True) +@click.argument('patch', type=click.Path(exists=True), nargs=-1, required=True) +def patch(package): + """Patch an existing package""" + category, name, _ = cpv_split(package) + pkg = pomu_active_repo().get_package(name=name, category=category).expect() + pkg.patch(patch).expect() + +@main.command() +@click.option('--single', is_flag=True, required=False, default=False) +def commit(single): + """Commit user changes""" + repo = pomu_active_repo() + change_map = process_changes(repo, single).expect() + +@main.command() @click.option('--uri', is_flag=True, help='Specify the package to remove by uri, instead of its name') @click.argument('package', required=True) @@ -97,7 +121,7 @@ def uninstall(uri, package): @click.option('--into', default=None, help='Specify fetch destination') def fetch(package, into): - """Fetch a package into a directory""" + """Fetch a package into a directory (or display its contents)""" pkg = dispatcher.get_package(package).expect() print('Fetched package', pkg, 'at', pkg.root) print('Contents:') @@ -121,12 +145,21 @@ def show(package): for f in pkg.files: print(' ', path.join(*f)) if pkg.backend: - print('Backend:', pkg.backend.__name__) + print('Backend:', pkg.backend.__cname__) print('Backend detailes:', pkg.backend) +@main.command() +@click.argument('query', required=True) +@click.option('--fetch-only', default=False, is_flag=True) +def search(query, fetch_only): + """Search gpo.zugaina.org""" + ds = ZugainaDataSource(query) + p = PSPrompt(ds) + packages = p.run() + def main_(): try: - main.main(standalone_mode=False) + main.main() except ResultException as e: print(str(e)) diff --git a/pomu/data/__init__.py b/pomu/data/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pomu/data/__init__.py diff --git a/pomu/data/datasource.py b/pomu/data/datasource.py new file mode 100644 index 0000000..53fcc62 --- /dev/null +++ b/pomu/data/datasource.py @@ -0,0 +1,19 @@ +""" +Base DataSource class +""" + +class DataSource(): + def __init__(self, query): + pass + + def page_count(self): + pass + + def get_page(self, page): + pass + + def list_items(self, ident): + pass + + def get_item(self, ident): + pass diff --git a/pomu/data/zugaina.py b/pomu/data/zugaina.py new file mode 100644 index 0000000..92accb8 --- /dev/null +++ b/pomu/data/zugaina.py @@ -0,0 +1,63 @@ +""" +gpo.zugaina.org searcher and fetcher +""" +import lxml.html +import requests + +from pomu.data.datasource import DataSource +from pomu.util.pkg import cpv_split + +BASE_URL = 'https://gpo.zugaina.org/' +SBASE_URL = BASE_URL + 'Search?search={}&page={}' + +class ZugainaDataSource(DataSource): + + def __init__(self, query): + self.query = query + self.pagecache = {} + self.itemcache = {} + self.pagecount = -1 + + def page_count(self): + if self.pagecount > 0: + return self.pagecount + text = self.fetch_page(1) + doc = lxml.html.document_fromstring(text) + field = doc.xpath('//div[@class="pager"]/span')[0].text + self.pagecount = (int(field.split(' ')[-1]) + 49) // 50 + return self.pagecount + + def get_page(self, page): + text = self.fetch_page(page) + doc = lxml.html.document_fromstring(text) + return [(x.text.strip(), x.getchildren()[0].text) + for x in doc.xpath('//div[@id="search_results"]/a/div')] + + def list_items(self, ident): + text = self.fetch_item(ident) + doc = lxml.html.document_fromstring(text) + res = [] + for div in doc.xpath('//div[@id="ebuild_list"]/ul/div'): + id_ = div.xpath('li/a')[0].get('href').split('/')[3] + pv = div.xpath('li/div/b')[0].text + v = cpv_split(pv)[2] + overlay = div.xpath('@id')[0] + res.append((id_, v, overlay)) + return res + + def get_item(self, ident): + return requests.get(BASE_URL + 'AJAX/Ebuild/' + str(ident)).text + + def fetch_item(self, ident): + if ident in self.itemcache: + return self.itemcache[ident] + res = requests.get(BASE_URL + ident).text + return res + + + def fetch_page(self, page): + if page in self.pagecache: + return self.pagecache[page] + res = requests.get(SBASE_URL.format(self.query, page)).text + self.pagecache[page] = res + return res diff --git a/pomu/package.py b/pomu/package.py index 5027569..a5b984c 100644 --- a/pomu/package.py +++ b/pomu/package.py @@ -13,10 +13,11 @@ import subprocess from patch import PatchSet from pomu.util.fs import strip_prefix +from pomu.util.misc import list_add from pomu.util.result import Result class Package(): - def __init__(self, name, root, backend=None, category=None, version=None, slot='0', d_path=None, files=None, filemap=None): + def __init__(self, name, root, backend=None, category=None, version=None, slot='0', d_path=None, files=None, filemap=None, patches=[]): """ Parameters: backend - specific source module object/class @@ -35,6 +36,7 @@ class Package(): self.version = version self.slot = slot self.filemap = {} + self.patches = patches if d_path is None and files is None and filemap is None: self.d_path = None self.read_path(self.root) @@ -57,6 +59,10 @@ class Package(): res.append(path.split(k)) return res + @property + def ebuild_path(self): + return path.join(self.category, self.name, '{}-{}.ebuild'.format(self.name, self.version)) + def strip_root(self, d_path): """Strip the root component of d_path""" # the path should be either relative, or a child of root @@ -75,21 +81,46 @@ class Package(): def merge_into(self, dst): """Merges contents of the package into a specified directory (dst)""" for trg, src in self.filemap.items(): - wd, _ = path.split(trg) + wd, filename = path.split(trg) dest = path.join(dst, wd) try: makedirs(dest, exist_ok=True) - copy2(src, dest) + if isinstance(src, bytes): + with open(path.join(dest, filename), 'wb') as f: + f.write(src) + else: + copy2(src, dest) except PermissionError: return Result.Err('You do not have enough permissions') - return Result.Ok() + return Result.Ok().and_(self.apply_patches()) - def apply_patches(self, location, patches): - """Applies a sequence of patches at the location""" + def patch(self, patch): + if patch: + list_add(self.patches, patch) + + def apply_patches(self, revert=False): + """Applies a sequence of patches at the root (after merging)""" ps = PatchSet() - for p in patches: + for p in self.patches: ps.parse(open(p, 'r')) - ps.apply(root=location) + for patch in ps: + if '.ebuild' in patch.target: + patch.source = self.ebuild_path + patch.target = self.ebuild_path + elif '/files/' in patch.target: + comps = patch.target.split('/') + comps = [self.category, self.name] + comps[comps.index('files'):] + patch.target = '/'.join(comps) + if not patch.source.split('/'[-2:] == ['dev', 'null']): + patch.source = '/'.join(comps) + else: + pass + + if revert: + ps.revert(root=self.root) + else: + ps.apply(root=self.root) + return Result.Ok() def gen_manifests(self, dst): """ @@ -121,3 +152,12 @@ class Package(): if self.slot != '0': s += self.slot return s + +class PatchList(): + """A class to represent a sequence of patches (can be merged in the repo)""" + def __init__(self, category, name, version, patches, slot='0'): + self.category = category + self.name = name + self.version = version + self.slot = slot + self.patches = patches diff --git a/pomu/patch/__init__.py b/pomu/patch/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pomu/patch/__init__.py diff --git a/pomu/patch/patch.py b/pomu/patch/patch.py new file mode 100644 index 0000000..7bb9cb8 --- /dev/null +++ b/pomu/patch/patch.py @@ -0,0 +1,123 @@ +""" +""" + +from os import path +from time import time + +from git.repo import Repo + +from pomu.util.pkg import cpv_split +from pomu.util.result import Result + +def process_changes(_repo, single): + # we only tackle repository changes so far + repo = Repo(_repo.root) + chans = repo.head.commit.diff(None, create_patch=True) + new_files = repo.untracked_files + all_pkgs = _repo.get_packages() + res = {x: [] for x in all_pkgs} + paths = {x: [] for x in all_pkgs} + multi = not single + chanpaks = ([],[],[]) # import, order, apply + + ## Process user-made changes to package files + for f in new_files: # process untracked files + pkpref = '/'.join(path.dirname(f).split('/')[0:2]) + if pkpref in res: + paths[pkpref].append(f) + res[pkpref].append(new_file_patch(_repo, f)) + for diff in chans: # changes in tracked files + pkpref = '/'.join(path.dirname(diff.a_path).split('/')[0:2]) + if pkpref in res: + paths[pkpref].append(diff.a_path) + res[pkpref].append('\n'.join(diff_header(diff.a_path, diff.b_path)) + + diff.diff.decode('utf-8')) + res = {x: res[x] for x in res if res[x]} + paths = {x: paths[x] for x in paths if res[x]} + for _pkg, diffs in res.items(): # add each change as its own patch + cat, name, _ = cpv_split(_pkg) + patch_contents = '\n'.join(diffs) + pkg = _repo.get_package(name, cat).expect() + patch_name = '{}-user_changes.patch'.format(int(time())) + had_order = path.exists(path.join(pkg.pkgdir, 'patches', 'PATCH_ORDER')) + pkg.add_patch(patch_contents, patch_name) + repo.index.add([p for ps in paths for p in paths[ps]]) + repo.index.add([path.join(pkg.pkgdir, 'patches', patch_name)]) + if not had_order: + repo.index.add([path.join(pkg.pkgdir, 'PATCH_ORDER')]) + if multi: + repo.index.commit('{}/{}: imported user changes'.format(cat, name)) + else: + chanpaks[0].append('{}/{}'.format(cat, name)) + + ## Process patch order changes + res = {x: [] for x in all_pkgs} + applied = {x: [] for x in all_pkgs} + for diff in chans: + if not len(diff.a_path.split('/')) == 4: + continue + _, cat, name, __ = diff.a_path.split('/') + if _ != 'metadata' and __ != 'PATCH_ORDER': + continue + if '/'.join([cat, name]) not in res: + continue + orig = repo.odb.stream(diff.a_blob.binsha).read().decode('utf-8') + pkg = _repo.get_package(name, cat) + orig_lines = [path.join(pkg.pkgdir, x.strip()) for x in orig.split('\n') if x.strip() != ''] + pkg.patches = orig_lines + pkg.apply_patches(revert=True) + pkg = _repo.get_package(name, cat) + pkg.patches = pkg.patch_list + applied['{}/{}'.format(cat, name)].extend(pkg.patches) + pkg.apply_patches() + repo.index.add([diff.a_path, pkg.root]) + if multi: + repo.index.commit('{}/{}: modified patch order'.format(cat, name)) + else: + chanpaks[1].append('{}/{}'.format(cat, name)) + + + ## Process new patch files + res = {x: [] for x in all_pkgs} + for f in new_files: + if not f.startswith('metadata/') or f.split('/')[-1] == 'PATCH_ORDER': + continue + pkpref = '/'.join(path.dirname(f).split('/')[2:4]) + if f.split('/')[-1] in applied[pkpref]: #skip, we've added the patch in the previous step + continue + if pkpref in res: + res[pkpref].append(f) + for _pkg, diffs in res.items(): # apply each newly added patch + cat, name, _ = cpv_split(_pkg) + pkg = _repo.get_package(name, cat).expect() + for d in diffs: + pkg.patch(d) + repo.index.add(diffs) + repo.index.add([path.join(cat, name)]) + if multi: + repo.index.commit('{}/{}: applied patches'.format(cat, name)) + else: + chanpaks[2].append('{}/{}'.format(cat, name)) + + if not multi: + msg = 'Synced modifications:\n' + if chanpaks[0]: + msg += '\nimported user changes:\n' + '\n'.join(chanpaks[0]) + '\n' + if chanpaks[1]: + msg += '\nmodified patch order:\n' + '\n'.join(chanpaks[1]) + '\n' + if chanpaks[2]: + msg += '\napplied patches:\n' + '\n'.join(chanpaks[2]) + '\n' + + return Result.Ok() + +def new_file_patch(repo, newf): + with open(path.join(repo.root, newf), 'r') as f: + lines = ['+' + x.strip('\n') for x in f.readlines()] + head = diff_header('/dev/null', newf, len(lines)) + return '\n'.join(head + lines) + '\n' + +def diff_header(a_path, b_path, lines=None): + header = ['--- ' + a_path, '+++ ' + 'b/' + b_path] + if lines: + header.append('@@ -0,0 +1,' + lines + ' @@') + return header diff --git a/pomu/repo/__init__.py b/pomu/repo/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pomu/repo/__init__.py diff --git a/pomu/repo/remote/__init__.py b/pomu/repo/remote/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pomu/repo/remote/__init__.py diff --git a/pomu/repo/remote/git.py b/pomu/repo/remote/git.py new file mode 100644 index 0000000..b6142b0 --- /dev/null +++ b/pomu/repo/remote/git.py @@ -0,0 +1,72 @@ +"""A class for remote git repos""" +from os import path +from shutil import rmtree +from subprocess import call +from tempfile import mkdtemp + +from git import Repo + +from pomu.repo.remote.remote import RemoteRepo, normalize_key +from pomu.util.git import parse_object +from pomu.util.result import Result + +class RemoteGitRepo(RemoteRepo): + """A class responsible for git remotes""" + def __init__(self, url): + self.uri = url + self.dir = mkdtemp() + if call(['git', 'clone', '--depth=1', '--bare', url, self.dir]) > 0: # we've a problem + raise RuntimeError() + self.repo = Repo(self.dir) + + def __enter__(self): + pass + + def __exit__(self, *_): + self.cleanup() + + def get_object(self, oid): + head, tail = oid[0:2], oid[2:] + opath = path.join(self.dir, 'objects', head, tail) + return open(opath, 'rb').read() + + def _fetch_tree(self, obj, tpath): + res = [] + ents = parse_object(self.get_object(obj), tpath).unwrap() + for is_dir, sha, opath in ents: + res.append((opath.decode('utf-8') + ('/' if is_dir else ''), sha)) + if is_dir: + res.extend(self._fetch_tree(sha, opath)) + return res + + def fetch_tree(self): + """Returns repos hierarchy""" + if hasattr(self, '_tree'): + return [x for x, y in self._tree] + tid = self.repo.tree().hexsha + res = self._fetch_tree(tid, b'') + self._tree = res + return [x for x, y in res] + + def fetch_subtree(self, key): + """Lists a subtree""" + k = normalize_key(key, True) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return Result.Err() + l = len(key) + return Result.Ok( + [tpath[l:] for tpath in self.fetch_tree() if tpath.startswith(k)]) + + def fetch_file(self, key): + """Fetches a file from the repo""" + k = normalize_key(key) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return Result.Err() + return parse_object(self.get_object(dic[k])) + + def cleanup(self): + rmtree(self.dir) diff --git a/pomu/repo/remote/hg.py b/pomu/repo/remote/hg.py new file mode 100644 index 0000000..4e5e8a9 --- /dev/null +++ b/pomu/repo/remote/hg.py @@ -0,0 +1,59 @@ +"""A class for remote hg repos""" +from os import chdir +from shutil import rmtree +from subprocess import call, run +from tempfile import mkdtemp + +from pomu.repo.remote.remote import RemoteRepo, normalize_key +from pomu.util.result import Result + +class RemoteHgRepo(RemoteRepo): + """A class responsible for hg remotes""" + def __init__(self, url): + self.uri = url + self.dir = mkdtemp() + chdir(self.dir) + if call('hg', 'clone', '-U', url, '.') > 0: # we've a problem + raise RuntimeError() + + def __enter__(self): + pass + + def __exit__(self, *_): + self.cleanup() + + def fetch_tree(self): + """Returns repos hierarchy""" + if hasattr(self, '_tree'): + return self._tree + p = run('hg', 'files', '-rdefault') + if p.returncode: + return [] + self._tree = ['/' + x for x in p.stdout.split('\n')] + return self._tree + + def fetch_subtree(self, key): + """Lists a subtree""" + k = normalize_key(key, True) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return Result.Err() + l = len(key) + return Result.Ok( + [tpath[l:] for tpath in self.fetch_tree() if tpath.startswith(k)]) + + def fetch_file(self, key): + """Fetches a file from the repo""" + k = normalize_key(key) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return Result.Err() + p = run('hg', 'cat', '-rdefault', k) + if p.returncode: + return Result.Err() + return Result.Ok(p.stdout) + + def cleanup(self): + rmtree(self.dir) diff --git a/pomu/repo/remote/remote.py b/pomu/repo/remote/remote.py new file mode 100644 index 0000000..0b216c1 --- /dev/null +++ b/pomu/repo/remote/remote.py @@ -0,0 +1,82 @@ +"""A template class for remote repos""" +from os import path +from urllib.parse import urlparse + +from pomu.package import Package +from pomu.util.remote import get_full_cpv, filelist_to_cpvs + +class RemoteRepo(): + """A class responsible for remotes""" + def __init__(self, url): + raise NotImplementedError() + + @classmethod + def from_url(cls, uri, type_=None): + tp = RemoteRepo.type_for_name(type_) + if not tp: + from pomu.repo.remote.git import RemoteGitRepo + # no custom schemes for hg, assume git for common ones + #from pomu.repo.remote.hg import RemoteHgRepo + from pomu.repo.remote.rsync import RemoteRsyncRepo + from pomu.repo.remote.svn import RemoteSvnRepo + try: + scheme, *_ = urlparse(uri) + except: + tp = RemoteGitRepo + if (scheme.startswith('http') or scheme.startswith('git') or + scheme.startswith('ssh')): + tp = RemoteGitRepo + elif scheme.startswith('svn'): + tp = RemoteSvnRepo + elif scheme.startswith('rsync'): + tp = RemoteRsyncRepo + return tp(uri) + + @classmethod + def type_for_name(cls, type_): + from pomu.repo.remote.git import RemoteGitRepo + from pomu.repo.remote.hg import RemoteHgRepo + from pomu.repo.remote.rsync import RemoteRsyncRepo + from pomu.repo.remote.svn import RemoteSvnRepo + res = {'git': RemoteGitRepo, 'mercurial': RemoteHgRepo, + 'rsync': RemoteRsyncRepo, 'svn': RemoteSvnRepo} + return res.get(type_) + + def fetch_package(self, name, category=None, version=None): + """Fetches a package, determined by the parametres""" + cat, n, ver = get_full_cpv(self.list_cpvs(), name, category, version).unwrap() + ebuild = '{}/{}/{}-{}.ebuild'.format(cat, n, n, ver) + subdir = '/{}/{}'.format(cat, name) + filemap = {} + filemap[ebuild] = self.fetch_file(ebuild).unwrap() + subtree = self.fetch_subtree('/{}/{}/'.format(cat, name)).unwrap() + for fpath in subtree: + if '/' in fpath: + parent, _, child = fpath.rpartition('/') + if parent != 'files': continue + if not fpath or fpath.endswith('.ebuild') or fpath.endswith('/'): continue + p = path.join(subdir, fpath) + filemap[p] = self.fetch_file(p).unwrap() + return Package(name, '/', None, category, version, filemap=filemap) + + def list_cpvs(self): + """Gets a list of all ebuilds in the repo""" + return filelist_to_cpvs(self.fetch_tree()) + + def fetch_tree(self): + """Returns repos hierarchy""" + raise NotImplementedError() + + def fetch_subtree(self, key): + """Lists a subtree""" + raise NotImplementedError() + + def fetch_file(self, key): + """Fetches a file from the repo""" + raise NotImplementedError() + +def normalize_key(key, trail=False): + k = '/' + key.lstrip('/') + if trail: + k = k.rstrip('/') + '/' + return k diff --git a/pomu/repo/remote/rsync.py b/pomu/repo/remote/rsync.py new file mode 100644 index 0000000..ae6ab76 --- /dev/null +++ b/pomu/repo/remote/rsync.py @@ -0,0 +1,59 @@ +"""A class for remote rsync repos""" +from os import rmdir, mkfifo, unlink, path +from subprocess import run +from tempfile import mkdtemp + +from pomu.repo.remote.remote import RemoteRepo, normalize_key +from pomu.util.result import Result + +class RemoteRsyncRepo(RemoteRepo): + """A class responsible for rsync remotes""" + def __init__(self, url): + self.uri = url + + def __enter__(self): + pass + + def __exit__(self, *_): + pass + + def fetch_tree(self): + """Returns repos hierarchy""" + if hasattr(self, '_tree'): + return self._tree + d = mkdtemp() + p = run('rsync', '-rn', '--out-format="%n"', self.uri, d) + rmdir(d) + if p.returncode: + return Result.Err() + self._tree = ['/' + x for x in p.stdout.split('\n')] + return self._tree + + def fetch_subtree(self, key): + """Lists a subtree""" + k = normalize_key(key, True) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return [] + l = len(key) + return Result.Ok( + [tpath[l:] for tpath in self.fetch_tree() if tpath.startswith(k)]) + + def fetch_file(self, key): + """Fetches a file from the repo""" + k = normalize_key(key) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return Result.Err() + d = mkdtemp() + fip = path.join(d, 'fifo') + mkfifo(fip) + p = run('rsync', self.uri.rstrip('/') + key, fip) + fout = fip.read() + unlink(fip) + rmdir(d) + if p.returncode: + return Result.Err() + return Result.Ok(fout) diff --git a/pomu/repo/remote/svn.py b/pomu/repo/remote/svn.py new file mode 100644 index 0000000..447a8a0 --- /dev/null +++ b/pomu/repo/remote/svn.py @@ -0,0 +1,49 @@ +"""A class for remote svn repos""" +from subprocess import run + +from pomu.repo.remote.remote import RemoteRepo, normalize_key +from pomu.util.result import Result + +class RemoteSvnRepo(RemoteRepo): + """A class responsible for svn remotes""" + def __init__(self, url): + self.uri = url + + def __enter__(self): + pass + + def __exit__(self, *_): + pass + + def fetch_tree(self): + """Returns repos hierarchy""" + if hasattr(self, '_tree'): + return self._tree + p = run('svn', 'ls', '-R', self.uri) + if p.returncode: + return [] + self._tree = p.stdout.split('\n') + return self._tree + + def fetch_subtree(self, key): + """Lists a subtree""" + k = normalize_key(key, True) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return Result.Err() + l = len(key) + return Result.Ok( + [tpath[l:] for tpath in self.fetch_tree() if tpath.startswith(k)]) + + def fetch_file(self, key): + """Fetches a file from the repo""" + k = normalize_key(key) + self.fetch_tree() + dic = dict(self._tree) + if k not in dic: + return Result.Err() + p = run('svn', 'cat', k) + if p.returncode: + return Result.Err() + return Result.Ok(p.stdout) diff --git a/pomu/repo/repo.py b/pomu/repo/repo.py index f41e0e4..9b17776 100644 --- a/pomu/repo/repo.py +++ b/pomu/repo/repo.py @@ -1,10 +1,13 @@ """Subroutines with repositories""" + from os import path, rmdir, makedirs +from shutil import copy2 from git import Repo +from patch import PatchSet import portage -from pomu.package import Package +from pomu.package import Package, PatchList from pomu.util.cache import cached from pomu.util.fs import remove_file, strip_prefix from pomu.util.result import Result @@ -29,7 +32,17 @@ class Repository(): def pomu_dir(self): return path.join(self.root, 'metadata/pomu') - def merge(self, package): + def merge(self, mergeable): + """Merges a package or a patchset into the repository""" + if isinstance(mergeable, Package): + return self.merge_pkg(mergeable) + elif isinstance(mergeable, PatchList): + pkg = self.get_package(mergeable.name, mergeable.category, + mergeable.slot).unwrap() + return pkg.patch(mergeable.patches) + return Result.Err() #unreachable yet + + def merge_pkg(self, package): """Merge a package (a pomu.package.Package package) into the repository""" r = self.repo pkgdir = path.join(self.pomu_dir, package.category, package.name) @@ -64,9 +77,16 @@ class Repository(): f.write('{}/{}\n'.format(wd, fil)) for m in manifests: f.write('{}\n'.format(strip_prefix(m, self.root))) + if package.patches: + patch_dir = path.join(pkgdir, 'patches') + makedirs(patch_dir, exist_ok=True) + with open(path.join(pkgdir, 'PATCH_ORDER'), 'w') as f: + for patch in package.patches: + copy2(patch, patch_dir) + f.write(path.basename(patch) + '\n') if package.backend: with open(path.join(pkgdir, 'BACKEND'), 'w+') as f: - f.write('{}\n'.format(package.backend.__name__)) + f.write('{}\n'.format(package.backend.__cname__)) package.backend.write_meta(pkgdir) with open(path.join(pkgdir, 'VERSION'), 'w+') as f: f.write(package.version) @@ -76,13 +96,13 @@ class Repository(): r = self.repo for wd, f in package.files: dst = path.join(self.root, wd) - remove_file(path.join(dst, f)) + remove_file(r, path.join(dst, f)) try: rmdir(dst) except OSError: pass pf = path.join(self.pomu_dir, package.name) if path.isfile(pf): - remove_file(pf) + remove_file(r, pf) r.commit('Removed package ' + package.name + ' successfully') return Result.Ok('Removed package ' + package.name + ' successfully') @@ -91,6 +111,12 @@ class Repository(): pkg = self.get_package(name).expect() return self.unmerge(pkg) + def update_package(self, category, name, new): + """Updates a package, replacing it by a newer version""" + pkg = self.get_package(category, name).expect() + self.unmerge(pkg).expect() + self.merge(new) + def _get_package(self, category, name, slot='0'): """Get an existing package (by category, name and slot), reading the manifest""" from pomu.source import dispatcher @@ -103,23 +129,38 @@ class Repository(): with open(path.join(pkgdir, 'BACKEND'), 'r') as f: bname = f.readline().strip() backend = dispatcher.backends[bname].from_meta_dir(pkgdir) + if backend.is_err(): + return backend + backend = backend.ok() with open(path.join(pkgdir, 'VERSION'), 'r') as f: version = f.readline().strip() with open(path.join(pkgdir, 'FILES'), 'r') as f: files = [x.strip() for x in f] - return Package(name, self.root, backend, category=category, version=version, slot=slot, files=files) + patches=[] + if path.isfile(path.join(pkgdir, 'PATCH_ORDER')): + with open(path.join(pkgdir, 'PATCH_ORDER'), 'r') as f: + patches = [x.strip() for x in f] + pkg = Package(name, self.root, backend, category=category, version=version, slot=slot, files=files, patches=[path.join(pkgdir, 'patches', x) for x in patches]) + pkg.__class__ = MergedPackage + return Result.Ok(pkg) def get_package(self, name, category=None, slot=None): """Get a package by name, category and slot""" with open(path.join(self.pomu_dir, 'world'), 'r') as f: for spec in f: + spec = spec.strip() cat, _, nam = spec.partition('/') nam, _, slo = nam.partition(':') if (not category or category == cat) and nam == name: if not slot or (slot == '0' and not slo) or slot == slo: - return self._get_package(category, name, slot) + return self._get_package(category, name, slot or '0') return Result.Err('Package not found') + def get_packages(self): + with open(path.join(self.pomu_dir, 'world'), 'r') as f: + lines = [x.strip() for x in f.readlines() if x.strip() != ''] + return lines + def portage_repos(): """Yield the repositories configured for portage""" @@ -161,3 +202,41 @@ def pomu_active_repo(no_portage=None, repo_path=None): if repo: return Result.Ok(Repository(portage_repo_path(repo), repo)) return Result.Err('pomu is not initialized') + +class MergedPackage(Package): + @property + def pkgdir(self): + ret = path.join(self.root, 'metadata', 'pomu', self.category, self.name) + if self.slot != '0': + ret = path.join(ret, self.slot) + return ret + + def patch(self, patch): + if isinstance(patch, list): + for x in patch: + self.patch(x) + return Result.Ok() + ps = PatchSet() + ps.parse(open(patch, 'r')) + ps.apply(root=self.root) + self.add_patch(patch) + return Result.Ok() + + @property + def patch_list(self): + with open(path.join(self.pkgdir, 'PATCH_ORDER'), 'r') as f: + lines = [x.strip() for x in f.readlines() if x.strip() != ''] + return lines + + def add_patch(self, patch, name=None): # patch is a path, unless name is passed + patch_dir = path.join(self.pkgdir, 'patches') + makedirs(patch_dir, exist_ok=True) + if name is None: + copy2(patch, patch_dir) + with open(path.join(self.pkgdir, 'PATCH_ORDER'), 'w+') as f: + f.write(path.basename(patch) + '\n') + else: + with open(path.join(patch_dir, name), 'w') as f: + f.write(patch) + with open(path.join(self.pkgdir, 'PATCH_ORDER'), 'w+') as f: + f.write(name + '\n') diff --git a/pomu/search.py b/pomu/search.py new file mode 100644 index 0000000..db8625c --- /dev/null +++ b/pomu/search.py @@ -0,0 +1,170 @@ +from enum import Enum +from pydoc import pager + +from curtsies import FullscreenWindow, fmtstr, fsarray +from curtsies.fmtfuncs import invert + +from pomu.util.iquery import Prompt, clamp + +class Entry: + def __init__(self, item, source, children=None): + self.expanded = False + self._source = source + self.item = item + self.children = children + + def toggle(self, idx=0): + if idx == 0: + self.expanded = not self.expanded + else: + child = self.children[idx - 1] + self.children[idx - 1] = (not child[0], child[1]) + if self.expanded and not self.children: + self.children = [(False, x) for x in self._source.list_items(self.item[0])] + + def get_idx(self, idx): + return ((self, None if idx == 0 or not self.children else self.children[idx-1])) + + def __len__(self): + return len(self.children) + 1 if self.expanded else 1 + + def selected(self): + if self.children: + return [child[1] for child in self.children if child[0]] + return [] + +class PromptState(Enum): + TOP_PREV=0 + TOP_OK=1 + LIST=2 + +class PSPrompt(Prompt): + def __init__(self, source): + super().__init__([]) + self.data = source + self.page_count = self.data.page_count() + self.pages = {} + self.state = PromptState.LIST + self.set_page(1) + + def results(self): + # (cp, v, overlay, data) + res = [] + for k, v in self.pages.items(): + for entry in v: + cp = entry.item[0] + for child in entry.selected(): + cid, v, overlay = child + data = self.data.get_item(cid) + res.append((cp, v, overlay, data)) + return res + + + def set_page(self, page): + self.idx = 0 + self.page = page + if page in self.pages: + self.entries = self.pages[page] + else: + self.entries = [self.process_entry(x) for x in self.data.get_page(page)] + self.pages[page] = self.entries + + def render(self): + title = str(self.page) + '//' + str(self.state.value) + if self.page > 1: + title = ('[ ' + + (invert('Prev') if self.state == 0 else ('Prev')) + + ' ] ' + title) + if self.page < self.page_count: + title = (title + ' [ ' + + (invert('Next') if self.state == 1 else ('Next')) + + ' ]') + title = self.center(title) + bottom = '[ ' + (invert('OK') if self.idx == len(self) else 'OK') + ' ]' + bottom = self.center(bottom) + items = [self.render_entry(e, idx) for idx, e in enumerate(self.lens())] + output = fsarray([title] + items + [bottom]) + self.window.render_to_terminal(output) + + def _refresh(self): + w, h = self.window.width, self.window.height + output = fsarray([' ' * w] * h) + self.window.render_to_terminal(output) + + def render_entry(self, entry, idx): + winw = self.window.width + if entry[1]: + hld, data = entry[1] + stt = '*' if hld else ' ' + text = ' [' + (invert(stt) if idx == 0 else stt) + '] ' + text += '{}::{}'.format(data[1], data[2]) + elif entry[0]: + data = entry[0].item + exp = 'v' if entry[0].expanded else '>' + text = '[' + (invert(exp) if idx == 0 else exp) + '] ' + text += data[0] + ' ' + strw = fmtstr(text).width + insw = fmtstr(data[1]).width + text += data[1][:winw - strw - 2] + ('..' if insw + strw > winw else data[1]) + else: + text = '' + return text + + def process_event(self, event): + res = super().process_event(event) + if res: + return res + elif event == '<TAB>': + self.state = PromptState((self.state.value + 1) % 2) + elif event in {'<Ctrl-j>', '<Ctrl-m>'}: + if self.state.value < 2: + return -1 + else: + return False + return True + + + def __len__(self): + return sum(len(y) for y in self.entries) + + def center(self, stri): + tw = fmtstr(stri).width + pw = (self.window.width - tw) // 2 + return ' ' * pw + stri + ' ' * pw + + def clamp(self, x): + return clamp(x, 0, len(self)) + + def toggle(self): + item, idx = self.get_idx(self.idx) + item[0].toggle(idx) + + def preview(self): + target = self.get_target() + if target[1]: + data = self.data.get_item(target[1][1][0]) + pager(data) + self._refresh() + self.render() + + def lens(self): + h = self.window.height - 2 + lst = [self.get_idx(i)[0] for i in range(self.idx, self.clamp(self.idx + h))] + lst += [(None, None)] * clamp(h - len(lst), 0, h) + return lst + + def get_target(self): + return self.get_idx(self.idx)[0] + + def get_idx(self, idx): + for entry in self.entries: + if len(entry) > idx: + break + idx -= len(entry) + return (entry.get_idx(idx), idx) + + def process_entry(self, item): + return Entry(item, self.data) + + def run(self): + return super().run(FullscreenWindow, hide_cursor=True) diff --git a/pomu/source/__init__.py b/pomu/source/__init__.py index 3e78150..7d587c0 100644 --- a/pomu/source/__init__.py +++ b/pomu/source/__init__.py @@ -4,3 +4,5 @@ dispatcher = PackageDispatcher() import pomu.source.portage import pomu.source.file +import pomu.source.url +import pomu.source.bugz diff --git a/pomu/source/base.py b/pomu/source/base.py new file mode 100644 index 0000000..fe4543d --- /dev/null +++ b/pomu/source/base.py @@ -0,0 +1,107 @@ +""" +A base package source module. +A package source module shall provide two classes: a class representing +a package (implementation details of the source, package-specific metadata, +fetching logic etc.), and the source per se: it is responsible for parsing +package specifications and fetching the specified package, as well as +instantiating specific packages from the metadata directory. +""" + +from os import path + +from pomu.source import dispatcher +from pomu.util.result import Result + +class PackageBase(): + """ + This is a base class for storing package-specific metadata. + It shall be subclassed explicitely. + The class is responsible for fetching the package, and reading/writing + the package-specific metadata. + + The implementation shall provide a name for the package type""" + __cname__ = None + + def __init__(self, category, name, version, slot='0'): + """ + Unified basic metadata storage for all the sources + """ + self.category = category + self.name = name + self.version = version + self.slot = slot + + def fetch(self): + """ + A method which is responsible for fetching the package: it should return a Package object (specifying set of files with sufficient metadata), and specify this package object as the backend for the Package object (to store source-specific metadata). + """ + raise NotImplementedError() + + @staticmethod + def from_data_dir(pkgdir): + """ + This method is responsible for instantiating source-specific metadata + from the metadata directory. + It shall return an instance of this class, and take a path to the meta + directory. + """ + try: + lines = [x.strip() for x in open(path.join(pkgdir, 'PACKAGE_BASE_DATA'), 'r')] + except: + return Result.Err('Could not read data file') + if len(lines) < 4: + return Result.Err('Invalid data provided') + category, name, version, slot, *_ = lines + return Result.Ok(PackageBase(category, name, version, slot)) + + def write_meta(self, pkgdir): + """ + This method shall write source-specific metadata to the provided + metadata directory. + """ + with open(path.join(pkgdir, 'PACKAGE_BASE_DATA'), 'w') as f: + f.write(self.category + '\n') + f.write(self.name + '\n') + f.write(self.version + '\n') + f.write(self.slot + '\n') + + def __str__(self): + """ + The implementation shall provide a method to pretty-print + package-specific metadata (displayed in show command). + """ + return '{}/{}:{}-{}'.format(self.category, self.name, '' if self.slot == '0' else ':' + self.slot, self.version) + +@dispatcher.source +class BaseSource: + """ + This is the base package source class. + It should be decorated with @dispatcher.source. + The implementation shall provide methods to parse the package specification, + which would be called by the dispatcher (see manager.py for details). + Parser methods shall be decorated with @dispatcher.handler(priority=...) + decorator (default is 5). + It shall provide a method to instantiate a package of this type from the + metadata directory. + """ + __cname__ = None + + @dispatcher.handler() + @staticmethod + def parse_full(uri): + """ + This method shall parse a full package specification (which starts with + the backend name, followed by a colon). + It shall return a package, wrapped in Result. + """ + raise NotImplementedError() + + @classmethod + def from_meta_dir(cls, metadir): + """ + This method is responsible for instantiating package-specific metadata + from the metadata directory. + Example: + return PackageBase.from_data_dir(metadir) + """ + raise NotImplementedError() diff --git a/pomu/source/bugz.py b/pomu/source/bugz.py new file mode 100644 index 0000000..f6d4ef7 --- /dev/null +++ b/pomu/source/bugz.py @@ -0,0 +1,116 @@ +""" +A package source module to import ebuilds and patches from bugzilla +""" + +import xmlrpc.client +from os import path +from urllib.parse import urlparse + +from pomu.package import Package +from pomu.source import dispatcher +from pomu.source.base import PackageBase, BaseSource +from pomu.util.iquery import EditSelectPrompt +from pomu.util.misc import extract_urls +from pomu.util.query import query, QueryContext +from pomu.util.result import Result + +class BzEbuild(PackageBase): + """A class to represent an ebuild from bugzilla""" + __cname__ = 'bugzilla' + + def __init__(self, bug_id, filemap, category, name, version, slot='0'): + super().__init__(category, name, version, slot) + self.bug_id = bug_id + self.filemap = filemap + + def fetch(self): + return Package(self.name, '/', self, self.category, self.version, filemap=self.filemap) + + @staticmethod + def from_data_dir(pkgdir): + pkg = PackageBase.from_data_dir(pkgdir) + if pkg.is_err(): + return pkg + pkg = pkg.unwrap() + + with QueryContext(category=pkg.category, name=pkg.name, version=pkg.version, slot=pkg.slot): + with open(path.join(pkgdir, 'BZ_BUG_ID'), 'r') as f: + return BugzillaSource.parse_bug(f.readline()).unwrap() + + def write_meta(self, pkgdir): + super().write_meta(pkgdir) + with open(path.join(pkgdir, 'BZ_BUG_ID'), 'w') as f: + f.write(self.bug_id + '\n') + + def __str__(self): + return super().__str__() + ' (from bug {})'.format(self.bug_id) + +CLIENT_BASE = 'https://bugs.gentoo.org/xmlrpc.cgi' + +@dispatcher.source +class BugzillaSource(BaseSource): + """The source module responsible for importing ebuilds and patches from bugzilla tickets""" + __cname__ = 'bugzilla' + + @dispatcher.handler(priority=1) + @staticmethod + def parse_bug(uri): + if not uri.isdigit(): + return Result.Err() + uri = int(uri) + proxy = xmlrpc.client.ServerProxy(CLIENT_BASE).Bug + payload = {'ids': [uri]} + try: + proxy.get(payload) + except (xmlrpc.client.Fault, OverflowError) as err: + return Result.Err(str(err)) + attachments = proxy.attachments(payload)['bugs'][str(uri)] + comments = proxy.comments(payload)['bugs'][str(uri)]['comments'] + comment_links = [] + for comment in comments: + comment_links.extend(extract_urls(comment['text'])) + items = [(x['file_name'], x['data'].data.decode('utf-8')) for x in attachments] + comment_links + if not items: + return Result.Err() + p = EditSelectPrompt(items) + files = p.run() + if not files: + return Result.Err() + category = query('category', 'Please enter package category').expect() + name = query('name', 'Please enter package name').expect() + ver = query('version', 'Please specify package version for {}'.format(name)).expect() + slot = query('slot', 'Please specify package slot', '0').expect() + fmap = {path.join(category, name, x[2]): x[1] for x in files} + return Result.Ok(BzEbuild(uri, fmap, category, name, ver, slot)) + + @dispatcher.handler(priority=2) + @staticmethod + def parse_link(uri): + res = urlparse(uri) + if res.netloc != 'bugs.gentoo.org': + return Result.Err() + if res.path == '/show_bugs.cgi': + ps = [x.split('=') for x in res.params.split('&') if x.startswith('id=')][1] + return BugzillaSource.parse_bug(ps) + if res.path.lstrip('/').isdigit(): + return BugzillaSource.parse_bug(res.path.lstrip('/')) + return Result.Err() + + + @dispatcher.handler() + @staticmethod + def parse_full(uri): + if not uri.startswith('bug:'): + return Result.Err() + rem = uri[4:] + if rem.isdigit(): + return BugzillaSource.parse_bug(rem) + return BugzillaSource.parse_link(rem) + + @classmethod + def fetch_package(self, pkg): + return pkg.fetch() + + @classmethod + def from_meta_dir(cls, metadir): + return BzEbuild.from_data_dir(metadir) diff --git a/pomu/source/file.py b/pomu/source/file.py index f42474d..67ecdb2 100644 --- a/pomu/source/file.py +++ b/pomu/source/file.py @@ -6,19 +6,17 @@ from os import path from pomu.package import Package from pomu.source import dispatcher +from pomu.source.base import PackageBase, BaseSource from pomu.util.pkg import cpv_split, ver_str -from pomu.util.query import query +from pomu.util.query import query, QueryContext from pomu.util.result import Result -class LocalEbuild(): +class LocalEbuild(PackageBase): """A class to represent a local ebuild""" - __name__ = 'fs' + __cname__ = 'fs' - # slots? - def __init__(self, category, name, version, path): - self.category = category - self.name = name - self.version = version + def __init__(self, path, category, name, version, slot='0'): + super().__init__(category, name, version, slot) self.path = path def fetch(self): @@ -32,42 +30,57 @@ class LocalEbuild(): @staticmethod def from_data_dir(pkgdir): - with open(path.join(pkgdir, 'FS_ORIG_PATH'), 'r') as f: - return LocalEbuildSource.parse_ebuild_path(f.readline()).unwrap() + pkg = PackageBase.from_data_dir(pkgdir) + if pkg.is_err(): + return pkg + pkg = pkg.unwrap() + + with QueryContext(category=pkg.category, name=pkg.name, version=pkg.version, slot=pkg.slot): + with open(path.join(pkgdir, 'FS_ORIG_PATH'), 'r') as f: + return LocalEbuildSource.parse_ebuild_path(f.readline()).unwrap() def write_meta(self, pkgdir): + super().write_meta(pkgdir) with open(path.join(pkgdir, 'FS_ORIG_PATH'), 'w') as f: f.write(self.path + '\n') def __str__(self): - return '{}/{}-{} (from {})'.format(self.category, self.name, self.version, self.path) + return super().__str__() + ' (from {})'.format(self.path) @dispatcher.source -class LocalEbuildSource(): +class LocalEbuildSource(BaseSource): """The source module responsible for importing local ebuilds""" + __cname__ = 'fs' + @dispatcher.handler(priority=5) + @staticmethod def parse_ebuild_path(uri): - if not path.isfile(uri) or not path.endswith('.ebuild'): + if not path.isfile(uri) or not uri.endswith('.ebuild'): return Result.Err() uri = path.abspath(uri) dirn, basen = path.split(uri) basen = basen[:-7] - _, name, v1, v2, v3 = cpv_split(basen) - ver = ver_str(v1, v2, v3) + _, name, ver = cpv_split(basen) parent = dirn.split('/')[-1] # we need to query the impure world # TODO: write a global option which would set the impure values non-interactively if not ver: ver = query('version', 'Please specify package version for {}'.format(basen)).expect() category = query('category', 'Please enter category for {}'.format(basen), parent).expect() - return Result.Ok(LocalEbuild(category, name, ver, uri)) + slot = query('slot', 'Please specify package slot', '0').expect() + return Result.Ok(LocalEbuild(uri, category, name, ver, slot)) @dispatcher.handler() + @staticmethod def parse_full(uri): if not uri.startswith('fs:'): return Result.Err() return LocalEbuildSource.parse_ebuild_path(uri[3:]) @classmethod + def fetch_package(self, pkg): + return pkg.fetch() + + @classmethod def from_meta_dir(cls, metadir): - return LocalEbuild.from_data_dir(cls, metadir) + return LocalEbuild.from_data_dir(metadir) diff --git a/pomu/source/manager.py b/pomu/source/manager.py index 600a987..60dfe38 100644 --- a/pomu/source/manager.py +++ b/pomu/source/manager.py @@ -42,7 +42,13 @@ class PackageDispatcher(): It would register all the methods of the class marked by @handler with the dispatcher. """ - self.backends[cls.__name__] = cls + try: + from pomu.source.base import BaseSource + except ImportError: #circular import + return cls + if cls == BaseSource: + return cls + self.backends[cls.__cname__] = cls for m, obj in inspect.getmembers(cls): if isinstance(obj, self.handler._handler): self.register_package_handler(cls, obj.handler, obj.priority) @@ -57,8 +63,8 @@ class PackageDispatcher(): class _handler(): def __init__(self, handler): self.handler = handler - def __call__(self, *args): - return self.handler(*args) + def __call__(self, *args, **kwargs): + return self.handler(*args, **kwargs) def __init__(self, priority=1000, *args, **kwargs): self.priority = priority @@ -66,7 +72,7 @@ class PackageDispatcher(): def __call__(self, func, *args, **kwargs): x = self._handler(func) x.priority = self.priority - return x + return staticmethod(x) def register_package_handler(self, source, handler, priority): """ @@ -75,7 +81,7 @@ class PackageDispatcher(): """ i = 0 for i in range(len(self.handlers)): - if self.handlers[0][0] > priority: + if self.handlers[i][0] > priority: break self.handlers.insert(i, (priority, source, handler)) diff --git a/pomu/source/patch.py b/pomu/source/patch.py deleted file mode 100644 index 69a0821..0000000 --- a/pomu/source/patch.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -A package source module to import packages from filesystem locations (ebuilds) -""" - -from os import path, mkdtemp - -from pomu.package import Package -from pomu.source import dispatcher -from pomu.util.pkg import cpv_split, ver_str -from pomu.util.query import query -from pomu.util.result import Result - -class Patcher(): - """A class to represent a local ebuild""" - __name__ = 'patch' - - def __init__(self, wrapped, *patches): - self.patches = patches - self.wrapped = wrapped - # nested patching - if wrapped is Patcher: - self.patches = wrapped.patches + self.patches - self.wrapped = wrapped.wrapped - - def fetch(self): - pkg = self.wrapped.fetch() - pkg.backend = self - pd = mkdtemp() - pkg.merge_into(pd) - pkg.apply_patches(pd, self.patches) - return Package(pkg.name, pd, self, pkg.category, pkg.version) - - @staticmethod - def from_data_dir(pkgdir): - with open(path.join(pkgdir, 'PATCH_ORIG_BACKEND'), 'r') as f: - wrapped = dispatcher.backends[bname].from_meta_dir(pkgdir) - patches = [path.join(pkgdir, 'patches', x.strip()) - for x in open(path.join(pkgdir, 'PATCH_PATCHES_ORDER'))] - - def write_meta(self, pkgdir): - with open(path.join(pkgdir, 'PATCH_ORIG_BACKEND'), 'w') as f: - f.write('{}\n'.format(self.wrapped.__name__)) - with open(path.join(pkgdir, 'PATCH_PATCHES_ORDER'), 'w') as f: - for p in self.patches: - f.write(path.basename(p) + '\n') - os.makedirs(path.join(pkgdir, 'patches'), exist_ok=True) - for p in self.patches: - shutil.copy2(p, path.join(pkgdir, 'patches')) - # write originals? - - def __str__(self): - return '{}/{}-{} (from {})'.format(self.category, self.name, self.version, self.path) - -@dispatcher.source -class LocalEbuildSource(): - """The source module responsible for importing and patching various ebuilds""" - @dispatcher.handler() - def parse_full(uri): - if not uri.startswith('patch:'): - return Result.Err() - uri = uri[6:] - patchf, _, pks = uri.partition(':') - if not path.isfile(patchf): - return Result.Err('Invalid patch file') - if not pks: - return Result.Err('Package not provided') - pkg = dispatcher.get_package(pks) - return Result.Ok(Patcher(patchf, pkg) - - @classmethod - def from_meta_dir(cls, metadir): - return Patcher.from_data_dir(cls, metadir) diff --git a/pomu/source/portage.py b/pomu/source/portage.py index f4f112c..9b5d57f 100644 --- a/pomu/source/portage.py +++ b/pomu/source/portage.py @@ -9,20 +9,18 @@ from portage.versions import vercmp from pomu.package import Package from pomu.repo.repo import portage_repos, portage_repo_path from pomu.source import dispatcher +from pomu.source.base import PackageBase, BaseSource from pomu.util.pkg import cpv_split, ver_str from pomu.util.portage import repo_pkgs from pomu.util.result import Result -class PortagePackage(): +class PortagePackage(PackageBase): """A class to represent a portage package""" - __name__ = 'portage' + __cname__ = 'portage' def __init__(self, repo, category, name, version, slot='0'): + super().__init__(category, name, version, slot) self.repo = repo - self.category = category - self.name = name - self.version = version - self.slot = slot def fetch(self): return Package(self.name, portage_repo_path(self.repo), self, @@ -31,36 +29,34 @@ class PortagePackage(): path.join(self.category, self.name, self.name + '-' + self.version + '.ebuild')]) def write_meta(self, pkgdir): + super().write_meta(pkgdir) with open(path.join(pkgdir, 'PORTAGE_DATA'), 'w') as f: f.write(self.repo + '\n') - f.write(self.category + '\n') - f.write(self.name + '\n') - f.write(self.version + '\n') - f.write(self.slot + '\n') @staticmethod def from_data_dir(pkgdir): - try: - lines = [x.strip() for x in open(path.join(pkgdir, 'PORTAGE_DATA'), 'r')] - except: - return Result.Err('Could not read data file') - if len(lines) < 5: - return Result.Err('Invalid data provided') - res = PortagePackage() - res.repo, res.category, res.name, res.version, res.slot, *_ = lines - if sanity_check(res.repo, res.category, res.name, None, None, None, res.slot, ver=res.version): - return Result.Ok(res) - return Result.Err('Package {} not found'.format(res)) + pkg = PackageBase.from_data_dir(pkgdir) + if pkg.is_err(): + return pkg + pkg = pkg.unwrap() + + with open(path.join(pkgdir, 'PORTAGE_DATA'), 'r') as f: + repo = f.readline().strip() + if sanity_check(repo, pkg.category, pkg.name, pkg.version, pkg.slot): + return Result.Ok(PortagePackage(repo, pkg.category, pkg.name, pkg.slot, pkg.version)) + return Result.Err('Package {} not found'.format(pkg)) def __str__(self): - return '{}/{}-{}{}::{}'.format(self.category, self.name, self.version, - '' if self.slot == '0' else ':' + self.slot, self.repo) + return super().__str__() + '::' + self.repo @dispatcher.source -class PortageSource(): +class PortageSource(BaseSource): """The source module responsible for fetching portage packages""" + __cname__ = 'portage' + @dispatcher.handler(priority=5) + @staticmethod def parse_spec(uri, repo=None): # dev-libs/openssl-0.9.8z_p8-r100:0.9.8::gentoo pkg, _, repo_ = uri.partition('::') # portage repo may be specified on the rhs as well @@ -70,13 +66,14 @@ class PortageSource(): pkg, _, slot = uri.partition(':') # slot may be omitted if not slot: slot = None - category, name, vernum, suff, rev = cpv_split(pkg) - res = sanity_check(repo, category, name, vernum, suff, rev, slot) + category, name, ver = cpv_split(pkg) + res = sanity_check(repo, category, name, ver, slot) if not res: return Result.Err() return Result.Ok(res) @dispatcher.handler() + @staticmethod def parse_full(uri): # portage/gentoo:dev-libs/openssl-0.9.8z_p8-r100:0.9.8::gentoo if not uri.startswith('portage'): @@ -94,6 +91,7 @@ class PortageSource(): return PortageSource.parse_spec(uri, repo) @dispatcher.handler(priority=4) + @staticmethod def parse_repo_ebuild(uri): if not path.exists(uri): return Result.Err() @@ -105,8 +103,7 @@ class PortageSource(): if path.isfile(uri): if not uri.endswith('.ebuild'): return Result.Err() - _, name, v1, v2, v3 = cpv_split(path.basename(uri)) - ver = ver_str(v1, v2, v3) + _, name, ver = cpv_split(path.basename(uri)) dircomps = path.dirname(uri)[len(repo_path):].split('/') if len(dircomps) != 2: return Result.Err() @@ -125,10 +122,10 @@ class PortageSource(): @classmethod def from_meta_dir(cls, metadir): - return PortagePackage.from_data_dir(cls, metadir) + return PortagePackage.from_data_dir(metadir) -def sanity_check(repo, category, name, vernum, suff, rev, slot, ver=None): +def sanity_check(repo, category, name, ver, slot): """ Checks whether a package descriptor is valid and corresponds to a package in a configured portage repository @@ -137,17 +134,10 @@ def sanity_check(repo, category, name, vernum, suff, rev, slot, ver=None): return False if repo and repo not in list(portage_repos()): return False - if not ver: - if (rev or suff) and not vernum: - return False - if vernum: - ver = ver_str(vernum, suff, rev) - else: - ver = None pkgs = repo_pkgs(repo, category, name, ver, slot) if not pkgs: return False pkg = sorted(pkgs, key=cmp_to_key(lambda x,y:vercmp(x[3],y[3])), reverse=True)[0] return PortagePackage(*pkg) -__all__ = [PortagePackage, PortageSource] +__all__ = ['PortagePackage', 'PortageSource'] diff --git a/pomu/source/url.py b/pomu/source/url.py new file mode 100644 index 0000000..bb4868c --- /dev/null +++ b/pomu/source/url.py @@ -0,0 +1,99 @@ +""" +A package source module to import packages from URLs +""" + +from os import path + +from pbraw import grab + +from pomu.package import Package +from pomu.source import dispatcher +from pomu.source.base import PackageBase, BaseSource +from pomu.util.query import query, QueryContext +from pomu.util.result import Result + +class URLEbuild(PackageBase): + """A class to represent an ebuild fetched from a url""" + __cname__ = 'url' + + def __init__(self, url, contents, category, name, version, slot): + self.url = url + self.contents = contents + self.category = category + self.name = name + self.version = version + self.slot = slot + + def fetch(self): + if self.contents: + if isinstance(self.contents, str): + self.content = self.contents.encode('utf-8') + else: + fs = grab(self.url) + self.content = fs[0][1].encode('utf-8') + return Package(self.name, '/', self, self.category, self.version, + filemap = { + path.join( + self.category, + self.name, + '{}-{}.ebuild'.format(self.name, self.version) + ) : self.content}) + + @staticmethod + def from_data_dir(pkgdir): + pkg = PackageBase.from_data_dir(pkgdir) + if pkg.is_err(): + return pkg + pkg = pkg.unwrap() + + with QueryContext(category=pkg.category, name=pkg.name, version=pkg.version, slot=pkg.slot): + with open(path.join(pkgdir, 'ORIG_URL'), 'r') as f: + return URLGrabberSource.parse_link(f.readline()).unwrap() + + def write_meta(self, pkgdir): + super().write_meta(pkgdir) + with open(path.join(pkgdir, 'ORIG_URL'), 'w') as f: + f.write(self.url + '\n') + + def __str__(self): + return super().__str__() + ' (from {})'.format(self.url) + +@dispatcher.source +class URLGrabberSource(BaseSource): + """ + The source module responsible for grabbing modules from URLs, + including pastebins + """ + __cname__ = 'url' + + @dispatcher.handler(priority=5) + @staticmethod + def parse_link(uri): + if not (uri.startswith('http://') or uri.startswith('https://')): + return Result.Err() + + name = query('name', 'Please specify package name').expect() + category, _, name = name.rpartition('/') + ver = query('version', 'Please specify package version for {}'.format(name)).expect() + if not category: + category = query('category', 'Please enter category for {}'.format(name)).expect() + files = grab(uri) + if not files: + return Result.Err() + slot = query('slot', 'Please specify package slot', '0').expect() + return Result.Ok(URLEbuild(uri, files[0][1], category, name, ver, slot)) + + @dispatcher.handler() + @staticmethod + def parse_full(url): + if not url.startswith('url:'): + return Result.Err() + return URLGrabberSource.parse_link(url[4:]) + + @classmethod + def fetch_package(self, pkg): + return pkg.fetch() + + @classmethod + def from_meta_dir(cls, metadir): + return URLEbuild.from_data_dir(metadir) diff --git a/pomu/util/__init__.py b/pomu/util/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pomu/util/__init__.py diff --git a/pomu/util/fs.py b/pomu/util/fs.py index 7016717..66716a7 100644 --- a/pomu/util/fs.py +++ b/pomu/util/fs.py @@ -5,6 +5,10 @@ import os def strip_prefix(string, prefix): """Returns a string, stripped from its prefix""" + if not prefix.endswith('/'): + aprefix = prefix + '/' + if string.startswith(aprefix): + return string[len(aprefix):] if string.startswith(prefix): return string[len(prefix):] else: diff --git a/pomu/util/git.py b/pomu/util/git.py new file mode 100644 index 0000000..985087c --- /dev/null +++ b/pomu/util/git.py @@ -0,0 +1,52 @@ +"""Miscellaneous utility functions for git structures""" + +from base64 import b16encode +import zlib + +from pomu.util.result import Result + +def parse_tree(blob, path=b''): + """Parses a git tree""" + res = [] + leng, _, tree = blob.partition(b'\0') + if path is str: + path = path.encode('utf-8') + if not tree: + return Result.Err('Invalid tree') + while len(tree) > 0: + mode, _, tree = tree.partition(b' ') + name, _, tree = tree.partition(b'\0') + sha = b16encode(tree[0:20]).decode('utf-8').lower() + tree = tree[20:] + if not name or not sha: + return Result.Err() + is_dir = mode[0:1] != b'1' + res.append((is_dir, sha, path + b'/' + name)) + return Result.Ok(res) + +def parse_blob(blob): + """Parses a git blob""" + leng, _, data = blob.partition(b'\0') + if not leng or not data: + return Result.Err() + return Result.Ok(data) + +def commit_head(blob): + if not blob[7:] == 'commit ': + return Result.Err() + l = blob.split('\n')[0] + cid, _, tid = l.partition('\0') + if not tid[0:5] == 'tree ': + return Result.Err() + return tid[5:] + +def parse_object(obj, tpath=b''): + """Parses a git object""" + if tpath is str: + tpath = tpath.encode('utf-8') + data = zlib.decompress(obj) + if data[0:5] == b'blob ': + return parse_blob(data[5:]) + elif data[0:5] == b'tree ': + return parse_tree(data[5:], tpath) + return Result.Err('Unsupported object type') diff --git a/pomu/util/iquery.py b/pomu/util/iquery.py new file mode 100644 index 0000000..05c923f --- /dev/null +++ b/pomu/util/iquery.py @@ -0,0 +1,210 @@ +"""A module to interactively query""" +from pydoc import pager + +from curtsies import CursorAwareWindow, Input, fsarray, fmtstr +from curtsies.fmtfuncs import invert +from pbraw import grab + + +class Position: + def __init__(self, row=0, column=0): + self.row = row + self.column = column + +def clamp(x, m, M): + return max(m, min(x, M)) + +def render_entry(name, state, value, width, active=False): + char = '*' if state else ' ' + w = 3 + fmtstr(name).width + 2 + if value: + text = fmtstr(value) + val = value[:width - w - 2] + '..' if text.width >= width - w else value + else: + val = '' + return fmtstr( + '[' + (invert(char) if active else char) + '] ' + + name + ' ' + val) + +class Prompt: + def __init__(self, entries): + self.entries = [self.process_entry(x) for x in entries] + self.idx = 0 + self.cursor_pos = Position() + + def run(self, window_type=CursorAwareWindow, **args): + with open('/dev/tty', 'r') as tty_in, \ + open('/dev/tty', 'w') as tty_out, \ + Input(in_stream=tty_in) as input_: + if window_type == CursorAwareWindow: + iargs = {'in_stream':tty_in, 'out_stream':tty_out, + 'hide_cursor':False, 'extra_bytes_callback':input_.unget_bytes} + else: + iargs = {'out_stream':tty_out} + iargs.update(args) + with window_type(**args) as window: + return self.event_loop(window, input_) + + def event_loop(self, window, input_): + self.window = window + self.render() + for event in input_: + if self.process_event(event) == -1: + break + self.render() + return self.results() + + def clamp(self, x): + return clamp(x, 0, len(self.entries)) + + def results(self): + raise NotImplementedError() + + def preview(self): + pass + + def extract_nsv(self, entry): + raise NotImplementedError() + + def toggle(self): + raise NotImplementedError() + + def process_entry(self, entry): + raise NotImplementedError() + + def process_event(self, event): + if event == '<UP>': + self.idx = self.clamp(self.idx - 1) + elif event == '<DOWN>': + self.idx = self.clamp(self.idx + 1) + elif event == '<SPACE>': + self.toggle() + elif event in {'p', 'P'}: + self.preview() + elif event in {'<ESC>', '<Ctrl-g>'}: + return -1 + else: + return False + return True + + def render(self): + output = fsarray( + [render_entry(*self.extract_nsv(x), self.window.width, i == self.idx) for i, x in enumerate(self.entries)] + + [' [ ' + + ('OK' if self.idx < len(self.entries) else invert('OK')) + + ' ] '], width=self.window.width) + self.window.render_to_terminal(output) + +class PickerPrompt(Prompt): + def process_entry(self, entry): + return (entry[0], False, entry[1]) + + def extract_nsv(self, entry): + return entry + + def results(self): + return [x[0] for x in self.entries if x[1]] + + def toggle(self): + if self.idx == len(self.entries): + return + e = self.entries[self.idx] + self.entries[self.idx] = (e[0], e[1], not e[2], e[3]) + + +class EditSelectPrompt(Prompt): + def __init__(self, items): + super().__init__(items) + self.text = '' + self.list = True + + def render(self): + if self.list: + super().render() + else: + self.cursor_pos.row = 1 + cur = self.entries[self.idx] + output = fsarray(['Please provide value for {}'.format(cur[0]), cur[3]], width=self.window.width) + self.window.render_to_terminal(output, (self.cursor_pos.row, self.cursor_pos.column)) + + def results(self): + return [(x[0], x[1], x[3]) for x in self.entries if x[2]] + + def extract_nsv(self, entry): + return (entry[0], entry[2], entry[3]) + + def process_event(self, event): + if self.list: + res = super().process_event(event) + if res == -1: + return -1 + elif res: + return True + elif event in {'e', '<Ctrl-j>', '<Ctrl-m>'}: + self.list = False + if self.idx == len(self.entries): + return -1 + self.cursor_pos.column = fmtstr(self.entries[self.idx][3]).width + else: + return False + return True + else: + if event in {'<ESC>', '<Ctrl-g>'}: + self.list = True + if event == '<BACKSPACE>': + self.delete_cur_char() + elif event == '<SPACE>': + self.add_char(' ') + elif event == '<HOME>': + self.cursor_pos.column = 0 + elif event == '<END>': + self.cursor_pos.column = fmtstr(self.entries[self.idx][3]).width + elif event == '<LEFT>': + self.cursor_pos.column = clamp(self.cursor_pos.column - 1, + 0, fmtstr(self.entries[self.idx][3]).width) + elif event == '<RIGHT>': + self.cursor_pos.column = clamp(self.cursor_pos.column + 1, + 0, fmtstr(self.entries[self.idx][3]).width) + elif event in {'<Ctrl-j>', '<Ctrl-m>'}: + self.list = True + elif isinstance(event, str) and not event.startswith('<'): + self.add_char(event) + return False + return True + + def process_entry(self, entry): + if isinstance(entry, str): + return (entry[0], None, False, None) + return (entry[0], entry[1], False, entry[0] if entry[0].endswith('.ebuild') else 'files/{}'.format(entry[0])) + + def preview(self): + entry = self.entries[self.idx] + if entry[0] is not None: + pager(entry[1]) + else: + gr = grab(entry) + if not gr: + del self.entries[self.idx] + self.idx = self.clamp(self.idx - 1) + pager('Error: could not fetch {}'.format(entry)) + self.entries[self.idx:self.idx+1] = [self.process_entry((x[0], x[1].encode('utf-8'))) for x in gr] + pager(self.entries[self.idx][1]) + + def toggle(self): + if self.idx == len(self.entries): + return + e = self.entries[self.idx] + self.entries[self.idx] = (e[0], e[1], not e[2], e[3]) + + def add_char(self, char): + e = self.entries[self.idx] + p = self.cursor_pos.column + self.entries[self.idx] = (e[0], e[1], e[2], e[3][:p] + char + e[3][p:]) + self.cursor_pos.column += 1 + + def delete_cur_char(self): + e = self.entries[self.idx] + p = self.cursor_pos.column + if e[3]: + self.entries[self.idx] = (e[0], e[1], e[2], e[3][:p - 1] + e[3][p:]) + self.cursor_pos.column -= 1 diff --git a/pomu/util/misc.py b/pomu/util/misc.py new file mode 100644 index 0000000..7cb1781 --- /dev/null +++ b/pomu/util/misc.py @@ -0,0 +1,69 @@ +"""Miscellaneous utility functions""" +import re + +from pomu.util.result import Result + +def list_add(dst, src): + """ + Extends the target list with a scalar, or contents of the given list + """ + if isinstance(src, list): + dst.extend(src) + else: + dst.append(src) + + +def pivot(string, idx, keep_pivot=False): + """ + A function to split a string in two, pivoting at string[idx]. + If keep_pivot is set, the pivot character is included in the second string. + Alternatively, it is omitted. + """ + if keep_pivot: + return (string[:idx], string[idx:]) + else: + return (string[:idx], string[idx+1:]) + +def extract_urls(text): + """Extracts URLs from arbitrary text""" + schemas = ['http://', 'https://', 'ftp://', 'ftps://'] + words = list(filter(lambda x: any(y in x for y in schemas), text.split())) + maxshift = lambda x: max(x.find(y) for y in schemas) + links = [x[maxshift(x):].rstrip('\'")>.,') for x in words] + return links + +def parse_range(text, max_num=None): + """Parses a numeric range (e.g. 1-2,5-16)""" + text = re.sub(r'\s*-\s*', '-', text) + subranges = [x.strip() for x in text.split(',')] + subs = [] + maxint = -1 + for sub in subranges: + l, _, r = sub.partition('-') + if not l and not _: + continue + if (l and not l.isdigit()) or (r and not r.isdigit()): + return Result.Err('Invalid subrange: {}'.format(sub)) + if l: + l = int(l) + maxint = max(l, maxint) + if r: + r = int(r) + maxint = max(r, maxint) + if _: + subs.append((l if l else 1, r if r else -1)) + else: + subs.append((l, None)) + if max_num: + maxint = max_num + res = set() + add = lambda x: res.add(x) if x <= maxint else None + for l, r in subs: + if not r: + add(l) + continue + if r == -1: + r = maxint + for x in range(l, r + 1): + add(x) + return Result.Ok(res) diff --git a/pomu/util/pkg.py b/pomu/util/pkg.py index 7544e42..b65ec88 100644 --- a/pomu/util/pkg.py +++ b/pomu/util/pkg.py @@ -6,15 +6,15 @@ import re from portage.versions import suffix_value -from pomu.util.str import pivot +from pomu.util.misc import pivot suffixes = [x[0] for x in sorted(suffix_value.items(), key=lambda x:x[1])] def ver_str(vernum, suff, rev): """Gets the string representation of the version (specified by number, suffix and rev)""" - return vernum + (suff if suff else '') + (rev if rev else '') + return vernum + (suff if suff else '') + ('-' + rev if rev else '') -def cpv_split(pkg): +def cpv_split(pkg, unified_ver=True): """ Extracts category, name, version number, suffix, revision from a package descriptor e.g. dev-libs/openssl-0.9.8z_p8-r100 -> dev-libs, openssl, 0.9.8z, p8, r100 @@ -41,4 +41,7 @@ def cpv_split(pkg): vernum = None # openssl name = pkg - return category, name, vernum, suff, rev + if unified_ver: + return category, name, ver_str(vernum, suff, rev) + else: + return category, name, vernum, suff, rev diff --git a/pomu/util/portage.py b/pomu/util/portage.py index 3491dfa..b505602 100644 --- a/pomu/util/portage.py +++ b/pomu/util/portage.py @@ -21,8 +21,10 @@ def best_ver(repo, category, name, ver=None): ebuilds = [category + '/' + name + x[len(name):-7] for x in os.listdir(path.join(portage_repo_path(repo), category, name)) if x.endswith('.ebuild')] - cat, name, vernum, suff, rev = cpv_split(best(ebuilds)) - return ver_str(vernum, suff, rev) + if not ebuilds: + return None + cat, name, ver = cpv_split(best(ebuilds)) + return ver def repo_pkgs(repo, category, name, ver=None, slot=None): """List of package occurences in the repo""" diff --git a/pomu/util/query.py b/pomu/util/query.py index 872e0a3..66ddf7a 100644 --- a/pomu/util/query.py +++ b/pomu/util/query.py @@ -56,6 +56,6 @@ class QueryContext: def __enter__(self): self.map_old = {x: query.set(x, self.map[x]) for x in self.map} - def __exit__(self): + def __exit__(self, ex_type, ex_val, tb): for x, y in self.map_old.items(): query.set(x, y) diff --git a/pomu/util/remote.py b/pomu/util/remote.py new file mode 100644 index 0000000..f992c1d --- /dev/null +++ b/pomu/util/remote.py @@ -0,0 +1,38 @@ +""" +Utilities for remotes +""" + +from portage.versions import best + +from pomu.util.pkg import ver_str, cpv_split +from pomu.util.portage import misc_dirs + +from pomu.util.result import Result + +def filelist_to_cpvs(tree): + """Converts a list of files to list of cpvs""" + res = [] + for opath in tree: + comps = opath.split('/')[1:] + if (opath.endswith('/') or + any(opath.startswith('/' + x + '/') for x in misc_dirs) or + len(comps) != 3 or + not comps[2].endswith('.ebuild')): + continue + category, name, ebuild = comps[0], comps[1], comps[2][:-7] + c, n, ver = cpv_split(ebuild) + if not category or n != name: + continue + res.append((category, name, ver)) + return res + +def get_full_cpv(cpvs, name, category=None, version=None): + cpvl = filter(lambda x: x[1] == name and (not category or x[0] == category), cpvs) + if not cpvl: return Result.Err() + if version: + cpvl = list(filter(lambda x: x[2] == version, cpvl))[:1] + b = best(list('{}/{}-{}'.format(c, n, v) for c, n, v in cpvl)) + if b: + cat, name, ver = cpv_split(b) + return Result.Ok((cat, name, ver)) + return Result.Err() diff --git a/pomu/util/result.py b/pomu/util/result.py index cd67fa0..d6933d8 100644 --- a/pomu/util/result.py +++ b/pomu/util/result.py @@ -55,6 +55,19 @@ class Result(): return self._val raise ResultException(msg + ': ' + self._val) + def and_(self, rhs): + if not self.is_ok(): + return Result.Err(self.err()) + return rhs + + def and_then(self, f): + return self.map(f) + + def or_(self, rhs): + if self.is_ok(): + return Result.Ok(self.ok()) + return rhs + def __iter__(self): if self._is_val: yield self._val diff --git a/pomu/util/str.py b/pomu/util/str.py deleted file mode 100644 index 11fc514..0000000 --- a/pomu/util/str.py +++ /dev/null @@ -1,11 +0,0 @@ -"""String processing utilities""" -def pivot(string, idx, keep_pivot=False): - """ - A function to split a string in two, pivoting at string[idx]. - If keep_pivot is set, the pivot character is included in the second string. - Alternatively, it is omitted. - """ - if keep_pivot: - return (string[:idx], string[idx:]) - else: - return (string[:idx], string[idx+1:]) @@ -9,7 +9,7 @@ setup( author_email='hilobakho@gmail.com', license='GNU GPLv2', packages=find_packages(exclude=['tests']), - install_requires=['Click', 'GitPython', 'patch'], + install_requires=['Click', 'GitPython', 'patch', 'pbraw', 'curtsies'], entry_points={ 'console_scripts':['pomu = pomu.cli:main_'] } |