path: root/g_sorcery/package_db.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
    package_db.py
    ~~~~~~~~~~~~~

    package database

    :copyright: (c) 2013-2015 by Jauhien Piatlicki
    :license: GPL-2, see LICENSE for more details.
"""

import glob
import os

import portage

from .compatibility import basestring, py2k, TemporaryDirectory

from .db_layout import DBLayout, JSON_FILE_SUFFIX
from .exceptions import DBError, DBStructureError, InvalidKeyError, SyncError
from .fileutils import FileJSON, load_remote_file, copy_all, wget
from .g_collections import Package
from .logger import Logger


class PackageDB(object):
    """
    Package database.
    It uses DBLayout class to manipulate files that
    contain DB stored on disk.

    There are two versions of DB layout now:
        0 -- legacy version
        1 -- new version that supports DB structure versioning

    DB structure itself has two versions:
        0 -- legacy version, categories contain dictionary package name: versions dict
        1 -- actual version, corresponds to the DB in memory:
            DB is a dictionary with categories as keys.
            Each category contains a dictionary with two entries:
                common_data -- fields common to all the packages
                packages -- dictionary with packages (content of category dictionary in v. 0)

    For DB layout v. 0 only DB structure v. 0 is possible.
    """

    class Iterator(object):
        """
        Iterator class over the package database.
        """
        def __init__(self, package_db):
            self.cats_iter = iter(package_db.database.items())
            try:
                self.cat_name, self.cat_data = next(self.cats_iter)
            except StopIteration:
                self.set_to_end()
                return

            if not self.cat_data:
                self.set_to_end()
                return

            self.pkgs_iter = iter(self.cat_data['packages'].items())
            try:
                self.pkg_name, self.pkg_data = next(self.pkgs_iter)
            except StopIteration:
                self.set_to_end()
                return

            if not self.pkg_data:
                self.set_to_end()
                return

            self.vers_iter = iter(self.pkg_data.items())

        def set_to_end(self):
            self.cat_name, self.cat_data = None, None
            self.pkgs_iter = None
            self.pkg_name, self.pkg_data = None, None
            self.vers_iter = None

        def __iter__(self):
            return self

        if py2k:
            def next(self):
                if not self.vers_iter or not self.pkgs_iter:
                    raise StopIteration

                ver, ebuild_data = None, None
                while not ver:
                    try:
                        ver, ebuild_data = next(self.vers_iter)
                    except StopIteration:
                        ver, ebuild_data = None, None
                    if not ver:
                        try:
                            self.pkg_name, self.pkg_data = next(self.pkgs_iter)
                            self.vers_iter = iter(self.pkg_data.items())
                        except StopIteration:
                            self.cat_name, self.cat_data = next(self.cats_iter)
                            self.pkgs_iter = iter(self.cat_data['packages'].items())
                            self.pkg_name, self.pkg_data = next(self.pkgs_iter)
                            self.vers_iter = iter(self.pkg_data.items())

                ebuild_data = dict(ebuild_data)
                ebuild_data.update(self.cat_data['common_data'])
                return (Package(self.cat_name, self.pkg_name, ver), ebuild_data)

        else:
            def __next__(self):
                if not self.vers_iter or not self.pkgs_iter:
                    raise StopIteration

                ver, ebuild_data = None, None
                while not ver:
                    try:
                        ver, ebuild_data = next(self.vers_iter)
                    except StopIteration:
                        ver, ebuild_data = None, None
                    if not ver:
                        try:
                            self.pkg_name, self.pkg_data = next(self.pkgs_iter)
                            self.vers_iter = iter(self.pkg_data.items())
                        except StopIteration:
                            self.cat_name, self.cat_data = next(self.cats_iter)
                            self.pkgs_iter = iter(self.cat_data['packages'].items())
                            self.pkg_name, self.pkg_data = next(self.pkgs_iter)
                            self.vers_iter = iter(self.pkg_data.items())

                ebuild_data = dict(ebuild_data)
                ebuild_data.update(self.cat_data['common_data'])
                return (Package(self.cat_name, self.pkg_name, ver), ebuild_data)


    def __init__(self, directory,
                 preferred_layout_version=1,
                 preferred_db_version=1,
                 preferred_category_format=JSON_FILE_SUFFIX):

        if preferred_layout_version == 0 \
           and preferred_db_version != 0:
            raise DBStructureError("Wrong DB version: " + preferred_db_version + \
                                   ", with DB layout version 0 it can be only 0")

        if not preferred_db_version in [0, 1]:
            raise DBStructureError("Unsupported DB version: " + preferred_db_version)

        self.logger = Logger()
        self.directory = os.path.abspath(directory)
        self.preferred_layout_version = preferred_layout_version
        self.preferred_db_version = preferred_db_version
        self.preferred_category_format = preferred_category_format
        self.db_layout = DBLayout(self.directory)
        self.reset_db()


    def __iter__(self):
        return PackageDB.Iterator(self)


    def reset_db(self):
        """
        Reset database.
        """
        self.database = {}
        self.categories = {}


    def sync(self, db_uri):
        """
        Synchronize local database with remote database.

        Args:
            db_uri: URI for synchronization with remote database.
        """
        real_db_uri = self.get_real_db_uri(db_uri)
        download_dir = TemporaryDirectory()
        if wget(real_db_uri, download_dir.name):
            raise SyncError('sync failed: ' + real_db_uri)

        temp_dir = TemporaryDirectory()
        for f_name in glob.iglob(os.path.join(download_dir.name, '*.tar.gz')):
            self.logger.info("unpacking " + f_name)
            os.system("tar -xvzf " + f_name + " -C " + temp_dir.name)

        tempdb_dir = os.path.join(temp_dir.name, os.listdir(temp_dir.name)[0])
        tempdb = PackageDB(tempdb_dir)

        tempdb.db_layout.check_manifest()

        self.logger.info("copy files to an actual database")
        self.clean()
        copy_all(tempdb_dir, self.directory)

        self.db_layout.check_manifest()

        del download_dir
        del temp_dir


    def get_real_db_uri(self, db_uri):
        """
        Convert self.db_uri to URI where remote database can be
        fetched from.

        Returns:
            URI of remote database file.
        """
        return db_uri


    def clean(self):
        """
        Clean database.
        """
        self.db_layout.clean()
        self.reset_db()


    def write(self):
        """
        Write and digest database.
        """
        if self.database:
            self.logger.info("writing database...")

        metadata = {'db_version': self.preferred_db_version,
                    'layout_version': self.preferred_layout_version,
                    'category_format': self.preferred_category_format}

        if self.preferred_db_version == 0:
            packages = dict(self.database)
            for category, cat_data in packages.items():
                for _, versions in cat_data['packages'].items():
                    for version, ebuild_data in versions.items():
                        ebuild_data.update(cat_data['common_data'])
                packages[category] = cat_data['packages']
        else:
            packages = dict(self.database)

        self.db_layout.write(metadata, self.categories, packages)

        if self.database:
            self.logger.info("database written")


    def read(self):
        """
        Read database.
        """
        metadata, self.categories, packages = self.db_layout.read()

        db_version = metadata['db_version']
        self.database = packages
        if db_version == 0:
            for category, cat_data in self.database.items():
                self.database[category] = {'common_data': {}, 'packages': cat_data}
        elif db_version == 1:
            pass
        else:
            raise DBStructureError("Unsupported DB version: " + db_version)


    def add_category(self, category, description=None):
        """
        Add a category.

        Args:
            category: Category name.
            description: Category description.
        """
        if not description:
            description = {}
        self.categories[category] = description


    def set_common_data(self, category, common_data):
        """
        Set common data for a category.

        Args:
            category: Category name.
            common_data: Category common data.
        """
        if not category in self.categories:
            raise InvalidKeyError('Non-existent category: ' + category)

        if not category in self.database:
            self.database[category] = {'common_data': common_data, 'packages': {}}
        else:
            self.database[category]['common_data'] = common_data


    def get_common_data(self, category):
        """
        Get common data for a category.

        Args:
            category: Category name.

        Returns:
            Dictionary with category common data.
        """
        if not category in self.categories:
            raise InvalidKeyError('Non-existent category: ' + category)

        if not category in self.database:
            return {}
        else:
            return self.database[category]['common_data']


    def add_package(self, package, ebuild_data=None):
        """
        Add a package.

        Args:
            package: package_db.Package instance.
            ebuild_data: Dictionary with package description.
        """
        if not ebuild_data:
            ebuild_data = {}

        category = package.category
        name = package.name
        version = package.version

        if not category or not name or not version:
            raise DBError("wrong package: " + str(package))

        if not category in self.categories:
            raise InvalidKeyError('Non-existent category: ' + category)

        if not category in self.database:
            self.database[category] = {'common_data': {}, 'packages': {}}

        if not name in self.database[category]['packages']:
            self.database[category]['packages'][name] = {}

        self.database[category]['packages'][name][version] = ebuild_data


    def list_categories(self):
        """
        List all categories.

        Returns:
            List with category names.
        """
        return list(self.categories)


    def in_category(self, category, name):
        """
        Tests whether a package is in a given category.

        Args:
            category: Category name.
            name: Package name.

        Returns:
            Boolean value.
        """
        if not category or (not category in self.categories):
            raise InvalidKeyError('No such category: ' + category)

        if not category in self.database:
            return False

        return name in self.database[category]['packages']


    def list_package_names(self, category):
        """
        List package names in a category.

        Args:
            category: Category name.

        Returns:
            List of package names.
        """
        if not category or (not category in self.categories):
            raise InvalidKeyError('No such category: ' + category)

        if not category in self.database:
            return []

        return list(self.database[category]['packages'])


    def list_catpkg_names(self):
        """
        List category/package entries.

        Returns:
            List with category/package entries.
        """
        result = []
        for category, cat_data in self.database.items():
            for name in cat_data['packages']:
                result.append(category + '/' + name)
        return result


    def list_package_versions(self, category, name):
        """
        List package versions.

        Args:
            category: Category name.
            name: package name.

        Returns:
            List of package versions.
        """
        if not category or (not category in self.categories):
            raise InvalidKeyError('No such category: ' + category)

        if not category in self.database \
           or not name in self.database[category]['packages']:
            raise InvalidKeyError('No such package: ' + category + '/' + name)

        return list(self.database[category]['packages'][name])


    def list_all_packages(self):
        """
        List all packages in a database.

        Returns:
            List of package_db.Package instances.
        """
        result = []
        for category, cat_data in self.database.items():
            for name, versions in cat_data['packages'].items():
                for version in versions:
                    result.append(Package(category, name, version))
        return result


    def get_package_description(self, package):
        """
        Get package ebuild data.

        Args:
            package: package_db.Package instance.

        Returns:
            Dictionary with package ebuild data.
        """
        #a possible exception should be catched in the caller
        desc = dict(self.database[package.category]['packages']\
                    [package.name][package.version])
        desc.update(self.database[package.category]['common_data'])
        return desc


    def get_max_version(self, category, name):
        """
        Get the recent available version of a package.

        Args:
            category: Category name.
            name: package name.

        Returns:
            The recent version of a package.
        """
        if not category or (not category in self.categories):
            raise InvalidKeyError('No such category: ' + category)

        if not category in self.database \
           or not name in self.database[category]['packages']:
            raise InvalidKeyError('No such package: ' + category + '/' + name)

        pkgname = category + '/' + name
        versions = list(self.database[category]['packages'][name])
        max_ver = versions[0]
        for version in versions[1:]:
            if portage.pkgcmp(portage.pkgsplit(pkgname + '-' + version),
                              portage.pkgsplit(pkgname + '-' + max_ver)) > 0:
                max_ver = version
        return max_ver


class DBGenerator(object):
    """
    Generator for package databases.
    Creates new databases or syncs with existing.
    """

    __slots__ = ('package_db_class')

    def __init__(self, package_db_class=PackageDB,
                 preferred_layout_version=1,
                 preferred_db_version=1,
                 preferred_category_format=JSON_FILE_SUFFIX):
        self.package_db_class = package_db_class
        self.preferred_layout_version = preferred_layout_version
        self.preferred_db_version = preferred_db_version
        self.preferred_category_format = preferred_category_format


    def __call__(self, directory, repository,
                 common_config=None, config=None, generate=True):
        """
        Get a package database.

        A directory layout looks like:
        backend directory
            config.json
            repo1 dir
                config.json
                db - directory with database
            repo2 dir
                config.json
                db - directory with database

        Args:
            directory: Directory for package databases (backend directory).
            repository: Repository name.
            common_config: Backend config.
            config: Repository config.
            generate: Whether package tree should be generated.

        Returns:
            Package database.
        """
        db_path = os.path.join(directory, repository, "db")
        pkg_db = self.package_db_class(db_path,
                                       preferred_layout_version=self.preferred_layout_version,
                                       preferred_db_version=self.preferred_db_version,
                                       preferred_category_format=self.preferred_category_format)

        config_f = FileJSON(os.path.join(directory, repository),
                            "config.json", [])
        if config:
            config_f.write(config)
        else:
            config = config_f.read()

        common_config_f = FileJSON(directory, "config.json", [])
        if common_config:
            common_config_f.write(common_config)
        else:
            common_config = common_config_f.read()

        if generate:
            pkg_db.clean()
            self.generate_tree(pkg_db, common_config, config)
            pkg_db.write()
        return pkg_db


    def generate_tree(self, pkg_db, common_config, config):
        """
        Generate package entries.

        Args:
            pkg_db: Package database.
            common_config: Backend config.
            config: Repository config.
        """
        data = self.download_data(common_config, config)
        self.process_data(pkg_db, data, common_config, config)

    def download_data(self, common_config, config):
        """
        Obtain data for database generation.

        Args:
            common_config: Backend config.
            config: Repository config.

        Returns:
            Downloaded data.
        """
        uries = self.get_download_uries(common_config, config)
        uries = self.decode_download_uries(uries)
        data = {}
        for uri in uries:
            self.process_uri(uri, data)
        return data

    def process_uri(self, uri, data):
        """
        Download and parse data from a given URI.

        Args:
            uri: URI.
            data: Data dictionary.
        """
        data.update(load_remote_file(**uri))

    def get_download_uries(self, common_config, config):
        """
        Get uries to download from.

        Args:
            common_config: Backend config.
            config: Repository config.

        Returns:
            List with URI entries.
            Each entry has one of the following formats:
            1. String with URI.
            2. A dictionary with entries:
                - uri: URI.
                - parser: Parser to be applied to downloaded data.
                - open_file: Whether parser accepts file objects.
                - open_mode: Open mode for a downloaded file.
                The only mandatory entry is uri.
        """
        return [config["repo_uri"]]

    def decode_download_uries(self, uries):
        """
        Convert URI list with incomplete and string entries
        into list with complete dictionary entries.

        Args:
            uries: List of URIes.

        Returns:
            List of URIes with dictionary entries.
        """
        decoded = []
        for uri in uries:
            decuri = {}
            if isinstance(uri, basestring):
                decuri["uri"] = uri
                decuri["parser"] = self.parse_data
                decuri["open_file"] = True
                decuri["open_mode"] = "r"
            else:
                decuri = uri
                if not "parser" in decuri:
                    decuri["parser"] = self.parse_data
                if not "open_file" in decuri:
                    decuri["open_file"] = True
                if not "open_mode" in decuri:
                    decuri["open_mode"] = "r"
            decoded.append(decuri)
        return decoded

    def parse_data(self, data_file):
        """
        Parse downloaded data.

        Args:
            data_file: File name of open file object.

        Returns:
            Parsed data.
        """
        #todo: implement reasonable default
        pass

    def process_data(self, pkg_db, data, common_config, config):
        """
        Process downloaded data and fill database.

        Args:
            pkg_db: Package database.
            data: Dictionary with data, keys are file names.
            common_config; Backend config.
            config: Repository config.
        """
        pass

    def convert(self, configs, dict_name, value):
        """
        Convert a value using configs.
        This function is aimed to be used for conversion
        of values such as license or package names.

        Args:
            configs: List of configs.
            dict_name: Name of a dictionary in config
        that should be used for conversion.
            value: Value to convert.

        Returns:
            Converted value.
        """
        result = value
        for config in configs:
            if config:
                if dict_name in config:
                    transform = config[dict_name]
                    if value in transform:
                        result = transform[value]
        return result

    def convert_dependency(self, configs, dependency, external=True):
        """
        Convert dependency.

        Args:
            configs: List of configs.
            dependency: Dependency to be converted.
            external: Whether external deps should be keeped.

        Returns:
            Right dependency name or None if dependency is external and
        external == False.
        """
        external_dep = ""
        for config in configs:
            if config:
                if "external" in config:
                    ext_deps = config["external"]
                    if dependency in ext_deps:
                        external_dep = ext_deps[dependency]
        if external_dep:
            if external:
                return self.convert_external_dependency(configs, external_dep)
            else:
                return None
        else:
            return self.convert_internal_dependency(configs, dependency)

    def convert_internal_dependency(self, configs, dependency):
        """
        Hook to convert internal dependencies.
        """
        return dependency

    def convert_external_dependency(self, configs, dependency):
        """
        Hook to convert external dependencies.
        """
        return dependency

    def in_config(self, configs, list_name, value):
        """
        Check whether value is in config.

        Args:
            configs: List of configs.
            list_name: Name of a list in config
        that should be used for checking.
            value: Value to look for.

        Returns:
            Boolean.
        """
        result = False
        for config in configs:
            if config:
                if list_name in config:
                    if value in config[list_name]:
                        result = True
                        break
        return result