diff options
Diffstat (limited to 'portage_with_autodep/pym/portage/cache')
21 files changed, 105 insertions, 189 deletions
diff --git a/portage_with_autodep/pym/portage/cache/__init__.pyo b/portage_with_autodep/pym/portage/cache/__init__.pyo Binary files differindex eb5a90e..735ab0b 100644 --- a/portage_with_autodep/pym/portage/cache/__init__.pyo +++ b/portage_with_autodep/pym/portage/cache/__init__.pyo diff --git a/portage_with_autodep/pym/portage/cache/anydbm.pyo b/portage_with_autodep/pym/portage/cache/anydbm.pyo Binary files differindex 5946da9..8422437 100644 --- a/portage_with_autodep/pym/portage/cache/anydbm.pyo +++ b/portage_with_autodep/pym/portage/cache/anydbm.pyo diff --git a/portage_with_autodep/pym/portage/cache/cache_errors.pyo b/portage_with_autodep/pym/portage/cache/cache_errors.pyo Binary files differindex 866088e..e323de7 100644 --- a/portage_with_autodep/pym/portage/cache/cache_errors.pyo +++ b/portage_with_autodep/pym/portage/cache/cache_errors.pyo diff --git a/portage_with_autodep/pym/portage/cache/ebuild_xattr.py b/portage_with_autodep/pym/portage/cache/ebuild_xattr.py index 0086e40..db6e177 100644 --- a/portage_with_autodep/pym/portage/cache/ebuild_xattr.py +++ b/portage_with_autodep/pym/portage/cache/ebuild_xattr.py @@ -1,4 +1,4 @@ -# -*- coding: UTF8 -*- +# -*- coding: utf-8 -*- # Copyright: 2009-2011 Gentoo Foundation # Author(s): Petteri Räty (betelgeuse@gentoo.org) # License: GPL2 diff --git a/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo b/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo Binary files differindex fe32dcc..405f372 100644 --- a/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo +++ b/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo diff --git a/portage_with_autodep/pym/portage/cache/flat_hash.py b/portage_with_autodep/pym/portage/cache/flat_hash.py index 2eae9f6..08dcbe8 100644 --- a/portage_with_autodep/pym/portage/cache/flat_hash.py +++ b/portage_with_autodep/pym/portage/cache/flat_hash.py @@ -1,7 +1,9 @@ -# Copyright: 2005-2011 Gentoo Foundation +# Copyright: 2005-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) +from __future__ import unicode_literals + from portage.cache import fs_template from portage.cache import cache_errors import errno @@ -11,16 +13,13 @@ import sys import os as _os from portage import os from portage import _encodings -from portage import _unicode_decode from portage import _unicode_encode +from portage.exception import InvalidData +from portage.versions import _pkg_str if sys.hexversion >= 0x3000000: long = int -# Coerce to unicode, in order to prevent TypeError when writing -# raw bytes to TextIOWrapper with python2. -_setitem_fmt = _unicode_decode("%s=%s\n") - class database(fs_template.FsBased): autocommits = True @@ -40,11 +39,10 @@ class database(fs_template.FsBased): # Don't use os.path.join, for better performance. fp = self.location + _os.sep + cpv try: - myf = io.open(_unicode_encode(fp, + with io.open(_unicode_encode(fp, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace') - try: + errors='replace') as myf: lines = myf.read().split("\n") if not lines[-1]: lines.pop() @@ -54,8 +52,6 @@ class database(fs_template.FsBased): # that uses mtime mangling. d['_mtime_'] = _os.fstat(myf.fileno())[stat.ST_MTIME] return d - finally: - myf.close() except (IOError, OSError) as e: if e.errno != errno.ENOENT: raise cache_errors.CacheCorruption(cpv, e) @@ -94,7 +90,10 @@ class database(fs_template.FsBased): v = values.get(k) if not v: continue - myf.write(_setitem_fmt % (k, v)) + # NOTE: This format string requires unicode_literals, so that + # k and v are coerced to unicode, in order to prevent TypeError + # when writing raw bytes to TextIOWrapper with Python 2. + myf.write("%s=%s\n" % (k, v)) finally: myf.close() self._ensure_access(fp) @@ -135,8 +134,6 @@ class database(fs_template.FsBased): del e continue for l in dir_list: - if l.endswith(".cpickle"): - continue p = os.path.join(dir_path, l) try: st = os.lstat(p) @@ -151,7 +148,11 @@ class database(fs_template.FsBased): if depth < 1: dirs.append((depth+1, p)) continue - yield p[len_base+1:] + + try: + yield _pkg_str(p[len_base+1:]) + except InvalidData: + continue class md5_database(database): diff --git a/portage_with_autodep/pym/portage/cache/flat_hash.pyo b/portage_with_autodep/pym/portage/cache/flat_hash.pyo Binary files differindex 4f568a8..90a1b36 100644 --- a/portage_with_autodep/pym/portage/cache/flat_hash.pyo +++ b/portage_with_autodep/pym/portage/cache/flat_hash.pyo diff --git a/portage_with_autodep/pym/portage/cache/flat_list.py b/portage_with_autodep/pym/portage/cache/flat_list.py deleted file mode 100644 index 7288307..0000000 --- a/portage_with_autodep/pym/portage/cache/flat_list.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright 2005-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.cache import fs_template -from portage.cache import cache_errors -from portage import os -from portage import _encodings -from portage import _unicode_decode -from portage import _unicode_encode -import errno -import io -import stat -import sys - -if sys.hexversion >= 0x3000000: - long = int - -# Coerce to unicode, in order to prevent TypeError when writing -# raw bytes to TextIOWrapper with python2. -_setitem_fmt = _unicode_decode("%s\n") - -# store the current key order *here*. -class database(fs_template.FsBased): - - autocommits = True - - # do not screw with this ordering. _eclasses_ needs to be last - auxdbkey_order=('DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI', - 'RESTRICT', 'HOMEPAGE', 'LICENSE', 'DESCRIPTION', - 'KEYWORDS', 'IUSE', 'REQUIRED_USE', - 'PDEPEND', 'PROVIDE', 'EAPI', 'PROPERTIES', 'DEFINED_PHASES') - - def __init__(self, *args, **config): - super(database,self).__init__(*args, **config) - self.location = os.path.join(self.location, - self.label.lstrip(os.path.sep).rstrip(os.path.sep)) - - if len(self._known_keys) > len(self.auxdbkey_order) + 2: - raise Exception("less ordered keys then auxdbkeys") - if not os.path.exists(self.location): - self._ensure_dirs() - - - def _getitem(self, cpv): - d = {} - try: - myf = io.open(_unicode_encode(os.path.join(self.location, cpv), - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') - for k,v in zip(self.auxdbkey_order, myf): - d[k] = v.rstrip("\n") - except (OSError, IOError) as e: - if errno.ENOENT == e.errno: - raise KeyError(cpv) - raise cache_errors.CacheCorruption(cpv, e) - - try: - d["_mtime_"] = os.fstat(myf.fileno())[stat.ST_MTIME] - except OSError as e: - myf.close() - raise cache_errors.CacheCorruption(cpv, e) - myf.close() - return d - - - def _setitem(self, cpv, values): - s = cpv.rfind("/") - fp=os.path.join(self.location,cpv[:s],".update.%i.%s" % (os.getpid(), cpv[s+1:])) - try: - myf = io.open(_unicode_encode(fp, - encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - except (OSError, IOError) as e: - if errno.ENOENT == e.errno: - try: - self._ensure_dirs(cpv) - myf = io.open(_unicode_encode(fp, - encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - except (OSError, IOError) as e: - raise cache_errors.CacheCorruption(cpv, e) - else: - raise cache_errors.CacheCorruption(cpv, e) - - - for x in self.auxdbkey_order: - myf.write(_setitem_fmt % (values.get(x, ""),)) - - myf.close() - self._ensure_access(fp, mtime=values["_mtime_"]) - #update written. now we move it. - new_fp = os.path.join(self.location,cpv) - try: - os.rename(fp, new_fp) - except (OSError, IOError) as e: - os.remove(fp) - raise cache_errors.CacheCorruption(cpv, e) - - - def _delitem(self, cpv): - try: - os.remove(os.path.join(self.location,cpv)) - except OSError as e: - if errno.ENOENT == e.errno: - raise KeyError(cpv) - else: - raise cache_errors.CacheCorruption(cpv, e) - - - def __contains__(self, cpv): - return os.path.exists(os.path.join(self.location, cpv)) - - - def __iter__(self): - """generator for walking the dir struct""" - dirs = [self.location] - len_base = len(self.location) - while len(dirs): - for l in os.listdir(dirs[0]): - if l.endswith(".cpickle"): - continue - p = os.path.join(dirs[0],l) - st = os.lstat(p) - if stat.S_ISDIR(st.st_mode): - dirs.append(p) - continue - yield p[len_base+1:] - dirs.pop(0) - - - def commit(self): pass diff --git a/portage_with_autodep/pym/portage/cache/flat_list.pyo b/portage_with_autodep/pym/portage/cache/flat_list.pyo Binary files differdeleted file mode 100644 index ab7dc82..0000000 --- a/portage_with_autodep/pym/portage/cache/flat_list.pyo +++ /dev/null diff --git a/portage_with_autodep/pym/portage/cache/fs_template.py b/portage_with_autodep/pym/portage/cache/fs_template.py index a82e862..0567c72 100644 --- a/portage_with_autodep/pym/portage/cache/fs_template.py +++ b/portage_with_autodep/pym/portage/cache/fs_template.py @@ -1,14 +1,14 @@ -# Copyright: 2005 Gentoo Foundation +# Copyright 2005-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) -# License: GPL2 +import os as _os import sys from portage.cache import template from portage import os from portage.proxy.lazyimport import lazyimport lazyimport(globals(), - 'portage.data:portage_gid', 'portage.exception:PortageException', 'portage.util:apply_permissions', ) @@ -22,14 +22,11 @@ class FsBased(template.database): attempt to ensure files have the specified owners/perms""" def __init__(self, *args, **config): - """throws InitializationError if needs args aren't specified - gid and perms aren't listed do to an oddity python currying mechanism - gid=portage_gid - perms=0665""" for x, y in (("gid", -1), ("perms", -1)): if x in config: - setattr(self, "_"+x, config[x]) + # Since Python 3.4, chown requires int type (no proxies). + setattr(self, "_" + x, int(config[x])) del config[x] else: setattr(self, "_"+x, y) @@ -78,7 +75,17 @@ class FsBased(template.database): if self._perms != -1: os.umask(um) - + def _prune_empty_dirs(self): + all_dirs = [] + for parent, dirs, files in os.walk(self.location): + for x in dirs: + all_dirs.append(_os.path.join(parent, x)) + while all_dirs: + try: + _os.rmdir(all_dirs.pop()) + except OSError: + pass + def gen_label(base, label): """if supplied label is a path, generate a unique label based upon label, and supplied base path""" if label.find(os.path.sep) == -1: diff --git a/portage_with_autodep/pym/portage/cache/fs_template.pyo b/portage_with_autodep/pym/portage/cache/fs_template.pyo Binary files differindex 6cbbc2f..de71ee6 100644 --- a/portage_with_autodep/pym/portage/cache/fs_template.pyo +++ b/portage_with_autodep/pym/portage/cache/fs_template.pyo diff --git a/portage_with_autodep/pym/portage/cache/mappings.py b/portage_with_autodep/pym/portage/cache/mappings.py index bc8ce9a..cd39a6e 100644 --- a/portage_with_autodep/pym/portage/cache/mappings.py +++ b/portage_with_autodep/pym/portage/cache/mappings.py @@ -199,10 +199,10 @@ class OrderedDict(UserDict): return iter(self._order) def __setitem__(self, key, item): - if key in self: - self._order.remove(key) + new_key = key not in self UserDict.__setitem__(self, key, item) - self._order.append(key) + if new_key: + self._order.append(key) def __delitem__(self, key): UserDict.__delitem__(self, key) diff --git a/portage_with_autodep/pym/portage/cache/mappings.pyo b/portage_with_autodep/pym/portage/cache/mappings.pyo Binary files differindex 1eb3f4f..c183246 100644 --- a/portage_with_autodep/pym/portage/cache/mappings.pyo +++ b/portage_with_autodep/pym/portage/cache/mappings.pyo diff --git a/portage_with_autodep/pym/portage/cache/metadata.py b/portage_with_autodep/pym/portage/cache/metadata.py index 9d2c3a5..6612e73 100644 --- a/portage_with_autodep/pym/portage/cache/metadata.py +++ b/portage_with_autodep/pym/portage/cache/metadata.py @@ -28,7 +28,8 @@ class database(flat_hash.database): auxdbkey_order=('DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI', 'RESTRICT', 'HOMEPAGE', 'LICENSE', 'DESCRIPTION', 'KEYWORDS', 'INHERITED', 'IUSE', 'REQUIRED_USE', - 'PDEPEND', 'PROVIDE', 'EAPI', 'PROPERTIES', 'DEFINED_PHASES') + 'PDEPEND', 'PROVIDE', 'EAPI', 'PROPERTIES', + 'DEFINED_PHASES', 'HDEPEND') autocommits = True serialize_eclasses = False diff --git a/portage_with_autodep/pym/portage/cache/metadata.pyo b/portage_with_autodep/pym/portage/cache/metadata.pyo Binary files differindex c98445b..649ec5b 100644 --- a/portage_with_autodep/pym/portage/cache/metadata.pyo +++ b/portage_with_autodep/pym/portage/cache/metadata.pyo diff --git a/portage_with_autodep/pym/portage/cache/sql_template.pyo b/portage_with_autodep/pym/portage/cache/sql_template.pyo Binary files differindex e2c5974..54636ed 100644 --- a/portage_with_autodep/pym/portage/cache/sql_template.pyo +++ b/portage_with_autodep/pym/portage/cache/sql_template.pyo diff --git a/portage_with_autodep/pym/portage/cache/sqlite.py b/portage_with_autodep/pym/portage/cache/sqlite.py index fcc62ff..40db070 100644 --- a/portage_with_autodep/pym/portage/cache/sqlite.py +++ b/portage_with_autodep/pym/portage/cache/sqlite.py @@ -1,6 +1,9 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + +import re import sys from portage.cache import fs_template from portage.cache import cache_errors @@ -20,7 +23,6 @@ class database(fs_template.FsBased): # to calculate the number of pages requested, according to the following # equation: cache_bytes = page_bytes * page_count cache_bytes = 1024 * 1024 * 10 - _db_table = None def __init__(self, *args, **config): super(database, self).__init__(*args, **config) @@ -28,6 +30,7 @@ class database(fs_template.FsBased): self._allowed_keys = ["_mtime_", "_eclasses_"] self._allowed_keys.extend(self._known_keys) self._allowed_keys.sort() + self._allowed_keys_set = frozenset(self._allowed_keys) self.location = os.path.join(self.location, self.label.lstrip(os.path.sep).rstrip(os.path.sep)) @@ -37,8 +40,8 @@ class database(fs_template.FsBased): config.setdefault("autocommit", self.autocommits) config.setdefault("cache_bytes", self.cache_bytes) config.setdefault("synchronous", self.synchronous) - # Timeout for throwing a "database is locked" exception (pysqlite - # default is 5.0 seconds). + # Set longer timeout for throwing a "database is locked" exception. + # Default timeout in sqlite3 module is 5.0 seconds. config.setdefault("timeout", 15) self._db_init_connection(config) self._db_init_structures() @@ -47,11 +50,8 @@ class database(fs_template.FsBased): # sqlite3 is optional with >=python-2.5 try: import sqlite3 as db_module - except ImportError: - try: - from pysqlite2 import dbapi2 as db_module - except ImportError as e: - raise cache_errors.InitializationError(self.__class__, e) + except ImportError as e: + raise cache_errors.InitializationError(self.__class__, e) self._db_module = db_module self._db_error = db_module.Error @@ -62,7 +62,6 @@ class database(fs_template.FsBased): # Avoid potential UnicodeEncodeError in python-2.x by # only calling str() when it's absolutely necessary. s = str(s) - # This is equivalent to the _quote function from pysqlite 1.1. return "'%s'" % s.replace("'", "''") def _db_init_connection(self, config): @@ -92,9 +91,6 @@ class database(fs_template.FsBased): self._db_table["packages"]["table_name"] = mytable self._db_table["packages"]["package_id"] = "internal_db_package_id" self._db_table["packages"]["package_key"] = "portage_package_key" - self._db_table["packages"]["internal_columns"] = \ - [self._db_table["packages"]["package_id"], - self._db_table["packages"]["package_key"]] create_statement = [] create_statement.append("CREATE TABLE") create_statement.append(mytable) @@ -109,15 +105,18 @@ class database(fs_template.FsBased): create_statement.append(")") self._db_table["packages"]["create"] = " ".join(create_statement) - self._db_table["packages"]["columns"] = \ - self._db_table["packages"]["internal_columns"] + \ - self._allowed_keys cursor = self._db_cursor for k, v in self._db_table.items(): if self._db_table_exists(v["table_name"]): create_statement = self._db_table_get_create(v["table_name"]) - if create_statement != v["create"]: + table_ok, missing_keys = self._db_validate_create_statement(create_statement) + if table_ok: + if missing_keys: + for k in sorted(missing_keys): + cursor.execute("ALTER TABLE %s ADD COLUMN %s TEXT" % + (self._db_table["packages"]["table_name"], k)) + else: writemsg(_("sqlite: dropping old table: %s\n") % v["table_name"]) cursor.execute("DROP TABLE %s" % v["table_name"]) cursor.execute(v["create"]) @@ -138,6 +137,37 @@ class database(fs_template.FsBased): self._db_escape_string(table_name)) return cursor.fetchall()[0][0] + def _db_validate_create_statement(self, statement): + missing_keys = None + if statement == self._db_table["packages"]["create"]: + return True, missing_keys + + m = re.match(r'^\s*CREATE\s*TABLE\s*%s\s*\(\s*%s\s*INTEGER\s*PRIMARY\s*KEY\s*AUTOINCREMENT\s*,(.*)\)\s*$' % + (self._db_table["packages"]["table_name"], + self._db_table["packages"]["package_id"]), + statement) + if m is None: + return False, missing_keys + + unique_constraints = set([self._db_table["packages"]["package_key"]]) + missing_keys = set(self._allowed_keys) + unique_re = re.compile(r'^\s*UNIQUE\s*\(\s*(\w*)\s*\)\s*$') + column_re = re.compile(r'^\s*(\w*)\s*TEXT\s*$') + for x in m.group(1).split(","): + m = column_re.match(x) + if m is not None: + missing_keys.discard(m.group(1)) + continue + m = unique_re.match(x) + if m is not None: + unique_constraints.discard(m.group(1)) + continue + + if unique_constraints: + return False, missing_keys + + return True, missing_keys + def _db_init_cache_size(self, cache_bytes): cursor = self._db_cursor cursor.execute("PRAGMA page_size") @@ -173,13 +203,17 @@ class database(fs_template.FsBased): raise KeyError(cpv) else: raise cache_errors.CacheCorruption(cpv, "key is not unique") + result = result[0] d = {} - internal_columns = self._db_table["packages"]["internal_columns"] - column_index = -1 - for k in self._db_table["packages"]["columns"]: - column_index +=1 - if k not in internal_columns: - d[k] = result[0][column_index] + allowed_keys_set = self._allowed_keys_set + for column_index, column_info in enumerate(cursor.description): + k = column_info[0] + if k in allowed_keys_set: + v = result[column_index] + if v is None: + # This happens after a new empty column has been added. + v = "" + d[k] = v return d diff --git a/portage_with_autodep/pym/portage/cache/sqlite.pyo b/portage_with_autodep/pym/portage/cache/sqlite.pyo Binary files differindex a82d25f..aba2f6c 100644 --- a/portage_with_autodep/pym/portage/cache/sqlite.pyo +++ b/portage_with_autodep/pym/portage/cache/sqlite.pyo diff --git a/portage_with_autodep/pym/portage/cache/template.py b/portage_with_autodep/pym/portage/cache/template.py index cf1e8ae..9b8a4d3 100644 --- a/portage_with_autodep/pym/portage/cache/template.py +++ b/portage_with_autodep/pym/portage/cache/template.py @@ -1,6 +1,6 @@ -# Copyright: 2005-2012 Gentoo Foundation +# Copyright 2005-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) -# License: GPL2 from portage.cache import cache_errors from portage.cache.cache_errors import InvalidRestriction @@ -164,7 +164,14 @@ class database(object): def commit(self): if not self.autocommits: - raise NotImplementedError + raise NotImplementedError(self) + + def __del__(self): + # This used to be handled by an atexit hook that called + # close_portdbapi_caches() for all portdbapi instances, but that was + # prone to memory leaks for API consumers that needed to create/destroy + # many portdbapi instances. So, instead we rely on __del__. + self.sync() def __contains__(self, cpv): """This method should always be overridden. It is provided only for diff --git a/portage_with_autodep/pym/portage/cache/template.pyo b/portage_with_autodep/pym/portage/cache/template.pyo Binary files differindex 45da015..3315771 100644 --- a/portage_with_autodep/pym/portage/cache/template.pyo +++ b/portage_with_autodep/pym/portage/cache/template.pyo diff --git a/portage_with_autodep/pym/portage/cache/volatile.pyo b/portage_with_autodep/pym/portage/cache/volatile.pyo Binary files differindex fac5d55..9ebb5b8 100644 --- a/portage_with_autodep/pym/portage/cache/volatile.pyo +++ b/portage_with_autodep/pym/portage/cache/volatile.pyo |