diff options
Diffstat (limited to 'portage_with_autodep/pym/portage/cache')
21 files changed, 111 insertions, 305 deletions
diff --git a/portage_with_autodep/pym/portage/cache/__init__.pyo b/portage_with_autodep/pym/portage/cache/__init__.pyo Binary files differnew file mode 100644 index 0000000..eb5a90e --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/__init__.pyo diff --git a/portage_with_autodep/pym/portage/cache/anydbm.pyo b/portage_with_autodep/pym/portage/cache/anydbm.pyo Binary files differnew file mode 100644 index 0000000..5946da9 --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/anydbm.pyo diff --git a/portage_with_autodep/pym/portage/cache/cache_errors.pyo b/portage_with_autodep/pym/portage/cache/cache_errors.pyo Binary files differnew file mode 100644 index 0000000..866088e --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/cache_errors.pyo diff --git a/portage_with_autodep/pym/portage/cache/ebuild_xattr.py b/portage_with_autodep/pym/portage/cache/ebuild_xattr.py index 6b388fa..0086e40 100644 --- a/portage_with_autodep/pym/portage/cache/ebuild_xattr.py +++ b/portage_with_autodep/pym/portage/cache/ebuild_xattr.py @@ -1,5 +1,6 @@ +# -*- coding: UTF8 -*- # Copyright: 2009-2011 Gentoo Foundation -# Author(s): Petteri Räty (betelgeuse@gentoo.org) +# Author(s): Petteri Räty (betelgeuse@gentoo.org) # License: GPL2 __all__ = ['database'] diff --git a/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo b/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo Binary files differnew file mode 100644 index 0000000..fe32dcc --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo diff --git a/portage_with_autodep/pym/portage/cache/flat_hash.py b/portage_with_autodep/pym/portage/cache/flat_hash.py index b6bc074..2eae9f6 100644 --- a/portage_with_autodep/pym/portage/cache/flat_hash.py +++ b/portage_with_autodep/pym/portage/cache/flat_hash.py @@ -31,7 +31,7 @@ class database(fs_template.FsBased): self.label.lstrip(os.path.sep).rstrip(os.path.sep)) write_keys = set(self._known_keys) write_keys.add("_eclasses_") - write_keys.add("_mtime_") + write_keys.add("_%s_" % (self.validation_chf,)) self._write_keys = sorted(write_keys) if not self.readonly and not os.path.exists(self.location): self._ensure_dirs() @@ -69,7 +69,6 @@ class database(fs_template.FsBased): raise cache_errors.CacheCorruption(cpv, e) def _setitem(self, cpv, values): -# import pdb;pdb.set_trace() s = cpv.rfind("/") fp = os.path.join(self.location,cpv[:s],".update.%i.%s" % (os.getpid(), cpv[s+1:])) try: @@ -153,3 +152,9 @@ class database(fs_template.FsBased): dirs.append((depth+1, p)) continue yield p[len_base+1:] + + +class md5_database(database): + + validation_chf = 'md5' + store_eclass_paths = False diff --git a/portage_with_autodep/pym/portage/cache/flat_hash.pyo b/portage_with_autodep/pym/portage/cache/flat_hash.pyo Binary files differnew file mode 100644 index 0000000..4f568a8 --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/flat_hash.pyo diff --git a/portage_with_autodep/pym/portage/cache/flat_list.pyo b/portage_with_autodep/pym/portage/cache/flat_list.pyo Binary files differnew file mode 100644 index 0000000..ab7dc82 --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/flat_list.pyo diff --git a/portage_with_autodep/pym/portage/cache/fs_template.pyo b/portage_with_autodep/pym/portage/cache/fs_template.pyo Binary files differnew file mode 100644 index 0000000..6cbbc2f --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/fs_template.pyo diff --git a/portage_with_autodep/pym/portage/cache/mappings.py b/portage_with_autodep/pym/portage/cache/mappings.py index 60a918e..bc8ce9a 100644 --- a/portage_with_autodep/pym/portage/cache/mappings.py +++ b/portage_with_autodep/pym/portage/cache/mappings.py @@ -316,7 +316,7 @@ def slot_dict_class(keys, prefix="_val_"): attribute names from keys @type prefix: String @rtype: SlotDict - @returns: A class that constructs SlotDict instances + @return: A class that constructs SlotDict instances having the specified keys. """ if isinstance(keys, frozenset): diff --git a/portage_with_autodep/pym/portage/cache/mappings.pyo b/portage_with_autodep/pym/portage/cache/mappings.pyo Binary files differnew file mode 100644 index 0000000..1eb3f4f --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/mappings.pyo diff --git a/portage_with_autodep/pym/portage/cache/metadata.py b/portage_with_autodep/pym/portage/cache/metadata.py index 4c735d7..9d2c3a5 100644 --- a/portage_with_autodep/pym/portage/cache/metadata.py +++ b/portage_with_autodep/pym/portage/cache/metadata.py @@ -6,6 +6,7 @@ import errno import re import stat import sys +from operator import attrgetter from portage import os from portage import _encodings from portage import _unicode_encode @@ -63,13 +64,14 @@ class database(flat_hash.database): if "INHERITED" in d: if self.ec is None: self.ec = portage.eclass_cache.cache(self.location[:-15]) + getter = attrgetter(self.validation_chf) try: - d["_eclasses_"] = self.ec.get_eclass_data( - d["INHERITED"].split()) + ec_data = self.ec.get_eclass_data(d["INHERITED"].split()) + d["_eclasses_"] = dict((k, (v.eclass_dir, getter(v))) + for k,v in ec_data.items()) except KeyError as e: # INHERITED contains a non-existent eclass. raise cache_errors.CacheCorruption(cpv, e) - del d["INHERITED"] else: d["_eclasses_"] = {} elif isinstance(d["_eclasses_"], basestring): diff --git a/portage_with_autodep/pym/portage/cache/metadata.pyo b/portage_with_autodep/pym/portage/cache/metadata.pyo Binary files differnew file mode 100644 index 0000000..c98445b --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/metadata.pyo diff --git a/portage_with_autodep/pym/portage/cache/metadata_overlay.py b/portage_with_autodep/pym/portage/cache/metadata_overlay.py deleted file mode 100644 index cfa0051..0000000 --- a/portage_with_autodep/pym/portage/cache/metadata_overlay.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 1999-2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.cache import template -from portage.cache.cache_errors import CacheCorruption -from portage.cache.flat_hash import database as db_rw -from portage.cache.metadata import database as db_ro - -class database(template.database): - - serialize_eclasses = False - - def __init__(self, location, label, auxdbkeys, db_rw=db_rw, db_ro=db_ro, - *args, **config): - super_config = config.copy() - super_config.pop("gid", None) - super_config.pop("perms", None) - super(database, self).__init__(location, label, auxdbkeys, - *args, **super_config) - self.db_rw = db_rw(location, label, auxdbkeys, **config) - self.commit = self.db_rw.commit - self.autocommits = self.db_rw.autocommits - if isinstance(db_ro, type): - ro_config = config.copy() - ro_config["readonly"] = True - self.db_ro = db_ro(label, "metadata/cache", auxdbkeys, **ro_config) - else: - self.db_ro = db_ro - - def __getitem__(self, cpv): - """funnel whiteout validation through here, since value needs to be fetched""" - try: - value = self.db_rw[cpv] - except KeyError: - return self.db_ro[cpv] # raises a KeyError when necessary - except CacheCorruption: - del self.db_rw[cpv] - return self.db_ro[cpv] # raises a KeyError when necessary - if self._is_whiteout(value): - if self._is_whiteout_valid(cpv, value): - raise KeyError(cpv) - else: - del self.db_rw[cpv] - return self.db_ro[cpv] # raises a KeyError when necessary - else: - return value - - def _setitem(self, name, values): - try: - value_ro = self.db_ro.get(name) - except CacheCorruption: - value_ro = None - if value_ro is not None and \ - self._are_values_identical(value_ro, values): - # we have matching values in the underlying db_ro - # so it is unnecessary to store data in db_rw - try: - del self.db_rw[name] # delete unwanted whiteout when necessary - except KeyError: - pass - return - self.db_rw[name] = values - - def _delitem(self, cpv): - value = self[cpv] # validates whiteout and/or raises a KeyError when necessary - if cpv in self.db_ro: - self.db_rw[cpv] = self._create_whiteout(value) - else: - del self.db_rw[cpv] - - def __contains__(self, cpv): - try: - self[cpv] # validates whiteout when necessary - except KeyError: - return False - return True - - def __iter__(self): - s = set() - for cpv in self.db_rw: - if cpv in self: # validates whiteout when necessary - yield cpv - # set includes whiteouts so they won't be yielded later - s.add(cpv) - for cpv in self.db_ro: - if cpv not in s: - yield cpv - - def _is_whiteout(self, value): - return value["EAPI"] == "whiteout" - - def _create_whiteout(self, value): - return {"EAPI":"whiteout","_eclasses_":value["_eclasses_"],"_mtime_":value["_mtime_"]} - - def _is_whiteout_valid(self, name, value_rw): - try: - value_ro = self.db_ro[name] - return self._are_values_identical(value_rw,value_ro) - except KeyError: - return False - - def _are_values_identical(self, value1, value2): - if value1['_mtime_'] != value2['_mtime_']: - return False - return value1["_eclasses_"] == value2["_eclasses_"] diff --git a/portage_with_autodep/pym/portage/cache/sql_template.pyo b/portage_with_autodep/pym/portage/cache/sql_template.pyo Binary files differnew file mode 100644 index 0000000..e2c5974 --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/sql_template.pyo diff --git a/portage_with_autodep/pym/portage/cache/sqlite.pyo b/portage_with_autodep/pym/portage/cache/sqlite.pyo Binary files differnew file mode 100644 index 0000000..a82d25f --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/sqlite.pyo diff --git a/portage_with_autodep/pym/portage/cache/template.py b/portage_with_autodep/pym/portage/cache/template.py index f84d8f4..cf1e8ae 100644 --- a/portage_with_autodep/pym/portage/cache/template.py +++ b/portage_with_autodep/pym/portage/cache/template.py @@ -1,4 +1,4 @@ -# Copyright: 2005 Gentoo Foundation +# Copyright: 2005-2012 Gentoo Foundation # Author(s): Brian Harring (ferringb@gentoo.org) # License: GPL2 @@ -7,10 +7,14 @@ from portage.cache.cache_errors import InvalidRestriction from portage.cache.mappings import ProtectedDict import sys import warnings +import operator if sys.hexversion >= 0x3000000: + _unicode = str basestring = str long = int +else: + _unicode = unicode class database(object): # this is for metadata/cache transfer. @@ -21,6 +25,8 @@ class database(object): autocommits = False cleanse_keys = False serialize_eclasses = True + validation_chf = 'mtime' + store_eclass_paths = True def __init__(self, location, label, auxdbkeys, readonly=False): """ initialize the derived class; specifically, store label/keys""" @@ -40,9 +46,15 @@ class database(object): self.updates = 0 d=self._getitem(cpv) if self.serialize_eclasses and "_eclasses_" in d: - d["_eclasses_"] = reconstruct_eclasses(cpv, d["_eclasses_"]) + d["_eclasses_"] = reconstruct_eclasses(cpv, d["_eclasses_"], + self.validation_chf, paths=self.store_eclass_paths) elif "_eclasses_" not in d: d["_eclasses_"] = {} + # Never return INHERITED, since portdbapi.aux_get() will + # generate it automatically from _eclasses_, and we want + # to omit it in comparisons between cache entries like + # those that egencache uses to avoid redundant writes. + d.pop("INHERITED", None) mtime = d.get('_mtime_') if mtime is None: raise cache_errors.CacheCorruption(cpv, @@ -60,22 +72,46 @@ class database(object): override this in derived classess""" raise NotImplementedError + @staticmethod + def _internal_eclasses(extern_ec_dict, chf_type, paths): + """ + When serialize_eclasses is False, we have to convert an external + eclass dict containing hashed_path objects into an appropriate + internal dict containing values of chf_type (and eclass dirs + if store_eclass_paths is True). + """ + if not extern_ec_dict: + return extern_ec_dict + chf_getter = operator.attrgetter(chf_type) + if paths: + intern_ec_dict = dict((k, (v.eclass_dir, chf_getter(v))) + for k, v in extern_ec_dict.items()) + else: + intern_ec_dict = dict((k, chf_getter(v)) + for k, v in extern_ec_dict.items()) + return intern_ec_dict + def __setitem__(self, cpv, values): """set a cpv to values This shouldn't be overriden in derived classes since it handles the readonly checks""" if self.readonly: raise cache_errors.ReadOnlyRestriction() + d = None if self.cleanse_keys: d=ProtectedDict(values) for k, v in list(d.items()): if not v: del d[k] - if self.serialize_eclasses and "_eclasses_" in values: - d["_eclasses_"] = serialize_eclasses(d["_eclasses_"]) - elif self.serialize_eclasses and "_eclasses_" in values: - d = ProtectedDict(values) - d["_eclasses_"] = serialize_eclasses(d["_eclasses_"]) - else: + if "_eclasses_" in values: + if d is None: + d = ProtectedDict(values) + if self.serialize_eclasses: + d["_eclasses_"] = serialize_eclasses(d["_eclasses_"], + self.validation_chf, paths=self.store_eclass_paths) + else: + d["_eclasses_"] = self._internal_eclasses(d["_eclasses_"], + self.validation_chf, self.store_eclass_paths) + elif d is None: d = values self._setitem(cpv, d) if not self.autocommits: @@ -159,6 +195,23 @@ class database(object): except KeyError: return x + def validate_entry(self, entry, ebuild_hash, eclass_db): + hash_key = '_%s_' % self.validation_chf + try: + entry_hash = entry[hash_key] + except KeyError: + return False + else: + if entry_hash != getattr(ebuild_hash, self.validation_chf): + return False + update = eclass_db.validate_and_rewrite_cache(entry['_eclasses_'], self.validation_chf, + self.store_eclass_paths) + if update is None: + return False + if update: + entry['_eclasses_'] = update + return True + def get_matches(self, match_dict): """generic function for walking the entire cache db, matching restrictions to filter what cpv's are returned. Derived classes should override this if they @@ -195,7 +248,9 @@ class database(object): keys = __iter__ items = iteritems -def serialize_eclasses(eclass_dict): +_keysorter = operator.itemgetter(0) + +def serialize_eclasses(eclass_dict, chf_type='mtime', paths=True): """takes a dict, returns a string representing said dict""" """The "new format", which causes older versions of <portage-2.1.2 to traceback with a ValueError due to failed long() conversion. This format @@ -206,27 +261,40 @@ def serialize_eclasses(eclass_dict): """ if not eclass_dict: return "" - return "\t".join(k + "\t%s\t%s" % eclass_dict[k] \ - for k in sorted(eclass_dict)) + getter = operator.attrgetter(chf_type) + if paths: + return "\t".join("%s\t%s\t%s" % (k, v.eclass_dir, getter(v)) + for k, v in sorted(eclass_dict.items(), key=_keysorter)) + return "\t".join("%s\t%s" % (k, getter(v)) + for k, v in sorted(eclass_dict.items(), key=_keysorter)) + -def reconstruct_eclasses(cpv, eclass_string): +def reconstruct_eclasses(cpv, eclass_string, chf_type='mtime', paths=True): """returns a dict when handed a string generated by serialize_eclasses""" eclasses = eclass_string.rstrip().lstrip().split("\t") if eclasses == [""]: # occasionally this occurs in the fs backends. they suck. return {} - - if len(eclasses) % 2 != 0 and len(eclasses) % 3 != 0: + + converter = _unicode + if chf_type == 'mtime': + converter = long + + if paths: + if len(eclasses) % 3 != 0: + raise cache_errors.CacheCorruption(cpv, "_eclasses_ was of invalid len %i" % len(eclasses)) + elif len(eclasses) % 2 != 0: raise cache_errors.CacheCorruption(cpv, "_eclasses_ was of invalid len %i" % len(eclasses)) d={} try: - if eclasses[1].isdigit(): - for x in range(0, len(eclasses), 2): - d[eclasses[x]] = ("", long(eclasses[x + 1])) - else: + i = iter(eclasses) + if paths: # The old format contains paths that will be discarded. - for x in range(0, len(eclasses), 3): - d[eclasses[x]] = (eclasses[x + 1], long(eclasses[x + 2])) + for name, path, val in zip(i, i, i): + d[name] = (path, converter(val)) + else: + for name, val in zip(i, i): + d[name] = converter(val) except IndexError: raise cache_errors.CacheCorruption(cpv, "_eclasses_ was of invalid len %i" % len(eclasses)) diff --git a/portage_with_autodep/pym/portage/cache/template.pyo b/portage_with_autodep/pym/portage/cache/template.pyo Binary files differnew file mode 100644 index 0000000..45da015 --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/template.pyo diff --git a/portage_with_autodep/pym/portage/cache/util.py b/portage_with_autodep/pym/portage/cache/util.py deleted file mode 100644 index b824689..0000000 --- a/portage_with_autodep/pym/portage/cache/util.py +++ /dev/null @@ -1,170 +0,0 @@ -# Copyright: 2005 Gentoo Foundation -# Author(s): Brian Harring (ferringb@gentoo.org) -# License: GPL2 - -from __future__ import print_function - -__all__ = ["mirror_cache", "non_quiet_mirroring", "quiet_mirroring"] - -from itertools import chain -from portage.cache import cache_errors -from portage.localization import _ - -def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, eclass_cache=None, verbose_instance=None): - - from portage import eapi_is_supported, \ - _validate_cache_for_unsupported_eapis - if not src_cache.complete_eclass_entries and not eclass_cache: - raise Exception("eclass_cache required for cache's of class %s!" % src_cache.__class__) - - if verbose_instance == None: - noise=quiet_mirroring() - else: - noise=verbose_instance - - dead_nodes = set(trg_cache) - count=0 - - if not trg_cache.autocommits: - trg_cache.sync(100) - - for x in valid_nodes_iterable: -# print "processing x=",x - count+=1 - dead_nodes.discard(x) - try: - entry = src_cache[x] - except KeyError as e: - noise.missing_entry(x) - del e - continue - except cache_errors.CacheError as ce: - noise.exception(x, ce) - del ce - continue - - eapi = entry.get('EAPI') - if not eapi: - eapi = '0' - eapi = eapi.lstrip('-') - eapi_supported = eapi_is_supported(eapi) - if not eapi_supported: - if not _validate_cache_for_unsupported_eapis: - noise.misc(x, _("unable to validate cache for EAPI='%s'") % eapi) - continue - - write_it = True - trg = None - try: - trg = trg_cache[x] - except (KeyError, cache_errors.CacheError): - pass - else: - if trg['_mtime_'] == entry['_mtime_'] and \ - eclass_cache.is_eclass_data_valid(trg['_eclasses_']) and \ - set(trg['_eclasses_']) == set(entry['_eclasses_']): - write_it = False - - for d in (entry, trg): - if d is not None and d.get('EAPI') in ('', '0'): - del d['EAPI'] - - if trg and not write_it: - """ We don't want to skip the write unless we're really sure that - the existing cache is identical, so don't trust _mtime_ and - _eclasses_ alone.""" - for k in set(chain(entry, trg)).difference( - ("_mtime_", "_eclasses_")): - if trg.get(k, "") != entry.get(k, ""): - write_it = True - break - - if write_it: - try: - inherited = entry.get("INHERITED", "") - eclasses = entry.get("_eclasses_") - except cache_errors.CacheError as ce: - noise.exception(x, ce) - del ce - continue - - if eclasses is not None: - if not eclass_cache.is_eclass_data_valid(entry["_eclasses_"]): - noise.eclass_stale(x) - continue - inherited = eclasses - else: - inherited = inherited.split() - - if inherited: - if src_cache.complete_eclass_entries and eclasses is None: - noise.corruption(x, "missing _eclasses_ field") - continue - - # Even if _eclasses_ already exists, replace it with data from - # eclass_cache, in order to insert local eclass paths. - try: - eclasses = eclass_cache.get_eclass_data(inherited) - except KeyError: - # INHERITED contains a non-existent eclass. - noise.eclass_stale(x) - continue - - if eclasses is None: - noise.eclass_stale(x) - continue - entry["_eclasses_"] = eclasses - - if not eapi_supported: - for k in set(entry).difference(("_mtime_", "_eclasses_")): - entry[k] = "" - entry["EAPI"] = "-" + eapi - - # by this time, if it reaches here, the eclass has been validated, and the entry has - # been updated/translated (if needs be, for metadata/cache mainly) - try: - trg_cache[x] = entry - except cache_errors.CacheError as ce: - noise.exception(x, ce) - del ce - continue - if count >= noise.call_update_min: - noise.update(x) - count = 0 - - if not trg_cache.autocommits: - trg_cache.commit() - - # ok. by this time, the trg_cache is up to date, and we have a dict - # with a crapload of cpv's. we now walk the target db, removing stuff if it's in the list. - for key in dead_nodes: - try: - del trg_cache[key] - except KeyError: - pass - except cache_errors.CacheError as ce: - noise.exception(ce) - del ce - noise.finish() - - -class quiet_mirroring(object): - # call_update_every is used by mirror_cache to determine how often to call in. - # quiet defaults to 2^24 -1. Don't call update, 'cept once every 16 million or so :) - call_update_min = 0xffffff - def update(self,key,*arg): pass - def exception(self,key,*arg): pass - def eclass_stale(self,*arg): pass - def missing_entry(self, key): pass - def misc(self,key,*arg): pass - def corruption(self, key, s): pass - def finish(self, *arg): pass - -class non_quiet_mirroring(quiet_mirroring): - call_update_min=1 - def update(self,key,*arg): print("processed",key) - def exception(self, key, *arg): print("exec",key,arg) - def missing(self,key): print("key %s is missing", key) - def corruption(self,key,*arg): print("corrupt %s:" % key,arg) - def eclass_stale(self,key,*arg):print("stale %s:"%key,arg) - diff --git a/portage_with_autodep/pym/portage/cache/volatile.py b/portage_with_autodep/pym/portage/cache/volatile.py index 0bf6bab..5516745 100644 --- a/portage_with_autodep/pym/portage/cache/volatile.py +++ b/portage_with_autodep/pym/portage/cache/volatile.py @@ -8,18 +8,23 @@ class database(template.database): autocommits = True serialize_eclasses = False + store_eclass_paths = False def __init__(self, *args, **config): config.pop("gid", None) config.pop("perms", None) super(database, self).__init__(*args, **config) self._data = {} - self.__iter__ = self._data.__iter__ self._delitem = self._data.__delitem__ - self.__contains__ = self._data.__contains__ def _setitem(self, name, values): self._data[name] = copy.deepcopy(values) - def _getitem(self, cpv): + def __getitem__(self, cpv): return copy.deepcopy(self._data[cpv]) + + def __iter__(self): + return iter(self._data) + + def __contains__(self, key): + return key in self._data diff --git a/portage_with_autodep/pym/portage/cache/volatile.pyo b/portage_with_autodep/pym/portage/cache/volatile.pyo Binary files differnew file mode 100644 index 0000000..fac5d55 --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/volatile.pyo |