1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
|
# Copyright 2010-2011 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
__all__ = ['digestgen']
import errno
import portage
portage.proxy.lazyimport.lazyimport(globals(),
'portage.package.ebuild._spawn_nofetch:spawn_nofetch',
)
from portage import os
from portage.const import MANIFEST2_REQUIRED_HASH
from portage.dbapi.porttree import FetchlistDict
from portage.dep import use_reduce
from portage.exception import InvalidDependString, FileNotFound, \
PermissionDenied, PortagePackageException
from portage.localization import _
from portage.output import colorize
from portage.package.ebuild.fetch import fetch
from portage.util import writemsg, writemsg_stdout
from portage.versions import catsplit
def digestgen(myarchives=None, mysettings=None, myportdb=None):
"""
Generates a digest file if missing. Fetches files if necessary.
NOTE: myarchives and mysettings used to be positional arguments,
so their order must be preserved for backward compatibility.
@param mysettings: the ebuild config (mysettings["O"] must correspond
to the ebuild's parent directory)
@type mysettings: config
@param myportdb: a portdbapi instance
@type myportdb: portdbapi
@rtype: int
@return: 1 on success and 0 on failure
"""
if mysettings is None or myportdb is None:
raise TypeError("portage.digestgen(): 'mysettings' and 'myportdb' parameter are required.")
try:
portage._doebuild_manifest_exempt_depend += 1
distfiles_map = {}
fetchlist_dict = FetchlistDict(mysettings["O"], mysettings, myportdb)
for cpv in fetchlist_dict:
try:
for myfile in fetchlist_dict[cpv]:
distfiles_map.setdefault(myfile, []).append(cpv)
except InvalidDependString as e:
writemsg("!!! %s\n" % str(e), noiselevel=-1)
del e
return 0
mytree = os.path.dirname(os.path.dirname(mysettings["O"]))
try:
mf = mysettings.repositories.get_repo_for_location(mytree)
except KeyError:
# backward compatibility
mytree = os.path.realpath(mytree)
mf = mysettings.repositories.get_repo_for_location(mytree)
mf = mf.load_manifest(mysettings["O"], mysettings["DISTDIR"],
fetchlist_dict=fetchlist_dict)
if not mf.allow_create:
writemsg_stdout(_(">>> Skipping creating Manifest for %s; "
"repository is configured to not use them\n") % mysettings["O"])
return 1
# Don't require all hashes since that can trigger excessive
# fetches when sufficient digests already exist. To ease transition
# while Manifest 1 is being removed, only require hashes that will
# exist before and after the transition.
required_hash_types = set()
required_hash_types.add("size")
required_hash_types.add(MANIFEST2_REQUIRED_HASH)
dist_hashes = mf.fhashdict.get("DIST", {})
# To avoid accidental regeneration of digests with the incorrect
# files (such as partially downloaded files), trigger the fetch
# code if the file exists and it's size doesn't match the current
# manifest entry. If there really is a legitimate reason for the
# digest to change, `ebuild --force digest` can be used to avoid
# triggering this code (or else the old digests can be manually
# removed from the Manifest).
missing_files = []
for myfile in distfiles_map:
myhashes = dist_hashes.get(myfile)
if not myhashes:
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError:
st = None
if st is None or st.st_size == 0:
missing_files.append(myfile)
continue
size = myhashes.get("size")
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError as e:
if e.errno != errno.ENOENT:
raise
del e
if size == 0:
missing_files.append(myfile)
continue
if required_hash_types.difference(myhashes):
missing_files.append(myfile)
continue
else:
if st.st_size == 0 or size is not None and size != st.st_size:
missing_files.append(myfile)
continue
if missing_files:
for myfile in missing_files:
uris = set()
all_restrict = set()
for cpv in distfiles_map[myfile]:
uris.update(myportdb.getFetchMap(
cpv, mytree=mytree)[myfile])
restrict = myportdb.aux_get(cpv, ['RESTRICT'],
mytree=mytree)[0]
# Here we ignore conditional parts of RESTRICT since
# they don't apply unconditionally. Assume such
# conditionals only apply on the client side where
# digestgen() does not need to be called.
all_restrict.update(use_reduce(restrict,
flat=True, matchnone=True))
# fetch() uses CATEGORY and PF to display a message
# when fetch restriction is triggered.
cat, pf = catsplit(cpv)
mysettings["CATEGORY"] = cat
mysettings["PF"] = pf
# fetch() uses PORTAGE_RESTRICT to control fetch
# restriction, which is only applied to files that
# are not fetchable via a mirror:// URI.
mysettings["PORTAGE_RESTRICT"] = " ".join(all_restrict)
try:
st = os.stat(os.path.join(
mysettings["DISTDIR"],myfile))
except OSError:
st = None
if not fetch({myfile : uris}, mysettings):
myebuild = os.path.join(mysettings["O"],
catsplit(cpv)[1] + ".ebuild")
spawn_nofetch(myportdb, myebuild)
writemsg(_("!!! Fetch failed for %s, can't update "
"Manifest\n") % myfile, noiselevel=-1)
if myfile in dist_hashes and \
st is not None and st.st_size > 0:
# stat result is obtained before calling fetch(),
# since fetch may rename the existing file if the
# digest does not match.
writemsg(_("!!! If you would like to "
"forcefully replace the existing "
"Manifest entry\n!!! for %s, use "
"the following command:\n") % myfile + \
"!!! " + colorize("INFORM",
"ebuild --force %s manifest" % \
os.path.basename(myebuild)) + "\n",
noiselevel=-1)
return 0
writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"])
try:
mf.create(assumeDistHashesSometimes=True,
assumeDistHashesAlways=(
"assume-digests" in mysettings.features))
except FileNotFound as e:
writemsg(_("!!! File %s doesn't exist, can't update "
"Manifest\n") % e, noiselevel=-1)
return 0
except PortagePackageException as e:
writemsg(("!!! %s\n") % (e,), noiselevel=-1)
return 0
try:
mf.write(sign=False)
except PermissionDenied as e:
writemsg(_("!!! Permission Denied: %s\n") % (e,), noiselevel=-1)
return 0
if "assume-digests" not in mysettings.features:
distlist = list(mf.fhashdict.get("DIST", {}))
distlist.sort()
auto_assumed = []
for filename in distlist:
if not os.path.exists(
os.path.join(mysettings["DISTDIR"], filename)):
auto_assumed.append(filename)
if auto_assumed:
cp = os.path.sep.join(mysettings["O"].split(os.path.sep)[-2:])
pkgs = myportdb.cp_list(cp, mytree=mytree)
pkgs.sort()
writemsg_stdout(" digest.assumed" + colorize("WARN",
str(len(auto_assumed)).rjust(18)) + "\n")
for pkg_key in pkgs:
fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree)
pv = pkg_key.split("/")[1]
for filename in auto_assumed:
if filename in fetchlist:
writemsg_stdout(
" %s::%s\n" % (pv, filename))
return 1
finally:
portage._doebuild_manifest_exempt_depend -= 1
|
GitWeb