mirrors-python
/
distlib
mirror of https://bitbucket.org/pypa/distlib.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity Software
Browse Source

Use resources rather than direct file access.

master
Vinay Sajip 9 years ago
parent
d00a7ad428
commit
55dc543a3b
7 changed files with 155 additions and 102 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      CONTRIBUTORS.txt
  2. 185
      distlib/database.py
  3. 6
      distlib/metadata.py
  4. 10
      distlib/util.py
  5. 2
      distlib/wheel.py
  6. 27
      tests/test_database.py
  7. 26
      tests/test_util.py

1
CONTRIBUTORS.txt
Unescape View File

@ -72,6 +72,7 @@ Thanks to:
- Sean Reifschneider
- Antoine Reversat
- Arc Riley
- C. Anthony Risinger
- Elson Rodriguez
- Luis Rojas
- Erik Rose

185
distlib/database.py
Unescape View File

@ -9,16 +9,17 @@ from __future__ import unicode_literals
import base64
import codecs
import contextlib
import hashlib
import logging
import os
import posixpath
import sys
import zipimport
from . import DistlibException
from .compat import StringIO, string_types
from . import DistlibException, resources
from .compat import StringIO
from .version import get_scheme, UnsupportedVersionError
from .markers import interpret
from .metadata import Metadata, METADATA_FILENAME
from .util import (parse_requirement, cached_property, parse_name_and_version,
read_exports, write_exports, CSVReader, CSVWriter)
@ -68,6 +69,7 @@ class _Cache(object):
self.path[dist.path] = dist
self.name.setdefault(dist.key, []).append(dist)
class DistributionPath(object):
"""
Represents a set of distributions installed on a path (typically sys.path).
@ -112,17 +114,39 @@ class DistributionPath(object):
"""
Yield .dist-info and/or .egg(-info) distributions.
"""
# We need to check if we've seen some resources already, because on
# some Linux systems (e.g. some Debian/Ubuntu variants) there are
# symlinks which alias other files in the environment.
seen = set()
for path in self.path:
realpath = os.path.realpath(path)
if not os.path.isdir(realpath):
finder = resources.finder_for_path(path)
if finder is None:
continue
r = finder.find('')
if not r or not r.is_container:
continue
for dir in os.listdir(realpath):
dist_path = os.path.join(realpath, dir)
if self._include_dist and dir.endswith(DISTINFO_EXT):
yield new_dist_class(dist_path, env=self)
elif self._include_egg and dir.endswith(('.egg-info',
'.egg')):
yield old_dist_class(dist_path, self)
rset = sorted(r.resources)
for entry in rset:
r = finder.find(entry)
if not r or r.path in seen:
continue
if self._include_dist and entry.endswith(DISTINFO_EXT):
metadata_path = posixpath.join(entry, METADATA_FILENAME)
pydist = finder.find(metadata_path)
if not pydist:
continue
metadata = Metadata(fileobj=pydist.as_stream(),
scheme='legacy')
logger.debug('Found %s', r.path)
seen.add(r.path)
yield new_dist_class(r.path, metadata=metadata,
env=self)
elif self._include_egg and entry.endswith(('.egg-info',
'.egg')):
logger.debug('Found %s', r.path)
seen.add(r.path)
yield old_dist_class(r.path, self)
def _generate_cache(self):
"""
@ -165,7 +189,6 @@ class DistributionPath(object):
name = name.replace('-', '_')
return '-'.join([name, version]) + DISTINFO_EXT
def get_distributions(self):
"""
Provides an iterator that looks for distributions and returns
@ -188,7 +211,6 @@ class DistributionPath(object):
for dist in self._cache_egg.path.values():
yield dist
def get_distribution(self, name):
"""
Looks for a named distribution on the path.
@ -278,6 +300,7 @@ class DistributionPath(object):
for v in d.values():
yield v
class Distribution(object):
"""
A base class for distributions, whether installed or from indexes.
@ -308,8 +331,8 @@ class Distribution(object):
self.version = metadata.version
self.locator = None
self.md5_digest = None
self.extras = None # additional features requested during installation
self.context = None # environment marker overrides
self.extras = None # additional features requested
self.context = None # environment marker overrides
@property
def source_url(self):
@ -452,7 +475,7 @@ class BaseInstalledDistribution(Distribution):
"""
super(BaseInstalledDistribution, self).__init__(metadata)
self.path = path
self.dist_path = env
self.dist_path = env
def get_hash(self, data, hasher=None):
"""
@ -485,34 +508,44 @@ class BaseInstalledDistribution(Distribution):
digest = base64.urlsafe_b64encode(digest).rstrip(b'=').decode('ascii')
return '%s%s' % (prefix, digest)
class InstalledDistribution(BaseInstalledDistribution):
"""Created with the *path* of the ``.dist-info`` directory provided to the
"""
Created with the *path* of the ``.dist-info`` directory provided to the
constructor. It reads the metadata contained in ``pydist.json`` when it is
instantiated., or uses a passed in Metadata instance (useful for when
dry-run mode is being used)."""
dry-run mode is being used).
"""
hasher = 'sha256'
def __init__(self, path, metadata=None, env=None):
self.finder = finder = resources.finder_for_path(path)
if finder is None:
import pdb; pdb.set_trace ()
if env and env._cache_enabled and path in env._cache.path:
metadata = env._cache.path[path].metadata
elif metadata is None:
metadata_path = os.path.join(path, METADATA_FILENAME)
r = finder.find(METADATA_FILENAME)
# Temporary - for legacy support
if not os.path.exists(metadata_path):
metadata_path = os.path.join(path, 'METADATA')
if not os.path.exists(metadata_path):
if r is None:
r = finder.find('METADATA')
if r is None:
raise ValueError('no %s found in %s' % (METADATA_FILENAME,
path))
metadata = Metadata(path=metadata_path, scheme='legacy')
with contextlib.closing(r.as_stream()) as stream:
metadata = Metadata(fileobj=stream, scheme='legacy')
super(InstalledDistribution, self).__init__(metadata, path, env)
if env and env._cache_enabled:
env._cache.add(self)
path = self.get_distinfo_file('REQUESTED')
self.requested = os.path.exists(path)
try:
r = finder.find('REQUESTED')
except AttributeError:
import pdb; pdb.set_trace ()
self.requested = r is not None
def __repr__(self):
return '<InstalledDistribution %r %s at %r>' % (
@ -529,18 +562,19 @@ class InstalledDistribution(BaseInstalledDistribution):
as stored in the file (which is as in PEP 376).
"""
results = []
path = self.get_distinfo_file('RECORD')
with CSVReader(path) as record_reader:
# Base location is parent dir of .dist-info dir
#base_location = os.path.dirname(self.path)
#base_location = os.path.abspath(base_location)
for row in record_reader:
missing = [None for i in range(len(row), 3)]
path, checksum, size = row + missing
#if not os.path.isabs(path):
# path = path.replace('/', os.sep)
# path = os.path.join(base_location, path)
results.append((path, checksum, size))
r = self.get_distinfo_resource('RECORD')
with contextlib.closing(r.as_stream()) as stream:
with CSVReader(stream=stream) as record_reader:
# Base location is parent dir of .dist-info dir
#base_location = os.path.dirname(self.path)
#base_location = os.path.abspath(base_location)
for row in record_reader:
missing = [None for i in range(len(row), 3)]
path, checksum, size = row + missing
#if not os.path.isabs(path):
# path = path.replace('/', os.sep)
# path = os.path.join(base_location, path)
results.append((path, checksum, size))
return results
@cached_property
@ -552,39 +586,34 @@ class InstalledDistribution(BaseInstalledDistribution):
export entries.
"""
result = {}
rf = self.get_distinfo_file(EXPORTS_FILENAME)
if os.path.exists(rf):
result = self.read_exports(rf)
r = self.get_distinfo_resource(EXPORTS_FILENAME)
if r:
result = self.read_exports()
return result
def read_exports(self, filename=None):
def read_exports(self):
"""
Read exports data from a file in .ini format.
:param filename: An absolute pathname of the file to read. If not
specified, the EXPORTS file in the .dist-info
directory of the distribution is read.
:return: A dictionary of exports, mapping an export category to a list
of :class:`ExportEntry` instances describing the individual
export entries.
"""
result = {}
rf = filename or self.get_distinfo_file(EXPORTS_FILENAME)
if os.path.exists(rf):
with open(rf, 'r') as f:
result = read_exports(f)
r = self.get_distinfo_resource(EXPORTS_FILENAME)
if r:
with contextlib.closing(r.as_stream()) as stream:
result = read_exports(stream)
return result
def write_exports(self, exports, filename=None):
def write_exports(self, exports):
"""
Write a dictionary of exports to a file in .ini format.
:param exports: A dictionary of exports, mapping an export category to
a list of :class:`ExportEntry` instances describing the
individual export entries.
:param filename: The absolute pathname of the file to write to. If not
specified, the EXPORTS file in the .dist-info
directory is written to.
"""
rf = filename or self.get_distinfo_file(EXPORTS_FILENAME)
rf = self.get_distinfo_file(EXPORTS_FILENAME)
with open(rf, 'w') as f:
write_exports(exports, f)
@ -599,11 +628,12 @@ class InstalledDistribution(BaseInstalledDistribution):
of interest.
:return: The absolute path where the resource is to be found.
"""
path = self.get_distinfo_file('RESOURCES')
with CSVReader(path) as resources_reader:
for relative, destination in resources_reader:
if relative == relative_path:
return destination
r = self.get_distinfo_resource('RESOURCES')
with contextlib.closing(r.as_stream()) as stream:
with CSVReader(stream=stream) as resources_reader:
for relative, destination in resources_reader:
if relative == relative_path:
return destination
raise KeyError('no resource file with relative path %r '
'is installed' % relative_path)
@ -628,7 +658,7 @@ class InstalledDistribution(BaseInstalledDistribution):
base = os.path.dirname(self.path)
base_under_prefix = base.startswith(prefix)
base = os.path.join(base, '')
record_path = os.path.join(self.path, 'RECORD')
record_path = self.get_distinfo_file('RECORD')
logger.info('creating %s', record_path)
if dry_run:
return None
@ -642,7 +672,7 @@ class InstalledDistribution(BaseInstalledDistribution):
with open(path, 'rb') as fp:
hash_value = self.get_hash(fp.read())
if path.startswith(base) or (base_under_prefix and
path.startswith(prefix)):
path.startswith(prefix)):
path = os.path.relpath(path, base)
writer.writerow((path, hash_value, size))
@ -663,7 +693,7 @@ class InstalledDistribution(BaseInstalledDistribution):
"""
mismatches = []
base = os.path.dirname(self.path)
record_path = os.path.join(self.path, 'RECORD')
record_path = self.get_distinfo_file('RECORD')
for path, hash_value, size in self.list_installed_files():
if not os.path.isabs(path):
path = os.path.join(base, path)
@ -739,6 +769,15 @@ class InstalledDistribution(BaseInstalledDistribution):
f.write('\n'.join(lines))
return shared_path
def get_distinfo_resource(self, path):
if path not in DIST_FILES:
raise DistlibException('invalid path for a dist-info file: '
'%r at %r' % (path, self.path))
finder = resources.finder_for_path(self.path)
if finder is None:
raise DistlibException('Unable to get a finder for %s' % self.path)
return finder.find(path)
def get_distinfo_file(self, path):
"""
Returns a path located under the ``.dist-info`` directory. Returns a
@ -749,7 +788,7 @@ class InstalledDistribution(BaseInstalledDistribution):
If *path* is an absolute path and doesn't start
with the ``.dist-info`` directory path,
a :class:`DistlibException` is raised
:type path: string
:type path: str
:rtype: str
"""
# Check if it is an absolute path # XXX use relpath, add tests
@ -808,7 +847,7 @@ class EggInfoDistribution(BaseInstalledDistribution):
s.version = v
self.path = path
self.dist_path = env
self.dist_path = env
if env and env._cache_enabled and path in env._cache_egg.path:
metadata = env._cache_egg.path[path].metadata
set_name_and_version(self, metadata.name, metadata.version)
@ -916,14 +955,14 @@ class EggInfoDistribution(BaseInstalledDistribution):
mismatches = []
record_path = os.path.join(self.path, 'installed-files.txt')
if os.path.exists(record_path):
for path, hash, size in self.list_installed_files():
for path, _, _ in self.list_installed_files():
if path == record_path:
continue
if not os.path.exists(path):
mismatches.append((path, 'exists', True, False))
return mismatches
def list_installed_files(self, local=False):
def list_installed_files(self):
"""
Iterates over the ``installed-files.txt`` entries and returns a tuple
``(path, hash, size)`` for each line.
@ -961,16 +1000,16 @@ class EggInfoDistribution(BaseInstalledDistribution):
result.append((record_path, None, None))
return result
def list_distinfo_files(self, local=False):
def list_distinfo_files(self, absolute=False):
"""
Iterates over the ``installed-files.txt`` entries and returns paths for
each line if the path is pointing to a file located in the
``.egg-info`` directory or one of its subdirectories.
:parameter local: If *local* is ``True``, each returned path is
:parameter absolute: If *absolute* is ``True``, each returned path is
transformed into a local absolute path. Otherwise the
raw value from ``installed-files.txt`` is returned.
:type local: boolean
:type absolute: boolean
:returns: iterator of paths
"""
record_path = os.path.join(self.path, 'installed-files.txt')
@ -984,7 +1023,7 @@ class EggInfoDistribution(BaseInstalledDistribution):
if not skip:
p = os.path.normpath(os.path.join(self.path, line))
if p.startswith(self.path):
if local:
if absolute:
yield p
else:
yield line
@ -1091,7 +1130,7 @@ class DependencyGraph(object):
for other, label in adjs:
if not label is None:
f.write('"%s" -> "%s" [label="%s"]\n' %
(dist.name, other.name, label))
(dist.name, other.name, label))
else:
f.write('"%s" -> "%s"\n' % (dist.name, other.name))
if not skip_disconnected and len(disconnected) > 0:
@ -1132,7 +1171,7 @@ class DependencyGraph(object):
for k, v in alist.items():
alist[k] = [(d, r) for d, r in v if d not in to_remove]
logger.debug('Moving to result: %s',
['%s (%s)' % (d.name, d.version) for d in to_remove])
['%s (%s)' % (d.name, d.version) for d in to_remove])
result.extend(to_remove)
return result, list(alist.keys())
@ -1223,6 +1262,7 @@ def get_dependent_dists(dists, dist):
dep.pop(0) # remove dist from dep, was there to prevent infinite loops
return dep
def get_required_dists(dists, dist):
"""Recursively generate a list of distributions from *dists* that are
required by *dist*.
@ -1247,6 +1287,7 @@ def get_required_dists(dists, dist):
return req
def make_dist(name, version, **kwargs):
"""
A convenience method for making a dist given just a name and version.

6
distlib/metadata.py
Unescape View File

@ -17,7 +17,7 @@ import re
from . import DistlibException, __version__
from .compat import StringIO, string_types
from .compat import StringIO, string_types, text_type
from .markers import interpret
from .util import extract_by_key
from .version import get_scheme, PEP426_VERSION_RE
@ -722,7 +722,7 @@ class Metadata(object):
else:
data = None
if path:
with codecs.open(path, 'r', 'utf-8') as f:
with open(path, 'rb') as f:
data = f.read()
elif fileobj:
data = fileobj.read()
@ -733,6 +733,8 @@ class Metadata(object):
'generator': self.GENERATOR,
}
else:
if not isinstance(data, text_type):
data = data.decode('utf-8')
try:
self._data = json.loads(data)
self._validate_mapping(self._data, scheme)

10
distlib/util.py
Unescape View File

@ -185,6 +185,9 @@ def extract_by_key(d, keys):
return result
def read_exports(stream):
if sys.version_info[0] >= 3:
# needs to be a text stream
stream = codecs.getreader('utf-8')(stream)
cp = configparser.ConfigParser()
if hasattr(cp, 'read_file'):
cp.read_file(stream)
@ -203,6 +206,9 @@ def read_exports(stream):
def write_exports(exports, stream):
if sys.version_info[0] >= 3:
# needs to be a text stream
stream = codecs.getwriter('utf-8')(stream)
cp = configparser.ConfigParser()
for k, v in exports.items():
# TODO check k, v for valid values
@ -1311,7 +1317,7 @@ class CSVBase(object):
class CSVReader(CSVBase):
def __init__(self, fn, **kwargs):
def __init__(self, **kwargs):
if 'stream' in kwargs:
stream = kwargs['stream']
if sys.version_info[0] >= 3:
@ -1319,7 +1325,7 @@ class CSVReader(CSVBase):
stream = codecs.getreader('utf-8')(stream)
self.stream = stream
else:
self.stream = _csv_open(fn, 'r')
self.stream = _csv_open(kwargs['path'], 'r')
self.reader = csv.reader(self.stream, **self.defaults)
def __iter__(self):

2
distlib/wheel.py
Unescape View File

@ -416,7 +416,7 @@ class Wheel(object):
libdir = paths['platlib']
records = {}
with zf.open(record_name) as bf:
with CSVReader(record_name, stream=bf) as reader:
with CSVReader(stream=bf) as reader:
for row in reader:
p = row[0]
records[p] = row

27
tests/test_database.py
Unescape View File

@ -25,7 +25,8 @@ from distlib.database import (InstalledDistribution, EggInfoDistribution,
BaseInstalledDistribution, EXPORTS_FILENAME,
DistributionPath, make_graph,
get_required_dists, get_dependent_dists)
from distlib.util import get_resources_dests, ExportEntry, CSVReader
from distlib.util import (get_resources_dests, ExportEntry, CSVReader,
read_exports, write_exports)
from test_util import GlobTestCaseBase
from support import LoggingCatcher, requires_zlib
@ -183,7 +184,7 @@ class TestDistribution(CommonDistributionTests, unittest.TestCase):
prefix = os.path.dirname(dist_location)
dist.write_installed_files(get_files(dist_location), prefix)
with CSVReader(record_file) as record_reader:
with CSVReader(path=record_file) as record_reader:
record_data = {}
for row in record_reader:
if row == []:
@ -582,28 +583,6 @@ class TestDatabase(LoggingCatcher,
import cgi
self.assertIs(e.value, cgi.print_directory)
def test_write_exports(self):
exports = {
'foo': {
'v1': ExportEntry('v1', 'p1', 's1', []),
'v2': ExportEntry('v2', 'p2', 's2', ['f2=a', 'g2']),
},
'bar': {
'v3': ExportEntry('v3', 'p3', 's3', ['f3', 'g3=h']),
'v4': ExportEntry('v4', 'p4', 's4', ['f4', 'g4']),
},
}
fd, fn = tempfile.mkstemp()
try:
os.close(fd)
d = DistributionPath().get_distribution('babar') # any one will do
d.write_exports(exports, fn)
actual = d.read_exports(fn)
self.assertEqual(actual, exports)
finally:
os.remove(fn)
def test_exports_iteration(self):
d = DistributionPath()
expected = set((

26
tests/test_util.py
Unescape View File

@ -25,7 +25,7 @@ from distlib.util import (get_export_entry, ExportEntry, resolve,
parse_credentials, ensure_slash, split_filename,
EventMixin, Sequencer, unarchive, Progress,
iglob, RICH_GLOB, parse_requirement,
Configurator,
Configurator, read_exports, write_exports,
FileOperator, is_string_sequence, get_package_data)
@ -836,5 +836,29 @@ class GlobTestCase(GlobTestCaseBase):
r = parse_requirement('a [%s]' % e)
validate(r, ('a', None, [e], 'a', None))
def test_write_exports(self):
exports = {
'foo': {
'v1': ExportEntry('v1', 'p1', 's1', []),
'v2': ExportEntry('v2', 'p2', 's2', ['f2=a', 'g2']),
},
'bar': {
'v3': ExportEntry('v3', 'p3', 's3', ['f3', 'g3=h']),
'v4': ExportEntry('v4', 'p4', 's4', ['f4', 'g4']),
},
}
fd, fn = tempfile.mkstemp()
try:
os.close(fd)
with open(fn, 'wb') as f:
write_exports(exports, f)
with open(fn, 'rb') as f:
actual = read_exports(f)
self.assertEqual(actual, exports)
finally:
os.remove(fn)
if __name__ == '__main__': # pragma: no cover
unittest.main()

Write Preview
Loading…
Cancel
Save