Browse Source

Run black on codebase (#336)

* run black on codebase

* add black check to travis ci

* add pyproject.toml, revert black on bottle.py

Co-authored-by: Pelle Koster <pelle.koster@nginfra.nl>
pull/342/head
PelleK 2 years ago committed by GitHub
parent
commit
8101cf9192
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 6
      .travis.yml
  2. 79
      bin/bumpver.py
  3. 104
      bootstrap.py
  4. 92
      pypiserver/__init__.py
  5. 127
      pypiserver/__main__.py
  6. 2
      pypiserver/_app.py
  7. 30
      pypiserver/cache.py
  8. 140
      pypiserver/core.py
  9. 60
      pypiserver/manage.py
  10. 27
      pyproject.toml
  11. 11
      tests/centodeps-setup.py
  12. 16
      tests/test_app.py
  13. 95
      tests/test_core.py
  14. 8
      tests/test_docs.py
  15. 43
      tests/test_init.py
  16. 65
      tests/test_main.py
  17. 236
      tests/test_manage.py
  18. 227
      tests/test_server.py

6
.travis.yml

@ -19,3 +19,9 @@ script:
branches:
except:
- standalone
jobs:
include:
- python: 3.8
install: pip install -U black
script: black --check .

79
bin/bumpver.py

@ -45,13 +45,13 @@ import docopt
my_dir = osp.dirname(__file__)
VFILE = osp.join(my_dir, '..', 'pypiserver', '__init__.py')
VFILE = osp.join(my_dir, "..", "pypiserver", "__init__.py")
VFILE_regex_v = re.compile(r'version *= *__version__ *= *"([^"]+)"')
VFILE_regex_d = re.compile(r'__updated__ *= *"([^"]+)"')
RFILE = osp.join(my_dir, '..', 'README.rst')
RFILE = osp.join(my_dir, "..", "README.rst")
PYTEST_ARGS = [osp.join('tests', 'test_docs.py')]
PYTEST_ARGS = [osp.join("tests", "test_docs.py")]
class CmdException(Exception):
@ -60,7 +60,7 @@ class CmdException(Exception):
@fnt.lru_cache()
def read_txtfile(fpath):
with open(fpath, 'rt', encoding='utf-8') as fp:
with open(fpath, "rt", encoding="utf-8") as fp:
return fp.read()
@ -75,9 +75,10 @@ def extract_file_regexes(fpath, regexes):
matches = [regex.search(txt) for regex in regexes]
if not all(matches):
raise CmdException("Failed extracting current versions with: %s"
"\n matches: %s" %
(regexes, matches))
raise CmdException(
"Failed extracting current versions with: %s"
"\n matches: %s" % (regexes, matches)
)
return [m.group(1) for m in matches]
@ -96,8 +97,7 @@ def replace_substrings(files, subst_pairs):
def format_syscmd(cmd):
if isinstance(cmd, (list, tuple)):
cmd = ' '.join('"%s"' % s if ' ' in s else s
for s in cmd)
cmd = " ".join('"%s"' % s if " " in s else s for s in cmd)
else:
assert isinstance(cmd, str), cmd
@ -107,7 +107,7 @@ def format_syscmd(cmd):
def strip_ver2_commonprefix(ver1, ver2):
cprefix = osp.commonprefix([ver1, ver2])
if cprefix:
striplen = cprefix.rfind('.')
striplen = cprefix.rfind(".")
if striplen > 0:
striplen += 1
else:
@ -123,7 +123,8 @@ def run_testcases():
retcode = pytest.main(PYTEST_ARGS)
if retcode:
raise CmdException(
"Doc TCs failed(%s), probably version-bumping has failed!" % retcode)
"Doc TCs failed(%s), probably version-bumping has failed!" % retcode
)
def exec_cmd(cmd):
@ -137,14 +138,14 @@ def exec_cmd(cmd):
def do_commit(new_ver, old_ver, dry_run, amend, ver_files):
import pathlib
#new_ver = strip_ver2_commonprefix(old_ver, new_ver)
cmt_msg = 'chore(ver): bump %s-->%s' % (old_ver, new_ver)
# new_ver = strip_ver2_commonprefix(old_ver, new_ver)
cmt_msg = "chore(ver): bump %s-->%s" % (old_ver, new_ver)
ver_files = [pathlib.Path(f).as_posix() for f in ver_files]
git_add = ['git', 'add'] + ver_files
git_cmt = ['git', 'commit', '-m', cmt_msg]
git_add = ["git", "add"] + ver_files
git_cmt = ["git", "commit", "-m", cmt_msg]
if amend:
git_cmt.append('--amend')
git_cmt.append("--amend")
commands = [git_add, git_cmt]
for cmd in commands:
@ -157,9 +158,9 @@ def do_commit(new_ver, old_ver, dry_run, amend, ver_files):
def do_tag(tag, tag_msg, dry_run, force):
cmd = ['git', 'tag', tag, '-s', '-m', tag_msg]
cmd = ["git", "tag", tag, "-s", "-m", tag_msg]
if force:
cmd.append('--force')
cmd.append("--force")
cmd_str = format_syscmd(cmd)
if dry_run:
yield "DRYRUN: %s" % cmd_str
@ -168,15 +169,16 @@ def do_tag(tag, tag_msg, dry_run, force):
exec_cmd(cmd)
def bumpver(new_ver, dry_run=False, force=False, amend=False,
tag_name_or_commit=None):
def bumpver(
new_ver, dry_run=False, force=False, amend=False, tag_name_or_commit=None
):
"""
:param tag_name_or_commit:
if true, do `git commit`, if string, also `git tag` with that as msg.
"""
if amend:
## Restore previous version before extracting it.
cmd = 'git checkout HEAD~ --'.split()
cmd = "git checkout HEAD~ --".split()
cmd.append(VFILE)
cmd.append(RFILE)
exec_cmd(cmd)
@ -199,7 +201,7 @@ def bumpver(new_ver, dry_run=False, force=False, amend=False,
from datetime import datetime
new_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S%z')
new_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S%z")
ver_files = [osp.normpath(f) for f in [VFILE, RFILE]]
subst_pairs = [(old_ver, new_ver), (old_date, new_date)]
@ -208,12 +210,12 @@ def bumpver(new_ver, dry_run=False, force=False, amend=False,
new_txt, fpath, replacements = repl
if not dry_run:
with open(fpath, 'wt', encoding='utf-8') as fp:
with open(fpath, "wt", encoding="utf-8") as fp:
fp.write(new_txt)
yield '%s: ' % fpath
yield "%s: " % fpath
for old, new, nrepl in replacements:
yield ' %i x (%24s --> %s)' % (nrepl, old, new)
yield " %i x (%24s --> %s)" % (nrepl, old, new)
yield "...now launching DocTCs..."
run_testcases()
@ -222,20 +224,21 @@ def bumpver(new_ver, dry_run=False, force=False, amend=False,
yield from do_commit(new_ver, old_ver, dry_run, amend, ver_files)
if isinstance(tag_name_or_commit, str):
tag = 'v%s' % new_ver
tag = "v%s" % new_ver
yield from do_tag(tag, tag_name_or_commit, dry_run, force)
def main(*args):
opts = docopt.docopt(__doc__, argv=args)
new_ver = opts['<new-ver>']
new_ver = opts["<new-ver>"]
assert not new_ver or new_ver[0] != 'v', (
"Version '%s' must NOT start with `v`!" % new_ver)
assert not new_ver or new_ver[0] != "v", (
"Version '%s' must NOT start with `v`!" % new_ver
)
commit = opts['--commit']
tag = opts['--tag']
commit = opts["--commit"]
tag = opts["--tag"]
if tag:
tag_name_or_commit = tag
elif commit:
@ -244,11 +247,13 @@ def main(*args):
tag_name_or_commit = None
try:
for i in bumpver(new_ver,
opts['--dry-run'],
opts['--force'],
opts['--amend'],
tag_name_or_commit):
for i in bumpver(
new_ver,
opts["--dry-run"],
opts["--force"],
opts["--amend"],
tag_name_or_commit,
):
print(i)
except CmdException as ex:
sys.exit(str(ex))
@ -256,5 +261,5 @@ def main(*args):
raise ex
if __name__ == '__main__':
if __name__ == "__main__":
main(*sys.argv[1:])

104
bootstrap.py

@ -23,7 +23,7 @@ from optparse import OptionParser
tmpeggs = tempfile.mkdtemp()
usage = '''\
usage = """\
[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
Bootstraps a buildout-based project.
@ -33,25 +33,34 @@ Python that you want bin/buildout to use.
Note that by using --setup-source and --download-base to point to
local resources, you can keep this script from going over the network.
'''
"""
parser = OptionParser(usage=usage)
parser.add_option("-v", "--version", help="use a specific zc.buildout version")
parser.add_option("-t", "--accept-buildout-test-releases",
dest='accept_buildout_test_releases',
action="store_true", default=False,
help=("Normally, if you do not specify a --version, the "
"bootstrap script and buildout gets the newest "
"*final* versions of zc.buildout and its recipes and "
"extensions for you. If you use this flag, "
"bootstrap and buildout will get the newest releases "
"even if they are alphas or betas."))
parser.add_option("-c", "--config-file",
help=("Specify the path to the buildout configuration "
"file to be used."))
parser.add_option("-f", "--find-links",
help="Specify a URL to search for buildout releases")
parser.add_option(
"-t",
"--accept-buildout-test-releases",
dest="accept_buildout_test_releases",
action="store_true",
default=False,
help=(
"Normally, if you do not specify a --version, the "
"bootstrap script and buildout gets the newest "
"*final* versions of zc.buildout and its recipes and "
"extensions for you. If you use this flag, "
"bootstrap and buildout will get the newest releases "
"even if they are alphas or betas."
),
)
parser.add_option(
"-c",
"--config-file",
help=("Specify the path to the buildout configuration " "file to be used."),
)
parser.add_option(
"-f", "--find-links", help="Specify a URL to search for buildout releases"
)
options, args = parser.parse_args()
@ -62,7 +71,8 @@ options, args = parser.parse_args()
to_reload = False
try:
import pkg_resources, setuptools
if not hasattr(pkg_resources, '_distribute'):
if not hasattr(pkg_resources, "_distribute"):
to_reload = True
raise ImportError
except ImportError:
@ -73,13 +83,14 @@ except ImportError:
except ImportError:
from urllib2 import urlopen
exec(urlopen('http://python-distribute.org/distribute_setup.py').read(), ez)
exec(urlopen("http://python-distribute.org/distribute_setup.py").read(), ez)
setup_args = dict(to_dir=tmpeggs, download_delay=0, no_fake=True)
ez['use_setuptools'](**setup_args)
ez["use_setuptools"](**setup_args)
if to_reload:
reload(pkg_resources)
import pkg_resources
# This does not (always?) update the default working set. We will
# do it.
for path in sys.path:
@ -89,37 +100,47 @@ except ImportError:
######################################################################
# Install buildout
ws = pkg_resources.working_set
ws = pkg_resources.working_set
cmd = [sys.executable, '-c',
'from setuptools.command.easy_install import main; main()',
'-mZqNxd', tmpeggs]
cmd = [
sys.executable,
"-c",
"from setuptools.command.easy_install import main; main()",
"-mZqNxd",
tmpeggs,
]
find_links = os.environ.get(
'bootstrap-testing-find-links',
options.find_links or
('http://downloads.buildout.org/'
if options.accept_buildout_test_releases else None)
)
"bootstrap-testing-find-links",
options.find_links
or (
"http://downloads.buildout.org/"
if options.accept_buildout_test_releases
else None
),
)
if find_links:
cmd.extend(['-f', find_links])
cmd.extend(["-f", find_links])
distribute_path = ws.find(
pkg_resources.Requirement.parse('distribute')).location
pkg_resources.Requirement.parse("distribute")
).location
requirement = 'zc.buildout'
requirement = "zc.buildout"
version = options.version
if version is None and not options.accept_buildout_test_releases:
# Figure out the most recent final version of zc.buildout.
import setuptools.package_index
_final_parts = '*final-', '*final'
_final_parts = "*final-", "*final"
def _final_version(parsed_version):
for part in parsed_version:
if (part[:1] == '*') and (part not in _final_parts):
if (part[:1] == "*") and (part not in _final_parts):
return False
return True
index = setuptools.package_index.PackageIndex(
search_path=[distribute_path])
index = setuptools.package_index.PackageIndex(search_path=[distribute_path])
if find_links:
index.add_find_links((find_links,))
req = pkg_resources.Requirement.parse(requirement)
@ -138,14 +159,13 @@ if version is None and not options.accept_buildout_test_releases:
best.sort()
version = best[-1].version
if version:
requirement = '=='.join((requirement, version))
requirement = "==".join((requirement, version))
cmd.append(requirement)
import subprocess
if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=distribute_path)) != 0:
raise Exception(
"Failed to execute command:\n%s",
repr(cmd)[1:-1])
raise Exception("Failed to execute command:\n%s", repr(cmd)[1:-1])
######################################################################
# Import and run buildout
@ -154,12 +174,12 @@ ws.add_entry(tmpeggs)
ws.require(requirement)
import zc.buildout.buildout
if not [a for a in args if '=' not in a]:
args.append('bootstrap')
if not [a for a in args if "=" not in a]:
args.append("bootstrap")
# if -c was provided, we push it back into args for buildout' main function
if options.config_file is not None:
args[0:0] = ['-c', options.config_file]
args[0:0] = ["-c", options.config_file]
zc.buildout.buildout.main(args)
shutil.rmtree(tmpeggs)

92
pypiserver/__init__.py

@ -3,7 +3,7 @@ import re as _re
import sys
version = __version__ = "1.4.0"
__version_info__ = tuple(_re.split('[.-]', __version__))
__version_info__ = tuple(_re.split("[.-]", __version__))
__updated__ = "2020-10-03 17:45:07"
__title__ = "pypiserver"
@ -20,11 +20,12 @@ class Configuration(object):
vars(self).update(kwds)
def __repr__(self, *args, **kwargs):
return 'Configuration(**%s)' % vars(self)
return "Configuration(**%s)" % vars(self)
def __str__(self, *args, **kwargs):
return 'Configuration:\n%s' % '\n'.join('%20s = %s' % (k, v)
for k, v in sorted(vars(self).items()))
return "Configuration:\n%s" % "\n".join(
"%20s = %s" % (k, v) for k, v in sorted(vars(self).items())
)
def update(self, props):
d = props if isinstance(props, dict) else vars(props)
@ -35,27 +36,28 @@ DEFAULT_SERVER = "auto"
def default_config(
root=None,
host="0.0.0.0",
port=8080,
server=DEFAULT_SERVER,
redirect_to_fallback=True,
fallback_url=None,
authenticated=['update'],
password_file=None,
overwrite=False,
hash_algo='md5',
verbosity=1,
log_file=None,
log_stream="stderr",
log_frmt="%(asctime)s|%(name)s|%(levelname)s|%(thread)d|%(message)s",
log_req_frmt="%(bottle.request)s",
log_res_frmt="%(status)s",
log_err_frmt="%(body)s: %(exception)s \n%(traceback)s",
welcome_file=None,
cache_control=None,
auther=None,
VERSION=__version__):
root=None,
host="0.0.0.0",
port=8080,
server=DEFAULT_SERVER,
redirect_to_fallback=True,
fallback_url=None,
authenticated=["update"],
password_file=None,
overwrite=False,
hash_algo="md5",
verbosity=1,
log_file=None,
log_stream="stderr",
log_frmt="%(asctime)s|%(name)s|%(levelname)s|%(thread)d|%(message)s",
log_req_frmt="%(bottle.request)s",
log_res_frmt="%(status)s",
log_err_frmt="%(body)s: %(exception)s \n%(traceback)s",
welcome_file=None,
cache_control=None,
auther=None,
VERSION=__version__,
):
"""
Fetch default-opts with overridden kwds, capable of starting-up pypiserver.
@ -126,19 +128,19 @@ def app(**kwds):
from . import core
_app = __import__("_app", globals(), locals(), ["."], 1)
sys.modules.pop('pypiserver._app', None)
sys.modules.pop("pypiserver._app", None)
kwds = default_config(**kwds)
config, packages = core.configure(**kwds)
_app.config = config
_app.packages = packages
_app.app.module = _app # HACK for testing.
_app.app.module = _app # HACK for testing.
return _app.app
def str2bool(s, default):
if s is not None and s != '':
if s is not None and s != "":
return s.lower() not in ("no", "off", "0", "false")
return default
@ -164,7 +166,7 @@ def paste_app_factory(global_config, **local_conf):
if value is not None:
conf[attr] = int(value)
def upd_conf_with_list_item(conf, attr, sdict, sep=' ', parse=_str_strip):
def upd_conf_with_list_item(conf, attr, sdict, sep=" ", parse=_str_strip):
values = sdict.pop(attr, None)
if values:
conf[attr] = list(filter(None, map(parse, values.split(sep))))
@ -177,21 +179,21 @@ def paste_app_factory(global_config, **local_conf):
c = default_config()
upd_conf_with_bool_item(c, 'overwrite', local_conf)
upd_conf_with_bool_item(c, 'redirect_to_fallback', local_conf)
upd_conf_with_list_item(c, 'authenticated', local_conf, sep=' ')
upd_conf_with_list_item(c, 'root', local_conf, sep='\n', parse=_make_root)
upd_conf_with_int_item(c, 'verbosity', local_conf)
upd_conf_with_bool_item(c, "overwrite", local_conf)
upd_conf_with_bool_item(c, "redirect_to_fallback", local_conf)
upd_conf_with_list_item(c, "authenticated", local_conf, sep=" ")
upd_conf_with_list_item(c, "root", local_conf, sep="\n", parse=_make_root)
upd_conf_with_int_item(c, "verbosity", local_conf)
str_items = [
'fallback_url',
'hash_algo',
'log_err_frmt',
'log_file',
'log_frmt',
'log_req_frmt',
'log_res_frmt',
'password_file',
'welcome_file'
"fallback_url",
"hash_algo",
"log_err_frmt",
"log_file",
"log_frmt",
"log_req_frmt",
"log_res_frmt",
"password_file",
"welcome_file",
]
for str_item in str_items:
upd_conf_with_str_item(c, str_item, local_conf)
@ -203,9 +205,9 @@ def paste_app_factory(global_config, **local_conf):
def _logwrite(logger, level, msg):
if msg:
line_endings = ['\r\n', '\n\r', '\n']
line_endings = ["\r\n", "\n\r", "\n"]
for le in line_endings:
if msg.endswith(le):
msg = msg[:-len(le)]
msg = msg[: -len(le)]
if msg:
logger.log(level, msg)

127
pypiserver/__main__.py

@ -13,10 +13,16 @@ import textwrap
import functools as ft
log = logging.getLogger('pypiserver.main')
log = logging.getLogger("pypiserver.main")
def init_logging(level=logging.NOTSET, frmt=None, filename=None, stream=sys.stderr, logger=None):
def init_logging(
level=logging.NOTSET,
frmt=None,
filename=None,
stream=sys.stderr,
logger=None,
):
logger = logger or logging.getLogger()
logger.setLevel(level)
@ -31,8 +37,10 @@ def init_logging(level=logging.NOTSET, frmt=None, filename=None, stream=sys.stde
handler.setFormatter(formatter)
logger.addHandler(handler)
def usage():
return textwrap.dedent("""\
return textwrap.dedent(
"""\
pypi-server [OPTIONS] [PACKAGES_DIRECTORY...]
start PyPI compatible package server serving packages from
PACKAGES_DIRECTORY. If PACKAGES_DIRECTORY is not given on the
@ -159,7 +167,8 @@ def usage():
containing arbitrary code.
Visit https://pypi.org/project/pypiserver/ for more information.
""")
"""
)
def main(argv=None):
@ -178,29 +187,33 @@ def main(argv=None):
update_blacklist_file = None
try:
opts, roots = getopt.getopt(argv[1:], "i:p:a:r:d:P:Uuvxoh", [
"interface=",
"passwords=",
"authenticate=",
"port=",
"root=",
"server=",
"fallback-url=",
"disable-fallback",
"overwrite",
"hash-algo=",
"blacklist-file=",
"log-file=",
"log-stream=",
"log-frmt=",
"log-req-frmt=",
"log-res-frmt=",
"log-err-frmt=",
"welcome=",
"cache-control=",
"version",
"help"
])
opts, roots = getopt.getopt(
argv[1:],
"i:p:a:r:d:P:Uuvxoh",
[
"interface=",
"passwords=",
"authenticate=",
"port=",
"root=",
"server=",
"fallback-url=",
"disable-fallback",
"overwrite",
"hash-algo=",
"blacklist-file=",
"log-file=",
"log-stream=",
"log-frmt=",
"log-req-frmt=",
"log-res-frmt=",
"log-err-frmt=",
"welcome=",
"cache-control=",
"version",
"help",
],
)
except getopt.GetoptError:
err = sys.exc_info()[1]
sys.exit("usage error: %s" % (err,))
@ -213,10 +226,10 @@ def main(argv=None):
err = sys.exc_info()[1]
sys.exit("Invalid port(%r) due to: %s" % (v, err))
elif k in ("-a", "--authenticate"):
c.authenticated = [a.lower()
for a in re.split("[, ]+", v.strip(" ,"))
if a]
if c.authenticated == ['.']:
c.authenticated = [
a.lower() for a in re.split("[, ]+", v.strip(" ,")) if a
]
if c.authenticated == ["."]:
c.authenticated = []
else:
actions = ("list", "download", "update")
@ -275,57 +288,75 @@ def main(argv=None):
print(usage())
sys.exit(0)
if (not c.authenticated and c.password_file != '.' or
c.authenticated and c.password_file == '.'):
if (
not c.authenticated
and c.password_file != "."
or c.authenticated
and c.password_file == "."
):
auth_err = "When auth-ops-list is empty (-a=.), password-file (-P=%r) must also be empty ('.')!"
sys.exit(auth_err % c.password_file)
if len(roots) == 0:
roots.append(os.path.expanduser("~/packages"))
roots=[os.path.abspath(x) for x in roots]
roots = [os.path.abspath(x) for x in roots]
c.root = roots
verbose_levels=[
logging.WARNING, logging.INFO, logging.DEBUG, logging.NOTSET]
log_level=list(zip(verbose_levels, range(c.verbosity)))[-1][0]
verbose_levels = [
logging.WARNING,
logging.INFO,
logging.DEBUG,
logging.NOTSET,
]
log_level = list(zip(verbose_levels, range(c.verbosity)))[-1][0]
valid_streams = {"none": None, "stderr": sys.stderr, "stdout": sys.stdout}
if c.log_stream not in valid_streams:
sys.exit("invalid log stream %s. choose one of %s" % (
c.log_stream, ", ".join(valid_streams.keys())))
sys.exit(
"invalid log stream %s. choose one of %s"
% (c.log_stream, ", ".join(valid_streams.keys()))
)
init_logging(
level=log_level,
filename=c.log_file,
frmt=c.log_frmt,
stream=valid_streams[c.log_stream]
stream=valid_streams[c.log_stream],
)
if command == "update":
from pypiserver.manage import update_all_packages
update_all_packages(
roots, update_directory,
dry_run=update_dry_run, stable_only=update_stable_only,
blacklist_file=update_blacklist_file
roots,
update_directory,
dry_run=update_dry_run,
stable_only=update_stable_only,
blacklist_file=update_blacklist_file,
)
return
# Fixes #49:
# The gevent server adapter needs to patch some
# modules BEFORE importing bottle!
if c.server and c.server.startswith('gevent'):
if c.server and c.server.startswith("gevent"):
import gevent.monkey # @UnresolvedImport
gevent.monkey.patch_all()
from pypiserver import bottle
if c.server not in bottle.server_names:
sys.exit("unknown server %r. choose one of %s" % (
c.server, ", ".join(bottle.server_names.keys())))
sys.exit(
"unknown server %r. choose one of %s"
% (c.server, ", ".join(bottle.server_names.keys()))
)
bottle.debug(c.verbosity > 1)
bottle._stderr = ft.partial(pypiserver._logwrite,
logging.getLogger(bottle.__name__), logging.INFO)
bottle._stderr = ft.partial(
pypiserver._logwrite, logging.getLogger(bottle.__name__), logging.INFO
)
app = pypiserver.app(**vars(c))
bottle.run(app=app, host=c.host, port=c.port, server=c.server)

2
pypiserver/_app.py

@ -174,7 +174,7 @@ def file_upload():
):
raise HTTPError(
400,
"Unrelated signature %r for package %r!" % (ufiles.sig, ufiles.pkg)
"Unrelated signature %r for package %r!" % (ufiles.sig, ufiles.pkg),
)
for uf in ufiles:

30
pypiserver/cache.py

@ -8,20 +8,21 @@ from os.path import dirname
from watchdog.observers import Observer
import threading
class CacheManager(object):
"""
A naive cache implementation for listdir and digest_file
The listdir_cache is just a giant list of PkgFile objects, and
for simplicity it is invalidated anytime a modification occurs
within the directory it represents. If we were smarter about
the way that the listdir data structure were created/stored,
then we could do more granular invalidation. In practice, this
is good enough for now.
The digest_cache exists on a per-file basis, because computing
hashes on large files can get expensive, and it's very easy to
invalidate specific filenames.
A naive cache implementation for listdir and digest_file
The listdir_cache is just a giant list of PkgFile objects, and
for simplicity it is invalidated anytime a modification occurs
within the directory it represents. If we were smarter about
the way that the listdir data structure were created/stored,
then we could do more granular invalidation. In practice, this
is good enough for now.
The digest_cache exists on a per-file basis, because computing
hashes on large files can get expensive, and it's very easy to
invalidate specific filenames.
"""
def __init__(self):
@ -85,8 +86,8 @@ class CacheManager(object):
self.watched.add(root)
self.observer.schedule(_EventHandler(self, root), root, recursive=True)
class _EventHandler(object):
class _EventHandler(object):
def __init__(self, cache, root):
self.cache = cache
self.root = root
@ -106,7 +107,7 @@ class _EventHandler(object):
# Digests are more expensive: invalidate specific paths
paths = []
if event.event_type == 'moved':
if event.event_type == "moved":
paths.append(event.src_path)
paths.append(event.dest_path)
else:
@ -117,4 +118,5 @@ class _EventHandler(object):
for path in paths:
subcache.pop(path, None)
cache_manager = CacheManager()

140
pypiserver/core.py

@ -32,7 +32,7 @@ def configure(**kwds):
log.info("+++Pypiserver invoked with: %s", c)
if c.root is None:
c. root = os.path.expanduser("~/packages")
c.root = os.path.expanduser("~/packages")
roots = c.root if isinstance(c.root, (list, tuple)) else [c.root]
roots = [os.path.abspath(r) for r in roots]
for r in roots:
@ -49,8 +49,9 @@ def configure(**kwds):
if not c.authenticated:
c.authenticated = []
if not callable(c.auther):
if c.password_file and c.password_file != '.':
if c.password_file and c.password_file != ".":
from passlib.apache import HtpasswdFile
htPsswdFile = HtpasswdFile(c.password_file)
else:
c.password_file = htPsswdFile = None
@ -61,13 +62,17 @@ def configure(**kwds):
if not c.welcome_file:
c.welcome_file = "welcome.html"
c.welcome_msg = pkg_resources.resource_string( # @UndefinedVariable
__name__, "welcome.html").decode("utf-8") # @UndefinedVariable
__name__, "welcome.html"
).decode(
"utf-8"
) # @UndefinedVariable
else:
with io.open(c.welcome_file, 'r', encoding='utf-8') as fd:
with io.open(c.welcome_file, "r", encoding="utf-8") as fd:
c.welcome_msg = fd.read()
except Exception:
log.warning(
"Could not load welcome-file(%s)!", c.welcome_file, exc_info=1)
"Could not load welcome-file(%s)!", c.welcome_file, exc_info=1
)
if c.fallback_url is None:
c.fallback_url = "https://pypi.org/simple"
@ -76,10 +81,10 @@ def configure(**kwds):
try:
halgos = hashlib.algorithms_available
except AttributeError:
halgos = ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512']
halgos = ["md5", "sha1", "sha224", "sha256", "sha384", "sha512"]
if c.hash_algo not in halgos:
sys.exit('Hash-algorithm %s not one of: %s' % (c.hash_algo, halgos))
sys.exit("Hash-algorithm %s not one of: %s" % (c.hash_algo, halgos))
log.info("+++Pypiserver started with: %s", c)
@ -100,32 +105,34 @@ mimetypes.add_type("text/plain", ".asc")
# ### Next 2 functions adapted from :mod:`distribute.pkg_resources`.
#
component_re = re.compile(r'(\d+ | [a-z]+ | \.| -)', re.I | re.VERBOSE)
replace = {'pre': 'c', 'preview': 'c', '-': 'final-', 'rc': 'c', 'dev': '@'}.get
component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.I | re.VERBOSE)
replace = {"pre": "c", "preview": "c", "-": "final-", "rc": "c", "dev": "@"}.get
def _parse_version_parts(s):
for part in component_re.split(s):
part = replace(part, part)
if part in ['', '.']:
if part in ["", "."]:
continue
if part[:1] in '0123456789':
if part[:1] in "0123456789":
yield part.zfill(8) # pad for numeric comparison
else:
yield '*' + part
yield "*" + part
yield '*final' # ensure that alpha/beta/candidate are before final
yield "*final" # ensure that alpha/beta/candidate are before final
def parse_version(s):
parts = []
for part in _parse_version_parts(s.lower()):
if part.startswith('*'):
if part.startswith("*"):
# remove trailing zeros from each series of numeric parts
while parts and parts[-1] == '00000000':
while parts and parts[-1] == "00000000":
parts.pop()
parts.append(part)
return tuple(parts)
#
#### -- End of distribute's code.
@ -133,16 +140,18 @@ def parse_version(s):
_archive_suffix_rx = re.compile(
r"(\.zip|\.tar\.gz|\.tgz|\.tar\.bz2|-py[23]\.\d-.*|"
r"\.win-amd64-py[23]\.\d\..*|\.win32-py[23]\.\d\..*|\.egg)$",
re.I)
re.I,
)
wheel_file_re = re.compile(
r"""^(?P<namever>(?P<name>.+?)-(?P<ver>\d.*?))
((-(?P<build>\d.*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
\.whl|\.dist-info)$""",
re.VERBOSE)
_pkgname_re = re.compile(r'-\d+[a-z_.!+]', re.I)
re.VERBOSE,
)
_pkgname_re = re.compile(r"-\d+[a-z_.!+]", re.I)
_pkgname_parts_re = re.compile(
r"[\.\-](?=cp\d|py\d|macosx|linux|sunos|solaris|irix|aix|cygwin|win)",
re.I)
r"[\.\-](?=cp\d|py\d|macosx|linux|sunos|solaris|irix|aix|cygwin|win)", re.I
)
def _guess_pkgname_and_version_wheel(basename):
@ -166,16 +175,16 @@ def guess_pkgname_and_version(path):
return _guess_pkgname_and_version_wheel(path)
if not _archive_suffix_rx.search(path):
return
path = _archive_suffix_rx.sub('', path)
if '-' not in path:
pkgname, version = path, ''
elif path.count('-') == 1:
pkgname, version = path.split('-', 1)
elif '.' not in path:
pkgname, version = path.rsplit('-', 1)
path = _archive_suffix_rx.sub("", path)
if "-" not in path:
pkgname, version = path, ""
elif path.count("-") == 1:
pkgname, version = path.split("-", 1)
elif "." not in path:
pkgname, version = path.rsplit("-", 1)
else:
pkgname = _pkgname_re.split(path)[0]
ver_spec = path[len(pkgname) + 1:]
ver_spec = path[len(pkgname) + 1 :]
parts = _pkgname_parts_re.split(ver_spec)
version = parts[0]
return pkgname, version
@ -198,15 +207,22 @@ def is_allowed_path(path_part):
class PkgFile(object):
__slots__ = ['fn', 'root', '_fname_and_hash',
'relfn', 'relfn_unix',
'pkgname_norm',
'pkgname',
'version',
'parsed_version',
'replaces']
def __init__(self, pkgname, version, fn=None, root=None, relfn=None, replaces=None):
__slots__ = [
"fn",
"root",
"_fname_and_hash",
"relfn",
"relfn_unix",
"pkgname_norm",
"pkgname",
"version",
"parsed_version",
"replaces",
]
def __init__(
self, pkgname, version, fn=None, root=None, relfn=None, replaces=None
):
self.pkgname = pkgname
self.pkgname_norm = normalize_pkgname(pkgname)
self.version = version
@ -220,14 +236,21 @@ class PkgFile(object):
def __repr__(self):
return "%s(%s)" % (
self.__class__.__name__,
", ".join(["%s=%r" % (k, getattr(self, k, 'AttributeError'))
for k in sorted(self.__slots__)]))
", ".join(
[
"%s=%r" % (k, getattr(self, k, "AttributeError"))
for k in sorted(self.__slots__)
]
),
)
def fname_and_hash(self, hash_algo):
if not hasattr(self, '_fname_and_hash'):
if not hasattr(self, "_fname_and_hash"):
if hash_algo:
self._fname_and_hash = '%s#%s=%s' % (
self.relfn_unix, hash_algo, digest_file(self.fn, hash_algo)
self._fname_and_hash = "%s#%s=%s" % (
self.relfn_unix,
hash_algo,
digest_file(self.fn, hash_algo),
)
else:
self._fname_and_hash = self.relfn_unix
@ -248,10 +271,13 @@ def _listdir(root):
continue
pkgname, version = res
if pkgname:
yield PkgFile(pkgname=pkgname,
version=version,
fn=fn, root=root,
relfn=fn[len(root) + 1:])
yield PkgFile(
pkgname=pkgname,
version=version,
fn=fn,
root=root,
relfn=fn[len(root) + 1 :],
)
def read_lines(filename):
@ -264,14 +290,15 @@ def read_lines(filename):
try:
with open(filename) as f:
lines = [
line
for line in (ln.strip() for ln in f.readlines())
if line and not line.startswith('#')
line
for line in (ln.strip() for ln in f.readlines())
if line and not line.startswith("#")
]
except Exception:
log.error('Failed to read package blacklist file "%s". '
'Aborting server startup, please fix this.'
% filename)
log.error(
'Failed to read package blacklist file "%s". '
"Aborting server startup, please fix this." % filename
)
raise
return lines
@ -310,7 +337,7 @@ def get_bad_url_redirect_path(request, prefix):
p = request.custom_fullpath
if p.endswith("/"):
p = p[:-1]
p = p.rsplit('/', 1)[0]
p = p.rsplit("/", 1)[0]
prefix = quote(prefix)
p += "/simple/{}/".format(prefix)
return p
@ -325,10 +352,10 @@ def _digest_file(fpath, hash_algo):
From http://stackoverflow.com/a/21565932/548792
"""
blocksize = 2**16
blocksize = 2 ** 16
digester = getattr(hashlib, hash_algo)()
with open(fpath, 'rb') as f:
for block in iter(lambda: f.read(blocksize), b''):
with open(fpath, "rb") as f:
for block in iter(lambda: f.read(blocksize), b""):
digester.update(block)
return digester.hexdigest()
@ -344,6 +371,7 @@ try:
# fpath must be absolute path
return cache_manager.digest_file(fpath, hash_algo, _digest_file)
except ImportError:
listdir = _listdir
digest_file = _digest_file

60
pypiserver/manage.py

@ -17,6 +17,8 @@ if sys.version_info >= (3, 0):
def make_pypi_client(url):
return Server(url)
else:
from xmlrpclib import Transport # @UnresolvedImport
from xmlrpclib import ServerProxy
@ -24,7 +26,6 @@ else:
import urllib
class ProxiedTransport(Transport):
def set_proxy(self, proxy):
self.proxy = proxy
@ -38,17 +39,19 @@ else:
def send_request(self, connection, handler, request_body):
connection.putrequest(
"POST", 'http://%s%s' % (self.realhost, handler))
"POST", "http://%s%s" % (self.realhost, handler)
)
def send_host(self, connection, host):
connection.putheader('Host', self.realhost)
connection.putheader("Host", self.realhost)
def make_pypi_client(url):
http_proxy_url = urllib.getproxies().get("http", "")
if http_proxy_url:
http_proxy_spec = urllib.splithost(
urllib.splittype(http_proxy_url)[1])[0]
urllib.splittype(http_proxy_url)[1]
)[0]
transport = ProxiedTransport()
transport.set_proxy(http_proxy_spec)
else:
@ -92,9 +95,7 @@ def build_releases(pkg, versions):
for x in versions:
parsed_version = core.parse_version(x)
if parsed_version > pkg.parsed_version:
yield core.PkgFile(pkgname=pkg.pkgname,
version=x,
replaces=pkg)
yield core.PkgFile(pkgname=pkg.pkgname, version=x, replaces=pkg)
def find_updates(pkgset, stable_only=True):
@ -108,7 +109,8 @@ def find_updates(pkgset, stable_only=True):
latest_pkgs = frozenset(filter_latest_pkgs(pkgset))
sys.stdout.write(
"checking %s packages for newer version\n" % len(latest_pkgs),)
"checking %s packages for newer version\n" % len(latest_pkgs),
)
need_update = set()
pypi = make_pypi_client("https://pypi.org/pypi/")
@ -135,8 +137,10 @@ def find_updates(pkgset, stable_only=True):
write("\n\n")
if no_releases:
sys.stdout.write("no releases found on pypi for %s\n\n" %
(", ".join(sorted(no_releases)),))
sys.stdout.write(
"no releases found on pypi for %s\n\n"
% (", ".join(sorted(no_releases)),)
)
return need_update
@ -148,20 +152,25 @@ class PipCmd(object):
def update_root(pip_version):
"""Yield an appropriate root command depending on pip version."""
# legacy_pip = StrictVersion(pip_version) < StrictVersion('10.0')
legacy_pip = LooseVersion(pip_version) < LooseVersion('10.0')
for part in ('pip', '-q'):
legacy_pip = LooseVersion(pip_version) < LooseVersion("10.0")
for part in ("pip", "-q"):
yield part
yield 'install' if legacy_pip else 'download'
yield "install" if legacy_pip else "download"
@staticmethod
def update(cmd_root, destdir, pkg_name, pkg_version,
index='https://pypi.org/simple'):
def update(
cmd_root,
destdir,
pkg_name,
pkg_version,
index="https://pypi.org/simple",
):
"""Yield an update command for pip."""
for part in cmd_root:
yield part
for part in ('--no-deps', '-i', index, '-d', destdir):
for part in ("--no-deps", "-i", index, "-d", destdir):
yield part
yield '{}=={}'.format(pkg_name, pkg_ve