ci: add mypy typecheck

pull/186/head
David Hewitt 10 months ago
parent 560b16c5a5
commit ea3a4e8f84
  1. 14
      .github/workflows/ci.yml
  2. 19
      mypy.ini
  3. 122
      setuptools_rust/build.py
  4. 6
      setuptools_rust/clean.py
  5. 20
      setuptools_rust/command.py
  6. 16
      setuptools_rust/extension.py
  7. 0
      setuptools_rust/py.typed
  8. 81
      setuptools_rust/setuptools_ext.py
  9. 12
      setuptools_rust/utils.py
  10. 13
      tox.ini

@ -20,6 +20,20 @@ jobs:
- run: black --check .
mypy:
runs-on: "ubuntu-latest"
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: "3.x"
- run: pip install tox
- run: tox -e mypy
build:
name: ${{ matrix.python-version }} ${{ matrix.platform.os }}-${{ matrix.platform.python-architecture }}
runs-on: ${{ matrix.platform.os }}

@ -0,0 +1,19 @@
# We have to manually ignore types for some packages. It would be great if
# interested users want to add these types to typeshed!
[mypy]
show_error_codes = True
disallow_untyped_defs = True
disallow_any_unimported = True
no_implicit_optional = True
check_untyped_defs = True
warn_return_any = True
warn_unused_ignores = True
[mypy-semantic_version.*]
ignore_missing_imports = True
[mypy-wheel.*]
ignore_missing_imports = True

@ -6,6 +6,7 @@ import shutil
import subprocess
import sys
import sysconfig
from distutils.command.build import build as CommandBuild
from distutils.errors import (
CompileError,
DistutilsExecError,
@ -14,8 +15,9 @@ from distutils.errors import (
)
from distutils.sysconfig import get_config_var
from subprocess import check_output
from typing import List, NamedTuple, Optional, Tuple
from typing import List, NamedTuple, Optional, cast
from setuptools.command.build_ext import build_ext as CommandBuildExt
from setuptools.command.build_ext import get_abi3_suffix
from .command import RustCommand
@ -52,7 +54,9 @@ class build_rust(RustCommand):
]
boolean_options = ["inplace", "debug", "release", "qbuild"]
def initialize_options(self):
plat_name: Optional[str]
def initialize_options(self) -> None:
super().initialize_options()
self.inplace = None
self.debug = None
@ -62,11 +66,14 @@ class build_rust(RustCommand):
self.plat_name = None
self.target = os.getenv("CARGO_BUILD_TARGET")
def finalize_options(self):
def finalize_options(self) -> None:
super().finalize_options()
if self.plat_name is None:
self.plat_name = self.get_finalized_command("build").plat_name
self.plat_name = cast(
CommandBuild, self.get_finalized_command("build")
).plat_name
assert isinstance(self.plat_name, str)
# Inherit settings from the `build_ext` command
self.set_undefined_options(
@ -77,6 +84,8 @@ class build_rust(RustCommand):
)
def get_target_info(self) -> "_TargetInfo":
assert self.plat_name is not None
# If we are on a 64-bit machine, but running a 32-bit Python, then
# we'll target a 32-bit Rust build.
# Automatic target detection can be overridden via the CARGO_BUILD_TARGET
@ -115,7 +124,8 @@ class build_rust(RustCommand):
% cross_compile_info.host_type
)
return _TargetInfo.for_triple(self.target)
# FIXME!
return _TargetInfo.for_triple(self.target) # type: ignore[arg-type]
def get_nix_cross_compile_info(self) -> Optional["_CrossCompileInfo"]:
# See https://github.com/PyO3/setuptools-rust/issues/138
@ -143,7 +153,9 @@ class build_rust(RustCommand):
return _CrossCompileInfo(host_type, cross_lib, linker, linker_args)
def run_for_extension(self, ext: RustExtension):
def run_for_extension(self, ext: RustExtension) -> None:
assert self.plat_name is not None
arch_flags = os.getenv("ARCHFLAGS")
universal2 = False
if self.plat_name.startswith("macosx-") and arch_flags:
@ -156,15 +168,15 @@ class build_rust(RustCommand):
arm64_dylib_paths, x86_64_dylib_paths
):
fat_dylib_path = arm64_dylib.replace("aarch64-apple-darwin/", "")
self.create_universal2_binary(
fat_dylib_path, [arm64_dylib, x86_64_dylib]
)
dylib_paths.append((target_fname, fat_dylib_path))
create_universal2_binary(fat_dylib_path, [arm64_dylib, x86_64_dylib])
dylib_paths.append(_BuiltModule(target_fname, fat_dylib_path))
else:
dylib_paths = self.build_extension(ext)
self.install_extension(ext, dylib_paths)
def build_extension(self, ext: RustExtension, target_triple=None):
def build_extension(
self, ext: RustExtension, target_triple: Optional[str] = None
) -> List["_BuiltModule"]:
executable = ext.binding == Binding.Exec
if target_triple is None:
@ -333,7 +345,7 @@ class build_rust(RustCommand):
path = os.path.join(artifactsdir, name)
if os.access(path, os.X_OK):
dylib_paths.append((dest, path))
dylib_paths.append(_BuiltModule(dest, path))
else:
raise DistutilsExecError(
"Rust build failed; "
@ -351,7 +363,7 @@ class build_rust(RustCommand):
try:
dylib_paths.append(
(
_BuiltModule(
ext.name,
next(glob.iglob(os.path.join(artifactsdir, wildcard_so))),
)
@ -362,15 +374,18 @@ class build_rust(RustCommand):
)
return dylib_paths
def install_extension(self, ext: RustExtension, dylib_paths: List[Tuple[str, str]]):
def install_extension(
self, ext: RustExtension, dylib_paths: List["_BuiltModule"]
) -> None:
executable = ext.binding == Binding.Exec
debug_build = ext.debug if ext.debug is not None else self.inplace
debug_build = self.debug if self.debug is not None else debug_build
if self.release:
debug_build = False
# Ask build_ext where the shared library would go if it had built it,
# then copy it there.
build_ext = self.get_finalized_command("build_ext")
build_ext = cast(CommandBuildExt, self.get_finalized_command("build_ext"))
build_ext.inplace = self.inplace
for module_name, dylib_path in dylib_paths:
@ -419,9 +434,9 @@ class build_rust(RustCommand):
os.chmod(ext_path, mode)
def get_dylib_ext_path(self, ext: RustExtension, target_fname: str) -> str:
build_ext = self.get_finalized_command("build_ext")
build_ext = cast(CommandBuildExt, self.get_finalized_command("build_ext"))
filename = build_ext.get_ext_fullpath(target_fname)
filename: str = build_ext.get_ext_fullpath(target_fname)
if (ext.py_limited_api == "auto" and self._py_limited_api()) or (
ext.py_limited_api
@ -429,45 +444,60 @@ class build_rust(RustCommand):
abi3_suffix = get_abi3_suffix()
if abi3_suffix is not None:
so_ext = get_config_var("EXT_SUFFIX")
assert isinstance(so_ext, str)
filename = filename[: -len(so_ext)] + get_abi3_suffix()
return filename
@staticmethod
def create_universal2_binary(output_path, input_paths):
# Try lipo first
command = ["lipo", "-create", "-output", output_path, *input_paths]
try:
subprocess.check_output(command)
except subprocess.CalledProcessError as e:
output = e.output
if isinstance(output, bytes):
output = e.output.decode("latin-1").strip()
raise CompileError("lipo failed with code: %d\n%s" % (e.returncode, output))
except OSError:
# lipo not found, try using the fat-macho library
try:
from fat_macho import FatWriter
except ImportError:
raise DistutilsExecError(
"failed to locate `lipo` or import `fat_macho.FatWriter`. "
"Try installing with `pip install fat-macho` "
)
fat = FatWriter()
for input_path in input_paths:
with open(input_path, "rb") as f:
fat.add(f.read())
fat.write_to(output_path)
def _py_limited_api(self) -> PyLimitedApi:
bdist_wheel = self.distribution.get_command_obj("bdist_wheel", create=0)
bdist_wheel = self.distribution.get_command_obj("bdist_wheel", create=False)
if bdist_wheel is None:
# wheel package is not installed, not building a limited-api wheel
return False
else:
bdist_wheel.ensure_finalized()
return bdist_wheel.py_limited_api
from wheel.bdist_wheel import bdist_wheel as CommandBdistWheel
bdist_wheel_command = cast(CommandBdistWheel, bdist_wheel) # type: ignore[no-any-unimported]
bdist_wheel_command.ensure_finalized()
return cast(PyLimitedApi, bdist_wheel_command.py_limited_api)
def create_universal2_binary(output_path: str, input_paths: List[str]) -> None:
# Try lipo first
command = ["lipo", "-create", "-output", output_path, *input_paths]
try:
subprocess.check_output(command)
except subprocess.CalledProcessError as e:
output = e.output
if isinstance(output, bytes):
output = e.output.decode("latin-1").strip()
raise CompileError("lipo failed with code: %d\n%s" % (e.returncode, output))
except OSError:
# lipo not found, try using the fat-macho library
try:
from fat_macho import FatWriter
except ImportError:
raise DistutilsExecError(
"failed to locate `lipo` or import `fat_macho.FatWriter`. "
"Try installing with `pip install fat-macho` "
)
fat = FatWriter()
for input_path in input_paths:
with open(input_path, "rb") as f:
fat.add(f.read())
fat.write_to(output_path)
class _BuiltModule(NamedTuple):
"""
Attributes:
- module_name: dotted python import path of the module
- path: the location the module has been installed at
"""
module_name: str
path: str
class _TargetInfo(NamedTuple):

@ -1,5 +1,5 @@
import sys
import subprocess
import sys
from .command import RustCommand
from .extension import RustExtension
@ -10,11 +10,11 @@ class clean_rust(RustCommand):
description = "clean Rust extensions (compile/link to build directory)"
def initialize_options(self):
def initialize_options(self) -> None:
super().initialize_options()
self.inplace = False
def run_for_extension(self, ext: RustExtension):
def run_for_extension(self, ext: RustExtension) -> None:
# build cargo command
args = ["cargo", "clean", "--manifest-path", ext.path]

@ -1,6 +1,9 @@
from abc import ABC, abstractmethod
from distutils.cmd import Command
from distutils.errors import DistutilsPlatformError
from typing import List
from setuptools.dist import Distribution
from .extension import RustExtension
from .utils import get_rust_version
@ -9,17 +12,22 @@ from .utils import get_rust_version
class RustCommand(Command, ABC):
"""Abstract base class for commands which interact with Rust Extensions."""
def initialize_options(self):
self.extensions = ()
# Types for distutils variables which exist on all commands but seem to be
# missing from https://github.com/python/typeshed/blob/master/stdlib/distutils/cmd.pyi
distribution: Distribution
verbose: int
def initialize_options(self) -> None:
self.extensions: List[RustExtension] = []
def finalize_options(self):
def finalize_options(self) -> None:
self.extensions = [
ext
for ext in self.distribution.rust_extensions
for ext in self.distribution.rust_extensions # type: ignore[attr-defined]
if isinstance(ext, RustExtension)
]
def run(self):
def run(self) -> None:
if not self.extensions:
return
@ -27,7 +35,7 @@ class RustCommand(Command, ABC):
try:
version = get_rust_version()
if version is None:
min_version = max(
min_version = max( # type: ignore[type-var]
filter(
lambda version: version is not None,
(ext.get_rust_version() for ext in self.extensions),

@ -27,7 +27,7 @@ class Binding(IntEnum):
NoBinding = auto()
Exec = auto()
def __repr__(self):
def __repr__(self) -> str:
return f"{self.__class__.__name__}.{self.name}"
@ -45,7 +45,7 @@ class Strip(IntEnum):
Debug = auto()
All = auto()
def __repr__(self):
def __repr__(self) -> str:
return f"{self.__class__.__name__}.{self.name}"
@ -144,7 +144,7 @@ class RustExtension:
path = os.path.relpath(path)
self.path = path
def get_lib_name(self):
def get_lib_name(self) -> str:
"""Parse Cargo.toml to get the name of the shared library."""
with open(self.path, "rb") as f:
cfg = tomli.load(f)
@ -157,10 +157,14 @@ class RustExtension:
"Cargo.toml missing value for 'name' key "
"in both the [package] section and the [lib] section"
)
if not isinstance(name, str):
raise Exception(
f"Expected string for Rust library name in Cargo.toml, got {name}"
)
name = re.sub(r"[./\\-]", "_", name)
return name
def get_rust_version(self) -> Optional[SimpleSpec]:
def get_rust_version(self) -> Optional[SimpleSpec]: # type: ignore[no-any-unimported]
if self.rust_version is None:
return None
try:
@ -170,7 +174,7 @@ class RustExtension:
"Can not parse rust compiler version: %s", self.rust_version
)
def entry_points(self):
def entry_points(self) -> List[str]:
entry_points = []
if self.script and self.binding == Binding.Exec:
for name, mod in self.target.items():
@ -180,7 +184,7 @@ class RustExtension:
return entry_points
def install_script(self, module_name: str, exe_path: str):
def install_script(self, module_name: str, exe_path: str) -> None:
if self.script and self.binding == Binding.Exec:
dirname, executable = os.path.split(exe_path)
file = os.path.join(dirname, "_gen_%s.py" % module_name)

@ -1,12 +1,17 @@
import os
import subprocess
import sys
from distutils import log
from distutils.command.clean import clean
from typing import List, Tuple, Type, cast
from setuptools.command.build_ext import build_ext
from setuptools.command.install import install
from setuptools.command.sdist import sdist
import sys
import subprocess
from setuptools.dist import Distribution
from typing_extensions import Literal
from .extension import RustExtension
try:
from wheel.bdist_wheel import bdist_wheel
@ -14,8 +19,8 @@ except ImportError:
bdist_wheel = None
def add_rust_extension(dist):
sdist_base_class = dist.cmdclass.get("sdist", sdist)
def add_rust_extension(dist: Distribution) -> None:
sdist_base_class = cast(Type[sdist], dist.cmdclass.get("sdist", sdist))
sdist_options = sdist_base_class.user_options.copy()
sdist_boolean_options = sdist_base_class.boolean_options.copy()
sdist_negative_opt = sdist_base_class.negative_opt.copy()
@ -32,16 +37,16 @@ def add_rust_extension(dist):
sdist_boolean_options.append("vendor-crates")
sdist_negative_opt["no-vendor-crates"] = "vendor-crates"
class sdist_rust_extension(sdist_base_class):
class sdist_rust_extension(sdist_base_class): # type: ignore[misc,valid-type]
user_options = sdist_options
boolean_options = sdist_boolean_options
negative_opt = sdist_negative_opt
def initialize_options(self):
def initialize_options(self) -> None:
super().initialize_options()
self.vendor_crates = 0
def make_distribution(self):
def make_distribution(self) -> None:
if self.vendor_crates:
manifest_paths = []
for ext in self.distribution.rust_extensions:
@ -101,18 +106,20 @@ def add_rust_extension(dist):
dist.cmdclass["sdist"] = sdist_rust_extension
build_ext_base_class = dist.cmdclass.get("build_ext", build_ext)
build_ext_base_class = cast(
Type[build_ext], dist.cmdclass.get("build_ext", build_ext)
)
build_ext_options = build_ext_base_class.user_options.copy()
build_ext_options.append(("target", None, "Build for the target triple"))
class build_ext_rust_extension(build_ext_base_class):
class build_ext_rust_extension(build_ext_base_class): # type: ignore[misc,valid-type]
user_options = build_ext_options
def initialize_options(self):
def initialize_options(self) -> None:
super().initialize_options()
self.target = os.getenv("CARGO_BUILD_TARGET")
def run(self):
def run(self) -> None:
if self.distribution.rust_extensions:
log.info("running build_rust")
build_rust = self.get_finalized_command("build_rust")
@ -126,22 +133,22 @@ def add_rust_extension(dist):
dist.cmdclass["build_ext"] = build_ext_rust_extension
clean_base_class = dist.cmdclass.get("clean", clean)
clean_base_class = cast(Type[clean], dist.cmdclass.get("clean", clean))
class clean_rust_extension(clean_base_class):
def run(self):
class clean_rust_extension(clean_base_class): # type: ignore[misc,valid-type]
def run(self) -> None:
clean_base_class.run(self)
if not self.dry_run:
self.run_command("clean_rust")
dist.cmdclass["clean"] = clean_rust_extension
install_base_class = dist.cmdclass.get("install", install)
install_base_class = cast(Type[install], dist.cmdclass.get("install", install))
# this is required because, install directly access distribution's
# ext_modules attr to check if dist has ext modules
class install_rust_extension(install_base_class):
def finalize_options(self):
class install_rust_extension(install_base_class): # type: ignore[misc,valid-type]
def finalize_options(self) -> None:
ext_modules = self.distribution.ext_modules
# all ext modules
@ -181,19 +188,21 @@ def add_rust_extension(dist):
dist.cmdclass["install"] = install_rust_extension
if bdist_wheel is not None:
bdist_wheel_base_class = dist.cmdclass.get("bdist_wheel", bdist_wheel)
bdist_wheel_base_class = cast( # type: ignore[no-any-unimported]
Type[bdist_wheel], dist.cmdclass.get("bdist_wheel", bdist_wheel)
)
bdist_wheel_options = bdist_wheel_base_class.user_options.copy()
bdist_wheel_options.append(("target", None, "Build for the target triple"))
# this is for console entries
class bdist_wheel_rust_extension(bdist_wheel_base_class):
class bdist_wheel_rust_extension(bdist_wheel_base_class): # type: ignore[misc,valid-type]
user_options = bdist_wheel_options
def initialize_options(self):
def initialize_options(self) -> None:
super().initialize_options()
self.target = os.getenv("CARGO_BUILD_TARGET")
def finalize_options(self):
def finalize_options(self) -> None:
scripts = []
for ext in self.distribution.rust_extensions:
scripts.extend(ext.entry_points())
@ -216,7 +225,7 @@ def add_rust_extension(dist):
bdist_wheel_base_class.finalize_options(self)
def get_tag(self):
def get_tag(self) -> Tuple[str, str, str]:
python, abi, plat = super().get_tag()
arch_flags = os.getenv("ARCHFLAGS")
universal2 = False
@ -237,7 +246,7 @@ def add_rust_extension(dist):
dist.cmdclass["bdist_wheel"] = bdist_wheel_rust_extension
def patch_distutils_build():
def patch_distutils_build() -> None:
"""Patch distutils to use `has_ext_modules()`
See https://github.com/pypa/distutils/pull/43
@ -245,26 +254,34 @@ def patch_distutils_build():
from distutils.command import build as _build
class build(_build.build):
def finalize_options(self):
# Missing type def from distutils.cmd.Command; add it here for now
distribution: Distribution
def finalize_options(self) -> None:
build_lib_user_specified = self.build_lib is not None
super().finalize_options()
if not build_lib_user_specified:
if self.distribution.has_ext_modules():
if self.distribution.has_ext_modules(): # type: ignore[attr-defined]
self.build_lib = self.build_platlib
else:
self.build_lib = self.build_purelib
_build.build = build
_build.build = build # type: ignore[misc]
def rust_extensions(dist, attr, value):
def rust_extensions(
dist: Distribution, attr: Literal["rust_extensions"], value: List[RustExtension]
) -> None:
assert attr == "rust_extensions"
has_rust_extensions = len(value) > 0
orig_has_ext_modules = dist.has_ext_modules
dist.has_ext_modules = lambda: (
orig_has_ext_modules() or bool(dist.rust_extensions)
)
# Monkey patch has_ext_modules to include Rust extensions; pairs with
# patch_distutils_build above.
#
# has_ext_modules is missing from Distribution typing.
orig_has_ext_modules = dist.has_ext_modules # type: ignore[attr-defined]
dist.has_ext_modules = lambda: (orig_has_ext_modules() or has_rust_extensions) # type: ignore[attr-defined]
if dist.rust_extensions:
if has_rust_extensions:
patch_distutils_build()
add_rust_extension(dist)

@ -1,13 +1,13 @@
import subprocess
from distutils.errors import DistutilsPlatformError
from typing import Optional, Set, Union
from typing import List, Optional, Set, Union
from semantic_version import Version
from typing_extensions import Literal
from .extension import Binding, RustExtension
PyLimitedApi = Union[Literal["cp36", "cp37", "cp38", "cp39"], bool]
PyLimitedApi = Literal["cp36", "cp37", "cp38", "cp39", True, False]
def binding_features(
@ -31,7 +31,7 @@ def binding_features(
raise DistutilsPlatformError(f"unknown Rust binding: '{ext.binding}'")
def get_rust_version() -> Optional[Version]:
def get_rust_version() -> Optional[Version]: # type: ignore[no-any-unimported]
try:
output = subprocess.check_output(["rustc", "-V"]).decode("latin-1")
return Version(output.split(" ")[1])
@ -39,18 +39,18 @@ def get_rust_version() -> Optional[Version]:
return None
def get_rust_target_info(target_triple=None):
def get_rust_target_info(target_triple: Optional[str] = None) -> List[str]:
cmd = ["rustc", "--print", "cfg"]
if target_triple:
cmd.extend(["--target", target_triple])
output = subprocess.check_output(cmd)
output = subprocess.check_output(cmd, universal_newlines=True)
return output.splitlines()
_rust_target_list = None
def get_rust_target_list():
def get_rust_target_list() -> List[str]:
global _rust_target_list
if _rust_target_list is None:
output = subprocess.check_output(

@ -0,0 +1,13 @@
[tox]
envlist = mypy
[testenv:mypy]
deps =
# Need next mypy after 0.910 to get correct types for distutils,
# but it's not released yet.
mypy @ git+https://github.com/python/mypy
fat_macho
types-setuptools
setenv =
MYPYPATH={toxinidir}
commands = mypy setuptools_rust {posargs}
Loading…
Cancel
Save