New packaging

- Pass from Pipenv to venv + pip
- Use of setup.cfg and pyproject.toml
- Replace use of atomicwrites and requests libraries with built-in
  alternatives
- Fix pre-commit hooks
This commit is contained in:
Franco Masotti 2023-01-08 19:31:49 +01:00
parent 162ddd3041
commit 5d291f8e16
Signed by: frnmst
GPG Key ID: 24116ED85666780A
20 changed files with 576 additions and 305 deletions

View File

@ -2,7 +2,7 @@
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.3.0
rev: 'v4.4.0'
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
@ -13,45 +13,51 @@ repos:
- id: check-ast
- id: check-case-conflict
- id: debug-statements
- id: fix-encoding-pragma
- id: forbid-submodules
- id: check-symlinks
- id: check-shebang-scripts-are-executable
- id: check-case-conflict
- id: check-added-large-files
args: ['--maxkb=4096']
- id: destroyed-symlinks
- repo: https://github.com/pre-commit/mirrors-yapf
rev: 'v0.32.0' # Use the sha / tag you want to point at
hooks:
- id: yapf
args: ['--style', '{based_on_style: pep8; indent_width: 4}']
additional_dependencies: [toml]
- repo: https://github.com/pycqa/flake8
rev: '5.0.4' # Use the sha / tag you want to point at
hooks:
- id: flake8
args: ['--ignore=E125,E126,F401,E501,W503,W504']
additional_dependencies: [flake8-docstrings]
- repo: https://github.com/PyCQA/bandit
rev: '1.7.4' # Use the sha / tag you want to point at
hooks:
- id: bandit
args: ['--skip', 'B320,B404,B410,B603', '--level', 'LOW']
args: ['--skip', 'B310,B320,B404,B410,B603', '--level', 'LOW']
- repo: https://github.com/pycqa/isort
rev: 5.10.1
rev: '5.11.4'
hooks:
- id: isort
- repo: https://codeberg.org/frnmst/licheck
rev: 1.0.0
hooks:
- id: licheck
args: ['--configuration-file', '.allowed_licenses.yml']
# - repo: https://codeberg.org/frnmst/licheck
# rev: 1.0.0
# hooks:
# - id: licheck
# args: ['--configuration-file', '.allowed_licenses.yml']
- repo: https://github.com/mgedmin/check-manifest
rev: "0.48"
rev: '0.49'
hooks:
- id: check-manifest
args: ['--ignore','docs/*,docs/assets/*,Pipfile,Makefile,asciinema/*,*.yaml,*.yml,assets/*,fattura_elettronica_reader/tests/*,packages/aur/*']
- repo: https://codeberg.org/frnmst/md-toc
rev: '8.1.5' # or a specific git tag from md-toc
rev: '8.1.8' # or a specific git tag from md-toc
hooks:
- id: md-toc
args: [-p, 'cmark', '-l6'] # CLI options
@ -67,6 +73,6 @@ repos:
pass_filenames: false
- repo: https://github.com/jorisroovers/gitlint
rev: 'v0.17.0'
rev: 'v0.18.0'
hooks:
- id: gitlint

View File

@ -1,3 +1,12 @@
global-include LICENSE.txt
global-include README.md
global-include CONTRIBUTING.md
global-exclude *.csv *.txt
prune assets
prune fattura_elettronica_reader/tests
prune docs
prune .venv
prune packages
prune asciinema
exclude *.yml *.yaml
exclude Makefile

View File

@ -1,8 +1,7 @@
#!/usr/bin/env make
#
# Makefile
#
# Copyright (C) 2019-2021 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
# Copyright (C) 2019-2023 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
#
# This file is part of fattura-elettronica-reader.
#
@ -22,10 +21,16 @@
export PACKAGE_NAME=fattura_elettronica_reader
default: doc
# See
# https://docs.python.org/3/library/venv.html#how-venvs-work
export VENV_CMD=. .venv/bin/activate
doc: clean
pipenv run $(MAKE) -C docs html
default: install-dev
doc:
$(VENV_CMD) \
&& $(MAKE) -C docs html \
&& deactivate
install:
pip3 install . --user
@ -34,40 +39,77 @@ uninstall:
pip3 uninstall --verbose --yes $(PACKAGE_NAME)
install-dev:
pipenv install --dev
pipenv run pre-commit install
pipenv run pre-commit install --hook-type commit-msg
pipenv graph
python3 -m venv .venv
$(VENV_CMD) \
&& pip install --requirement requirements-freeze.txt \
&& deactivate
$(VENV_CMD) \
&& pre-commit install \
&& deactivate
$(VENV_CMD) \
&& pre-commit install --hook-type commit-msg \
&& deactivate
regenerate-freeze: uninstall-dev
python3 -m venv .venv
$(VENV_CMD) \
&& pip install --requirement requirements.txt --requirement requirements-dev.txt \
&& pip freeze --local > requirements-freeze.txt \
&& deactivate
uninstall-dev:
rm -f Pipfile.lock
pipenv --rm
rm -rf .venv
update: install-dev
pipenv run pre-commit autoupdate
$(VENV_CMD) \
&& pre-commit autoupdate \
--repo https://github.com/pre-commit/pre-commit-hooks \
--repo https://github.com/PyCQA/bandit \
--repo https://github.com/pycqa/isort \
--repo https://codeberg.org/frnmst/licheck \
--repo https://codeberg.org/frnmst/md-toc \
--repo https://github.com/mgedmin/check-manifest \
--repo https://github.com/jorisroovers/gitlint \
&& deactivate
# --repo https://github.com/pre-commit/mirrors-mypy \
test:
pipenv run python -m unittest $(PACKAGE_NAME).tests.tests --failfast --locals --verbose
$(VENV_CMD) \
&& python -m unittest $(PACKAGE_NAME).tests.tests --failfast --locals --verbose \
&& deactivate
pre-commit:
$(VENV_CMD) \
&& pre-commit run --all \
&& deactivate
dist:
pipenv run python setup.py sdist
# Create a reproducible archve at least on the wheel.
# Create a reproducible archive at least on the wheel.
# See
# https://bugs.python.org/issue31526
# https://bugs.python.org/issue38727
# https://github.com/pypa/setuptools/issues/1468
# https://github.com/pypa/setuptools/issues/2133
# https://reproducible-builds.org/docs/source-date-epoch/
SOURCE_DATE_EPOCH=$$(git -c log.showSignature='false' log -1 --pretty=%ct) pipenv run python setup.py bdist_wheel
pipenv run twine check dist/*
$(VENV_CMD) \
&& SOURCE_DATE_EPOCH=$$(git -c log.showSignature='false' log -1 --pretty=%ct) \
python -m build \
&& deactivate
$(VENV_CMD) \
&& twine check --strict dist/* \
&& deactivate
upload:
pipenv run twine upload dist/*
$(VENV_CMD) \
&& twine upload dist/* \
&& deactivate
clean:
rm -rf build dist *.egg-info
# Remove all markdown files except the readme.
rm -rf build dist *.egg-info tests/benchmark-results
# Remove all markdown files except the readmes.
find -regex ".*\.[mM][dD]" ! -name 'README.md' ! -name 'CONTRIBUTING.md' -type f -exec rm -f {} +
pipenv run $(MAKE) -C docs clean
$(VENV_CMD) \
&& $(MAKE) -C docs clean \
&& deactivate
.PHONY: default doc install uninstall install-dev uninstall-dev update test clean
.PHONY: default doc install uninstall install-dev uninstall-dev update test clean pre-commit

49
docs/_static/css/custom.css vendored Normal file
View File

@ -0,0 +1,49 @@
@media (min-width:540px) {
.container,
.container-sm {
max-width:100%;
}
}
@media (min-width:720px) {
.container,
.container-md,
.container-sm {
max-width:100%;
}
}
@media (min-width:960px) {
.container,
.container-lg,
.container-md,
.container-sm {
max-width:100%;
}
}
@media (min-width:1200px) {
.container,
.container-lg,
.container-md,
.container-sm,
.container-xl {
max-width:100%;
}
}
/* set the background color to black when client requests dark mode and invert the colors
* from post by user lewisl9029: https://news.ycombinator.com/item?id=26472246
* See also https://github.com/pages-themes/primer/issues/64
*/
@media (prefers-color-scheme: dark) {
body {
background-color: black;
filter: brightness(.9) contrast(1.2) hue-rotate(180deg) invert(80%);
}
img {
filter: brightness(.6) contrast(1.2) invert(-80%) grayscale(50%);
}
}
@media (prefers-color-scheme: light) {
img {
filter:grayscale(50%)
}
}

View File

@ -18,11 +18,10 @@ import sys
sys.path.insert(0, os.path.abspath('..'))
# -- Project information -----------------------------------------------------
project = 'fattura-elettronica-reader'
copyright = '2019-2022, Franco Masotti'
copyright = '2019-2023, Franco Masotti'
author = 'Franco Masotti'
# The short X.Y version
@ -30,7 +29,6 @@ version = '3.0.3'
# The full version, including alpha/beta/rc tags
release = '3.0.3'
# -- General configuration ---------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
@ -43,8 +41,7 @@ release = '3.0.3'
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.coverage',
'sphinx.ext.viewcode',
'sphinx.ext.githubpages',
'sphinx_copybutton',
]
# Add any paths that contain templates here, relative to this directory.
@ -72,25 +69,14 @@ language = None
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = None
pygments_style = 'default'
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'sphinx_rtd_theme'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
html_theme_options = {
'github_user': 'frnmst',
'github_repo': 'fattura-elettronica-reader',
'github_banner': True,
}
html_theme = 'sphinx_book_theme'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
@ -107,13 +93,11 @@ html_static_path = ['_static']
#
# html_sidebars = {}
# -- Options for HTMLHelp output ---------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'fattura-elettronica-readerdoc'
# -- Options for LaTeX output ------------------------------------------------
latex_elements = {
@ -138,20 +122,16 @@ latex_elements = {
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'fattura-elettronica-reader.tex', 'fattura-elettronica-reader Documentation',
'Franco Masotti', 'manual'),
(master_doc, 'fattura-elettronica-reader.tex',
'fattura-elettronica-reader Documentation', 'Franco Masotti', 'manual'),
]
# -- Options for manual page output ------------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'fattura-elettronica-reader', 'fattura-elettronica-reader Documentation',
[author], 1)
]
man_pages = [(master_doc, 'fattura-elettronica-reader',
'fattura-elettronica-reader Documentation', [author], 1)]
# -- Options for Texinfo output ----------------------------------------------
@ -159,58 +139,38 @@ man_pages = [
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'fattura-elettronica-reader', 'fattura-elettronica-reader Documentation',
author, 'Franco Masotti', 'manual',
'Miscellaneous'),
(master_doc, 'fattura-elettronica-reader',
'fattura-elettronica-reader Documentation', author, 'Franco Masotti',
'manual', 'Miscellaneous'),
]
# -- Options for Epub output -------------------------------------------------
# Bibliographic Dublin Core info.
epub_title = project
# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#
# epub_identifier = ''
# A unique identification for the text.
#
# epub_uid = ''
# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']
# -- Extension configuration -------------------------------------------------
# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# This is required for the alabaster theme
# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
html_sidebars = {
'**': [
'about.html',
'navigation.html',
'relations.html', # needs 'show_related': True theme option to display
'searchbox.html',
'donate.html',
]
}
html_theme_options = {
'canonical_url': '',
'logo_only': False,
'display_version': True,
'prev_next_buttons_location': 'bottom',
'style_external_links': False,
'style_nav_header_background': 'light-blue',
# Toc options
'collapse_navigation': True,
'sticky_navigation': True,
'navigation_depth': 6,
'includehidden': True,
'titles_only': False
'repository_url':
'https://software.franco.net.eu.org/frnmst/fattura-elettronica-reader',
'use_repository_button': True,
'use_download_button': True,
'use_issues_button': True,
}
html_baseurl = 'https://docs.franco.net.eu.org/fattura-elettronica-reader/'
# These paths are either relative to html_static_path
# or fully qualified paths (eg. https://...)
html_css_files = [
'css/custom.css',
]
pygments_style = 'default'
html_last_updated_fmt = '%Y-%m-%d %H:%M:%S %z'
copybutton_line_continuation_character = '\\'
# Epub.
epub_theme = 'epub'
epub_author = 'Franco Masotti'
epub_theme_options = {
"relbar1": False,
"footer": False,
}
epub_css_style = [
'css/epub.css',
]

View File

@ -16,3 +16,8 @@ Distribution packages
- A ``PKGBUILD`` for Arch Linux like distributions is available under
the ``./packages/aur`` directory as well as on the AUR website.
Dependencies
------------
See the ``./setup.cfg`` file.

View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
#
# __init__.py
#
@ -21,20 +22,36 @@
#
"""Python discovery file."""
from .api import (assert_data_structure, asset_checksum_matches,
create_appdirs, define_appdirs_user_config_dir_file_path,
define_appdirs_user_data_dir_file_path,
extract_attachments_from_invoice_file, get_ca_certificates,
get_invoice_as_html, get_invoice_filename, get_remote_file,
invoice_file_checksum_matches, is_p7m_file_authentic,
is_p7m_file_signed, is_xml_file_conforming_to_schema,
parse_xml_file, patch_invoice_schema_file, pipeline,
remove_signature_from_p7m_file, write_configuration_file)
from .api import (
assert_data_structure,
asset_checksum_matches,
create_appdirs,
define_appdirs_user_config_dir_file_path,
define_appdirs_user_data_dir_file_path,
extract_attachments_from_invoice_file,
get_ca_certificates,
get_invoice_as_html,
get_invoice_filename,
get_remote_file,
invoice_file_checksum_matches,
is_p7m_file_authentic,
is_p7m_file_signed,
is_xml_file_conforming_to_schema,
parse_xml_file,
patch_invoice_schema_file,
pipeline,
remove_signature_from_p7m_file,
write_configuration_file,
)
from .cli import CliInterface
from .exceptions import (AssetsChecksumDoesNotMatch,
CannotExtractOriginalP7MFile,
ExtractedAttachmentNotInExtensionWhitelist,
ExtractedAttachmentNotInFileTypeWhitelist,
InvoiceFileChecksumFailed, MissingTagInMetadataFile,
P7MFileDoesNotHaveACoherentCryptographicalSignature,
P7MFileNotAuthentic, XMLFileNotConformingToSchema)
from .exceptions import (
AssetsChecksumDoesNotMatch,
CannotExtractOriginalP7MFile,
ExtractedAttachmentNotInExtensionWhitelist,
ExtractedAttachmentNotInFileTypeWhitelist,
InvoiceFileChecksumFailed,
MissingTagInMetadataFile,
P7MFileDoesNotHaveACoherentCryptographicalSignature,
P7MFileNotAuthentic,
XMLFileNotConformingToSchema,
)

View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
#
# __main__.py
#

View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
#
# api.py
#
@ -23,27 +24,34 @@
import base64
import hashlib
import io
import os
import pathlib
import re
import shlex
import shutil
import tempfile
import urllib.parse
import urllib.request
import appdirs
import atomicwrites
import filetype
import fpyutils
import lxml.etree as ET
import requests
import yaml
from . import constants as const
from .exceptions import (AssetsChecksumDoesNotMatch,
CannotExtractOriginalP7MFile,
ExtractedAttachmentNotInExtensionWhitelist,
ExtractedAttachmentNotInFileTypeWhitelist,
InvoiceFileChecksumFailed, MissingTagInMetadataFile,
P7MFileDoesNotHaveACoherentCryptographicalSignature,
P7MFileNotAuthentic, XMLFileNotConformingToSchema)
from .exceptions import (
AssetsChecksumDoesNotMatch,
CannotExtractOriginalP7MFile,
ExtractedAttachmentNotInExtensionWhitelist,
ExtractedAttachmentNotInFileTypeWhitelist,
InvoiceFileChecksumFailed,
MissingTagInMetadataFile,
P7MFileDoesNotHaveACoherentCryptographicalSignature,
P7MFileNotAuthentic,
XMLFileNotConformingToSchema,
)
#######
@ -110,7 +118,8 @@ def is_p7m_file_signed(p7m_file: str) -> bool:
"""
command = 'openssl pkcs7 -print_certs -text -noout -inform DER -in {}'.format(
shlex.quote(p7m_file))
return True if fpyutils.execute_command_live_output(command) == 0 else False
return True if fpyutils.execute_command_live_output(
command) == 0 else False
def invoice_file_checksum_matches(metadata_file_xml_root, invoice_file: str,
@ -156,17 +165,26 @@ def get_remote_file(destination: str, url: str):
:type url: str
:returns: None
:rtype: None
:raises: a built-in exception or a requests error.
:raises: ValueError or a built-in exception.
.. note: requests also checks that the url is in a valid form.
"""
r = requests.get(url)
if r.ok:
with atomicwrites.atomic_write(destination, mode='wb',
overwrite=True) as f:
f.write(r.content)
else:
r.raise_for_status()
# Check if the computed string is a valid URL and if it starts with
# http{,s}.
url_string: str = urllib.parse.urlparse(url)
if (url_string.scheme == str() or url_string.netloc == str()
or not re.match('^http(|s)', url_string.scheme)):
raise ValueError
with urllib.request.urlopen(url) as response:
content: io.BytesIO = response.read()
# Atomic write.
# See
# https://stupidpythonideas.blogspot.com/2014/07/getting-atomic-writes-right.html
with tempfile.NamedTemporaryFile('wb', delete=False) as f:
f.flush()
os.fsync(f.fileno())
f.write(content)
shutil.move(f.name, destination)
def get_ca_certificates(trusted_list_xml_root: str,
@ -197,9 +215,11 @@ def get_ca_certificates(trusted_list_xml_root: str,
preeb = '-----BEGIN CERTIFICATE-----'
posteb = '-----END CERTIFICATE-----'
max_line_len = 64
with atomicwrites.atomic_write(ca_certificate_pem_file,
mode='w',
overwrite=True) as f:
# Atomic write.
# See
# https://stupidpythonideas.blogspot.com/2014/07/getting-atomic-writes-right.html
with tempfile.NamedTemporaryFile('w', delete=False) as f:
# See https://lxml.de/tutorial.html#elementpath
# for the exception that gets raised.
for e in trusted_list_xml_root.iter(
@ -214,13 +234,18 @@ def get_ca_certificates(trusted_list_xml_root: str,
strictbase64finl = str()
strictbase64text = base64fullline + strictbase64finl
stricttextualmsg = preeb + eol + strictbase64text + posteb + eol
f.flush()
os.fsync(f.fileno())
f.write(stricttextualmsg)
shutil.move(f.name, ca_certificate_pem_file)
def is_p7m_file_authentic(p7m_file: str,
ca_certificate_pem_file: str,
ignore_signature_check: bool = False,
ignore_signers_certificate_check: bool = False) -> bool:
def is_p7m_file_authentic(
p7m_file: str,
ca_certificate_pem_file: str,
ignore_signature_check: bool = False,
ignore_signers_certificate_check: bool = False) -> bool:
r"""Check authenticity of the invoice file on various levels.
:param p7m_file: the path of the signed invoice file.
@ -247,7 +272,8 @@ def is_p7m_file_authentic(p7m_file: str,
' -CAfile {}'.format(shlex.quote(ca_certificate_pem_file)) +
' -in {}'.format(shlex.quote(p7m_file)) +
' -inform DER -out /dev/null')
return True if fpyutils.execute_command_live_output(command) == 0 else False
return True if fpyutils.execute_command_live_output(
command) == 0 else False
def remove_signature_from_p7m_file(p7m_file: str, output_file: str) -> bool:
@ -264,20 +290,21 @@ def remove_signature_from_p7m_file(p7m_file: str, output_file: str) -> bool:
command = ('openssl smime -nosigs -verify -noverify -in {}'.format(
shlex.quote(p7m_file)) +
' -inform DER -out {}'.format(shlex.quote(output_file)))
return True if fpyutils.execute_command_live_output(command) == 0 else False
return True if fpyutils.execute_command_live_output(
command) == 0 else False
def extract_attachments_from_invoice_file(
invoice_file_xml_root,
invoice_file_xml_attachment_xpath: str,
invoice_file_xml_attachment_tag: str,
invoice_file_xml_attachment_filename_tag: str,
invoice_file_text_encoding: str,
ignore_attachment_extension_whitelist: bool = False,
ignore_attachment_filetype_whitelist: bool = False,
attachment_extension_whitelist: list = list(),
attachment_filetype_whitelist: list = list(),
destination_directory: str = '.'):
invoice_file_xml_root,
invoice_file_xml_attachment_xpath: str,
invoice_file_xml_attachment_tag: str,
invoice_file_xml_attachment_filename_tag: str,
invoice_file_text_encoding: str,
ignore_attachment_extension_whitelist: bool = False,
ignore_attachment_filetype_whitelist: bool = False,
attachment_extension_whitelist: list = list(),
attachment_filetype_whitelist: list = list(),
destination_directory: str = '.'):
r"""Extract, decode and save possible attachments within the invoice file.
:param invoice_file_xml_root: the original invoice file.
@ -312,13 +339,15 @@ def extract_attachments_from_invoice_file(
"""
for at in invoice_file_xml_root.findall(invoice_file_xml_attachment_xpath):
attachment_content = at.find(invoice_file_xml_attachment_tag).text
attachment_relative = pathlib.Path(at.find(invoice_file_xml_attachment_filename_tag).text).name
attachment_dest_path = str(pathlib.Path(destination_directory, attachment_relative))
attachment_relative = pathlib.Path(
at.find(invoice_file_xml_attachment_filename_tag).text).name
attachment_dest_path = str(
pathlib.Path(destination_directory, attachment_relative))
if not ignore_attachment_extension_whitelist:
if not attachment_dest_path.endswith(
tuple(attachment_extension_whitelist)):
raise ExtractedAttachmentNotInExtensionWhitelist
if not (ignore_attachment_extension_whitelist
and not attachment_dest_path.endswith(
tuple(attachment_extension_whitelist))):
raise ExtractedAttachmentNotInExtensionWhitelist
# b64decode accepts any bytes-like object. There should not be any
# character encoding problems since base64 characters are represented
@ -326,17 +355,22 @@ def extract_attachments_from_invoice_file(
# Just in case that there are alien characters in the base64 string
# (sic, it happened!) we use validate=False as an option to skip them.
decoded = base64.b64decode(
attachment_content.encode(invoice_file_text_encoding), validate=False)
attachment_content.encode(invoice_file_text_encoding),
validate=False)
if not ignore_attachment_filetype_whitelist:
# See https://h2non.github.io/filetype.py/1.0.0/filetype.m.html#filetype.filetype.get_type
if filetype.guess(
decoded).mime not in attachment_filetype_whitelist:
raise ExtractedAttachmentNotInFileTypeWhitelist
with atomicwrites.atomic_write(attachment_dest_path,
mode='wb',
overwrite=True) as f:
# Atomic write.
# See
# https://stupidpythonideas.blogspot.com/2014/07/getting-atomic-writes-right.html
with tempfile.NamedTemporaryFile('wb', delete=False) as f:
f.flush()
os.fsync(f.fileno())
f.write(decoded)
shutil.move(f.name, attachment_dest_path)
def get_invoice_as_html(invoice_file_xml_root,
@ -359,17 +393,23 @@ def get_invoice_as_html(invoice_file_xml_root,
:type destination_directory: str
:returns: None
:rtype: None
:raises: an lxml, atomicwrites, or a built-in exception.
:raises: an lxml or a built-in exception.
"""
transform = ET.XSLT(invoice_file_xml_stylesheet_root)
newdom = transform(invoice_file_xml_root)
html_output_file_relative = pathlib.Path(html_output_file).name
html_output_file = str(pathlib.Path(destination_directory, html_output_file_relative))
with atomicwrites.atomic_write(html_output_file, mode='w',
overwrite=True) as f:
html_output_file = str(
pathlib.Path(destination_directory, html_output_file_relative))
# Atomic write.
# See
# https://stupidpythonideas.blogspot.com/2014/07/getting-atomic-writes-right.html
with tempfile.NamedTemporaryFile('w', delete=False) as f:
f.flush()
os.fsync(f.fileno())
f.write(
ET.tostring(newdom,
pretty_print=True).decode(invoice_file_text_encoding))
shutil.move(f.name, html_output_file)
def patch_invoice_schema_file(invoice_schema_file: str, offending_line: str,
@ -398,11 +438,12 @@ def patch_invoice_schema_file(invoice_schema_file: str, offending_line: str,
save.append(fix_line)
else:
save.append(line)
with atomicwrites.atomic_write(invoice_schema_file,
mode='w',
overwrite=True) as f:
for s in save:
f.write(s)
with tempfile.NamedTemporaryFile('w', delete=False) as f:
f.flush()
os.fsync(f.fileno())
f.write(save)
shutil.move(f.name, invoice_schema_file)
##############################
@ -486,9 +527,12 @@ def write_configuration_file(configuration_file: str):
const.xml['metadata_file']['tags']['system_id']
}
config['trusted_list_file'] = {
'xml_namespace': const.xml['trusted_list_file']['namespaces']['default'],
'xml_certificate_tag': const.xml['trusted_list_file']['tags']['certificate'],
'download': const.downloads['trusted_list_file']['default'],
'xml_namespace':
const.xml['trusted_list_file']['namespaces']['default'],
'xml_certificate_tag':
const.xml['trusted_list_file']['tags']['certificate'],
'download':
const.downloads['trusted_list_file']['default'],
}
config['invoice_file'] = {
'xml_namespace':
@ -504,7 +548,8 @@ def write_configuration_file(configuration_file: str):
'xsd_download':
const.downloads['invoice_file']['xsd']['default'],
'w3c_xsd_download':
const.downloads['invoice_file']['xsd']['w3c_schema_for_xml_signatures'],
const.downloads['invoice_file']['xsd']
['w3c_schema_for_xml_signatures'],
'xslt_ordinaria_download':
const.downloads['invoice_file']['xslt']['ordinaria'],
'xslt_pa_download':
@ -728,11 +773,13 @@ def pipeline(source: str, file_type: str, data: dict):
if configuration_file == str():
configuration_file = define_appdirs_user_config_dir_file_path(
project_name, const.paths['configuration_file'])
if not pathlib.Path(configuration_file).is_file() or data['write_default_configuration_file']:
if not pathlib.Path(configuration_file).is_file(
) or data['write_default_configuration_file']:
write_configuration_file(configuration_file)
if source != 'NOOP' and file_type != 'NOOP':
config = yaml.load(open(configuration_file, 'r'), Loader=yaml.SafeLoader)
config = yaml.load(open(configuration_file, 'r'),
Loader=yaml.SafeLoader)
# Define all the paths for the static elements.
trusted_list_file = define_appdirs_user_data_dir_file_path(
@ -740,8 +787,8 @@ def pipeline(source: str, file_type: str, data: dict):
ca_certificate_pem_file = define_appdirs_user_data_dir_file_path(
project_name, const.paths['ca_certificate_pem_file'])
w3c_schema_file_for_xml_signatures = define_appdirs_user_data_dir_file_path(
project_name,
const.paths['invoice_file']['xsd']['w3c_schema_for_xml_signatures'])
project_name, const.paths['invoice_file']['xsd']
['w3c_schema_for_xml_signatures'])
if source == 'invoice':
invoice_schema_file = define_appdirs_user_data_dir_file_path(
@ -800,21 +847,25 @@ def pipeline(source: str, file_type: str, data: dict):
if not data['ignore_assets_checksum']:
if not asset_checksum_matches(trusted_list_file):
raise AssetsChecksumDoesNotMatch("Run the program with the '--ignore-assets-checksum' option, contact the developer or open a pull request. Have a look at " + const.docs['assets_url'])
raise AssetsChecksumDoesNotMatch(
"Run the program with the '--ignore-assets-checksum' option, contact the developer or open a pull request. Have a look at "
+ const.docs['assets_url'])
trusted_list_xml_root = parse_xml_file(trusted_list_file)
get_ca_certificates(trusted_list_xml_root, ca_certificate_pem_file,
config['trusted_list_file']['xml_namespace'],
config['trusted_list_file']['xml_certificate_tag'])
get_ca_certificates(
trusted_list_xml_root, ca_certificate_pem_file,
config['trusted_list_file']['xml_namespace'],
config['trusted_list_file']['xml_certificate_tag'])
if (not (source == 'invoice' and file_type == 'plain')) or (
source == 'invoice'
and file_type == 'p7m') or (source == 'generic'
and file_type == 'p7m'):
if not is_p7m_file_authentic(file_to_consider, ca_certificate_pem_file,
data['ignore_signature_check'],
data['ignore_signers_certificate_check']):
if not is_p7m_file_authentic(
file_to_consider, ca_certificate_pem_file,
data['ignore_signature_check'],
data['ignore_signers_certificate_check']):
raise P7MFileNotAuthentic
if source == 'invoice' or ('no_invoice_xml_validation' in data and
@ -837,7 +888,9 @@ def pipeline(source: str, file_type: str, data: dict):
# Verify the checksum of the patched file.
if not data['ignore_assets_checksum']:
if not asset_checksum_matches(invoice_schema_file):
raise AssetsChecksumDoesNotMatch("Run the program with the '--ignore-assets-checksum' option, contact the developer or open a pull request. Have a look at https://frnmst.github.io/fattura-elettronica-reader/assets.html")
raise AssetsChecksumDoesNotMatch(
"Run the program with the '--ignore-assets-checksum' option, contact the developer or open a pull request. Have a look at https://frnmst.github.io/fattura-elettronica-reader/assets.html"
)
# Create a temporary directory to store the original XML invoice file.
with tempfile.TemporaryDirectory() as tmpdirname:
@ -879,8 +932,9 @@ def pipeline(source: str, file_type: str, data: dict):
if not data['no_invoice_xml_validation']:
if not is_xml_file_conforming_to_schema(
str(
pathlib.Path(tmpdirname,
file_to_consider_original_relative)),
pathlib.Path(
tmpdirname,
file_to_consider_original_relative)),
invoice_schema_file):
raise XMLFileNotConformingToSchema
@ -898,8 +952,10 @@ def pipeline(source: str, file_type: str, data: dict):
config['invoice_file']['text_encoding'],
data['ignore_attachment_extension_whitelist'],
data['ignore_attachment_filetype_whitelist'],
config['invoice_file']['attachment_extension_whitelist'],
config['invoice_file']['attachment_filetype_whitelist'],
config['invoice_file']
['attachment_extension_whitelist'],
config['invoice_file']
['attachment_filetype_whitelist'],
data['destination_directory'])
if data['generate_html_output']:
@ -913,15 +969,17 @@ def pipeline(source: str, file_type: str, data: dict):
if not data['ignore_assets_checksum']:
if not asset_checksum_matches(invoice_xslt_file):
raise AssetsChecksumDoesNotMatch("Run the program with the '--ignore-assets-checksum' option, contact the developer or open a pull request. Have a look at https://frnmst.github.io/fattura-elettronica-reader/assets.html")
raise AssetsChecksumDoesNotMatch(
"Run the program with the '--ignore-assets-checksum' option, contact the developer or open a pull request. Have a look at https://frnmst.github.io/fattura-elettronica-reader/assets.html"
)
invoice_xslt_root = parse_xml_file(invoice_xslt_file)
html_output = file_to_consider + '.html'
get_invoice_as_html(invoice_root, invoice_xslt_root,
html_output,
config['invoice_file']['text_encoding'],
data['destination_directory'])
get_invoice_as_html(
invoice_root, invoice_xslt_root, html_output,
config['invoice_file']['text_encoding'],
data['destination_directory'])
if (source == 'invoice'
and file_type == 'p7m') or (source == 'generic'
@ -931,7 +989,9 @@ def pipeline(source: str, file_type: str, data: dict):
str(
pathlib.Path(tmpdirname,
file_to_consider_original_relative)),
str(pathlib.Path(data['destination_directory'], file_to_consider_original_relative)))
str(
pathlib.Path(data['destination_directory'],
file_to_consider_original_relative)))
if __name__ == '__main__':

View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
#
# cli.py
#
@ -25,18 +26,25 @@ import argparse
import sys
import textwrap
from pkg_resources import DistributionNotFound, get_distribution
# See
# https://packaging.python.org/en/latest/guides/single-sourcing-package-version/
if sys.version_info >= (3, 8):
from importlib import metadata
else:
import importlib_metadata as metadata
from .api import pipeline
PROGRAM_DESCRIPTION = 'fattura-elettronica-reader: Validate, extract, and generate printables\nof electronic invoice files received from the "Sistema di Interscambio"\nas well as other P7M files'
VERSION_NAME = 'fattura_elettronica_reader'
try:
VERSION_NUMBER = str(
get_distribution('fattura_elettronica_reader').version)
except DistributionNotFound:
dist = metadata.distribution('fattura_elettronica_reader')
VERSION_NUMBER = dist.version
except metadata.PackageNotFoundError:
VERSION_NUMBER = 'vDevel'
VERSION_COPYRIGHT = 'Copyright (C) 2019 Franco Masotti, frnmst'
VERSION_COPYRIGHT = 'Copyright (C) 2019-2023 Franco Masotti, frnmst'
VERSION_LICENSE = 'License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>\nThis is free software: you are free to change and redistribute it.\nThere is NO WARRANTY, to the extent permitted by law.'
RETURN_VALUES = 'Return values: 0 ok, 1 error, 2 invalid command'
PROGRAM_EPILOG = RETURN_VALUES + '\n\n' + VERSION_COPYRIGHT + '\n' + VERSION_LICENSE
@ -48,16 +56,12 @@ class CliToApi():
def run(self, args):
r"""Run the pipeline."""
common_data = {
'patched':
False,
'configuration_file':
args.configuration_file,
'patched': False,
'configuration_file': args.configuration_file,
'write_default_configuration_file':
args.write_default_configuration_file,
'ignore_assets_checksum':
args.ignore_assets_checksum,
'destination_directory':
args.destination_directory,
'ignore_assets_checksum': args.ignore_assets_checksum,
'destination_directory': args.destination_directory,
}
# Prepare the data structure.
@ -333,10 +337,11 @@ class CliInterface():
default='.',
help='the output directory for all files')
parser.add_argument('-k',
'--ignore-assets-checksum',
action='store_true',
help='avoid running checksums for the downloadable assets')
parser.add_argument(
'-k',
'--ignore-assets-checksum',
action='store_true',
help='avoid running checksums for the downloadable assets')
parser.add_argument('-v',
'--version',

View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
#
# constants.py
#
@ -132,21 +133,26 @@ file['invoice']['attachment'] = {
#############
# SHA-512 checksum of the assets.
checksum = dict()
checksum[paths['invoice_file']['xslt']['pa']] = 'a93dbd93fe8f3beac9ab1ea6ef322c0fdcc27b47e911a4a598c6c12c2abfb1d2ff41c406373d36ccb5d4613c36e21d09421983b5616b778573305f9bb6e3456b'
checksum[paths['invoice_file']['xslt']['ordinaria']] = '2c315cbb04126e98192c0afa585fe3b264ed4fada044504cf9ad77f2272e26106916239e86238dc250f15f5b22a33383e2e690ae28a5f5eb7a8a3b84d3f393b3'
checksum[paths['invoice_file']['xslt'][
'pa']] = 'a93dbd93fe8f3beac9ab1ea6ef322c0fdcc27b47e911a4a598c6c12c2abfb1d2ff41c406373d36ccb5d4613c36e21d09421983b5616b778573305f9bb6e3456b'
checksum[paths['invoice_file']['xslt'][
'ordinaria']] = '2c315cbb04126e98192c0afa585fe3b264ed4fada044504cf9ad77f2272e26106916239e86238dc250f15f5b22a33383e2e690ae28a5f5eb7a8a3b84d3f393b3'
# checksum of the patched schema file, not of the original one which is
# 2a7c3f2913ee390c167e41ae5618c303b481f548f9b2a8d60dddc36804ddd3ebf7cb5003e5cc6996480c67d085b82b438aff7cc0f74d7c104225449785cb575b
#
# The xml schema file for FatturaPA version 1.2.1 needs to be patched. fattura_elettronica_reader
# runs the SHA-512 checksum on the patched version of that file which corresponds to:
checksum[paths['invoice_file']['xsd']['default']] = 'a1b02818f81ac91f35358260dd12e1bf4480e1545bb457caffa0d434200a1bd05bedd88df2d897969485a989dda78922850ebe978b92524778a37cb0afacba27'
checksum[paths['invoice_file']['xsd'][
'default']] = 'a1b02818f81ac91f35358260dd12e1bf4480e1545bb457caffa0d434200a1bd05bedd88df2d897969485a989dda78922850ebe978b92524778a37cb0afacba27'
# TSL-IT.xml
checksum[paths['trusted_list_file']] = '9b76bb0b319449286e51f51c5732f1f002b8aed40ad3e932f719728a7b8cf4b7681bfea1b94af11acae6f729f24d770a7284b2887585b7cdf6b6cf6f33e7710f'
checksum[paths[
'trusted_list_file']] = '6c3ac28d370d363dafedab42a794368608eda716339058f43dd604589dad38769cd88d54f15e384f406debb24fb6e1d1cfd7d78a2f33bbe7368e5ec7888e3348'
docs = dict()
docs['assets_url'] = 'https://docs.franco.net.eu.org/fattura-elettronica-reader/assets.html'
docs[
'assets_url'] = 'https://docs.franco.net.eu.org/fattura-elettronica-reader/assets.html'
if __name__ == '__main__':
pass

View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
#
# exceptions.py
#

View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
#
# __init__.py
#

View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
#
# tests.py
#

3
pyproject.toml Normal file
View File

@ -0,0 +1,3 @@
[build-system]
requires = ["setuptools>=39.2.0"]
build-backend = "setuptools.build_meta"

View File

@ -1,7 +1,6 @@
# requirements-dev.txt
#
# Pipfile
#
# Copyright (C) 2020-2021 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
# Copyright (C) 2023 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
#
# This file is part of fattura-elettronica-reader.
#
@ -19,25 +18,12 @@
# along with fattura-elettronica-reader. If not, see <http://www.gnu.org/licenses/>.
#
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
# Documentation.
Sphinx = ">=4,<5"
sphinx-rtd-theme = ">=1,<2"
sphinx-book-theme>=0.3.3,<0.4
Sphinx==4.5.0
sphinx-copybutton>=0.5,<0.6
# Tools.
twine = ">=3,<4"
pre-commit = ">=2,<3"
[packages]
appdirs = ">=1.4,<1.5"
atomicwrites = ">=1.4,<2"
filetype = ">=1,<2"
fpyutils = ">=2.2,<3"
lxml = ">=4.9,<4.10"
PyYAML = ">=6,<7"
requests = ">=2.28,<3"
twine>=3,<4
build>=0.9,<0.10
pre-commit>=2,<3

63
requirements-freeze.txt Normal file
View File

@ -0,0 +1,63 @@
alabaster==0.7.12
appdirs==1.4.4
Babel==2.11.0
beautifulsoup4==4.11.1
bleach==5.0.1
build==0.9.0
certifi==2022.12.7
cffi==1.15.1
cfgv==3.3.1
charset-normalizer==2.1.1
colorama==0.4.6
cryptography==39.0.0
distlib==0.3.6
docutils==0.17.1
filelock==3.9.0
filetype==1.2.0
fpyutils==3.0.1
identify==2.5.12
idna==3.4
imagesize==1.4.1
importlib-metadata==6.0.0
jaraco.classes==3.2.3
jeepney==0.8.0
Jinja2==3.1.2
keyring==23.13.1
lxml==4.9.2
MarkupSafe==2.1.1
more-itertools==9.0.0
nodeenv==1.7.0
packaging==22.0
pep517==0.13.0
pkginfo==1.9.5
platformdirs==2.6.2
pre-commit==2.21.0
pycparser==2.21
pydata-sphinx-theme==0.8.1
Pygments==2.14.0
pytz==2022.7
PyYAML==6.0
readme-renderer==37.3
requests==2.28.1
requests-toolbelt==0.10.1
rfc3986==2.0.0
SecretStorage==3.3.3
six==1.16.0
snowballstemmer==2.2.0
soupsieve==2.3.2.post1
Sphinx==4.5.0
sphinx-book-theme==0.3.3
sphinx-copybutton==0.5.1
sphinxcontrib-applehelp==1.0.3
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
tomli==2.0.1
tqdm==4.64.1
twine==3.8.0
urllib3==1.26.13
virtualenv==20.17.1
webencodings==0.5.1
zipp==3.11.0

5
requirements.txt Normal file
View File

@ -0,0 +1,5 @@
appdirs>=1.4,<1.5
filetype>=1,<2
fpyutils>=3.0.1,<4
lxml>=4.9,<4.10
PyYAML>=6,<7

86
setup.cfg Normal file
View File

@ -0,0 +1,86 @@
# -*- coding: utf-8 -*-
#
# setup.cfg
#
# Copyright (C) 2022-2023 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
#
# This file is part of md-toc.
#
# md-toc is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# md-toc is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with md-toc. If not, see <http://www.gnu.org/licenses/>.
#
[metadata]
name = fattura_elettronica_reader
# 'version' needs setuptools >= 39.2.0.
version = 3.0.3
license = GPLv3+,
description = Check and extract electronic invoices received from the Sistema di Interscambio
long_description=file: README.md
long_description_content_type = text/markdown
author = Franco Masotti
author_email = franco.masotti@tutanota.com
keywords=
invoice
reader
SDI
url = https://blog.franco.net.eu.org/software/#fattura-elettronica-reader
classifiers=
Development Status :: 2 - Pre-Alpha
Topic :: Utilities
Intended Audience :: End Users/Desktop
Environment :: Console
License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
Programming Language :: Python :: 3
[options]
python_requires = >=3.5, <4
install_requires=
appdirs >=1.4, <1.5
filetype >=1, <2
fpyutils >=3.0.1, <4
lxml >=4.9, <4.10
PyYAML >=6, <7
importlib-metadata >= 1.0 ; python_version < "3.8"
packages=find:
[options.entry_points]
console_scripts =
fattura_elettronica_reader = fattura_elettronica_reader.__main__:main
[options.packages.find]
exclude=
*tests*
[options.package_data]
* = *.txt, *.rst
[yapf]
based_on_style = pep8
indent_width = 4
[flake8]
ignore =
E125
E126
E131
E501
W503
W504
F401
[isort]
# See
# https://github.com/ESMValGroup/ESMValCore/issues/777
multi_line_output = 3
include_trailing_comma = true

View File

@ -1,66 +1,31 @@
# -*- coding: utf-8 -*-
#
# setup.py
#
# Copyright (C) 2019-2022 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
# Copyright (C) 2017-2023 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
#
# This file is part of fattura-elettronica-reader.
# This file is part of md-toc.
#
# fattura-elettronica-reader is free software: you can redistribute it and/or modify
# md-toc is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# fattura-elettronica-reader is distributed in the hope that it will be useful,
# md-toc is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with fattura-elettronica-reader. If not, see <http://www.gnu.org/licenses/>.
# along with md-toc. If not, see <http://www.gnu.org/licenses/>.
#
"""setup."""
r"""setup.py."""
from setuptools import find_packages, setup
import setuptools
with open('README.md', 'r', encoding='utf-8') as f:
readme = f.read()
# See
# https://importlib-metadata.readthedocs.io/en/latest/migration.html#pkg-resources-require
# import pkg_resources
# pkg_resources.require('setuptools>=39.2.0')
setup(
name='fattura_elettronica_reader',
version='3.0.3',
packages=find_packages(exclude=['*tests*']),
license='GPL',
description='A utility that is able to check and extract electronic invoice received from the Sistema di Interscambio.',
long_description=readme,
long_description_content_type='text/markdown',
package_data={
'': ['*.txt', '*.rst'],
},
author='Franco Masotti',
author_email='franco.masotti@tutanota.com',
keywords='invoice reader SDI',
url='https://blog.franco.net.eu.org/software/#fattura-elettronica-reader',
python_requires='>=3.5,<4',
entry_points={
'console_scripts': [
'fattura_elettronica_reader=fattura_elettronica_reader.__main__:main',
],
},
classifiers=[
'Development Status :: 2 - Pre-Alpha',
'Topic :: Utilities',
'Intended Audience :: End Users/Desktop',
'Environment :: Console',
'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
'Programming Language :: Python :: 3',
],
install_requires=[
'appdirs>=1.4,<1.5',
'atomicwrites>=1.4,<2',
'filetype>=1,<2',
'fpyutils>=2.2,<3'
'lxml>=4.9,<4.10',
'PyYAML>=6,<7',
'requests>=2.28,<3',
],
)
setuptools.setup()