frnmst-archives
/
python-packages-source
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity Software Blog

Update repository to new specs 

- Change configuration file structure
- Use of venv and requirements file instead of Pipenv
- Move some pre-commit options to setup.cfg
This commit is contained in:
Franco Masotti 2023-01-14 18:44:48 +01:00
parent 7c44e2759c
commit 6e08b374ac
Signed by: frnmst
GPG Key ID: 24116ED85666780A
15 changed files with 257 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
.pre-commit-config.yaml
View File

@ -1,34 +1,42 @@
# IMPORTANT:
# Do not deal with submodules.
files: '(README.md|configuration.yaml|Makefile|scripts/.*)'
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.3.0
rev: 'v4.4.0'
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
- id: check-added-large-files
- id: destroyed-symlinks
- id: detect-private-key
- id: check-ast
- id: check-case-conflict
- id: debug-statements
- id: fix-encoding-pragma
- id: forbid-submodules
- id: check-symlinks
- id: check-shebang-scripts-are-executable
- id: check-case-conflict
- id: check-added-large-files
args: ['--maxkb=32768']
- id: destroyed-symlinks
- repo: https://codeberg.org/frnmst/md-toc
rev: '8.1.5' # or a specific git tag from md-toc
rev: '8.1.8' # or a specific git tag from md-toc
hooks:
- id: md-toc
args: [-p, 'github', '-l6'] # CLI options
- repo: https://codeberg.org/frnmst/licheck
rev: 1.0.0
hooks:
- id: licheck
args: ['--configuration-file', '.allowed_licenses.yml']
# - repo: https://codeberg.org/frnmst/licheck
# rev: 1.0.0
# hooks:
# - id: licheck
# args: ['--configuration-file', '.allowed_licenses.yml']
- repo: https://github.com/pycqa/isort
rev: 5.10.1
rev: '5.11.4'
hooks:
- id: isort
@ -42,14 +50,12 @@ repos:
rev: '5.0.4' # Use the sha / tag you want to point at
hooks:
- id: flake8
args: ['--ignore=E501,W503,W504,F401,D401']
additional_dependencies: [flake8-docstrings]
- repo: https://github.com/pre-commit/mirrors-yapf
rev: 'v0.32.0' # Use the sha / tag you want to point at
hooks:
- id: yapf
args: ['--style', '{based_on_style: pep8; indent_width: 4}']
- repo: local
hooks:

104
Makefile
View File

@ -1,8 +1,7 @@
#!/usr/bin/env make
#
# Makefile
#
# Copyright (C) 2021-2022 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
# Copyright (C) 2017-2023 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
#
# This file is part of python-packages-source.
#
@ -20,39 +19,96 @@
# along with python-packages-source. If not, see <http://www.gnu.org/licenses/>.
#
export PACKAGE_NAME=python_packages_source
# See
# https://docs.python.org/3/library/venv.html#how-venvs-work
export VENV_CMD=. .venv/bin/activate
default: install-dev submodules-update submodules-add-gitea
stats:
cd scripts; pipenv run python3 -m collect_data
plot:
cd scripts; pipenv run python3 -m plot_data $(OUTPUT)
submodules-add:
cd scripts; ./add_submodules.sh $(SUBMODULES)
submodules-add-gitea:
cd scripts; pipenv run python3 -m add_submodules_gitea
submodules-update:
cd scripts; ./update_submodules.sh
doc:
$(VENV_CMD) \
&& $(MAKE) -C docs html \
&& deactivate
install:
pipenv install
pip3 install . --user
uninstall:
pip3 uninstall --verbose --yes $(PACKAGE_NAME)
install-dev:
pipenv install --dev
pipenv run pre-commit install
pipenv graph
python3 -m venv .venv
$(VENV_CMD) \
&& pip install --requirement requirements-freeze.txt \
&& deactivate
$(VENV_CMD) \
&& pre-commit install \
&& deactivate
$(VENV_CMD) \
&& pre-commit install --hook-type commit-msg \
&& deactivate
regenerate-freeze: uninstall-dev
python3 -m venv .venv
$(VENV_CMD) \
&& pip install --requirement requirements.txt --requirement requirements-dev.txt \
&& pip freeze --local > requirements-freeze.txt \
&& deactivate
uninstall-dev:
rm -f Pipfile.lock
pipenv --rm
rm -rf .venv
update: install-dev
pipenv run pre-commit autoupdate
$(VENV_CMD) \
&& pre-commit autoupdate \
--repo https://github.com/pre-commit/pre-commit-hooks \
--repo https://github.com/PyCQA/bandit \
--repo https://github.com/pycqa/isort \
--repo https://codeberg.org/frnmst/licheck \
--repo https://codeberg.org/frnmst/md-toc \
--repo https://github.com/mgedmin/check-manifest \
--repo https://github.com/jorisroovers/gitlint \
&& deactivate
# --repo https://github.com/pre-commit/mirrors-mypy \
test:
$(VENV_CMD) \
&& python -m unittest $(PACKAGE_NAME).tests.tests --failfast --locals --verbose \
&& deactivate
pre-commit:
$(VENV_CMD) \
&& pre-commit run --all \
&& deactivate
clean:
rm -rf build dist *.egg-info
.PHONY: default stats plot submodules-add submodules-add-gitea submodules-update install-dev uninstall-dev update clean
stats:
$(VENV_CMD) \
&& cd scripts \
&& python3 -m collect_data
plot:
$(VENV_CMD) \
&& cd scripts \
&& python3 -m plot_data $(OUTPUT)
submodules-add:
$(VENV_CMD) \
&& cd scripts \
&& ./add_submodules.sh $(SUBMODULES)
submodules-add-gitea:
$(VENV_CMD) \
&& cd scripts \
&& python3 -m add_submodules_gitea
submodules-update:
$(VENV_CMD) \
&& cd scripts \
&& ./update_submodules.sh
.PHONY: default doc install uninstall install-dev uninstall-dev update test clean pre-commit stats plot submodules-add submodules-add-gitea submodules-update

12
Pipfile
View File

@ -1,12 +0,0 @@
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[dev-packages]
pre-commit = '>=2,<3'
[packages]
fpyutils = '>=2,<3'
requests = '>=2.26,<2.27'
matplotlib = '>=3,<4'

8
README.md
View File

@ -200,7 +200,7 @@ Changelogs, instructions, sources and keys can be found at [blog.franco.net.eu.o
## Crypto donations
- Bitcoin: bc1qnkflazapw3hjupawj0lm39dh9xt88s7zal5mwu
- Monero: 84KHWDTd9hbPyGwikk33Qp5GW7o7zRwPb8kJ6u93zs4sNMpDSnM5ZTWVnUp2cudRYNT6rNqctnMQ9NbUewbj7MzCBUcrQEY
- Dogecoin: DMB5h2GhHiTNW7EcmDnqkYpKs6Da2wK3zP
- Vertcoin: vtc1qd8n3jvkd2vwrr6cpejkd9wavp4ld6xfu9hkhh0
- Bitcoin: `bc1qnkflazapw3hjupawj0lm39dh9xt88s7zal5mwu`
- Monero: `84KHWDTd9hbPyGwikk33Qp5GW7o7zRwPb8kJ6u93zs4sNMpDSnM5ZTWVnUp2cudRYNT6rNqctnMQ9NbUewbj7MzCBUcrQEY`
- Dogecoin: `DMB5h2GhHiTNW7EcmDnqkYpKs6Da2wK3zP`
- Vertcoin: `vtc1qd8n3jvkd2vwrr6cpejkd9wavp4ld6xfu9hkhh0`

71
configuration.yaml
View File

@ -3,7 +3,7 @@
#
# configuration.yaml
#
# Copyright (C) 2021-2022 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
# Copyright (C) 2021-2023 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
#
# This file is part of python-packages-source.
#
@ -22,29 +22,54 @@
#
submodules:
base_directory_override:
# Key is the repository name.
# Key is the repository name.
typing:
# Value is the relative path of the directory in the repository.
typing: 'typing_extensions'
geographiclib: 'python'
snowball: 'python'
flit-core: 'flit_core'
pytz: 'src'
psycopg2-binary: 'psycopg'
hypothesis: 'hypothesis-python'
core-workflow: 'blurb'
checkout:
base_directory_override: 'typing_extensions'
override_commands: {}
ref_checkout: []
geographiclib:
base_directory_override: 'python'
override_commands: {}
ref_checkout: []
snowball:
base_directory_override: 'python'
override_commands: {}
ref_checkout: []
flit-core:
base_directory_override: 'flit_core'
override_commands: {}
ref_checkout: []
pytz:
base_directory_override: 'src'
override_commands: {}
ref_checkout: []
psycopg2-binary:
base_directory_override: 'psycopg'
override_commands: {}
ref_checkout: []
hypothesis:
base_directory_override: 'hypothesis-python'
override_commands: {}
ref_checkout: []
core-workflow:
base_directory_override: 'blurb'
override_commands: {}
ref_checkout: []
MarkupPy:
base_directory_override: ''
override_commands:
# Checkout only the specified tags or commits.
MarkupPy:
ref_checkout:
- 9af6454566d59a6e81bad8c4f58bb828431b07b6
# Run these commands for the marked repository
# WARNING: these are arbitrary commands. It is your
# responsability to check!
# One argument for each list element.
build:
pre_commands:
babel:
babel:
base_directory_override: ''
# Run these commands for the marked repository
# WARNING: these are arbitrary commands. It is your
# responsability to check!
# One argument for each list element.
override_commands:
pre:
install_pytz:
- 'pip3'
- 'install'
@ -53,4 +78,6 @@ submodules:
- 'python3'
- 'setup.py'
- 'import_cldr'
post_commands: {}
build: {}
post: {}
ref_checkout: []

1
requirements-dev.txt Normal file
View File

@ -0,0 +1 @@
pre-commit>=2,<3

26
requirements-freeze.txt Normal file
View File

@ -0,0 +1,26 @@
certifi==2022.12.7
cfgv==3.3.1
charset-normalizer==2.0.12
contourpy==1.0.7
cycler==0.11.0
distlib==0.3.6
filelock==3.9.0
fonttools==4.38.0
fpyutils==3.0.1
identify==2.5.13
idna==3.4
kiwisolver==1.4.4
matplotlib==3.6.3
nodeenv==1.7.0
numpy==1.24.1
packaging==23.0
Pillow==9.4.0
platformdirs==2.6.2
pre-commit==2.21.0
pyparsing==3.0.9
python-dateutil==2.8.2
PyYAML==6.0
requests==2.26.0
six==1.16.0
urllib3==1.26.14
virtualenv==20.17.1

3
requirements.txt Normal file
View File

@ -0,0 +1,3 @@
fpyutils>=3.0.1,<4
requests>=2.26,<2.27
matplotlib>=3,<4

8
scripts/add_submodules_gitea.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# add_submodules_gitea.py
#
@ -45,8 +46,9 @@ def _pushd(new_dir):
if __name__ == '__main__':
def main():
r"""Main."""
r"""Run main."""
results = gitea.get_org_repos()
# Get the original URLs of mirrors only.
@ -54,6 +56,8 @@ if __name__ == '__main__':
for r in results:
for rr in r:
if rr['mirror'] and not rr['empty']:
fpyutils.shell.execute_command_live_output('git submodule add ' + shlex.quote(rr['clone_url']))
fpyutils.shell.execute_command_live_output(
'git submodule add ' +
shlex.quote(rr['clone_url']))
main()

4
scripts/collect_data.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# collect_data.py
#
@ -34,8 +35,9 @@ REGEX = r'http(|s)://(|www\.)github.com/.*/.*'
PLOT_DAYS_SENSIBILITY = 1
if __name__ == '__main__':
def main():
r"""main."""
r"""Run main."""
now = datetime.datetime.now(datetime.timezone.utc)
results = gitea.get_org_repos()

25
scripts/gitea.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# gitea.py
#
@ -43,14 +44,13 @@ def get_org_repos() -> list:
# https://try.gitea.io/api/swagger#/organization/orgListRepos
# for Gitea's API.
args = {'page': str(i)}
parse = urllib.parse.ParseResult(
scheme='https',
netloc=GITEA_BASE_URL,
path='api/v1/orgs/' + GITEA_ORG + '/repos',
params='',
query=urllib.parse.urlencode(args),
fragment=''
)
parse = urllib.parse.ParseResult(scheme='https',
netloc=GITEA_BASE_URL,
path='api/v1/orgs/' + GITEA_ORG +
'/repos',
params='',
query=urllib.parse.urlencode(args),
fragment='')
repos = urllib.parse.urlunparse(parse)
# print(repos)
r = requests.get(repos)
@ -81,7 +81,8 @@ def read_csv(file: str) -> tuple:
for row in reader:
total.append(int(row[0]))
match.append(int(row[1]))
date.append(datetime.datetime.strptime(row[2], '%Y-%m-%d %H:%M:%S %z'))
date.append(
datetime.datetime.strptime(row[2], '%Y-%m-%d %H:%M:%S %z'))
runs.append(i)
i += 1
@ -91,6 +92,8 @@ def read_csv(file: str) -> tuple:
def write_csv(file: str, total: list, match: list, date: list):
r"""Write a CSV file."""
with open(file, 'a') as csvfile:
writer = csv.writer(csvfile, delimiter=',',
quotechar='|', quoting=csv.QUOTE_MINIMAL)
writer = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
writer.writerow([total, match, date])

33
scripts/plot_data.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# plot_data.py
#
@ -35,8 +36,9 @@ INPUT_FILE = '../stats/repo_stats.csv'
OUTPUT_FILE = sys.argv[1]
if __name__ == '__main__':
def main():
r"""main."""
r"""Run main."""
########
# Data #
########
@ -64,7 +66,9 @@ if __name__ == '__main__':
# Plot #
########
# Get and resize graph.
fig, (frquency_plot, ratio_plot) = plt.subplots(2, figsize=(15, 10), dpi=160)
fig, (frquency_plot, ratio_plot) = plt.subplots(2,
figsize=(15, 10),
dpi=160)
# Set grids.
frquency_plot.grid()
@ -77,15 +81,26 @@ if __name__ == '__main__':
frquency_plot.set_yticks(range(0, y_ticks_rage_max, y_ticks_step))
# X
frquency_plot.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
frquency_plot.xaxis.set_major_formatter(
mdates.DateFormatter('%Y-%m-%d'))
for label in frquency_plot.get_xticklabels(which='major'):
label.set(rotation=30, horizontalalignment='right')
frquency_plot.plot(date, total, linestyle='solid', marker='o', label='Total mirrored repositories')
frquency_plot.plot(date, match, linestyle='solid', marker='o', label='Repositories mirrored from GitHub')
frquency_plot.plot(date,
total,
linestyle='solid',
marker='o',
label='Total mirrored repositories')
frquency_plot.plot(date,
match,
linestyle='solid',
marker='o',
label='Repositories mirrored from GitHub')
frquency_plot.legend()
frquency_plot.set_ylim(bottom=0, top=y_ticks_rage_max + (y_ticks_rage_max * 0.10))
frquency_plot.set_ylim(bottom=0,
top=y_ticks_rage_max +
(y_ticks_rage_max * 0.10))
# Y
ratio_plot.set_ylabel('Ratio')
@ -99,7 +114,11 @@ if __name__ == '__main__':
for label in ratio_plot.get_xticklabels(which='major'):
label.set(rotation=30, horizontalalignment='right')
ratio_plot.plot(date, ratio, linestyle='solid', marker='o', label='GitHub / Total repositories')
ratio_plot.plot(date,
ratio,
linestyle='solid',
marker='o',
label='GitHub / Total repositories')
ratio_plot.legend()
ratio_plot.set_ylim(bottom=0, top=1.1)

18
setup.cfg Normal file
View File

@ -0,0 +1,18 @@
[yapf]
based_on_style = pep8
indent_width = 4
[flake8]
ignore =
E125
E131
E501
W503
W504
F401
[isort]
# See
# https://github.com/ESMValGroup/ESMValCore/issues/777
multi_line_output = 3
include_trailing_comma = true

9
setup.py Normal file
View File

@ -0,0 +1,9 @@
import setuptools
# See
# https://importlib-metadata.readthedocs.io/en/latest/migration.html#pkg-resources-require
# import pkg_resources
# pkg_resources.require('setuptools>=39.2.0')
setuptools.setup()

1
stats/repo_stats.csv
View File

@ -62,3 +62,4 @@
583,542,2022-12-30 16:43:53 +0000
583,542,2023-01-01 23:11:09 +0000
583,542,2023-01-08 23:14:35 +0000
584,543,2023-01-14 17:28:03 +0000

1 21 21 2021-10-07 00:00:00 +0000
62 583 542 2022-12-30 16:43:53 +0000
63 583 542 2023-01-01 23:11:09 +0000
64 583 542 2023-01-08 23:14:35 +0000
65 584 543 2023-01-14 17:28:03 +0000