Added submodules.

Moved scripts to separate directory.
Added scripts that automatically retrieves submodules URLs from a Gitea instance.
Updated dependencies.
Updated readme.

.gitignore

@@ -1 +1,106 @@
 Pipfile.lock
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+.static_storage/
+.media/
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/

.gitmodules

@@ -1102,3 +1102,21 @@
 [submodule "submodules/numpydoc"]
 	path = submodules/numpydoc
 	url = https://software.franco.net.eu.org/mirrors-python/numpydoc.git
+[submodule "submodules/MyST-Parser"]
+	path = submodules/MyST-Parser
+	url = https://software.franco.net.eu.org/mirrors-python/MyST-Parser.git
+[submodule "submodules/pytest-regressions"]
+	path = submodules/pytest-regressions
+	url = https://software.franco.net.eu.org/mirrors-python/pytest-regressions.git
+[submodule "submodules/jupyter-sphinx"]
+	path = submodules/jupyter-sphinx
+	url = https://software.franco.net.eu.org/mirrors-python/jupyter-sphinx.git
+[submodule "submodules/plotly.py"]
+	path = submodules/plotly.py
+	url = https://software.franco.net.eu.org/mirrors-python/plotly.py.git
+[submodule "submodules/xarray"]
+	path = submodules/xarray
+	url = https://software.franco.net.eu.org/mirrors-python/xarray.git
+[submodule "submodules/nox"]
+	path = submodules/nox
+	url = https://software.franco.net.eu.org/mirrors-python/nox.git

Makefile

@@ -22,6 +22,15 @@
 
 default: install-dev
 
+submodules-add:
+	cd scripts; ./add_submodules.sh $(SUBMODULES)
+
+submodules-add-gitea:
+	cd scripts; pipenv run python3 -m add_submodules_gitea
+
+submodules-update:
+	cd scripts; ./update_submodules.sh
+
 install-dev:
 	pipenv install --dev
 	pipenv run pre-commit install
@@ -35,4 +44,7 @@ uninstall-dev:
 update: install-dev
 	pipenv run pre-commit autoupdate
 
-.PHONY: default install-dev uninstall-dev update
+clean:
+	rm -rf build dist *.egg-info
+
+.PHONY: default install-dev uninstall-dev update clean

Pipfile

@@ -6,3 +6,4 @@ name = "pypi"
 [dev-packages]
 pre-commit = '>=2,<3'
 fpyutils = '>=2,<3'
+requests = '>=2.26,<2.27'

README.md

@@ -11,7 +11,6 @@ Git repository pointers and configurations to build Python packages from source.
   - [Description](#description)
   - [Script](#script)
   - [Submodules](#submodules)
-    - [Clone](#clone)
     - [Update](#update)
     - [Add](#add)
     - [Important](#important)
@@ -76,11 +75,31 @@ have been built using the [build_python_packages.py](https://docs.franco.net.eu.
 
 ## Submodules
 
-### Clone
+1. clone this repository normally (without recrursive submodules)
 
-```shell
-git clone https://software.franco.net.eu.org/frnmst/python-packages-source.git
-```
+   ```shell
+   git clone https://software.franco.net.eu.org/frnmst/python-packages-source.git
+   ```
+
+2. to add submodules from external repositories run
+
+   ```shell
+   make submodules-add SUBMODULES="{submodule_URL_0} ${submodule_URL_1} ... ${submodule_URL_n}"
+   ```
+
+3. to add all submodules from a Gitea organization through Gitea's API run
+
+   ```shell
+   make submodules-add-gitea
+   ```
+
+   https://software.franco.net.eu.org/mirrors-python is hardcoded in the script as variables
+
+4. to update all submodules in the index run
+
+   ```shell
+   make submodules-update
+   ```
 
 ### Update
 
@@ -90,8 +109,10 @@ git clone https://software.franco.net.eu.org/frnmst/python-packages-source.git
 
 ### Add
 
+
+
 ```shell
-./0000_add_submodules.sh ${submodule_URL_0} ${submodule_URL_1} ${submodule_URL_n}
+./0000_add_submodules.sh ${submodule_URL_0} ${submodule_URL_1} ... ${submodule_URL_n}
 ```
 
 ### Important

scripts/add_submodules.sh

@@ -1,28 +1,27 @@
 #!/usr/bin/env bash
 #
-# 0000_add_submodules.sh
+# add_submodules.sh
 #
-# Copyright (C) 2021 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
+# Copyright (C) 2021-2022 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
 #
-# This file is part of pip-packages-source.
+# This file is part of python-packages-source.
 #
-# pip-packages-source is free software: you can redistribute it and/or modify
+# python-packages-source is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# pip-packages-source is distributed in the hope that it will be useful,
+# python-packages-source is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with pip-packages-source.  If not, see <http://www.gnu.org/licenses/>.
-#
+# along with python-packages-source.  If not, see <http://www.gnu.org/licenses/>.
 
-# Usage: ./0000_add_submodules.sh [GIT_URL] [GIT_URL] ...
+# Usage: ./add_submodules.sh [GIT_URL] [GIT_URL] ...
 
-pushd submodules
+pushd ../submodules
 for url in "$@"; do
     # Always add '.git'
     if [ "${url}" = "${url%%.git}" ]; then

scripts/add_submodules_gitea.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+#
+# add_submodules_gitea.py
+#
+# Copyright (C) 2021-2022 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
+#
+# This file is part of python-packages-source.
+#
+# python-packages-source is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# python-packages-source is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with python-packages-source.  If not, see <http://www.gnu.org/licenses/>.
+
+import shlex
+import contextlib
+import fpyutils
+import gitea
+import os
+
+SUBMODULES_DIRECTORY='../submodules'
+
+# See
+# https://stackoverflow.com/a/13847807
+# CC BY-SA 4.0
+# spiralman, bryant1410
+@contextlib.contextmanager
+def pushd(new_dir):
+    previous_dir = os.getcwd()
+    os.chdir(new_dir)
+    try:
+        yield
+    finally:
+        os.chdir(previous_dir)
+
+
+if __name__ == '__main__':
+    def main():
+        results = gitea.get_org_repos()
+
+        # Get the original URLs of mirrors only.
+        with pushd(SUBMODULES_DIRECTORY):
+            for r in results:
+                for rr in r:
+                    if rr['mirror'] and not rr['empty']:
+                        fpyutils.shell.execute_command_live_output('git submodule add ' + shlex.quote(rr['clone_url']))
+
+    main()

scripts/gitea.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+#
+# gitea.py
+#
+# Copyright (C) 2021-2022 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
+#
+# This file is part of python-packages-source.
+#
+# python-packages-source is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# python-packages-source is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with python-packages-source.  If not, see <http://www.gnu.org/licenses/>.
+
+import requests
+import json
+import urllib
+import os
+
+GITEA_BASE_URL = 'software.franco.net.eu.org'
+GITEA_ORG = 'mirrors-python'
+
+def get_org_repos() -> list:
+    results = list()
+    go = True
+
+    i = 1
+    while go:
+        # See also
+        # https://try.gitea.io/api/swagger#/organization/orgListRepos
+        # for Gitea's API.
+        args = {'page': str(i)}
+        parse = urllib.parse.ParseResult(
+            scheme='https',
+            netloc=GITEA_BASE_URL,
+            path='api/v1/orgs/' + GITEA_ORG + '/repos',
+            params='',
+            query=urllib.parse.urlencode(args),
+            fragment=''
+        )
+        repos = urllib.parse.urlunparse(parse)
+        # print(repos)
+        r = requests.get(repos)
+
+        if json.loads(r.text) == list():
+            # No more repos: stop condition.
+            go = False
+        else:
+            results.append(json.loads(r.text))
+
+        i += 1
+
+    return results

scripts/update_submodules.sh

@@ -1,26 +1,26 @@
 #!/usr/bin/env bash
 #
-# 0000_update_submodules.sh
+# update_submodules.sh
 #
 # Copyright (C) 2021 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
 #
-# This file is part of pip-packages-source.
+# This file is part of python-packages-source.
 #
-# pip-packages-source is free software: you can redistribute it and/or modify
+# python-packages-source is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# pip-packages-source is distributed in the hope that it will be useful,
+# python-packages-source is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with pip-packages-source.  If not, see <http://www.gnu.org/licenses/>.
+# along with python-packages-source.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-# Usage: ./0000_update_submodules.sh
+# Usage: ./update_submodules.sh
 
 git submodule sync
 git submodule update --init --remote

submodules/MyST-Parser

@@ -0,0 +1 @@
+Subproject commit 0f2902ab5eb6236805ed24dc387b34c00fe41744

submodules/jupyter-sphinx

@@ -0,0 +1 @@
+Subproject commit 34aaa66379cff9e53ceb148f11043fbf718510fb

submodules/nox

@@ -0,0 +1 @@
+Subproject commit 8261d2ad0a8e3d720c093c30ac56488f53605f85

submodules/plotly.py

@@ -0,0 +1 @@
+Subproject commit cfad7862594b35965c0e000813bd7805e8494a5b

submodules/pytest-regressions

@@ -0,0 +1 @@
+Subproject commit fb8b283410e1ccdb8e164ffe3b9748bf0e77bd9a

submodules/xarray

@@ -0,0 +1 @@
+Subproject commit 60754fdbc4ecd9eb3c0978e82635c6d43e8d485b