Make one script for automatic updates

scripts/update_all

@@ -3,13 +3,17 @@
 # Dependencies:
 #  pip3 install PyGithub
 #  pip3 install python-gitlab
+#  pip3 install internetarchive
 
 # Before running, place the authentication token for a user with push access to
-# the Github repository in an environment variable 'GITHUB_AUTH' and a token to
-# access the GitLab repository in an environment variable 'GITLAB_AUTH'
+# the Github repository in an environment variable 'GITHUB_AUTH', a token to
+# access the GitLab repository in an environment variable 'GITLAB_AUTH' and the
+# Internet Archive S3 Access and Secret tokens in the 'IA_ACCESS' and 'IA_SECRET'
+# environment variables
 
 from github import Github
 from gitlab import Gitlab
+import internetarchive as ia
 
 import sys
 import json
@@ -90,6 +94,33 @@ class GithubRemote:
     def upload(self, filename, arch):
         self.release.upload_asset(filename)
 
+class GdriveRemote:
+    def clean_up(self):
+        try:
+            subprocess.check_call(["rclone", "delete gdrive:releases"])
+            subprocess.check_call(["rclone", "cleanup gdrive:releases"])
+        except:
+            print("Error: failed to delete or cleanup with rclone", file=sys.stderr)
+            return 1
+
+    def upload(self, filename, arch):
+        try:
+            subprocess.check_call(["rclone", "copy", filename, "gdrive:releases"])
+        except:
+            print("Error: failed to rclone "+filename+" to Google Drive", file=sys.stderr)
+
+class IARemote:
+    def __init__(self, token):
+        self.access = token.split("\n")[0]
+        self.secret = token.split("\n")[1]
+
+    def upload(self, filename):
+
+        md = dict(collection="open_source_software", title=filename, mediatype="software")
+        item = ia.get_item("Tor_Browser_"+version)
+        item.modify_metadata(dict(title="Tor Browser "+version))
+        r = item.upload(files=[filename], metadata=md, access_key=self.access, secret_key=self.secret)
+
 #Download list of tor browser releases and upload them to remote repositories
 def upload_files(remotes):
     if len(remotes) == 0:
@@ -162,7 +193,21 @@ def main():
         gitlab.create_projects()
         remotes.append(gitlab)
 
-    return upload_files(remotes)
+    if 'IA_ACCESS' not in os.environ and 'IA_SECRET' not in os.environ:
+        print("WARNING: No Internet Archive S3 authentication tokens given", file=sys.stderr)
+        print("The authentication tokens for Internet Archive should be placed in the environment" 
+                    "variables 'IA_ACCESS' and 'IA_SECRET'", file=sys.stderr)
+    else:
+        ia = IARemote(os.environ['IA_ACCESS']+"\n"+os.environ['IA_SECRET'])
+        remotes.append(ia)
+
+    gdrive = GdriveRemote()
+    gdrive.clean_up()
+    remotes.append(gdrive)
+        
+    upload_files(remotes)
+
+    return 0
 
 if __name__ == "__main__":
     sys.exit(main())

scripts/update_files

@@ -1,44 +0,0 @@
-#!/bin/bash
-#
-# This file is part of GetTor, a Tor Browser distribution system.
-#
-# :authors: hiro <hiro@torproject.org>
-#           cecylia <cohosh@torproject.org>
-#           see also AUTHORS file
-#
-# :copyright:   (c) 2008-2019, The Tor Project, Inc.
-#
-# :license: This is Free Software. See LICENSE for license information.
-#
-# Updates tor browser binaries at Google Drive and Internet Archive
-#
-
-cd ~/releases
-rclone delete gdrive:releases
-rclone cleanup gdrive:releases
-
-for row in $(
-    curl -s 'https://aus1.torproject.org/torbrowser/update_3/release/downloads.json' |
-    jq -r '.downloads'
-  ); do
-    r=$(
-      echo ${row} |
-      egrep -o 'https?://[^ ]+' |
-      tr -d '",'
-    );
-    if [[ $r = *[!\ ]* ]]; then
-      wget $r
-
-      f=${r##*/}
-
-      # Update Google Drive
-      rclone copy $f gdrive:releases
-
-      # Update Internet Archive
-      ia upload ${f} $f --remote-name=$f --metadata="title:${f}" --metadata="mediatype:software" --metadata="collection:open_source_software"
-
-      rm $f
-
-    fi;
-done
-

setup.py

@@ -28,8 +28,7 @@ setuptools.setup(
                  'scripts/create_db',
                  'scripts/export_stats',
                  'scripts/process_email',
-                 'scripts/update_files',
-                 'scripts/update_git'],
+                 'scripts/update_all'],
         install_requires=['twisted',
                           'pytest',
                           'requests_oauthlib',