Browse Source

Import ilv PR introduce major refactoring and restructure to use Twisted

Update code and repository structure
Lay foundation to add testing and testing coverage
merge-requests/14/head
hiro 4 years ago
parent
commit
bff816b0d1
  1. 34
      .coveragerc
  2. 5
      .gitignore
  3. 1
      AUTHORS
  4. 34
      Makefile
  5. 44745
      TAGS
  6. 39
      bin/gettor_service
  7. 6
      config/blacklist.cfg
  8. 12
      config/core.cfg
  9. 17
      config/smtp.cfg
  10. 0
      config/tor-mirrors.json
  11. 21
      config/xmpp.cfg
  12. 58
      dev_scripts/create_db.py
  13. 0
      doc/INSTALL-SMTP
  14. 10
      gettor.conf.json
  15. 18
      gettor/__init__.py
  16. 481
      gettor/core.py
  17. 38
      gettor/main.py
  18. 1
      gettor/parse/__init__.py
  19. 217
      gettor/parse/email.py
  20. 63
      gettor/services/__init__.py
  21. 226
      gettor/services/email/sendmail.py
  22. 2
      gettor/services/twitter/twitter.py
  23. 2
      gettor/services/xmpp/xmpp.py
  24. 535
      gettor/smtp.py
  25. 131
      gettor/utils.py
  26. 2
      gettor/utils/blacklist.py
  27. 58
      gettor/utils/commons.py
  28. 7
      gettor/utils/db.py
  29. 30
      gettor/utils/options.py
  30. 62
      gettor/utils/settings.py
  31. 122
      gettor/utils/strings.py
  32. 2
      gettor/web/http.py
  33. 154
      scripts/create_db
  34. 30
      scripts/gettor
  35. 257
      setup.py
  36. 0
      share/css/bootstrap.min.css
  37. 0
      share/css/gettor.css
  38. 3
      share/locale/available_locales.json
  39. 12
      share/locale/en.json
  40. 1
      share/version.txt
  41. 0
      test/__init__.py
  42. 27
      test/test_email_service.py

34
.coveragerc

@ -0,0 +1,34 @@
[run]
source = gettor
branch = True
#parallel = True
timid = False
[report]
omit =
*/_langs*
*/_version*
*/__init__*
*/sitecustomize*
*/test/*
# Regexes for lines to exclude from report generation:
exclude_lines =
pragma: no cover
# don't complain if the code doesn't hit unimplemented sections:
raise NotImplementedError
pass
# don't complain if non-runnable or debuging code isn't run:
if 0:
if False:
if self[.verbosity.]
if options[.verbosity.]
def __repr__
if __name__ == .__main__.:
except Exception as impossible:
# Ignore source code which cannot be found:
ignore_errors = True
# Exit with status code 2 if under this percentage is covered:
fail_under = 80
[html]
directory = doc/coverage-html

5
.gitignore vendored

@ -0,0 +1,5 @@
venv
__pycache__
*.pyc
log
gettor.db

1
AUTHORS

@ -1,4 +1,5 @@
Current maintainer/core developers:
hiro <hiro@torproject.org>
Israel Leiva <ilv@torproject.org> 4096R/540BFC0E
Past core developers:

34
Makefile

@ -0,0 +1,34 @@
.PHONY: install test
.DEFAULT: install test
TRIAL:=$(shell which trial)
VERSION:=$(shell git describe)
define PYTHON_WHICH
import platform
import sys
sys.stdout.write(platform.python_implementation())
endef
PYTHON_IMPLEMENTATION:=$(shell python3 -c '$(PYTHON_WHICH)')
test:
python3 setup.py test
coverage-test:
ifeq ($(PYTHON_IMPLEMENTATION),PyPy)
@echo "Detected PyPy... not running coverage."
python setup.py test
else
coverage run --rcfile=".coveragerc" $(TRIAL) ./test/test_*.py
coverage report --rcfile=".coveragerc"
endif
coverage-html:
coverage html --rcfile=".coveragerc"
coverage: coverage-test coverage-html
tags:
find ./gettor -type f -name "*.py" -print | xargs etags

44745
TAGS

File diff suppressed because it is too large Load Diff

39
bin/gettor_service

@ -0,0 +1,39 @@
#!/bin/bash
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: isra <ilv@torproject.org>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014-2018, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
source venv/bin/activate
case "$1" in
start)
twistd3 --python=scripts/gettor --logfile=log/gettor.log --pidfile=gettor.pid
;;
stop)
kill -INT `cat gettor.pid`
;;
restart)
$0 stop
sleep 2;
$0 start
;;
status)
if [ -e gettor.pid ]; then
echo gettor is running with pid=`cat gettor.pid`
else
echo gettor is NOT running
exit 1
fi
;;
*)
echo "Usage: $0 {start|stop|status|restart}"
esac
exit 0

6
config/blacklist.cfg

@ -1,6 +0,0 @@
[general]
db: /path/to/gettor.db
[log]
level: DEBUG
dir: /path/to/log

12
config/core.cfg

@ -1,12 +0,0 @@
[general]
basedir: /path/to/gettor
db: gettor.db
[links]
dir: /path/to/providers/
os: linux,windows,osx
locales: es,en
[log]
dir: /path/to/log/
level: DEBUG

17
config/smtp.cfg

@ -1,17 +0,0 @@
[general]
basedir: /path/to/gettor/smtp
mirrors: /path/to/mirrors
our_domain: torproject.org
core_cfg: /path/to/core.cfg
[blacklist]
cfg: /path/to/blacklist.cfg
max_requests: 3
wait_time: 20
[i18n]
dir: /path/to/i18n/
[log]
level: DEBUG
dir: /path/to/log/

0
config/tor-mirrors → config/tor-mirrors.json

21
config/xmpp.cfg

@ -1,21 +0,0 @@
[account]
user: account@domain
password:
[general]
basedir: /path/to/gettor/xmpp/
core_cfg: /path/to/core.cfg
max_words: 10
db: /path/to/gettor.db
[blacklist]
cfg: /path/to/blacklist.cfg
max_requests: 3
wait_time: 20
[i18n]
dir: /path/to/i18n/
[log]
level: DEBUG
dir: /path/to/log/

58
dev_scripts/create_db.py

@ -1,58 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import os
import sqlite3
import argparse
def main():
"""Create/delete GetTor database for managing stats and blacklisting.
Database file (.db) must be empty. If it doesn't exist, it will be
created. See argparse usage for more details.
"""
parser = argparse.ArgumentParser(description='Utility for GetTor'
' database')
parser.add_argument('-c', '--create', default=None,
metavar='path_to_database.db',
help='create database')
parser.add_argument('-d', '--delete', default=None,
metavar='path_to_database.db',
help='delete database')
args = parser.parse_args()
if args.create:
con = sqlite3.connect(args.create)
with con:
cur = con.cursor()
# table for handling users (i.e. blacklist)
cur.execute(
"CREATE TABLE users(id TEXT, service TEXT, times INT,"
"blocked INT, last_request TEXT)"
)
cur.execute(
"CREATE TABLE requests(date TEXT, request TEXT, os TEXT,"
" locale TEXT, channel TEXT, PRIMARY KEY (date, channel))"
)
print "Database %s created" % os.path.abspath(args.create)
elif args.delete:
os.remove(os.path.abspath(args.delete))
print "Database %s deleted" % os.path.abspath(args.delete)
else:
print "See --help for details on usage."
if __name__ == "__main__":
main()

0
INSTALL-SMTP → doc/INSTALL-SMTP

10
gettor.conf.json

@ -0,0 +1,10 @@
{
"platforms": ["linux", "osx", "windows"],
"dbname": "gettor.db",
"email_parser_logfile": "email_parser.log",
"email_requests_limit": 5,
"sendmail_interval": 10,
"sendmail_addr": "email@addr",
"sendmail_host": "host",
"sendmail_port": 587
}

18
gettor/__init__.py

@ -1 +1,17 @@
# yes it's empty, of such a fullness
# -*- coding: utf-8 -*-
"""
This file is part of GetTor, a service providing alternative methods to download
the Tor Browser.
:authors: Hiro <hiro@torproject.org>
please also see AUTHORS file
:copyright: (c) 2008-2014, The Tor Project, Inc.
(c) 2014, all entities within the AUTHORS file
:license: see included LICENSE for information
"""
from .utils import strings
__version__ = strings.get_version()
__locales__ = strings.get_locales()

481
gettor/core.py

@ -1,481 +0,0 @@
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import os
import re
import logging
import gettext
import tempfile
import ConfigParser
import db
import utils
"""Core module for getting links from providers."""
class ConfigError(Exception):
pass
class NotSupportedError(Exception):
pass
class LinkFormatError(Exception):
pass
class LinkFileError(Exception):
pass
class InternalError(Exception):
pass
class Core(object):
"""Get links from providers and deliver them to other modules.
Public methods:
get_links(): Get the links for the OS and locale requested.
create_links_file(): Create a file to store links of a provider.
add_link(): Add a link to a links file of a provider.
get_supported_os(): Get a list of supported operating systems.
get_supported_lc(): Get a list of supported locales.
Exceptions:
UnsupportedOSError: OS and/or locale not supported.
ConfigError: Something's misconfigured.
LinkFormatError: The link added doesn't seem legit.
LinkFileError: Error related to the links file of a provider.
InternalError: Something went wrong internally.
"""
def __init__(self, cfg=None):
"""Create a new core object by reading a configuration file.
:param: cfg (string) the path of the configuration file.
:raise: ConfigurationError if the configuration file doesn't exists
or if something goes wrong while reading options from it.
"""
default_cfg = 'core.cfg'
config = ConfigParser.ConfigParser()
if cfg is None or not os.path.isfile(cfg):
cfg = default_cfg
try:
with open(cfg) as f:
config.readfp(f)
except IOError:
raise ConfigError("File %s not found!" % cfg)
try:
self.supported_lc = config.get('links', 'locales')
self.supported_os = config.get('links', 'os')
basedir = config.get('general', 'basedir')
self.linksdir = config.get('links', 'dir')
self.linksdir = os.path.join(basedir, self.linksdir)
self.i18ndir = config.get('i18n', 'dir')
loglevel = config.get('log', 'level')
logdir = config.get('log', 'dir')
logfile = os.path.join(logdir, 'core.log')
dbname = config.get('general', 'db')
dbname = os.path.join(basedir, dbname)
self.db = db.DB(dbname)
except ConfigParser.Error as e:
raise ConfigError("Configuration error: %s" % str(e))
except db.Exception as e:
raise InternalError("%s" % e)
# logging
log = logging.getLogger(__name__)
logging_format = utils.get_logging_format()
date_format = utils.get_date_format()
formatter = logging.Formatter(logging_format, date_format)
log.info('Redirecting CORE logging to %s' % logfile)
logfileh = logging.FileHandler(logfile, mode='a+')
logfileh.setFormatter(formatter)
logfileh.setLevel(logging.getLevelName(loglevel))
log.addHandler(logfileh)
# stop logging on stdout from now on
log.propagate = False
self.log = log
def _get_msg(self, msgid, lc):
"""Get message identified by msgid in a specific locale.
:param: msgid (string) the identifier of a string.
:param: lc (string) the locale.
:return: (string) the message from the .po file.
"""
# obtain the content in the proper language
try:
t = gettext.translation(lc, self.i18ndir, languages=[lc])
_ = t.ugettext
msgstr = _(msgid)
return msgstr
except IOError as e:
raise ConfigError("%s" % str(e))
def get_links(self, service, os, lc):
"""Get links for OS in locale.
This method should be called from the services modules of
GetTor (e.g. SMTP). To make it easy we let the module calling us
specify the name of the service (for stats purpose).
:param: service (string) the service trying to get the links.
:param: os (string) the operating system.
:param: lc (string) tthe locale.
:raise: InternalError if something goes wrong while internally.
:return: (string) the links.
"""
# english and windows by default
if lc not in self.supported_lc:
self.log.debug("Request for locale not supported. Default to en")
lc = 'en'
if os not in self.supported_os:
self.log.debug("Request for OS not supported. Default to windows")
os = 'windows'
# this could change in the future, let's leave it isolated.
self.log.debug("Trying to get the links...")
try:
links = self._get_links(os, lc)
self.log.debug("OK")
except InternalError as e:
self.log.debug("FAILED")
raise InternalError("%s" % str(e))
if links is None:
self.log.debug("No links found")
raise InternalError("No links. Something is wrong.")
return links
def _get_links(self, osys, lc):
"""Internal method to get the links.
Looks for the links inside each provider file. This should only be
called from get_links() method.
:param: osys (string) the operating system.
:param: lc (string) the locale.
:return: (string/None) links on success, None otherwise.
"""
# read the links files using ConfigParser
# see the README for more details on the format used
links_files = []
links32 = {}
links64 = {}
# for the message to be sent
if osys == 'windows':
arch = '32/64'
elif osys == 'osx':
arch = '64'
else:
arch = '32'
# look for files ending with .links
p = re.compile('.*\.links$')
for name in os.listdir(self.linksdir):
path = os.path.abspath(os.path.join(self.linksdir, name))
if os.path.isfile(path) and p.match(path):
links_files.append(path)
# let's create a dictionary linking each provider with the links
# found for os and lc. This way makes it easy to check if no
# links were found
providers = {}
# separator
spt = '=' * 72
# reading links from providers directory
for name in links_files:
# we're reading files listed on linksdir, so they must exist!
config = ConfigParser.ConfigParser()
# but just in case they don't
try:
with open(name) as f:
config.readfp(f)
except IOError:
raise InternalError("File %s not found!" % name)
try:
pname = config.get('provider', 'name')
# check if current provider pname has links for os in lc
providers[pname] = config.get(osys, lc)
except ConfigParser.Error as e:
# we should at least have the english locale available
self.log.error("Request for %s, returning 'en' instead" % lc)
providers[pname] = config.get(osys, 'en')
try:
#test = providers[pname].split("$")
#self.log.debug(test)
if osys == 'linux':
t32, t64 = [t for t in providers[pname].split(",") if t]
link, signature, chs32 = [l for l in t32.split("$") if l]
link = " %s: %s" % (pname, link)
links32[link] = signature
link, signature, chs64 = [l for l in t64.split("$") if l]
link = " %s: %s" % (pname, link.lstrip())
links64[link] = signature
else:
link, signature, chs32 = [l for l in providers[pname].split("$") if l]
link = " %s: %s" % (pname, link)
links32[link] = signature
#providers[pname] = providers[pname].replace(",", "")
#providers[pname] = providers[pname].replace("$", "\n\n")
### We will improve and add the verification section soon ###
# all packages are signed with same key
# (Tor Browser developers)
# fingerprint = config.get('key', 'fingerprint')
# for now, english messages only
# fingerprint_msg = self._get_msg('fingerprint', 'en')
# fingerprint_msg = fingerprint_msg % fingerprint
except ConfigParser.Error as e:
raise InternalError("%s" % str(e))
# create the final links list with all providers
all_links = []
msg = "Tor Browser %s-bit:\n" % arch
for link in links32:
msg = "%s\n%s" % (msg, link)
all_links.append(msg)
if osys == 'linux':
msg = "\n\n\nTor Browser 64-bit:\n"
for link in links64:
msg = "%s\n%s" % (msg, link)
all_links.append(msg)
### We will improve and add the verification section soon ###
"""
msg = "\n\n\nTor Browser's signature %s-bit:" %\
arch
for link in links32:
msg = "%s\n%s" % (msg, links32[link])
all_links.append(msg)
if osys == 'linux':
msg = "\n\n\nTor Browser's signature 64-bit:"
for link in links64:
msg = "%s%s" % (msg, links64[link])
all_links.append(msg)
msg = "\n\n\nSHA256 of Tor Browser %s-bit (advanced): %s\n" %\
(arch, chs32)
all_links.append(msg)
if osys == 'linux':
msg = "SHA256 of Tor Browser 64-bit (advanced): %s\n" % chs64
all_links.append(msg)
"""
### end verification ###
"""
for key in providers.keys():
# get more friendly description of the provider
try:
# for now, english messages only
provider_desc = self._get_msg('provider_desc', 'en')
provider_desc = provider_desc % key
all_links.append(
"%s\n%s\n\n%s%s\n\n\n" %
(provider_desc, spt, ''.join(providers[key]), spt)
)
except ConfigError as e:
raise InternalError("%s" % str(e))
"""
### We will improve and add the verification section soon ###
# add fingerprint after the links
# all_links.append(fingerprint_msg)
if all_links:
return "".join(all_links)
else:
# we're trying to get supported os an lc
# but there aren't any links!
return None
def get_supported_os(self):
"""Public method to get the list of supported operating systems.
:return: (list) the supported operating systems.
"""
return self.supported_os.split(',')
def get_supported_lc(self):
"""Public method to get the list of supported locales.
:return: (list) the supported locales.
"""
return self.supported_lc.split(',')
def create_links_file(self, provider, fingerprint):
"""Public method to create a links file for a provider.
This should be used by all providers since it writes the links
file with the proper format. It backs up the old links file
(if exists) and creates a new one.
:param: provider (string) the provider (links file will use this
name in slower case).
:param: fingerprint (string) the fingerprint of the key that signed
the packages to be uploaded to the provider.
"""
linksfile = os.path.join(self.linksdir, provider.lower() + '.links')
linksfile_backup = ""
self.log.debug("Request to create a new links file")
if os.path.isfile(linksfile):
self.log.debug("Trying to backup the old one...")
try:
# backup the old file in case something fails
linksfile_backup = linksfile + '.backup'
os.rename(linksfile, linksfile_backup)
except OSError as e:
self.log.debug("FAILED %s" % str(e))
raise LinkFileError(
"Error while creating new links file: %s" % str(e)
)
self.log.debug("Creating empty links file...")
try:
# this creates an empty links file
content = ConfigParser.RawConfigParser()
content.add_section('provider')
content.set('provider', 'name', provider)
content.add_section('key')
content.set('key', 'fingerprint', fingerprint)
content.add_section('linux')
content.add_section('windows')
content.add_section('osx')
with open(linksfile, 'w+') as f:
content.write(f)
except Exception as e:
self.log.debug("FAILED: %s" % str(e))
# if we passed the last exception, then this shouldn't
# be a problem...
if linksfile_backup:
os.rename(linksfile_backup, linksfile)
raise LinkFileError(
"Error while creating new links file: %s" % str(e)
)
def add_link(self, provider, osys, lc, link):
"""Public method to add a link to a provider's links file.
Use ConfigParser to add a link into the os section, under the lc
option. It checks for valid format; the provider's script should
use the right format (see design).
:param: provider (string) the provider.
:param: os (string) the operating system.
:param: lc (string) the locale.
:param: link (string) link to be added.
:raise: NotsupportedError if the OS and/or locale is not supported.
:raise: LinkFileError if there is no links file for the provider.
:raise: LinkFormatError if the link format doesn't seem legit.
:raise: InternalError if the links file doesn't have a section for
the OS requested. This *shouldn't* happen because it means
the file wasn't created correctly.
"""
linksfile = os.path.join(self.linksdir, provider.lower() + '.links')
self.log.debug("Request to add a new link")
# don't try to add unsupported stuff
if lc not in self.supported_lc:
self.log.debug("Request for locale %s not supported" % lc)
raise NotSupportedError("Locale %s not supported" % lc)
if osys not in self.supported_os:
self.log.debug("Request for OS %s not supported" % osys)
raise NotSupportedError("OS %s not supported" % osys)
self.log.debug("Opening links file...")
if os.path.isfile(linksfile):
content = ConfigParser.RawConfigParser()
try:
with open(linksfile) as f:
content.readfp(f)
except IOError as e:
self.log.debug("FAILED %s" % str(e))
raise LinksFileError("File %s not found!" % linksfile)
# check if exists and entry for locale; if not, create it
self.log.debug("Trying to add the link...")
try:
links = content.get(osys, lc)
links = "%s,\n%s" % (links, link)
content.set(osys, lc, links)
self.log.debug("Link added")
with open(linksfile, 'w') as f:
content.write(f)
except ConfigParser.NoOptionError:
content.set(osys, lc, link)
self.log.debug("Link added (with new locale created)")
with open(linksfile, 'w') as f:
content.write(f)
except ConfigParser.NoSectionError as e:
# this shouldn't happen, but just in case
self.log.debug("FAILED (OS not found)")
raise InternalError("Unknown section %s" % str(e))
else:
self.log.debug("FAILED (links file doesn't seem legit)")
raise LinkFileError("No links file for %s" % provider)

38
gettor/main.py

@ -0,0 +1,38 @@
# -*- coding: utf-8 -*-
"""
This file is part of GetTor, a service providing alternative methods to download
the Tor Browser.
:authors: Hiro <hiro@torproject.org>
please also see AUTHORS file
:copyright: (c) 2008-2014, The Tor Project, Inc.
(c) 2014, all entities within the AUTHORS file
:license: see included LICENSE for information
"""
"""This module sets up GetTor and starts the servers running."""
import sys
from .utils.commons import log
from .utils import options
from .services import BaseService
from .services.email.sendmail import Sendmail
def run(gettor, app):
"""
This is GetTor's main entry point and main runtime loop.
"""
settings = options.parse_settings()
sendmail = Sendmail(settings)
log.info("Starting services.")
sendmail_service = BaseService(
"sendmail", sendmail.get_interval(), sendmail
)
gettor.addService(sendmail_service)
gettor.setServiceParent(app)

1
gettor/parse/__init__.py

@ -0,0 +1 @@
# empty

217
gettor/parse/email.py

@ -0,0 +1,217 @@
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: isra <ilv@torproject.org>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014-2018, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
from __future__ import absolute_import
import re
import dkim
import hashlib
import validate_email
from datetime import datetime
import configparser
from email import message_from_string
from email.utils import parseaddr
from twisted.python import log
from twisted.internet import defer
from twisted.enterprise import adbapi
from .. import PLATFORMS, EMAIL_REQUESTS_LIMIT
from ..db import SQLite3
class AddressError(Exception):
"""
Error if email address is not valid or it can't be normalized.
"""
pass
class DKIMError(Exception):
"""
Error if DKIM signature verification fails.
"""
pass
class EmailParser(object):
"""Class for parsing email requests."""
def __init__(self, to_addr=None, dkim=False):
"""
Constructor.
param (Boolean) dkim: Set dkim verification to True or False.
"""
self.dkim = dkim
self.to_addr = to_addr
def parse(self, msg_str):
"""
Parse message content. Check if email address is well formed, if DKIM
signature is valid, and prevent service flooding. Finally, look for
commands to process the request. Current commands are:
- links: request links for download.
- help: help request.
:param msg_str (str): incomming message as string.
:return dict with email address and command (`links` or `help`).
"""
log.msg("Building email message from string.", system="email parser")
msg = message_from_string(msg_str)
# Normalization will convert <Alice Wonderland> alice@wonderland.net
# into alice@wonderland.net
name, norm_addr = parseaddr(msg['From'])
to_name, norm_to_addr = parseaddr(msg['To'])
log.msg(
"Normalizing and validating FROM email address.",
system="email parser"
)
# Validate_email will do a bunch of regexp to see if the email address
# is well address. Additional options for validate_email are check_mx
# and verify, which check if the SMTP host and email address exist.
# See validate_email package for more info.
if norm_addr and validate_email.validate_email(norm_addr):
log.msg(
"Email address normalized and validated.",
system="email parser"
)
else:
log.err(
"Error normalizing/validating email address.",
system="email parser"
)
raise AddressError("Invalid email address {}".format(msg['From']))
hid = hashlib.sha256(norm_addr)
log.msg(
"Request from {}".format(hid.hexdigest()), system="email parser"
)
if self.to_addr:
if self.to_addr != norm_to_addr:
log.msg("Got request for a different instance of gettor")
log.msg("Intended recipient: {}".format(norm_to_addr))
return {}
# DKIM verification. Simply check that the server has verified the
# message's signature
if self.dkim:
log.msg("Checking DKIM signature.", system="email parser")
# Note: msg.as_string() changes the message to conver it to
# string, so DKIM will fail. Use the original string instead
if dkim.verify(msg_str):
log.msg("Valid DKIM signature.", system="email parser")
else:
log.msg("Invalid DKIM signature.", system="email parser")
username, domain = norm_addr.split("@")
raise DkimError(
"DKIM failed for {} at {}".format(
hid.hexdigest(), domain
)
)
# Search for commands keywords
subject_re = re.compile(r"Subject: (.*)\r\n")
subject = subject_re.search(msg_str)
request = {
"id": norm_addr,
"command": None, "platform": None,
"service": "email"
}
if subject:
subject = subject.group(1)
for word in re.split(r"\s+", subject.strip()):
if word.lower() in PLATFORMS:
request["command"] = "links"
request["platform"] = word.lower()
break
if word.lower() == "help":
request["command"] = "help"
break
if not request["command"]:
for word in re.split(r"\s+", body_str.strip()):
if word.lower() in PLATFORMS:
request["command"] = "links"
request["platform"] = word.lower()
break
if word.lower() == "help":
request["command"] = "help"
break
return request
@defer.inlineCallbacks
def parse_callback(self, request):
"""
Callback invoked when the message has been parsed. It stores the
obtained information in the database for further processing by the
Sendmail service.
:param (dict) request: the built request based on message's content.
It contains the `email_addr` and command `fields`.
:return: deferred whose callback/errback will log database query
execution details.
"""
log.msg(
"Found request for {}.".format(request['command']),
system="email parser"
)
if request["command"]:
now_str = datetime.now().strftime("%Y%m%d%H%M%S")
conn = SQLite3()
hid = hashlib.sha256(request['id'])
# check limits first
num_requests = yield conn.get_num_requests(
id=hid.hexdigest(), service=request['service']
)
if num_requests[0][0] > EMAIL_REQUESTS_LIMIT:
log.msg(
"Discarded. Too many requests from {}.".format(
hid.hexdigest
), system="email parser"
)
else:
conn.new_request(
id=request['id'],
command=request['command'],
platform=request['platform'],
service=request['service'],
date=now_str,
status="ONHOLD",
)
def parse_errback(self, error):
"""
Errback if we don't/can't parse the message's content.
"""
log.msg(
"Error while parsing email content: {}.".format(error),
system="email parser"
)

63
gettor/services/__init__.py

@ -0,0 +1,63 @@
# -*- coding: utf-8 -*-
"""
This file is part of GetTor, a service providing alternative methods to download
the Tor Browser.
:authors: Hiro <hiro@torproject.org>
please also see AUTHORS file
:copyright: (c) 2008-2014, The Tor Project, Inc.
(c) 2014, all entities within the AUTHORS file
:license: see included LICENSE for information
"""
from __future__ import absolute_import
from twisted.application import internet
from ..utils.commons import log
class BaseService(internet.TimerService):
"""
Base service for Accounts, Messages and Fetchmail. It extends the
TimerService providing asynchronous connection to database by default.
"""
def __init__(self, name, step, instance, *args, **kwargs):
"""
Constructor. Initiate connection to database and link one of Accounts,
Messages or Fetchmail instances to TimerService behavour.
:param name (str): name of the service being initiated (just for log
purposes).
:param step (float): time interval for TimerService, in seconds.
:param instance (object): instance of Accounts, Messages, or
Fetchmail classes.
"""
log.info("SERVICE:: Initializing {} service.".format(name))
self.name = name
self.instance = instance
log.debug("SERVICE:: Initializing TimerService.")
internet.TimerService.__init__(
self, step, self.instance.get_new, **kwargs
)
def startService(self):
"""
Starts the service. Overridden from parent class to add extra logging
information.
"""
log.info("SERVICE:: Starting {} service.".format(self.name))
internet.TimerService.startService(self)
log.info("SERVICE:: Service started.")
def stopService(self):
"""
Stop the service. Overridden from parent class to close connection to
database, shutdown the service and add extra logging information.
"""
log.info("SERVICE:: Stopping {} service.".format(self.name))
log.debug("SERVICE:: Calling shutdown on {}".format(self.name))
self.instance.shutdown()
log.debug("SERVICE:: Shutdown for {} done".format(self.name))
internet.TimerService.stopService(self)
log.info("SERVICE:: Service stopped.")

226
gettor/services/email/sendmail.py

@ -0,0 +1,226 @@
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: isra <ilv@torproject.org>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014-2018, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
from __future__ import absolute_import
import gettext
import hashlib
import configparser
from email import encoders
from email import mime
from twisted.internet import defer
from twisted.mail.smtp import sendmail
from ...utils.db import DB
from ...utils.commons import log
class SMTPError(Exception):
"""
Error if we can't send emails.
"""
pass
class Sendmail(object):
"""
Class for sending email replies to `help` and `links` requests.
"""
def __init__(self, settings):
"""
Constructor. It opens and stores a connection to the database.
:dbname: reads from configs
"""
self.settings = settings
dbname = self.settings.get("dbname")
self.conn = DB(dbname)
def get_interval(self):
"""
Get time interval for service periodicity.
:return: time interval (float) in seconds.
"""
return self.settings.get("sendmail_interval")
def sendmail_callback(self, message):
"""
Callback invoked after an email has been sent.
:param message (string): Success details from the server.
"""
log.info("Email sent successfully.")
def sendmail_errback(self, error):
"""
Errback if we don't/can't send the message.
"""
log.debug("Could not send email.")
raise SMTPError("{}".format(error))
def sendmail(self, email_addr, subject, body):
"""
Send an email message. It creates a plain text message, set headers
and content and finally send it.
:param email_addr (str): email address of the recipient.
:param subject (str): subject of the message.
:param content (str): content of the message.
:return: deferred whose callback/errback will handle the SMTP
execution details.
"""
log.debug("Creating plain text email")
message = MIMEText(body)
message['Subject'] = subject
message['From'] = SENDMAIL_ADDR
message['To'] = email_addr
log.debug("Calling asynchronous sendmail.")
return sendmail(
SENDMAIL_HOST, SENDMAIL_ADDR, email_addr, message,
port=SENDMAIL_PORT,
requireAuthentication=True, requireTransportSecurity=True
).addCallback(self.sendmail_callback).addErrback(self.sendmail_errback)
@defer.inlineCallbacks
def get_new(self):
"""
Get new requests to process. This will define the `main loop` of
the Sendmail service.
"""
# Manage help and links messages separately
help_requests = yield self.conn.get_requests(
status="ONHOLD", command="help", service="email"
)
link_requests = yield self.conn.get_requests(
status="ONHOLD", command="links", service="email"
)
if help_requests:
try:
log.info("Got new help request.")
# for now just english
en = gettext.translation(
'email', localedir='locales', languages=['en']
)
en.install()
_ = en.gettext
for request in help_requests:
id = request[0]
date = request[4]
hid = hashlib.sha256(id)
log.info(
"Sending help message to {}.".format(
hid.hexdigest()
)
)
yield self.sendmail(
email_addr=id,
subject=_("help_subject"),
body=_("help_body")
)
yield self.conn.update_stats(
command="help", service="email"
)
yield self.conn.update_request(
id=id, hid=hid.hexdigest(), status="SENT",
service="email", date=date
)
except SMTPError as e:
log.info("Error sending email: {}.".format(e))
elif link_requests:
try:
log.info("Got new links request.")
# for now just english
en = gettext.translation(
'email', localedir='locales', languages=['en']
)
en.install()
_ = en.gettext
for request in link_requests:
id = request[0]
date = request[4]
platform = request[2]
log.info("Getting links for {}.".format(platform))
links = yield self.conn.get_links(
platform=platform, status="ACTIVE"
)
# build message
link_msg = None
for link in links:
provider = link[4]
version = link[3]
arch = link[2]
url = link[0]
link_str = "Tor Browser {} for {}-{} ({}): {}".format(
version, platform, arch, provider, url
)
if link_msg:
link_msg = "{}\n{}".format(link_msg, link_str)
else:
link_msg = link_str
body_msg = _("links_body")
body_msg = body_msg.format(links=link_msg)
subject_msg = _("links_subject")
hid = hashlib.sha256(id)
log.info(
"Sending links to {}.".format(
hid.hexdigest()
)
)
yield self.sendmail(
email_addr=id,
subject=subject_msg,
body=body_msg
)
yield self.conn.update_stats(
command="links", platform=platform, service="email"
)
yield self.conn.update_request(
id=id, hid=hid.hexdigest(), status="SENT",
service="email", date=date
)
except SMTPError as e:
log.info("Error sending email: {}.".format(e))
else:
log.debug("No pending email requests. Keep waiting.")

2
gettor/twitter.py → gettor/services/twitter/twitter.py

@ -16,7 +16,7 @@ import re
import tweepy
import logging
import gettext
import ConfigParser
import configparser
import core
import utils

2
gettor/xmpp.py → gettor/services/xmpp/xmpp.py

@ -17,7 +17,7 @@ import time
import gettext
import hashlib
import logging
import ConfigParser
import configparser
from sleekxmpp import ClientXMPP
from sleekxmpp.xmlstream.stanzabase import JID

535
gettor/smtp.py

@ -1,535 +0,0 @@
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import os
import re
import sys
import time
import email
import gettext
import logging
import smtplib
import datetime
import ConfigParser
from email import Encoders
from email.MIMEBase import MIMEBase
from email.mime.text import MIMEText
from email.MIMEMultipart import MIMEMultipart
import core
import utils
import blacklist
"""SMTP module for processing email requests."""
OS = {
'osx': 'Mac OS X',
'linux': 'Linux',
'windows': 'Windows'
}