diff --git a/.dockerignore b/.dockerignore index 81bbbae..ff25b35 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,4 @@ keys/ +*.egg-info +.tox/ +__cache__ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eaf482c --- /dev/null +++ b/.gitignore @@ -0,0 +1,107 @@ + +# Created by https://www.gitignore.io/api/python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# End of https://www.gitignore.io/api/python + +# more +key/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..9eef6f1 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,21 @@ +- repo: git://github.com/pre-commit/pre-commit-hooks + sha: v0.9.1 + hooks: + - id: check-added-large-files + - id: check-docstring-first + - id: check-merge-conflict + - id: check-yaml + - id: end-of-file-fixer + - id: flake8 + args: + - --exclude=__init__.py + language_version: python3 + - id: autopep8-wrapper + language_version: python3 + - id: requirements-txt-fixer + - id: trailing-whitespace +- repo: git://github.com/asottile/reorder_python_imports + sha: v0.3.5 + hooks: + - id: reorder-python-imports + language_version: python3 diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..bd531f2 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +sudo: false +language: python +python: + - "3.4" + - "3.5" + - "3.6" +install: pip install tox-travis pre-commit +script: + - pre-commit run --all-files + - tox diff --git a/Dockerfile b/Dockerfile index b6d278c..1a23e75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,13 +22,17 @@ RUN apk add --no-cache git libevent-dev openssl-dev gcc make automake ca-cer apk del git libevent-dev openssl-dev make automake python3-dev gcc autoconf musl-dev coreutils && \ apk add --no-cache libevent openssl +RUN mkdir -p /etc/tor/ + +RUN pip install pyentrypoint==0.5.0 + +# Delete me +RUN pip install 'Jinja2>=2.8' 'pycrypto' + ADD assets/entrypoint-config.yml / ADD assets/onions /usr/local/src/onions ADD assets/torrc /var/local/tor/torrc.tpl -RUN mkdir -p /etc/tor/ - -RUN pip install pyentrypoint==0.5.0 RUN cd /usr/local/src/onions && python3 setup.py install diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..afd18f5 --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +test: + tox + +check: + pre-commit run --all-files + +build: + docker-compose build + +run: build + docker-compose up + +build-v2: + docker-compose -f docker-compose.v2.yml build + +run-v2: build-v2 + docker-compose -f docker-compose.v2.yml up + +build-v3: + docker-compose -f docker-compose.v3.yml build + +run-v3: build-v3 + docker-compose -f docker-compose.v3.yml up diff --git a/README.md b/README.md index 664f58d..5d0bdef 100644 --- a/README.md +++ b/README.md @@ -90,10 +90,46 @@ To increase security, it's possible to setup your service through socket between __Warning__: Due to a bug in `tor` configuration parser, it's not possible to mix network link and socket link in the same `tor` configuration. +### Group services + +Multiple services can be hosted behind the same onion address. + +```yaml +links: + - hello + - world + - hey +environment: + # Set mapping ports + HELLO_PORTS: 80:80 + + # Multiple ports can be coma separated + WORLD_PORTS: 8000:80,8888:80,22:22 + + # Socket mapping is supported + HEY_PORTS: 80:unix:/var/run/socket.sock + + # hello and world will share the same onion address + # Service name can be any string as long there is not special char + HELLO_SERVICE_NAME: foo + WORLD_SERVICE_NAME: foo + +``` + +__Warning__: Be carefull to not use the same exposed ports for grouped services. + ### Compose v2 support Links setting are required when using docker-compose v2. See `docker-compose.v2.yml` for example. +### Copose v3 support and secrets + +Links setting are required when using docker-compose v3. See `docker-compose.v3.yml` for example. + +#### Secrets + +Secret key can be set through docker `secrets`, see `docker-compose.v3.yml` for example. + ### Tools A command line tool `onions` is available in container to get `.onion` url when container is running. diff --git a/assets/entrypoint-config.yml b/assets/entrypoint-config.yml index 92cd4fa..0266ec9 100644 --- a/assets/entrypoint-config.yml +++ b/assets/entrypoint-config.yml @@ -6,13 +6,12 @@ group: tor secret_env: - '*_KEY' - '*_PORTS' + - '*_SERVICE_NAME' pre_conf_commands: - onions --setup-hosts post_conf_commands: - - timeout -t 3 tor > /dev/null || true - - onions - chown -R tor:tor $HOME reload: diff --git a/assets/onions/onions/Onions.py b/assets/onions/onions/Onions.py index 4954a57..ed51ea7 100644 --- a/assets/onions/onions/Onions.py +++ b/assets/onions/onions/Onions.py @@ -1,22 +1,17 @@ #!/usr/bin/env python3 - -import os -from json import dumps - -from re import match - -from pyentrypoint import DockerLinks - import argparse +import logging +import os +import sys +from json import dumps +from re import match from jinja2 import Environment from jinja2 import FileSystemLoader +from pyentrypoint import DockerLinks -import socket - -from Crypto.PublicKey import RSA -from hashlib import sha1 -from base64 import b32encode +from .Service import Service +from .Service import ServicesGroup class Setup(object): @@ -25,18 +20,6 @@ class Setup(object): torrc = '/etc/tor/torrc' torrc_template = '/var/local/tor/torrc.tpl' - def onion_url_gen(self, key): - "Get onion url from private key" - - # Convert private RSA to public DER - priv = RSA.importKey(key.strip()) - der = priv.publickey().exportKey("DER") - - # hash key, keep first half of sha1, base32 encode - onion = b32encode(sha1(der[22:]).digest()[:10]) - - return '{onion}.onion'.format(onion=onion.decode().lower()) - def _add_host(self, host): if host not in self.setup: self.setup[host] = {} @@ -44,7 +27,9 @@ class Setup(object): def _get_ports(self, host, ports): self._add_host(host) if 'ports' not in self.setup[host]: - self.setup[host]['ports'] = [] + self.setup[host]['ports'] = {host: []} + if host not in self.setup[host]['ports']: + self.setup[host]['ports'][host] = [] ports_l = [ [ int(v) if not v.startswith('unix:') else v @@ -53,18 +38,82 @@ class Setup(object): ] for port in ports_l: assert len(port) == 2 - if port not in self.setup[host]['ports']: - self.setup[host]['ports'].append(port) + if port not in self.setup[host]['ports'][host]: + self.setup[host]['ports'][host].append(port) def _get_key(self, host, key): self._add_host(host) assert len(key) > 800 self.setup[host]['key'] = key - def _get_setup_from_env(self): + def _get_service(self, host, service): + self._add_host(host) + self.setup[host]['service'] = service + + def find_group_by_service(self, service): + for group in self.services: + if service in group.services: + return group + + def find_group_by_name(self, name): + for group in self.services: + if name == group.name: + return group + + def find_service_by_host(self, host): + for group in self.services: + service = group.get_service_by_host(host) + if service: + return service + + def add_empty_group(self, name): + if self.find_group_by_name(name): + raise Exception('Group {name} already exists'.format(name=name)) + group = ServicesGroup(name=name) + self.services.append(group) + return group + + def add_new_service(self, host, name=None, ports=None, key=None): + group = self.find_group_by_name(name) + service = self.find_service_by_host(host) + if not service: + service = Service(host=host) + if not group: + group = ServicesGroup( + service=service, + name=name, + hidden_service_dir=self.hidden_service_dir + ) + else: + group.add_service(service) + if group not in self.services: + self.services.append(group) + else: + group = self.find_group_by_service(service) + if key: + group.add_key(key) + if ports: + service.add_ports(ports) + return service + + def _set_service_names(self): + 'Create groups for services, should be run first' + reg = r'([A-Z0-9]*)_SERVICE_NAME' + for key, val in os.environ.items(): + m = match(reg, key) + if m: + self.add_new_service(host=m.groups()[0].lower(), name=val) + + def _set_ports(self, host, ports): + self.add_new_service(host=host, ports=ports) + + def _set_key(self, host, key): + self.add_new_service(host=host, key=key) + + def _setup_from_env(self): match_map = ( - (r'([A-Z0-9]*)_PORTS', self._get_ports), - (r'([A-Z0-9]*)_KEY', self._get_key), + (r'([A-Z0-9]*)_PORTS', self._set_ports), + (r'([A-Z0-9]*)_KEY', self._set_key), ) for key, val in os.environ.items(): for reg, call in match_map: @@ -72,100 +121,133 @@ class Setup(object): if m: call(m.groups()[0].lower(), val) + def _get_setup_from_env(self): + self._set_service_names() + self._setup_from_env() + def _get_setup_from_links(self): containers = DockerLinks().to_containers() if not containers: return for container in containers: host = container.names[0] - self._add_host(host) + self.add_new_service(host=host) for link in container.links: if link.protocol != 'tcp': continue port_map = os.environ.get('PORT_MAP') - self._get_ports(host, '{exposed}:{internal}'.format( + self._set_ports(host, '{exposed}:{internal}'.format( exposed=port_map or link.port, internal=link.port, )) - def _set_keys(self): - for link, conf in self.setup.items(): - if 'key' in conf: - serv_dir = os.path.join(self.hidden_service_dir, link) - os.makedirs(serv_dir, exist_ok=True) - os.chmod(serv_dir, 0o700) - with open(os.path.join(serv_dir, 'private_key'), 'w') as f: - f.write(conf['key']) - os.fchmod(f.fileno(), 0o600) - with open(os.path.join(serv_dir, 'hostname'), 'w') as f: - f.write(self.onion_url_gen(conf['key'])) + def apply_conf(self): + self._write_keys() + self._write_torrc() - def _set_conf(self): + def _write_keys(self): + for service in self.services: + service.write_key() + + def _write_torrc(self): env = Environment(loader=FileSystemLoader('/')) temp = env.get_template(self.torrc_template) with open(self.torrc, mode='w') as f: - f.write(temp.render(setup=self.setup, + f.write(temp.render(services=self.services, env=os.environ, type=type, int=int)) def setup_hosts(self): self.setup = {} - try: - self._get_setup_from_env() - self._get_setup_from_links() - self._set_keys() - self._set_conf() - except: - raise Exception('Something wrongs with setup') + self._get_setup_from_env() + self._get_setup_from_links() + self.check_services() + self.apply_conf() + + def check_services(self): + for group in self.services: + for service in group.services: + if not service.ports: + raise Exception( + 'Service {name} has not ports set'.format( + name=service.host + ) + ) + if len(group.services) > 1 and [ + True for p in service.ports if p.is_socket + ]: + raise Exception( + 'Cannot use socket and ports ' + 'in the same service'.format( + name=service.host + ) + ) + if len(set(dict(group)['urls'])) != len(dict(group)['urls']): + raise Exception( + 'Same port for multiple services in {name} group'.format( + name=group.name + ) + ) class Onions(Setup): """Onions""" def __init__(self): + self.services = [] if 'HIDDEN_SERVICE_DIR' in os.environ: self.hidden_service_dir = os.environ['HIDDEN_SERVICE_DIR'] - def _get_port_from_service(self, service, filename): + def torrc_parser(self): - with open(filename, 'r') as hostfile: - onion = str(hostfile.read()).strip() + def parse_dir(line): + _, path = line.split() + group_name = os.path.basename(path) + group = (self.find_group_by_name(group_name) + or self.add_empty_group(group_name)) + return group - with open(self.torrc, 'r') as torfile: - self.onions[service] = [] - for line in torfile.readlines(): - find = '# PORT {name}'.format(name=service) - if line.startswith(find): - self.onions[service].append( - '{onion}:{port}'.format( - onion=onion, - port=line[len(find):].strip() - ) - ) + def parse_port(line, service_group): + _, port_from, dest = line.split() + service_host, port = dest.split(':') + ports_str = '{port_from}:{dest}' + name = service_host + ports_param = ports_str.format(port_from=port_from, + dest=port) + if port.startswith('/'): + name = service_group.name + ports_param = ports_str.format(port_from=port_from, + dest=dest) + service = (service_group.get_service_by_host(name) + or Service(name)) + service.add_ports(ports_param) + if service not in service_group.services: + service_group.add_service(service) - def get_onions(self): - self.onions = {} - for root, dirs, _ in os.walk(self.hidden_service_dir, - topdown=False): - for service in dirs: - filename = "{root}{service}/hostname".format( - service=service, - root=root - ) - self._get_port_from_service(service, filename) + if not os.path.exists(self.torrc): + return + with open(self.torrc, 'r') as f: + for line in f.readlines(): + if line.startswith('HiddenServiceDir'): + service_group = parse_dir(line) + if line.startswith('HiddenServicePort'): + parse_port(line, service_group) def __str__(self): - if not self.onions: + if not self.services: return 'No onion site' - return '\n'.join(['%s: %s' % (service, ', '.join(onion)) - for (service, onion) in self.onions.items()]) + return '\n'.join([str(service) for service in self.services]) def to_json(self): - return dumps(self.onions) + service_lst = [dict(service) for service in self.services] + return dumps({ + service['name']: service['urls'] for service in service_lst + }) def main(): + logging.basicConfig() parser = argparse.ArgumentParser(description='Display onion sites', prog='onions') parser.add_argument('--json', dest='json', action='store_true', @@ -175,14 +257,26 @@ def main(): help='Setup hosts') args = parser.parse_args() - onions = Onions() - if args.setup: - onions.setup_hosts() - return - onions.get_onions() + try: + onions = Onions() + if args.setup: + onions.setup_hosts() + else: + onions.torrc_parser() + except BaseException as e: + error_msg = str(e) + else: + error_msg = None if args.json: + if error_msg: + print(dumps({'error': error_msg})) + sys.exit(1) + logging.getLogger().setLevel(logging.ERROR) print(onions.to_json()) else: + if error_msg: + logging.error(error_msg) + sys.exit(1) print(onions) diff --git a/assets/onions/onions/Service.py b/assets/onions/onions/Service.py new file mode 100644 index 0000000..6b0d605 --- /dev/null +++ b/assets/onions/onions/Service.py @@ -0,0 +1,177 @@ +'This class define a service link' +import logging +import os +import re +from base64 import b32encode +from hashlib import sha1 + +from Crypto.PublicKey import RSA + + +class ServicesGroup(object): + + name = None + _priv_key = None + _key_in_secrets = False + + hidden_service_dir = "/var/lib/tor/hidden_service/" + + def __init__(self, name=None, service=None, hidden_service_dir=None): + + name_regex = r'^[a-zA-Z0-9-_]+$' + + self.hidden_service_dir = hidden_service_dir or self.hidden_service_dir + if not name and not service: + raise Exception( + 'Init service group with a name or service at least' + ) + self.services = [] + self.name = name or service.host + if not re.match(name_regex, self.name): + raise Exception( + 'Group {name} has invalid name'.format(name=self.name) + ) + if service: + self.add_service(service) + + self.load_key() + if not self._priv_key: + self.gen_key() + + def add_service(self, service): + if service not in self.services: + if self.get_service_by_host(service.host): + raise Exception('Duplicate service name') + self.services.append(service) + + def get_service_by_host(self, host): + for service in self.services: + if host == service.host: + return service + + def add_key(self, key): + if self._key_in_secrets: + logging.warning('Secret key already set, overriding') + self._priv_key = key + self._key_in_secrets = False + + def __iter__(self): + yield 'name', self.name + yield 'onion', self.onion_url + yield 'urls', list(self.urls) + + def __str__(self): + return '{name}: {urls}'.format(name=self.name, + urls=', '.join(self.urls)) + + @property + def onion_url(self): + "Get onion url from private key" + + # Convert private RSA to public DER + priv = RSA.importKey(self._priv_key.strip()) + der = priv.publickey().exportKey("DER") + + # hash key, keep first half of sha1, base32 encode + onion = b32encode(sha1(der[22:]).digest()[:10]) + + return '{onion}.onion'.format(onion=onion.decode().lower()) + + @property + def urls(self): + for service in self.services: + for ports in service.ports: + yield '{onion}:{port}'.format(onion=self.onion_url, + port=ports.port_from) + + def write_key(self, hidden_service_dir=None): + 'Write key on disk and set tor service' + if not hidden_service_dir: + hidden_service_dir = self.hidden_service_dir + serv_dir = os.path.join(hidden_service_dir, self.name) + os.makedirs(serv_dir, exist_ok=True) + os.chmod(serv_dir, 0o700) + with open(os.path.join(serv_dir, 'private_key'), 'w') as f: + f.write(self._priv_key) + os.fchmod(f.fileno(), 0o600) + with open(os.path.join(serv_dir, 'hostname'), 'w') as f: + f.write(self.onion_url) + + def _load_key(self, key_file): + if os.path.exists(key_file): + with open(key_file, 'r') as f: + key = f.read().encode() + if not len(key): + return + try: + rsa = RSA.importKey(key) + self._priv_key = rsa.exportKey("PEM").decode() + except BaseException: + raise('Fail to load key for {name} services'.format( + name=self.name + )) + + def load_key(self): + self.load_key_from_secrets() + self.load_key_from_conf() + + def load_key_from_secrets(self): + 'Load key from docker secret using service name' + secret_file = os.path.join('/run/secrets', self.name) + if not os.path.exists(secret_file): + return + try: + self._load_key(secret_file) + self._key_in_secrets = True + except BaseException: + logging.warning('Fail to load key from secret, ' + 'check the key or secret name collision') + + def load_key_from_conf(self, hidden_service_dir=None): + 'Load key from disk if exists' + if not hidden_service_dir: + hidden_service_dir = self.hidden_service_dir + key_file = os.path.join(hidden_service_dir, + self.name, + 'private_key') + self._load_key(key_file) + + def gen_key(self): + 'Generate new 1024 bits RSA key for hidden service' + self._priv_key = RSA.generate( + bits=1024, + ).exportKey("PEM").decode() + + +class Ports: + + port_from = None + dest = None + + def __init__(self, port_from, dest): + self.port_from = int(port_from) + self.dest = dest if dest.startswith('unix:') else int(dest) + + @property + def is_socket(self): + return self.dest and type(self.dest) is not int + + def __iter__(self): + yield 'port_from', str(self.port_from) + yield 'dest', str(self.dest) + yield 'is_socket', self.is_socket + + +class Service: + + def __init__(self, host): + self.host = host + self.ports = [] + + def add_ports(self, ports): + p = [Ports(*sp.split(':', 1)) for sp in ports.split(',')] + self.ports.extend(p) + + def __iter__(self): + yield 'host', self.host + yield 'ports', [dict(p) for p in self.ports] diff --git a/assets/onions/onions/__init__.py b/assets/onions/onions/__init__.py index 9068b26..d3b0b70 100644 --- a/assets/onions/onions/__init__.py +++ b/assets/onions/onions/__init__.py @@ -1 +1,5 @@ -from .Onions import Onions, main +from .Onions import main +from .Onions import Onions +from .Service import Ports +from .Service import Service +from .Service import ServicesGroup diff --git a/assets/onions/setup.py b/assets/onions/setup.py index 5fa5962..467e13c 100644 --- a/assets/onions/setup.py +++ b/assets/onions/setup.py @@ -6,7 +6,7 @@ from setuptools import setup setup( name='onions', - version='0.2', + version='0.4', packages=find_packages(), @@ -31,9 +31,9 @@ setup( "Topic :: System :: Installation/Setup", ], - install_requires=['pyentrypoint', + install_requires=['pyentrypoint==0.5.0', 'Jinja2>=2.8', - 'pycrypto',], + 'pycrypto', ], entry_points={ 'console_scripts': [ diff --git a/assets/onions/tests/onions_test.py b/assets/onions/tests/onions_test.py new file mode 100644 index 0000000..9ac85c5 --- /dev/null +++ b/assets/onions/tests/onions_test.py @@ -0,0 +1,408 @@ +import json +import os +import re +from base64 import b32encode +from hashlib import sha1 + +import pytest +from Crypto.PublicKey import RSA +from onions import Onions + + +def get_key_and_onion(): + key = ''' +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQCsMP4gl6g1Q313miPhb1GnDr56ZxIWGsO2PwHM1infkbhlBakR +6DGQfpE31L1ZKTUxY0OexKbW088v8qCOfjD9Zk1i80JP4xzfWQcwFZ5yM/0fkhm3 +zLXqXdEahvRthmFsS8OWusRs/04U247ryTm4k5S0Ch5OTBuvMLzQ8W0yDwIDAQAB +AoGAAZr3U5B2ZgC6E7phKUHjbf5KMlPxrDkVqAZQWvuIKmhuYqq518vlYmZ7rhyS +o1kqAMrfH4TP1WLmJJlLe+ibRk2aonR4e0GbW4x151wcJdT1V3vdWAsVSzG3+dqX +PiGT//DIe0OPSH6ecI8ftFRLODd6f5iGkF4gsUSTcVzAFgkCQQDTY67dRpOD9Ozw +oYH48xe0B9NQCw7g4NSH85jPurJXnpn6lZ6bcl8x8ioAdgLyomR7fO/dJFYLw6uV +LZLqZsVbAkEA0Iei3QcpsJnYgcQG7l5I26Sq3LwoiGRDFKRI6k0e+en9JQJgA3Ay +tsLpyCHv9jQ762F6AVXFru5DmZX40F6AXQJBAIHoKac8Xx1h4FaEuo4WPkPZ50ey +dANIx/OAhTFrp3vnMPNpDV60K8JS8vLzkx4vJBcrkXDSirqSFhkIN9grLi8CQEO2 +l5MQPWBkRKK2pc2Hfj8cdIMi8kJ/1CyCwE6c5l8etR3sbIMRTtZ76nAbXRFkmsRv +La/7Syrnobngsh/vX90CQB+PSSBqiPSsK2yPz6Gsd6OLCQ9sdy2oRwFTasH8sZyl +bhJ3M9WzP/EMkAzyW8mVs1moFp3hRcfQlZHl6g1U9D8= +-----END RSA PRIVATE KEY----- + ''' + + onion = b32encode( + sha1( + RSA.importKey( + key.strip() + ).publickey().exportKey( + "DER" + )[22:] + ).digest()[:10] + ).decode().lower() + '.onion' + + return key.strip(), onion + + +def get_torrc_template(): + return r''' +{% for service_group in services %} +HiddenServiceDir /var/lib/tor/hidden_service/{{service_group.name}} +{% for service in service_group.services %} +{% for port in service.ports %} +{% if port.is_socket %} +HiddenServicePort {{port.port_from}} {{port.dest}} +{% endif %} +{% if not port.is_socket %} +HiddenServicePort {{port.port_from}} {{service.host}}:{{port.dest}} +{% endif %} +{% endfor %} +{% endfor %} +{% endfor %} + +{% if 'RELAY' in env %} +ORPort 9001 +{% endif %} + +SocksPort 0 + +# useless line for Jinja bug + '''.strip() + + +def test_ports(monkeypatch): + env = { + 'SERVICE1_PORTS': '80:80', + 'SERVICE2_PORTS': '80:80,81:8000', + 'SERVICE3_PORTS': '80:unix://unix.socket', + } + + monkeypatch.setattr(os, 'environ', env) + + onion = Onions() + onion._get_setup_from_env() + assert len(os.environ) == 3 + assert len(onion.services) == 3 + check = 0 + for service_group in onion.services: + assert len(service_group.services) == 1 + service = service_group.services[0] + if service.host == 'service1': + check += 1 + assert len(service.ports) == 1 + assert service.ports[0].port_from == 80 + assert service.ports[0].dest == 80 + assert not service.ports[0].is_socket + if service.host == 'service2': + check += 3 + assert len(service.ports) == 2 + assert service.ports[0].port_from == 80 + assert service.ports[0].dest == 80 + assert service.ports[1].port_from == 81 + assert service.ports[1].dest == 8000 + if service.host == 'service3': + check += 6 + assert len(service.ports) == 1 + assert service.ports[0].port_from == 80 + assert service.ports[0].dest == 'unix://unix.socket' + assert service.ports[0].is_socket + + assert check == 10 + + +def test_docker_links(fs, monkeypatch): + + env = { + 'HOSTNAME': 'test_env', + 'COMPOSE_SERVICE1_1_PORT': 'tcp://172.17.0.2:80', + 'COMPOSE_SERVICE1_1_PORT_80_TCP': 'tcp://172.17.0.2:80', + 'COMPOSE_SERVICE1_1_PORT_80_TCP_ADDR': '172.17.0.2', + 'COMPOSE_SERVICE1_1_PORT_80_TCP_PORT': '80', + 'COMPOSE_SERVICE1_1_PORT_80_TCP_PROTO': 'tcp', + 'COMPOSE_SERVICE1_1_PORT_8000_TCP': 'tcp://172.17.0.2:8000', + 'COMPOSE_SERVICE1_1_PORT_8000_TCP_ADDR': '172.17.0.2', + 'COMPOSE_SERVICE1_1_PORT_8000_TCP_PORT': '8000', + 'COMPOSE_SERVICE1_1_PORT_8000_TCP_PROTO': 'tcp', + 'COMPOSE_SERVICE1_1_NAME': '/compose_env_1/compose_service1_1', + 'SERVICE1_PORT': 'tcp://172.17.0.2:80', + 'SERVICE1_PORT_80_TCP': 'tcp://172.17.0.2:80', + 'SERVICE1_PORT_80_TCP_ADDR': '172.17.0.2', + 'SERVICE1_PORT_80_TCP_PORT': '80', + 'SERVICE1_PORT_80_TCP_PROTO': 'tcp', + 'SERVICE1_PORT_8000_TCP': 'tcp://172.17.0.2:8000', + 'SERVICE1_PORT_8000_TCP_ADDR': '172.17.0.2', + 'SERVICE1_PORT_8000_TCP_PORT': '8000', + 'SERVICE1_PORT_8000_TCP_PROTO': 'tcp', + 'SERVICE1_NAME': '/compose_env_1/service1', + 'SERVICE1_1_PORT': 'tcp://172.17.0.2:80', + 'SERVICE1_1_PORT_80_TCP': 'tcp://172.17.0.2:80', + 'SERVICE1_1_PORT_80_TCP_ADDR': '172.17.0.2', + 'SERVICE1_1_PORT_80_TCP_PORT': '80', + 'SERVICE1_1_PORT_80_TCP_PROTO': 'tcp', + 'SERVICE1_1_PORT_8000_TCP': 'tcp://172.17.0.2:8000', + 'SERVICE1_1_PORT_8000_TCP_ADDR': '172.17.0.2', + 'SERVICE1_1_PORT_8000_TCP_PORT': '8000', + 'SERVICE1_1_PORT_8000_TCP_PROTO': 'tcp', + 'SERVICE1_1_NAME': '/compose_env_1/service1_1', + } + + etc_host = ''' +127.0.0.1 localhost +::1 localhost ip6-localhost ip6-loopback +fe00::0 ip6-localnet +ff00::0 ip6-mcastprefix +ff02::1 ip6-allnodes +ff02::2 ip6-allrouters +172.17.0.2 service1 bf447f22cdba compose_service1_1 +172.17.0.2 service1_1 bf447f22cdba compose_service1_1 +172.17.0.2 compose_service1_1 bf447f22cdba + '''.strip() + + fs.CreateFile('/etc/hosts', contents=etc_host) + + monkeypatch.setattr(os, 'environ', env) + + onion = Onions() + onion._get_setup_from_links() + + assert len(onion.services) == 1 + group = onion.services[0] + assert len(group.services) == 1 + service = group.services[0] + assert len(service.ports) == 2 + assert set( + (port.port_from, port.dest) for port in service.ports + ) == set([(80, 80), (8000, 8000)]) + + +def test_key(monkeypatch): + + key, onion_url = get_key_and_onion() + env = { + 'SERVICE1_KEY': key + } + + monkeypatch.setattr(os, 'environ', env) + + onion = Onions() + onion._get_setup_from_env() + + assert len(os.environ) == 1 + assert len(onion.services) == 1 + + assert onion.services[0].onion_url == onion_url + + +def test_key_in_secret(fs, monkeypatch): + env = { + 'SERVICE1_SERVICE_NAME': 'group1', + 'SERVICE2_SERVICE_NAME': 'group1', + 'SERVICE3_SERVICE_NAME': 'group2', + 'SERVICE1_PORTS': '80:80', + 'SERVICE2_PORTS': '81:80,82:8000', + 'SERVICE3_PORTS': '80:unix://unix.socket', + } + + monkeypatch.setattr(os, 'environ', env) + + key, onion_url = get_key_and_onion() + + fs.CreateFile('/run/secrets/group1', contents=key) + + onion = Onions() + onion._get_setup_from_env() + + group1 = onion.find_group_by_name('group1') + group2 = onion.find_group_by_name('group2') + + # assert group._priv_key == key + assert group1.onion_url == onion_url + assert group2.onion_url != onion_url + + +def test_configuration(fs, monkeypatch): + env = { + 'SERVICE1_SERVICE_NAME': 'group1', + 'SERVICE2_SERVICE_NAME': 'group1', + 'SERVICE3_SERVICE_NAME': 'group2', + 'SERVICE1_PORTS': '80:80', + 'SERVICE2_PORTS': '81:80,82:8000', + 'SERVICE3_PORTS': '80:unix://unix.socket', + } + + monkeypatch.setattr(os, 'environ', env) + monkeypatch.setattr(os, 'fchmod', lambda x, y: None) + + key, onion_url = get_key_and_onion() + torrc_tpl = get_torrc_template() + + fs.CreateFile('/var/local/tor/torrc.tpl', contents=torrc_tpl) + fs.CreateFile('/etc/tor/torrc') + + onion = Onions() + onion._get_setup_from_env() + onion.apply_conf() + + with open('/etc/tor/torrc', 'r') as f: + torrc = f.read() + + assert 'HiddenServiceDir /var/lib/tor/hidden_service/group1' in torrc + assert 'HiddenServicePort 80 service1:80' in torrc + assert 'HiddenServicePort 81 service2:80' in torrc + assert 'HiddenServicePort 82 service2:8000' in torrc + assert 'HiddenServiceDir /var/lib/tor/hidden_service/group2' in torrc + assert 'HiddenServicePort 80 unix://unix.socket' in torrc + + # Check parser + onion2 = Onions() + onion2.torrc_parser() + + assert len(onion2.services) == 2 + + assert set( + group.name for group in onion2.services + ) == set(['group1', 'group2']) + + for group in onion2.services: + if group.name == 'group1': + assert len(group.services) == 2 + assert set( + service.host for service in group.services + ) == set(['service1', 'service2']) + for service in group.services: + if service.host == 'service1': + assert len(service.ports) == 1 + assert set( + (port.port_from, port.dest) for port in service.ports + ) == set([(80, 80)]) + if service.host == 'service2': + assert len(service.ports) == 2 + assert set( + (port.port_from, port.dest) for port in service.ports + ) == set([(81, 80), (82, 8000)]) + if group.name == 'group2': + assert len(group.services) == 1 + assert set( + service.host for service in group.services + ) == set(['group2']) + service = group.services[0] + assert len(service.ports) == 1 + assert set( + (port.port_from, port.dest) for port in service.ports + ) == set([(80, 'unix://unix.socket')]) + + +def test_groups(monkeypatch): + env = { + 'SERVICE1_SERVICE_NAME': 'group1', + 'SERVICE2_SERVICE_NAME': 'group1', + 'SERVICE3_SERVICE_NAME': 'group2', + 'SERVICE1_PORTS': '80:80', + 'SERVICE2_PORTS': '81:80,82:8000', + 'SERVICE3_PORTS': '80:unix://unix.socket', + } + + monkeypatch.setattr(os, 'environ', env) + + onion = Onions() + onion._get_setup_from_env() + + onion_match = r'^[a-z2-7]{16}.onion$' + + assert len(os.environ) == 6 + assert len(onion.services) == 2 + + assert set( + group.name for group in onion.services + ) == set(['group1', 'group2']) + + for group in onion.services: + if group.name == 'group1': + assert len(group.services) == 2 + assert set( + service.host for service in group.services + ) == set(['service1', 'service2']) + + if group.name == 'group2': + assert len(group.services) == 1 + assert set( + service.host for service in group.services + ) == set(['service3']) + + assert re.match(onion_match, group.onion_url) + + +def test_json(monkeypatch): + env = { + 'SERVICE1_SERVICE_NAME': 'group1', + 'SERVICE2_SERVICE_NAME': 'group1', + 'SERVICE3_SERVICE_NAME': 'group2', + 'SERVICE1_PORTS': '80:80', + 'SERVICE2_PORTS': '81:80,82:8000', + 'SERVICE3_PORTS': '80:unix://unix.socket', + } + + monkeypatch.setattr(os, 'environ', env) + + onion = Onions() + onion._get_setup_from_env() + onion.check_services() + + jsn = json.loads(onion.to_json()) + + assert len(jsn) == 2 + assert len(jsn['group1']) == 3 + assert len(jsn['group2']) == 1 + + +def test_output(monkeypatch): + env = { + 'SERVICE1_SERVICE_NAME': 'group1', + 'SERVICE2_SERVICE_NAME': 'group1', + 'SERVICE3_SERVICE_NAME': 'group2', + 'SERVICE1_PORTS': '80:80', + 'SERVICE2_PORTS': '81:80,82:8000', + 'SERVICE3_PORTS': '80:unix://unix.socket', + } + + monkeypatch.setattr(os, 'environ', env) + + onion = Onions() + onion._get_setup_from_env() + + for item in ['group1', 'group2', '.onion', ',']: + assert item in str(onion) + + +def test_not_valid_share_port(monkeypatch): + env = { + 'SERVICE1_SERVICE_NAME': 'group1', + 'SERVICE2_SERVICE_NAME': 'group1', + 'SERVICE3_SERVICE_NAME': 'group2', + 'SERVICE1_PORTS': '80:80', + 'SERVICE2_PORTS': '80:80,82:8000', + 'SERVICE3_PORTS': '80:unix://unix.socket', + } + + monkeypatch.setattr(os, 'environ', env) + + onion = Onions() + onion._get_setup_from_env() + + with pytest.raises(Exception) as excinfo: + onion.check_services() + assert 'Same port for multiple services' in str(excinfo.value) + + +def test_not_valid_no_services(monkeypatch): + env = { + 'SERVICE1_SERVICE_NAME': 'group1', + 'SERVICE2_SERVICE_NAME': 'group1', + 'SERVICE3_SERVICE_NAME': 'group2', + } + + monkeypatch.setattr(os, 'environ', env) + + onion = Onions() + onion._get_setup_from_env() + + with pytest.raises(Exception) as excinfo: + onion.check_services() + assert 'has not ports set' in str(excinfo.value) diff --git a/assets/torrc b/assets/torrc index 0f03255..76eaeea 100644 --- a/assets/torrc +++ b/assets/torrc @@ -1,9 +1,14 @@ -{% for service, conf in setup.items() %} -HiddenServiceDir /var/lib/tor/hidden_service/{{service}} -{% for ports in conf['ports'] %} -{% set map = ports[1] if type(ports[1]) != int else '{service}:{port}'.format(service=service, port=ports[1]) %} -# PORT {{service}} {{ports[0]}} -HiddenServicePort {{ports[0]}} {{map}} +{% for service_group in services %} +HiddenServiceDir /var/lib/tor/hidden_service/{{service_group.name}} +{% for service in service_group.services %} +{% for port in service.ports %} +{% if port.is_socket %} +HiddenServicePort {{port.port_from}} {{port.dest}} +{% endif %} +{% if not port.is_socket %} +HiddenServicePort {{port.port_from}} {{service.host}}:{{port.dest}} +{% endif %} +{% endfor %} {% endfor %} {% endfor %} @@ -12,3 +17,5 @@ ORPort 9001 {% endif %} SocksPort 0 + +# useless line for Jinja bug diff --git a/docker-compose.v2.yml b/docker-compose.v2.yml index cff8c48..709154c 100644 --- a/docker-compose.v2.yml +++ b/docker-compose.v2.yml @@ -9,6 +9,7 @@ services: links: - hello - world + - again environment: # Set mapping ports HELLO_PORTS: 80:80,800:80,8888:80 @@ -32,6 +33,12 @@ services: WORLD_PORTS: 8000:80 + AGAIN_PORTS: 88:80 + + # hello and again will share the same onion_adress + AGAIN_SERVICE_NAME: foo + HELLO_SERVICE_NAME: foo + # Keep keys in volumes volumes: - tor-keys:/var/lib/tor/hidden_service/ @@ -44,6 +51,10 @@ services: image: tutum/hello-world hostname: world + again: + image: tutum/hello-world + hostname: again + volumes: tor-keys: driver: local diff --git a/docker-compose.v3.yml b/docker-compose.v3.yml new file mode 100644 index 0000000..293c902 --- /dev/null +++ b/docker-compose.v3.yml @@ -0,0 +1,53 @@ +# docker version 3 example + +version: "3.1" + +services: + tor: + image: goldy/tor-hidden-service + build: . + links: + - hello + - world + - again + environment: + # Set mapping ports + HELLO_PORTS: 80:80,800:80,8888:80 + + WORLD_PORTS: 8000:80 + + AGAIN_PORTS: 88:80 + + # hello and again will share the same onion_adress + AGAIN_SERVICE_NAME: foo + HELLO_SERVICE_NAME: foo + + # Keep keys in volumes + volumes: + - tor-keys:/var/lib/tor/hidden_service/ + + # Set secret for key, use the same name as the service + secrets: + - source: foo + target: foo + mode: 0400 + + hello: + image: tutum/hello-world + hostname: hello + + world: + image: tutum/hello-world + hostname: world + + again: + image: tutum/hello-world + hostname: again + +volumes: + tor-keys: + driver: local + +secrets: + foo: + file: ./foo_private_key diff --git a/foo_private_key b/foo_private_key new file mode 100644 index 0000000..48d2330 --- /dev/null +++ b/foo_private_key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDR8TdQF9fDlGhy1SMgfhMBi9TaFeD12/FK27TZE/tYGhxXvs1C +NmFJy1hjVxspF5unmUsCk0yEsvEdcAdp17Vynz6W41VdinETU9yXHlUJ6NyI32AH +dnFnHEcsllSEqD1hPAAvMUWwSMJaNmBEFtl8DUMS9tPX5fWGX4w5Xx8dZwIDAQAB +AoGBAMb20jMHxaZHWg2qTRYYJa8LdHgS0BZxkWYefnBUbZn7dOz7mM+tddpX6raK +8OSqyQu3Tc1tB9GjPLtnVr9KfVwhUVM7YXC/wOZo+u72bv9+4OMrEK/R8xy30XWj +GePXEu95yArE4NucYphxBLWMMu2E4RodjyJpczsl0Lohcn4BAkEA+XPaEKnNA3AL +1DXRpSpaa0ukGUY/zM7HNUFMW3UP00nxNCpWLSBmrQ56Suy7iSy91oa6HWkDD/4C +k0HslnMW5wJBANdz4ehByMJZmJu/b5y8wnFSqep2jmJ1InMvd18BfVoBTQJwGMAr ++qwSwNXXK2YYl9VJmCPCfgN0o7h1AEzvdYECQAM5UxUqDKNBvHVmqKn4zShb1ugY +t1RfS8XNbT41WhoB96MT9P8qTwlniX8UZiwUrvNp1Ffy9n4raz8Z+APNwvsCQQC9 +AuaOsReEmMFu8VTjNh2G+TQjgvqKmaQtVNjuOgpUKYv7tYehH3P7/T+62dcy7CRX +cwbLaFbQhUUUD2DCHdkBAkB6CbB+qhu67oE4nnBCXllI9EXktXgFyXv/cScNvM9Y +FDzzNAAfVc5Nmbmx28Nw+0w6pnpe/3m0Tudbq3nHdHfQ +-----END RSA PRIVATE KEY----- diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..7ddf9e0 --- /dev/null +++ b/tox.ini @@ -0,0 +1,12 @@ +[tox] +envlist = py34, py35, py36 +changedir=assets/onions/ +setupdir=assets/onions/ +skip_missing_interpreters = true + +[testenv] +deps= + pytest + pyfakefs + pytest-mock +commands=pytest -v