diff --git a/.dockerignore b/.dockerignore index 942f938..b94de4f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,3 @@ .git* -Dockerfile +docker .Dockerfile.swp diff --git a/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md similarity index 100% rename from ISSUE_TEMPLATE.md rename to .github/ISSUE_TEMPLATE.md diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml index f9eb6a1..f4373e3 100644 --- a/.github/workflows/publish-package.yml +++ b/.github/workflows/publish-package.yml @@ -26,7 +26,7 @@ jobs: python -m pip install --upgrade pip pip install pytest flake8 pip install . - python compile_locales.py + python scripts/compile_locales.py - name: Check code style with flake8 (lint) run: | @@ -61,6 +61,6 @@ jobs: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - python compile_locales.py + python scripts/compile_locales.py python setup.py sdist bdist_wheel twine upload dist/* diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index eba163c..9e59bfd 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] + python-version: ['3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v2 @@ -51,7 +51,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Docker build - run: docker build -t libretranslate . + run: docker build -f docker/Dockerfile -t libretranslate . - name: Docker build with some models - run: docker build -t libretranslate --build-arg models=en,es . + run: docker build -f docker/Dockerfile -t libretranslate --build-arg models=en,es . diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29..0000000 diff --git a/README.md b/README.md index a354594..1d4e3ae 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ Then open a web browser to http://localhost:5000 ### Build with Docker ```bash -docker build [--build-arg with_models=true] -t libretranslate . +docker build -f docker/Dockerfile [--build-arg with_models=true] -t libretranslate . ``` If you want to run the Docker image in a complete offline environment, you need to add the `--build-arg with_models=true` parameter. Then the language models are downloaded during the build process of the image. Otherwise these models get downloaded on the first run of the image/container. @@ -218,7 +218,7 @@ If you're using docker: Start the program with the `--update-models` argument. For example: `libretranslate --update-models` or `./run.sh --update-models`. -Alternatively you can also run the `install_models.py` script. +Alternatively you can also run the `scripts/install_models.py` script. ## Run with WSGI and Gunicorn @@ -307,7 +307,7 @@ If you use Gunicorn, make sure to create a directory for storing multiprocess da mkdir -p /tmp/prometheus_data rm /tmp/prometheus_data/* export PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus_data -gunicorn -c gunicorn_conf.py --bind 0.0.0.0:5000 'wsgi:app(metrics=True)' +gunicorn -c scripts/gunicorn_conf.py --bind 0.0.0.0:5000 'wsgi:app(metrics=True)' ``` ## Language Bindings @@ -384,7 +384,7 @@ First you need to collect data, for example from [Opus](http://opus.nlpl.eu/), t ## Localization -The LibreTranslate Web UI is available in all the languages for which LibreTranslate can translate to. It can also (roughly) [translate itself!](https://github.com/LibreTranslate/LibreTranslate/blob/main/update_locales.py) Some languages might not appear in the UI since they haven't been reviewed by a human yet. You can enable all languages by turning on `--debug` mode. +The LibreTranslate Web UI is available in all the languages for which LibreTranslate can translate to. It can also (roughly) [translate itself!](https://github.com/LibreTranslate/LibreTranslate/blob/main/scripts/update_locales.py) Some languages might not appear in the UI since they haven't been reviewed by a human yet. You can enable all languages by turning on `--debug` mode. To help improve or review the UI translations: - Go to https://hosted.weblate.org/projects/libretranslate/app/#translations. All changes are automatically pushed to this repository. diff --git a/docker-compose.cuda.yml b/docker-compose.cuda.yml index 4535243..46dfeea 100644 --- a/docker-compose.cuda.yml +++ b/docker-compose.cuda.yml @@ -5,10 +5,10 @@ services: container_name: libretranslate-cuda build: context: . - dockerfile: docker/Dockerfile.cuda + dockerfile: docker/cuda.Dockerfile restart: unless-stopped ports: - - 5000:5000 + - "5000:5000" deploy: resources: reservations: diff --git a/docker-compose.yml b/docker-compose.yml index bc9bbe2..63bd509 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,10 +3,12 @@ version: "3" services: libretranslate: container_name: libretranslate - build: . + build: + context: . + dockerfile: docker/cuda.Dockerfile restart: unless-stopped ports: - - 5000:5000 + - "5000:5000" ## Uncomment above command and define your args if necessary # command: --ssl --ga-id MY-GA-ID --req-limit 100 --char-limit 500 ## Uncomment this section and the `volumes` section if you want to backup your API keys @@ -16,4 +18,4 @@ services: # - libretranslate_api_keys:/app/db/api_keys.db # volumes: -# libretranslate_api_keys: \ No newline at end of file +# libretranslate_api_keys: diff --git a/Dockerfile b/docker/Dockerfile similarity index 75% rename from Dockerfile rename to docker/Dockerfile index c68b1d3..fc99af1 100644 --- a/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8.14-slim-bullseye as builder +FROM python:3.10.9-slim-bullseye as builder WORKDIR /app @@ -15,14 +15,14 @@ RUN python -mvenv venv && ./venv/bin/pip install --upgrade pip COPY . . # Install package from source code, compile translations -RUN ./venv/bin/pip install Babel==2.11.0 && ./venv/bin/python compile_locales.py \ +RUN ./venv/bin/pip install Babel==2.11.0 && ./venv/bin/python scripts/compile_locales.py \ && ./venv/bin/pip install . \ && ./venv/bin/pip cache purge -FROM python:3.8.14-slim-bullseye +FROM python:3.10.9-slim-bullseye ARG with_models=false -ARG models= +ARG models="" RUN addgroup --system --gid 1032 libretranslate && adduser --system --uid 1032 libretranslate && mkdir -p /home/libretranslate/.local && chown -R libretranslate:libretranslate /home/libretranslate/.local USER libretranslate @@ -33,9 +33,9 @@ WORKDIR /app RUN if [ "$with_models" = "true" ]; then \ # initialize the language models if [ ! -z "$models" ]; then \ - ./venv/bin/python install_models.py --load_only_lang_codes "$models"; \ + ./venv/bin/python scripts/install_models.py --load_only_lang_codes "$models"; \ else \ - ./venv/bin/python install_models.py; \ + ./venv/bin/python scripts/install_models.py; \ fi \ fi diff --git a/docker/Dockerfile.cuda b/docker/cuda.Dockerfile similarity index 87% rename from docker/Dockerfile.cuda rename to docker/cuda.Dockerfile index bd94cf0..03b89b3 100644 --- a/docker/Dockerfile.cuda +++ b/docker/cuda.Dockerfile @@ -2,7 +2,7 @@ FROM nvidia/cuda:11.2.2-devel-ubuntu20.04 ENV ARGOS_DEVICE_TYPE cuda ARG with_models=true -ARG models= +ARG models="" WORKDIR /app @@ -27,14 +27,14 @@ RUN if [ "$with_models" = "true" ]; then \ pip3 install -e .; \ # initialize the language models if [ ! -z "$models" ]; then \ - ./install_models.py --load_only_lang_codes "$models"; \ + ./scripts/install_models.py --load_only_lang_codes "$models"; \ else \ - ./install_models.py; \ + ./scripts/install_models.py; \ fi \ fi # Install package from source code -RUN pip3 install Babel==2.11.0 && python3 compile_locales.py \ +RUN pip3 install Babel==2.11.0 && python3 scripts/compile_locales.py \ && pip3 install . \ && pip3 cache purge diff --git a/run.bat b/run.bat index d83c78b..3e32b5f 100644 --- a/run.bat +++ b/run.bat @@ -33,9 +33,9 @@ docker run -ti --rm -p %LT_PORT%:%LT_PORT% %DB_VOLUME% -v lt-local:/home/libretr GOTO :done :install_docker -ECHO Cannot find docker! Go to https://docs.docker.com/desktop/install/windows-install/ and install docker before running this script (pressing Enter will open the page) +ECHO Cannot find docker! Go to https://docs.docker.com/desktop/install/windows-install/ and install docker before running this script (pressing Enter will open the page) pause start "" https://docs.docker.com/desktop/install/windows-install/ GOTO :done -:done \ No newline at end of file +:done diff --git a/compile_locales.py b/scripts/compile_locales.py similarity index 92% rename from compile_locales.py rename to scripts/compile_locales.py index 9825237..5aff9af 100755 --- a/compile_locales.py +++ b/scripts/compile_locales.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import sys import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from babel.messages.frontend import main as pybabel if __name__ == "__main__": diff --git a/gunicorn_conf.py b/scripts/gunicorn_conf.py similarity index 100% rename from gunicorn_conf.py rename to scripts/gunicorn_conf.py diff --git a/install_models.py b/scripts/install_models.py similarity index 82% rename from install_models.py rename to scripts/install_models.py index 8fbae93..d844afb 100755 --- a/install_models.py +++ b/scripts/install_models.py @@ -1,4 +1,7 @@ #!/usr/bin/env python +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) import argparse from libretranslate.init import check_and_install_models diff --git a/suggestions-to-jsonl.py b/scripts/suggestions-to-jsonl.py similarity index 94% rename from suggestions-to-jsonl.py rename to scripts/suggestions-to-jsonl.py index 7878d52..6cd8e62 100755 --- a/suggestions-to-jsonl.py +++ b/scripts/suggestions-to-jsonl.py @@ -11,7 +11,7 @@ if __name__ == "__main__": type=str, nargs=1, help="Path to suggestions.db file", - default='suggestions.db' + default='db/suggestions.db' ) parser.add_argument( "--clear", diff --git a/update_locales.py b/scripts/update_locales.py similarity index 98% rename from update_locales.py rename to scripts/update_locales.py index 79dfb4e..1fd2853 100755 --- a/update_locales.py +++ b/scripts/update_locales.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import sys import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) import re import polib import json diff --git a/setup.py b/setup.py index 813a0b5..aaf2f88 100644 --- a/setup.py +++ b/setup.py @@ -22,16 +22,15 @@ setup( ], }, - python_requires='>=3.7.0', + python_requires='>=3.8.0', long_description=open('README.md').read(), long_description_content_type="text/markdown", install_requires=open("requirements.txt", "r").readlines(), - tests_require=['pytest==7.1.2'], + tests_require=['pytest==7.2.0'], setup_requires=['pytest-runner'], classifiers=[ "License :: OSI Approved :: GNU Affero General Public License v3 ", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10"