diff --git a/libretranslate/tests/test_api/conftest.py b/libretranslate/tests/test_api/conftest.py
index a1db034..3d7640f 100644
--- a/libretranslate/tests/test_api/conftest.py
+++ b/libretranslate/tests/test_api/conftest.py
@@ -3,14 +3,12 @@ import sys
import pytest
from libretranslate.app import create_app
-from libretranslate.default_values import DEFAULT_ARGUMENTS
from libretranslate.main import get_args
@pytest.fixture()
def app():
- sys.argv = ['']
- DEFAULT_ARGUMENTS['LOAD_ONLY'] = "en,es"
+ sys.argv = ['', '--load-only', 'en,es']
app = create_app(get_args())
yield app
diff --git a/pyproject.toml b/pyproject.toml
index c19d647..e8b1d42 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -33,8 +33,8 @@ classifiers = [
dynamic = ["version"]
dependencies = [
- "argostranslate ==1.8.2",
- "Flask ==2.2.2",
+ "argostranslate ==1.9.6",
+ "Flask ==2.2.5",
"flask-swagger ==0.2.14",
"flask-swagger-ui ==4.11.1",
"Flask-Limiter ==2.6.3",
@@ -42,19 +42,22 @@ dependencies = [
"Flask-Session ==0.4.0",
"waitress ==2.1.2",
"expiringdict ==1.2.2",
- " LTpycld2==0.42",
+ "langdetect==1.0.9",
+ "lexilang==1.0.6",
"morfessor ==2.0.6",
"appdirs ==1.4.4",
"APScheduler ==3.9.1",
"translatehtml ==1.5.2",
- "argos-translate-files ==1.1.4",
+ "argos-translate-files ==1.3.0",
"itsdangerous ==2.1.2",
- "Werkzeug ==2.2.2",
- "requests ==2.28.1",
- "redis ==4.3.4",
+ "Werkzeug ==2.3.8",
+ "requests ==2.31.0",
+ "redis ==4.4.4",
+ "numpy <2",
"prometheus-client ==0.15.0",
"polib ==1.1.1",
- "packaging ==23.1"
+ "packaging ==23.1",
+ "torch ==2.2.0"
]
[project.scripts]
@@ -68,6 +71,7 @@ test = [
"pytest-cov",
"pre-commit >=3.0.0",
"types-requests",
+ "pip-audit"
]
@@ -78,6 +82,8 @@ Documentation = "https://github.com/LibreTranslate/LibreTranslate"
Tracker = "https://github.com/LibreTranslate/LibreTranslate/issues"
History = "https://github.com/LibreTranslate/LibreTranslate/releases"
+[tool.hatch.build]
+artifacts = ["*.mo"]
# ENVIRONMENTS AND SCRIPTS
[tool.hatch.envs.default]
@@ -124,36 +130,6 @@ addopts = [
"--cov-report=term-missing",
]
-
-# https://beta.ruff.rs/docs/rules
-[tool.ruff]
-src = ["libretranslate", "scripts"]
-target-version = "py38"
-line-length = 136
-select = [
- "I", # isort
- "N", # pep8-naming
- "S", # bandit
- "A", # flake8-builtins
- "YTT", # flake8-2020
- "B", # flake8-bugbear
- # "C", # flake8-comprehensions
- "ICN", # flake8-import-conventions
- # "SIM", # flake8-simplify
- "TID", # flake8-tidy-imports
- # "Q", # flake8-quotes
- # "FBT", # flake8-boolean-trap
- "F", # pyflakes
- "UP", # pyupgrade
- # "E", # pycodestyle errors
- # "W", # pycodestyle warnings
- # "PLC", # pylint convention
- "PLE", # pylint error
- # "PLR", # pylint refactor
- # "PLW", # pylint warning
- # "RUF", # ruff specific
-]
-
ignore = [
"E501", # line too long
"A003", # Class attribute is shadowing a python builtin
@@ -162,10 +138,3 @@ ignore = [
"T201", "T203", # remove print and pprint
"E402", # Module level import not at top of file
]
-
-[tool.ruff.per-file-ignores]
-"__init__.py" = ["I", "F401"] # module imported but unused
-
-
-[tool.ruff.mccabe]
-max-complexity = 12
diff --git a/scripts/gunicorn_conf.py b/scripts/gunicorn_conf.py
index 6909d38..b1eacc7 100644
--- a/scripts/gunicorn_conf.py
+++ b/scripts/gunicorn_conf.py
@@ -12,7 +12,7 @@ def on_starting(server):
proc_name = server.cfg.default_proc_name
kwargs = {}
if proc_name.startswith("wsgi:app"):
- str_args = re.sub('wsgi:app\s*\(\s*(.*)\s*\)', '\\1', proc_name).strip().split(",")
+ str_args = re.sub(r'wsgi:app\s*\(\s*(.*)\s*\)', '\\1', proc_name).strip().split(",")
for a in str_args:
if "=" in a:
k,v = a.split("=")
@@ -21,12 +21,15 @@ def on_starting(server):
if v.lower() in ["true", "false"]:
v = v.lower() == "true"
+ if not v:
+ continue
elif v[0] == '"':
v = v[1:-1]
kwargs[k] = v
from libretranslate.main import get_args
sys.argv = ['--wsgi']
+
for k in kwargs:
ck = k.replace("_", "-")
if isinstance(kwargs[k], bool) and kwargs[k]:
diff --git a/scripts/healthcheck.py b/scripts/healthcheck.py
index 07bcaa4..10bf315 100644
--- a/scripts/healthcheck.py
+++ b/scripts/healthcheck.py
@@ -1,7 +1,9 @@
import requests
+import os
+port = os.environ.get('LT_PORT', '5000')
response = requests.post(
- url='http://0.0.0.0:5000/translate',
+ url=f'http://localhost:{port}/translate',
headers={'Content-Type': 'application/json'},
json={
'q': 'Hello World!',
diff --git a/scripts/update_locales.py b/scripts/update_locales.py
index 42e5490..365e119 100755
--- a/scripts/update_locales.py
+++ b/scripts/update_locales.py
@@ -9,11 +9,12 @@ import re
import polib
from babel.messages.frontend import main as pybabel
from flask_swagger import swagger
+from translatehtml import translate_html
+
from libretranslate.app import create_app, get_version
from libretranslate.language import improve_translation_formatting, load_languages
from libretranslate.locales import get_available_locale_codes, swag_eval
from libretranslate.main import get_args
-from translatehtml import translate_html
# Update strings
if __name__ == "__main__":
@@ -30,9 +31,17 @@ if __name__ == "__main__":
# Dump language list so it gets picked up by pybabel
langs_file = os.path.join(locales_dir, ".langs.py")
+ lang_names = {}
with open(langs_file, 'w') as f:
for l in languages:
f.write("_(%s)\n" % json.dumps(l.name))
+ lang_names[l.name] = True
+
+ # Additional names that might be missing
+ for l in ["Serbian", "Ukrainian", "Vietnamese"]:
+ if not l in lang_names:
+ f.write("_(%s)\n" % json.dumps(l))
+ lang_names[l] = True
print("Wrote %s" % langs_file)
# Dump swagger strings
@@ -61,34 +70,47 @@ if __name__ == "__main__":
"-o", messagespot, "libretranslate"]
pybabel()
- lang_codes = [l.code for l in languages if l.code != "en"]
+ lang_map = {
+ 'zt': 'zh_Hant'
+ }
+ lang_codes = [lang_map.get(l.code, l.code) for l in languages if l.code != "en"]
+ all_folders = [d for d in os.listdir(locales_dir) if os.path.isdir(os.path.join(locales_dir, d))]
+ review_map = {}
# Init/update
- for l in lang_codes:
+ for l in all_folders:
cmd = "init"
if os.path.isdir(os.path.join(locales_dir, l, "LC_MESSAGES")):
cmd = "update"
- sys.argv = ["", cmd, "-i", messagespot, "-d", locales_dir, "-l", l]
+ sys.argv = ["", cmd, "-i", messagespot, "-d", locales_dir, "-l", l] + (["--no-fuzzy-matching"] if cmd == "update" else [])
pybabel()
meta_file = os.path.join(locales_dir, l, "meta.json")
+ reviewed = False
if not os.path.isfile(meta_file):
with open(meta_file, 'w') as f:
f.write(json.dumps({
- 'name': next(lang.name for lang in languages if lang.code == l),
+ 'name': l if l not in lang_codes else next(lang.name for lang in languages if lang_map.get(lang.code, lang.code) == l),
'reviewed': False
}, indent=4))
print("Wrote %s" % meta_file)
+ else:
+ with open(meta_file) as f:
+ reviewed = json.loads(f.read()).get('reviewed', False)
+
+ review_map[l] = reviewed
# Automatically translate strings with libretranslate
# when a language model is available and a string is empty
locales = get_available_locale_codes(only_reviewed=False)
- print(locales)
for locale in locales:
if locale == 'en':
continue
+ if review_map.get(locale):
+ # Don't automatically translate reviewed languages
+ continue
tgt_lang = next((l for l in languages if l.code == locale), None)
diff --git a/wsgi.py b/wsgi.py
index 87413ae..3b5799a 100644
--- a/wsgi.py
+++ b/wsgi.py
@@ -4,10 +4,12 @@ from libretranslate import main
def app(*args, **kwargs):
import sys
sys.argv = ['--wsgi']
+
for k in kwargs:
ck = k.replace("_", "-")
- if isinstance(kwargs[k], bool) and kwargs[k]:
- sys.argv.append("--" + ck)
+ if isinstance(kwargs[k], bool):
+ if kwargs[k]:
+ sys.argv.append("--" + ck)
else:
sys.argv.append("--" + ck)
sys.argv.append(kwargs[k])