mover scripts in a scripts dir

scripts/compile_locales.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+import sys
+import os
+from babel.messages.frontend import main as pybabel
+
+if __name__ == "__main__":
+    if len(sys.argv) >= 2 and sys.argv[1] == 'mdtable':
+        from libretranslate.locales import get_available_locales
+        locales = get_available_locales(only_reviewed=False, sort_by_name=True)
+        print("Language | Reviewed | Weblate Link")
+        print("-------- | -------- | ------------")
+
+        for l in locales:
+            link = "https://hosted.weblate.org/translate/libretranslate/app/%s/" % l['code']
+            if l['code'] == 'en':
+                link = "https://hosted.weblate.org/projects/libretranslate/app/"
+            print("%s | %s | %s" % (l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link))
+    else:
+        locales_dir = os.path.join("libretranslate", "locales")
+        if not os.path.isdir(locales_dir):
+            os.makedirs(locales_dir)
+
+        print("Compiling locales")
+        sys.argv = ["", "compile", "-f", "-d", locales_dir]
+        pybabel()
+
+
+

scripts/install_models.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+import argparse
+from libretranslate.init import check_and_install_models
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--load_only_lang_codes", type=str, default="")
+    args = parser.parse_args()
+    lang_codes = args.load_only_lang_codes.split(",")
+    if len(lang_codes) == 0 or lang_codes[0] == '':
+        lang_codes = None
+    check_and_install_models(force=True, load_only_lang_codes=lang_codes)

scripts/run.bat

@@ -0,0 +1,41 @@
+@ECHO OFF
+
+SETLOCAL
+
+SET LT_PORT=5000
+
+:loop
+IF NOT "%1"=="" (
+    IF "%1"=="--port" (
+        SET LT_PORT=%2
+        SHIFT
+    )
+    IF "%1"=="--help" (
+        echo Usage: scripts/run.bat [--port N]
+        echo:
+        echo Run LibreTranslate using docker.
+        echo:
+        GOTO :done
+    )
+    IF "%1"=="--api-keys" (
+        SET DB_VOLUME=-v lt-db:/app/db
+        SHIFT
+    )
+    SHIFT
+    GOTO :loop
+)
+
+WHERE /Q docker
+IF %ERRORLEVEL% NEQ 0 GOTO :install_docker
+
+docker run -ti --rm -p %LT_PORT%:%LT_PORT% %DB_VOLUME% -v lt-local:/home/libretranslate/.local libretranslate/libretranslate %*
+
+GOTO :done
+
+:install_docker
+ECHO Cannot find docker! Go to https://docs.docker.com/desktop/install/windows-install/ and install docker before running this script (pressing Enter will open the page)
+pause
+start "" https://docs.docker.com/desktop/install/windows-install/
+GOTO :done
+
+:done

scripts/run.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+set -eo pipefail
+__dirname=$(cd "$(dirname "$0")"; pwd -P)
+cd "${__dirname}"
+
+platform="Linux" # Assumed
+uname=$(uname)
+case $uname in
+    "Darwin")
+    platform="MacOS / OSX"
+    ;;
+    MINGW*)
+    platform="Windows"
+    ;;
+esac
+
+usage(){
+  echo "Usage: $0 [--port N]"
+  echo
+  echo "Run LibreTranslate using docker."
+  echo
+  exit
+}
+
+export LT_PORT=5000
+
+# Parse args for overrides
+ARGS=()
+while [[ $# -gt 0 ]]
+do
+key="$1"
+case $key in
+    --port)
+    export LT_PORT="$2"
+    ARGS+=("$1")
+    ARGS+=("$2") # save it in an array for later
+    shift # past argument
+    shift # past value
+    ;;
+    --debug)
+    export LT_DEBUG=YES
+    ARGS+=("$1")
+    shift # past argument
+    ;;
+    --api-keys)
+    export DB_VOLUME="-v lt-db:/app/db"
+    ARGS+=("$1")
+    shift # past argument
+    ;;
+    --help)
+    usage
+    ;;
+    *)    # unknown option
+    ARGS+=("$1")
+    shift # past argument
+    ;;
+esac
+done
+
+# $1 = command | $2 = help_text | $3 = install_command (optional)
+check_command(){
+    hash "$1" 2>/dev/null || not_found=true
+    if [[ $not_found ]]; then
+        check_msg_prefix="Checking for $1... "
+
+        # Can we attempt to install it?
+        if [[ -n "$3" ]]; then
+            echo -e "$check_msg_prefix \033[93mnot found, we'll attempt to install\033[39m"
+            $3 || sudo $3
+
+            # Recurse, but don't pass the install command
+            check_command "$1" "$2"
+        else
+            check_msg_result="\033[91m can't find $1! Check that the program is installed and that you have added the proper path to the program to your PATH environment variable before launching the program. If you change your PATH environment variable, remember to close and reopen your terminal. $2\033[39m"
+        fi
+    fi
+
+    echo -e "$check_msg_prefix $check_msg_result"
+    if [[ $not_found ]]; then
+        return 1
+    fi
+}
+
+environment_check(){
+    check_command "docker" "https://www.docker.com/"
+}
+
+environment_check
+docker run -ti --rm -p $LT_PORT:$LT_PORT $DB_VOLUME -v lt-local:/home/libretranslate/.local libretranslate/libretranslate ${ARGS[@]}

scripts/suggestions-to-jsonl.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+import argparse
+import time
+import sqlite3
+import json
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Program to generate JSONL files from a LibreTranslate's suggestions.db")
+    parser.add_argument(
+        "--db",
+        type=str,
+        nargs=1,
+        help="Path to suggestions.db file",
+        default='suggestions.db'
+    )
+    parser.add_argument(
+        "--clear",
+        action='store_true',
+        help="Clear suggestions.db after generation",
+        default=False
+    )
+    args = parser.parse_args()
+
+    output_file = str(int(time.time())) + ".jsonl"
+
+    con = sqlite3.connect(args.db, check_same_thread=False)
+    cur = con.cursor()
+    
+    with open(output_file, 'w', encoding="utf-8") as f:
+        for row in cur.execute('SELECT q, s, source, target FROM suggestions WHERE source != "auto" ORDER BY source'):
+            q, s, source, target = row
+            obj = {
+                'q': q,
+                's': s, 
+                'source': source,
+                'target': target
+            }
+            json.dump(obj, f, ensure_ascii=False)
+            f.write('\n')
+    
+    print("Wrote %s" % output_file)
+
+    if args.clear:
+        cur.execute("DELETE FROM suggestions")
+        con.commit()
+        print("Cleared " + args.db)

scripts/update_locales.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+import sys
+import os
+import re
+import polib
+import json
+from babel.messages.frontend import main as pybabel
+from libretranslate.language import load_languages, improve_translation_formatting
+from libretranslate.locales import get_available_locale_codes, swag_eval
+from translatehtml import translate_html
+from libretranslate.app import get_version, create_app
+from libretranslate.main import get_args
+from flask_swagger import swagger
+
+# Update strings
+if __name__ == "__main__":
+    print("Loading languages")
+    languages = load_languages()
+    en_lang = next((l for l in languages if l.code == 'en'), None)
+    if en_lang is None:
+        print("Error: English model not found. You need it to run this script.")
+        exit(1)
+
+    locales_dir = os.path.join("libretranslate", "locales")
+    if not os.path.isdir(locales_dir):
+        os.makedirs(locales_dir)
+
+    # Dump language list so it gets picked up by pybabel
+    langs_file = os.path.join(locales_dir, ".langs.py")
+    with open(langs_file, 'w') as f:
+        for l in languages:
+            f.write("_(%s)\n" % json.dumps(l.name))
+    print("Wrote %s" % langs_file)
+
+    # Dump swagger strings
+    args = get_args()
+    app = create_app(args)
+    swag = swagger(app)
+
+    swag_strings = []
+    def add_swag_string(s):
+        if not s in swag_strings:
+            swag_strings.append(s)
+    swag_eval(swag, add_swag_string)
+
+    swag_file = os.path.join(locales_dir, ".swag.py")
+    with open(swag_file, 'w') as f:
+        for ss in swag_strings:
+            f.write("_(%s)\n" % json.dumps(ss))
+    print("Wrote %s" % swag_file)
+
+    messagespot = os.path.join(locales_dir, "messages.pot")
+    print("Updating %s" % messagespot)
+    sys.argv = ["", "extract", "-F", "babel.cfg", "-k", "_e _h", 
+                "--copyright-holder", "LibreTranslate Authors",
+                "--project", "LibreTranslate",
+                "--version", get_version(),
+                "-o", messagespot, "libretranslate"]
+    pybabel()
+
+    lang_codes = [l.code for l in languages if l.code != "en"]
+
+    # Init/update
+    for l in lang_codes:
+        cmd = "init"
+        if os.path.isdir(os.path.join(locales_dir, l, "LC_MESSAGES")):
+            cmd = "update"
+
+        sys.argv = ["", cmd, "-i", messagespot, "-d", locales_dir, "-l", l]
+        pybabel()
+
+        meta_file = os.path.join(locales_dir, l, "meta.json")
+        if not os.path.isfile(meta_file):
+            with open(meta_file, 'w') as f:
+                f.write(json.dumps({
+                    'name': next((lang.name for lang in languages if lang.code == l)),
+                    'reviewed': False
+                }, indent=4))
+                print("Wrote %s" % meta_file)
+    
+    # Automatically translate strings with libretranslate
+    # when a language model is available and a string is empty
+
+    locales = get_available_locale_codes(only_reviewed=False)
+    print(locales)
+    for locale in locales:
+        if locale == 'en':
+            continue
+
+        tgt_lang = next((l for l in languages if l.code == locale), None)
+
+        if tgt_lang is None:
+            # We cannot translate
+            continue
+
+        translator = en_lang.get_translation(tgt_lang)
+
+        messages_file = os.path.join(locales_dir, locale, "LC_MESSAGES", 'messages.po')
+        if os.path.isfile(messages_file):
+            print("Translating '%s'" % locale)
+            pofile = polib.pofile(messages_file)
+            c = 0
+            
+            for entry in pofile.untranslated_entries():
+                text = entry.msgid
+                
+                # Extract placeholders
+                placeholders = re.findall(r'%\(?[^\)]*\)?s', text)
+
+                for p in range(0, len(placeholders)):
+                    text = text.replace(placeholders[p], "<x>%s</x>" % p)
+
+                if len(placeholders) > 0:
+                    translated = str(translate_html(translator, text))
+                else:
+                    translated = improve_translation_formatting(text, translator.translate(text))
+
+                # Restore placeholders
+                for p in range(0, len(placeholders)):
+                    tag = "<x>%s</x>" % p
+                    if tag in translated:
+                        translated = translated.replace(tag, placeholders[p])
+                    else:
+                        # Meh, append
+                        translated += " " + placeholders[p]
+                
+                print(entry.msgid, " --> ", translated)
+                entry.msgstr = translated
+                c += 1
+            
+            if c > 0:
+                pofile.save(messages_file)
+                print("Saved %s" % messages_file)
+
+
+