compress and move dictionaries to assets

compressed files can't be read directly, so they are copied to files directorey before accessing
this still saves space, except if a user decides to use most of the available dictionaries

app/build.gradle

@@ -36,7 +36,8 @@ android {
 
     ndkVersion '21.3.6528147'
     androidResources {
-        noCompress 'dict'
+        noCompress 'main.dict'
+        noCompress 'empty.dict'
     }
 }
 

app/src/main/java/org/dslul/openboard/inputmethod/latin/BinaryDictionaryGetter.java

@@ -29,7 +29,9 @@ import org.dslul.openboard.inputmethod.latin.utils.BinaryDictionaryUtils;
 import org.dslul.openboard.inputmethod.latin.utils.DictionaryInfoUtils;
 
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.BufferUnderflowException;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -62,6 +64,9 @@ final public class BinaryDictionaryGetter {
     public static final String MAIN_DICTIONARY_CATEGORY = "main";
     public static final String ID_CATEGORY_SEPARATOR = ":";
 
+    public static final String MAIN_DICTIONARY_FILE_NAME = MAIN_DICTIONARY_CATEGORY + ".dict";
+    public static final String ASSETS_DICTIONARY_FOLDER = "dicts";
+
     // The key considered to read the version attribute in a dictionary file.
     private static String VERSION_KEY = "version";
 
@@ -265,9 +270,16 @@ final public class BinaryDictionaryGetter {
         }
 
         if (!foundMainDict && dictPackSettings.isWordListActive(mainDictId)) {
+            final File dict = loadDictionaryFromAssets(locale.toString(), context);
+            final AssetFileAddress fallbackAsset;
+            if (dict == null) {
+                // fall back to the old way (maybe remove? will not work if files are compressed)
                 final int fallbackResId =
                         DictionaryInfoUtils.getMainDictionaryResourceId(context.getResources(), locale);
-            final AssetFileAddress fallbackAsset = loadFallbackResource(context, fallbackResId);
+                fallbackAsset = loadFallbackResource(context, fallbackResId);
+            } else {
+                fallbackAsset = AssetFileAddress.makeFromFileName(dict.getPath());
+            }
             if (null != fallbackAsset) {
                 fileList.add(fallbackAsset);
             }
@@ -275,4 +287,76 @@ final public class BinaryDictionaryGetter {
 
         return fileList;
     }
+
+    /**
+     * Returns the best matching main dictionary from assets.
+     *
+     * Actually copies the dictionary to cache folder, and then returns that file. This allows
+     * the dictionaries to be stored in a compressed way, reducing APK size.
+     * On next load, the dictionary in cache folder is found by getCachedWordLists
+     *
+     * Returns null on IO errors or if no matching dictionary is found
+     */
+    public static File loadDictionaryFromAssets(final String locale, final Context context) {
+        final String[] dictionaryList;
+        try {
+            dictionaryList = context.getAssets().list(ASSETS_DICTIONARY_FOLDER);
+        } catch (IOException e) {
+            return null;
+        }
+        if (null == dictionaryList) return null;
+        String bestMatchName = null;
+        int bestMatchLevel = 0;
+        for (String dictionary : dictionaryList) {
+            final String dictLocale =
+                    extractLocaleFromAssetsDictionaryFile(dictionary);
+            if (dictLocale == null) continue;
+            final int matchLevel = LocaleUtils.getMatchLevel(dictLocale, locale);
+            if (LocaleUtils.isMatch(matchLevel) && matchLevel > bestMatchLevel) {
+                bestMatchName = dictionary;
+            }
+        }
+        if (bestMatchName == null) return null;
+
+        // we have a match, now copy contents of the dictionary to "cached" word lists folder
+        File outfile = new File(DictionaryInfoUtils.getWordListCacheDirectory(context) +
+                File.separator + extractLocaleFromAssetsDictionaryFile(bestMatchName) + File.separator +
+                BinaryDictionaryGetter.MAIN_DICTIONARY_FILE_NAME);
+        File parentFile = outfile.getParentFile();
+        if (parentFile == null || (!parentFile.exists() && !parentFile.mkdirs())) {
+            return null;
+        }
+        try {
+            InputStream in = context.getAssets().open(ASSETS_DICTIONARY_FOLDER + File.separator + bestMatchName);
+            FileOutputStream out = new FileOutputStream(outfile);
+            byte[] buf = new byte[1024];
+            int len;
+            while ((len = in.read(buf)) > 0) {
+                out.write(buf, 0, len);
+            }
+            out.flush();
+            return outfile;
+        } catch (IOException e) {
+            Log.e(TAG, "exception while looking for locale " + locale, e);
+            return null;
+        }
+    }
+
+    /**
+     * Returns the locale for a dictionary file name stored in assets.
+     *
+     * Assumes file name main_[locale].dict
+     *
+     * Returns the locale, or null if file name does not match the pattern
+     */
+    private static String extractLocaleFromAssetsDictionaryFile(final String dictionaryFileName) {
+        if (dictionaryFileName.startsWith(BinaryDictionaryGetter.MAIN_DICTIONARY_CATEGORY)
+                && dictionaryFileName.endsWith(".dict")) {
+            return dictionaryFileName.substring(
+                    BinaryDictionaryGetter.MAIN_DICTIONARY_CATEGORY.length() + 1,
+                    dictionaryFileName.lastIndexOf('.')
+            );
+        }
+        return null;
+    }
 }

app/src/main/java/org/dslul/openboard/inputmethod/latin/utils/DictionaryInfoUtils.java

@@ -151,7 +151,7 @@ public class DictionaryInfoUtils {
     /**
      * Helper method to get the top level cache directory.
      */
-    private static String getWordListCacheDirectory(final Context context) {
+    public static String getWordListCacheDirectory(final Context context) {
         return context.getFilesDir() + File.separator + "dicts";
     }
 
@@ -242,7 +242,7 @@ public class DictionaryInfoUtils {
         // An id is supposed to be in format category:locale, so splitting on the separator
         // should yield a 2-elements array
         if (2 != idArray.length) {
-            return false;
+            return id.startsWith(BinaryDictionaryGetter.MAIN_DICTIONARY_CATEGORY);
         }
         return BinaryDictionaryGetter.MAIN_DICTIONARY_CATEGORY.equals(idArray[0]);
     }