fix a bunch of mostly small issues with the new parser and layouts

write up some issues for tablet layout (to be fixed later)

app/src/main/assets/language_key_texts/ar.txt

@@ -7,7 +7,7 @@
 ي ئ ى
 ب پ
 ل ﻻ|لا ﻷ|لأ ﻹ|لإ ﻵ|لآ
-ا آ ء أ إ ٱ
+ا !fixedColumnOrder!5 آ ء أ إ ٱ
 ك گ ک
 ى ئ
 ز ژ

app/src/main/assets/language_key_texts/bg.txt

@@ -1,6 +1,5 @@
 [morekeys]
 и ѝ
-" ” „ “
 
 [labels]
 alphabet: АБВ

app/src/main/assets/language_key_texts/bn_bd.txt

@@ -1,71 +1,29 @@
 [morekeys]
-ৌ ঔ
 ৈ ঐ
-্র য
-ো ও
-ে এ
-র ল র
-ন ণ
-স ষ
-ম শ
-ঙ ং
-য য়
-ড ঢ
-প ফ
-ট ঠ
-চ ছ
-হ ঞ
-গ ঘ
-ড় ঢ়
-ূ ঊ
-ী ঈ
-ঁ ় ৺ ঽ ৗ ঌ ৡ ৠ ৱ ৢ ৣ ৄ ৰ
-্ ঁ
-ৃ ঋ
-ু উ
-ি ই
-া আ অ
-ব ভ
-ক খ
-দ ধ
-punctuation !autoColumnOrder!8 \, ॥ ? ! !icon/zwnj_key|\u200C !icon/zwj_key|\u200D # @ ( ) / ; : - + \%
-ঔ ৌ
-ঐ ৈ
-আ া
-ঈ ী
-ঊ ূ
-জ ঝ জ্ঞ
-ড ড়
-ও ো
-এ ে
-অ ্
-ই ি
-উ ু
-র ৃ ঋ ত্র
-ত ৎ থ ত্ত
-ঁ ঃ ং
-ঢ ঢ়
-ঞ ঙ ঞ্জ
-ষ ক্ষ
-স শ
-য় য
-। ॥
 ড় ়
 ট ৳ ট্ট ট্ব ট্ম
 ঞ ঞ্চ ঞ্ছ ঞ্জ
+ূ ঊ
+ী ঈ
 ৌ চ ৗ
 ফ ফ্ট ফ্য ফ্র ফ্ল
 ঋ ঌ ৡ ৴ ৵ ৶ ৢ ৣ
 ধ ধ্ব ধ্য ধ্র
 থ থ্য থ্র
-র র‍্য
+ে এ
+র র‍্য ল
 ত ৎ ত্ত ত্ন ত্ব ত্ম
+ু উ
+ি ই
+ো ও
 প প্ত প্ন প্প প্য প্র প্ল প্স
 শ শ্চ শ্ছ শ্ত শ্ন শ্ব শ্ম শ্র শ্ল
 ড ড্ড
 হ ঽ হ্ণ হ্ন হ্ব হ্ম হ্র হ্ল
+া আ
 স স্ব স্ত স্ট স্ক স্প
 দ দ্দ দ্ধ দ্ব দ্ভ দ্ম দ্য দ্র
+ৃ ঋ
 গ গ্ধ গ্র গ্ল গ্ন
 জ জ্ঞ জ্জ জ্য জ্ব জ্র
 ক ক্ষ ক্ক ক্ট ক্ত ক্র ক্স ক্ল
@@ -79,6 +37,29 @@ punctuation !autoColumnOrder!8 \, ॥ ? ! !icon/zwnj_key|\u200C !icon/zwj_key|\u
 ব ্ব ব্র ব্জ ব্দ ব্ধ ব্ব ব্ল ৱ ৰ
 ন ন্ত ন্থ ন্ট ন্দ ন্ধ ন্ড ন্ন ন্ব ন্ম ন্স
 ম  ম্ন ম্প ম্ব ম্ভ ম্ম ম্র ম্ল
+punctuation !autoColumnOrder!8 \, ॥ ? ! !icon/zwnj_key|‌ !icon/zwj_key|‍ # @ ( ) / ; : - + \%
+ঙ ং
+য য়
+ড ঢ
+প ফ
+ট ঠ
+চ ছ
+জ ঝ
+হ ঞ
+গ ঘ
+ড় ঢ়
+ঁ ় ৺ ঽ ৗ ঌ ৡ ৠ ৱ ৢ ৣ ৄ ৰ
+া অ
+্ ঁ
+ব ভ
+ক খ
+ত থ
+দ ধ
+ৌ ঔ
+্র ্য
+ন ণ
+স ষ
+ম শ
 
 [labels]
 alphabet: কখগ

app/src/main/assets/language_key_texts/bn_in.txt

@@ -0,0 +1,36 @@
+[morekeys]
+ব ভ %
+গ ঘ %
+দ ধ %
+জ ঝ জ্ঞ %
+ড ড়
+ও ো
+এ ে
+অ ্
+ই ি
+উ ু
+প ফ
+র ৃ ঋ ত্র
+ক খ
+ত ৎ থ ত্ত
+চ ছ
+ট ঠ
+ঁ ঃ ং
+ঢ ঢ়
+ন ণ
+ঞ ঙ ঞ্জ
+ষ ক্ষ
+স শ
+য় য
+। ॥
+ঔ ৌ
+ঐ ৈ
+আ া
+ঈ ী
+ঊ ূ
+
+[labels]
+alphabet: কখগ
+
+[number_row]
+১ ২ ৩ ৪ ৫ ৬ ৭ ৮ ৯ ০

app/src/main/assets/language_key_texts/de_de.txt

@@ -9,7 +9,7 @@ n ñ ń
 " ” „ “ » «
 
 [extra_keys]
-1: ü è
+1: ü
-2: ö é
+2: ö
-2: ä à
+2: ä
 3: ß

app/src/main/assets/language_key_texts/fa.txt

@@ -1,7 +1,7 @@
 [morekeys]
 ه ﻫ|ه‍ هٔ ة
 ی ئ ي ﯨ|ى
-ا ٱ ء آ أ إ
+ا !fixedColumnOrder!5 ٱ ء آ أ إ
 ت ة
 ک ك
 و ؤ

app/src/main/assets/language_key_texts/hi.txt

@@ -64,6 +64,7 @@ punctuation !autoColumnOrder!9 \, . ? ! # ) ( / ; ' @ : - " + \% &
 म ॐ
 न ण ञ ङ
 स श ष श्र
+ऑ ॉ
 
 [labels]
 alphabet: कखग

app/src/main/assets/language_key_texts/iw.txt

@@ -1,6 +1,4 @@
 [morekeys]
-' ‘ ’ ‚
-" “ ” „
 + ﬩
 ג ג׳
 י ײַ

app/src/main/assets/language_key_texts/lo.txt

@@ -1,14 +1,2 @@
-[morekeys]
-ຢ ໑
-ຟ ໒
-ໂ ໓
-ຖ ໔
-ຄ ໕
-ຕ ໖
-ຈ ໗
-ຂ ໘
-ຊ ໙
-ໄ ໐
-
 [labels]
 alphabet: ກຂຄ

app/src/main/assets/layouts/bengali_akkhor.json

@@ -93,7 +93,7 @@
   ],
   [
     { "$": "shift_state_selector",
-      "manualOrLocked": { "label": "।" },
+      "manualOrLocked": { "label": "।", "popup": { "main": { "label": "॥" } }, "labelFlags": 1073741824 },
       "default": { "label": "য" }
     },
     { "$": "shift_state_selector",

app/src/main/assets/layouts/bengali_unijoy.json

@@ -81,8 +81,8 @@
   ],
   [
     { "$": "shift_state_selector",
-      "manualOrLocked": { "label": "য" },
+      "manualOrLocked": { "label": "্য" },
-      "default": { "label": "্" }
+      "default": { "label": "্র" }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "ৌ" },

app/src/main/assets/layouts/hebrew.json

@@ -3,7 +3,7 @@
     { "$": "variation_selector",
       "email": { "label": "-" },
       "url": { "label": "-" },
-      "default": { "label": "'" }
+      "default": { "label": "'", "popup": { "main": { "label": "\"" } } }
     },
     { "$": "variation_selector",
       "email": { "label": "_" },

app/src/main/assets/layouts/khmer.json

@@ -1,7 +1,7 @@
 [
   [
     { "$": "shift_state_selector",
-      "manualOrLocked": { "label": "!" },
+      "manualOrLocked": { "label": "!", "labelFlags": 48 },
       "default": { "label": "១" }
     },
     { "$": "shift_state_selector",
@@ -9,7 +9,7 @@
       "default": { "label": "២" }
     },
     { "$": "shift_state_selector",
-      "manualOrLocked": { "label": "\"" },
+      "manualOrLocked": { "label": "\"", "labelFlags": 48 },
       "default": { "label": "៣" }
     },
     { "$": "shift_state_selector",
@@ -17,7 +17,7 @@
       "default": { "label": "៤" }
     },
     { "$": "shift_state_selector",
-      "manualOrLocked": { "label": "%" },
+      "manualOrLocked": { "label": "%", "labelFlags": 48 },
       "default": { "label": "៥" }
     },
     { "$": "shift_state_selector",
@@ -33,11 +33,11 @@
       "default": { "label": "៨" }
     },
     { "$": "shift_state_selector",
-      "manualOrLocked": { "label": "(" },
+      "manualOrLocked": { "label": "(", "labelFlags": 48 },
       "default": { "label": "៩" }
     },
     { "$": "shift_state_selector",
-      "manualOrLocked": { "label": ")" },
+      "manualOrLocked": { "label": ")", "labelFlags": 48 },
       "default": { "label": "០" }
     },
     { "$": "shift_state_selector",
@@ -187,7 +187,7 @@
       "default": { "label": "។" }
     },
     { "$": "shift_state_selector",
-      "manualOrLocked": { "label": "\\?" },
+      "manualOrLocked": { "label": "\\?", "labelFlags": 48 },
       "default": { "label": "៊" }
     }
   ]

app/src/main/assets/layouts/lao.json

@@ -2,19 +2,19 @@
   [
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "໑" },
-      "default": { "label": "ຢ" }
+      "default": { "label": "ຢ", "popup": { "relevant": [ { "label": "1" }, { "label": "໑" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "໒" },
-      "default": { "label": "ຟ" }
+      "default": { "label": "ຟ", "popup": { "relevant": [ { "label": "2" }, { "label": "໒" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "໓" },
-      "default": { "label": "ໂ" }
+      "default": { "label": "ໂ", "popup": { "relevant": [ { "label": "3" }, { "label": "໓" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "໔" },
-      "default": { "label": "ຖ" }
+      "default": { "label": "ຖ", "popup": { "relevant": [ { "label": "4" }, { "label": "໔" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "໌" },
@@ -26,23 +26,23 @@
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "໕" },
-      "default": { "label": "ຄ" }
+      "default": { "label": "ຄ", "popup": { "relevant": [ { "label": "5" }, { "label": "໕" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "໖" },
-      "default": { "label": "ຕ" }
+      "default": { "label": "ຕ", "popup": { "relevant": [ { "label": "6" }, { "label": "໖" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "໗" },
-      "default": { "label": "ຈ" }
+      "default": { "label": "ຈ", "popup": { "relevant": [ { "label": "7" }, { "label": "໗" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "໘" },
-      "default": { "label": "ຂ" }
+      "default": { "label": "ຂ", "popup": { "relevant": [ { "label": "8" }, { "label": "໘" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "໙" },
-      "default": { "label": "ຊ" }
+      "default": { "label": "ຊ", "popup": { "relevant": [ { "label": "9" }, { "label": "໙" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "ໍ່", "labelFlags": 128 },
@@ -56,7 +56,7 @@
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "໐" },
-      "default": { "label": "ໄ" }
+      "default": { "label": "ໄ", "popup": { "relevant": [ { "label": "0" }, { "label": "໐" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "ຳ້", "labelFlags": 128 },

app/src/main/assets/layouts/symbols.txt

@@ -16,8 +16,8 @@ $$$
 &
 - _ – — ·
 + ±
-( < { [
+( !fixedColumnOrder!3 < { [
-) > } ]
+) !fixedColumnOrder!3 > } ]
 
 * † ‡ ★
 "

app/src/main/assets/layouts/symbols_arabic.txt

@@ -16,12 +16,12 @@ $$$
 &
 - _ – — ·
 + ±
-( ﴾ < { [
+( !fixedColumnOrder!4 ﴾ < { [
-) ﴿ > } ]
+) !fixedColumnOrder!4 ﴿ > } ]
 
 * ٭ ★ † ‡
-« „ “ ”
+« !fixedColumnOrder!3 „ “ ”
-» ‚ ‘ ’ ‹ ›
+» !fixedColumnOrder!5 ‚ ‘ ’ ‹ ›
 :
 ؛ ;
 !

app/src/main/assets/layouts/thai.json

@@ -6,19 +6,19 @@
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "๑" },
-      "default": { "label": "/", "labelFlags": 48 }
+      "default": { "label": "/", "labelFlags": 48, "popup": { "relevant": [ { "label": "1" }, { "label": "๑" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "๒" },
-      "default": { "label": "_", "labelFlags": 48 }
+      "default": { "label": "_", "labelFlags": 48, "popup": { "relevant": [ { "label": "2" }, {"label": "๒" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "๓" },
-      "default": { "label": "ภ" }
+      "default": { "label": "ภ", "popup": { "relevant": [ { "label": "3" }, { "label": "๓" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "๔" },
-      "default": { "label": "ถ" }
+      "default": { "label": "ถ", "popup": { "relevant": [ { "label": "4" }, { "label": "๔" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": " ู|ู", "labelFlags": 128 },
@@ -30,29 +30,29 @@
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "๕" },
-      "default": { "label": "ค" }
+      "default": { "label": "ค", "popup": { "relevant": [ { "label": "5" }, { "label": "๕" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "๖" },
-      "default": { "label": "ต" }
+      "default": { "label": "ต", "popup": { "relevant": [ { "label": "6" }, { "label": "๖" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "๗" },
-      "default": { "label": "จ" }
+      "default": { "label": "จ", "popup": { "relevant": [ { "label": "7" }, { "label": "๗" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "๘" },
-      "default": { "label": "ข" }
+      "default": { "label": "ข", "popup": { "relevant": [ { "label": "8" }, { "label": "๘" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "๙" },
-      "default": { "label": "ช" }
+      "default": { "label": "ช", "popup": { "relevant": [ { "label": "9" }, { "label": "๙" } ] } }
     }
   ],
   [
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "๐" },
-      "default": { "label": "ๆ" }
+      "default": { "label": "ๆ", "popup": { "relevant": [ { "label": "0" }, { "label": "๐" } ] } }
     },
     { "$": "shift_state_selector",
       "manualOrLocked": { "label": "\"", "labelFlags": 48 },

app/src/main/java/org/dslul/openboard/inputmethod/keyboard/Key.java

@@ -132,8 +132,8 @@ public class Key implements Comparable<Key> {
     private static final int MORE_KEYS_FLAGS_NO_PANEL_AUTO_MORE_KEY = 0x10000000;
     // TODO: Rename these specifiers to !autoOrder! and !fixedOrder! respectively.
     public static final String MORE_KEYS_AUTO_COLUMN_ORDER = "!autoColumnOrder!";
-    private static final String MORE_KEYS_FIXED_COLUMN_ORDER = "!fixedColumnOrder!";
+    public static final String MORE_KEYS_FIXED_COLUMN_ORDER = "!fixedColumnOrder!";
-    private static final String MORE_KEYS_HAS_LABELS = "!hasLabels!";
+    public static final String MORE_KEYS_HAS_LABELS = "!hasLabels!";
     private static final String MORE_KEYS_NEEDS_DIVIDERS = "!needsDividers!";
     private static final String MORE_KEYS_NO_PANEL_AUTO_MORE_KEY = "!noPanelAutoMoreKey!";
 
@@ -960,7 +960,7 @@ public class Key implements Comparable<Key> {
         @Nullable public final String mHintLabel;
         public final int mLabelFlags;
         public final int mIconId;
-        public final MoreKeySpec[] mMoreKeys;
+        @Nullable public final MoreKeySpec[] mMoreKeys;
         public final int mMoreKeysColumnAndFlags;
         public final int mBackgroundType;
         public final int mActionFlags;
@@ -1193,6 +1193,8 @@ public class Key implements Comparable<Key> {
             int actionFlags = 0;
 
             final String[] languageMoreKeys = params.mLocaleKeyTexts.getMoreKeys(keySpec);
+            if (languageMoreKeys != null && layoutMoreKeys != null && languageMoreKeys[0].startsWith("!fixedColumnOrder!"))
+                languageMoreKeys[0] = null; // we change the number of keys, so better not use fixedColumnOrder to avoid awkward layout
             // todo: after removing old parser this could be done in a less awkward way without almostFinalMoreKeys
             final String[] almostFinalMoreKeys = MoreKeySpec.insertAdditionalMoreKeys(languageMoreKeys, layoutMoreKeys);
             mMoreKeysColumnAndFlags = getMoreKeysColumnAndFlagsAndSetNullInArray(params, almostFinalMoreKeys);

app/src/main/java/org/dslul/openboard/inputmethod/keyboard/internal/KeyboardBuilder.kt

@@ -64,11 +64,6 @@ open class KeyboardBuilder<KP : KeyboardParams>(protected val mContext: Context,
 
         // todo: further plan
         //  migrate other languages/layouts to this style
-        //   thai and lao number rows... they should probably have none, can't do it generically
-        //    so handle it like korean
-        //   languageMoreKeys for bengali and hindi layouts are completely mixed up -> maybe need to use layoutMoreKeys... but that's not nice
-        //   bangla (india) has different period & symbols label (should it really be latin?)
-        //    maybe need separate key text files for _IN and _BD
         //   test whether the layouts really are the same
         //    comparing params with both parsers looks good, see list of detected differences below
         //    still need to check moreKeys, there will be many more differences that might just be minor
@@ -105,6 +100,7 @@ open class KeyboardBuilder<KP : KeyboardParams>(protected val mContext: Context,
         //   write another parser, it should already consider split
         //   add a setting to display all emojis (and use emojiv2 or emojicompat or whatever is necessary)
         //    mention in subtitle that they may not be displayed properly, depending on the app you're writing in
+        //  more settings for localized number row, so it can be different in shift or symbols
         //  migrate moreKeys and moreSuggestions to this style?
         //   at least they should not make use of the KeyTextsSet/Table (and of the XmlKeyboardParser?)
         //  remove the old parser
@@ -193,32 +189,68 @@ open class KeyboardBuilder<KP : KeyboardParams>(protected val mContext: Context,
                         return@forEachIndexed
                     }
                     xmlRow2.forEachIndexed { index1, xmlParams ->
-                        // todo: compare moreKeys (and if different, check whether it's just the order)
+                        // todo: compare tablet layouts (how to best force it for both parsers?)
-                        //  also check holo, there might be different default parameters
+                        //   just rename the sw600 folders to sw 360
-                        //  and compare tablet layouts (how to best force it for both parsers?)
+                        // ->
+                        //  to shift symbols label should be ~ [ <
+                        //  last symbols row should be \ = * " ' : ; ! ? (but is * " ' : ; ! ? ! ?)
+                        //  last shift symbols row should have inverted ! and ?
+                        //  some different label flags
+                        //  ar: last symbols row should be \ = * " ' : ; ! ؟ (but is * « » : ; ! ؟ ! ?)
+                        //  ar: layout should not have ! and ? added (just empty space here...)
+                        //  ru, sr (both), others don't have a right shift key (come on...)
+                        //  but bulgarian (default) has -> not even per language
+                        //  armenian (and probably other 4 row layouts) messed up (delete key should be in first for, not x from bottom)
                         val keyParams = row[index1]
                         if (keyParams.mLabel != xmlParams.mLabel)
                             // currency keys (shift symbol) arranged differently
                             // obviously number row differences with possibly localized variants
                             Log.w(TAG, "label different: ${keyParams.mLabel} vs ${xmlParams.mLabel}")
+                        if (keyParams.mMoreKeys == null && xmlParams.mMoreKeys != null)
+                            Log.w(TAG, "moreKeys null for ${keyParams.mLabel} / ${keyParams.mCode}, but xml not null")
+                        else if (xmlParams.mMoreKeys == null && keyParams.mMoreKeys != null)
+                            // for ?123 key, wtf why are there moreKeys? can't see them anyway...
+                            Log.w(TAG, "moreKeys not null for ${keyParams.mLabel} / ${keyParams.mCode}, but xml null")
+                        else if (xmlParams.mMoreKeys == null || keyParams.mMoreKeys == null || keyParams.mMoreKeys.contentEquals(xmlParams.mMoreKeys))
+                            Unit
+                        else if (keyParams.mMoreKeys.size < xmlParams.mMoreKeys.size) {
+                            if (keyParams.mMoreKeys.size - xmlParams.mMoreKeys.size == -1 && keyParams.mCode.toChar().lowercase() == "s")
+                                Log.i(TAG, "missing moreKeys for ${keyParams.mLabel} / ${keyParams.mCode}")
+                            else
+                                Log.w(TAG, "missing moreKeys for ${keyParams.mLabel} / ${keyParams.mCode}")
+                        } else if (keyParams.mMoreKeys.size > xmlParams.mMoreKeys.size) {
+                            if (keyParams.mMoreKeys.toList().containsAll(xmlParams.mMoreKeys.toList()))
+                                Log.i(TAG, "more moreKeys for ${keyParams.mLabel} / ${keyParams.mCode}, first same: ${keyParams.mMoreKeys.firstOrNull() == xmlParams.mMoreKeys.firstOrNull() }" +
+                                        ", contains all original: true") // not really an issue i would say
+                            else
+                                Log.w(TAG, "more moreKeys for ${keyParams.mLabel} / ${keyParams.mCode}, first same: ${keyParams.mMoreKeys.firstOrNull() == xmlParams.mMoreKeys.firstOrNull() }" +
+                                        ", contains all original: false")
+                        } else if (!keyParams.mMoreKeys.toList().containsAll(xmlParams.mMoreKeys.toList()))
+                            Log.w(TAG, "same size but missing moreKeys for ${keyParams.mLabel} / ${keyParams.mCode}")
                         if (keyParams.mCode != xmlParams.mCode)
                             Log.w(TAG, "code different: ${keyParams.mCode} vs ${xmlParams.mCode}")
                         if (keyParams.mIconId != xmlParams.mIconId)
                             Log.w(TAG, "icon different: ${keyParams.mIconId} vs ${xmlParams.mIconId}")
-                        if (keyParams.mHintLabel != xmlParams.mHintLabel)
+                        if (keyParams.mMoreKeysColumnAndFlags != xmlParams.mMoreKeysColumnAndFlags)
+                            // symbols parentheses, symbols shift,
+                            Log.w(TAG, "mMoreKeysColumnAndFlags different for ${keyParams.mLabel} / ${keyParams.mCode}: ${keyParams.mMoreKeysColumnAndFlags} vs ${xmlParams.mMoreKeysColumnAndFlags}")
+                        if (keyParams.mHintLabel != xmlParams.mHintLabel
+                                && keyParams.mCode.toChar().lowercase() !in listOf("ö", "ä", "ü", "å", "ø", "æ", "é", "è", "à") // known, and imo irrelevant resp even better (but could be changed)
+                                && keyParams.mCode != '.'.code // happens for arabic, but really... hint label on period?
+                            )
                             // extra and number keys are the difference so far
                             // persian has small difference
                             // khmer has some difference
                             // urdu has a lot of difference
-                            Log.w(TAG, "hint label different: ${keyParams.mHintLabel} vs ${xmlParams.mHintLabel}")
+                            Log.w(TAG, "hint label different for ${keyParams.mLabel} / ${keyParams.mCode}: ${keyParams.mHintLabel} vs ${xmlParams.mHintLabel}")
-                        if (keyParams.mLabelFlags != xmlParams.mLabelFlags && keyParams.mCode != 10)
+                        if (keyParams.mLabelFlags != xmlParams.mLabelFlags
-                            // in symbol layout
+                                && !(keyParams.mLabelFlags - xmlParams.mLabelFlags == 0x40000000 && (mParams.mId.mElementId == KeyboardId.ELEMENT_SYMBOLS_SHIFTED || mParams.mId.mElementId == KeyboardId.ELEMENT_SYMBOLS)) // ignore the disableHintLabel flag
-                            //  my version has disableHintLabel for all
+                                && !(keyParams.mCode == -5 && keyParams.mLabelFlags - xmlParams.mLabelFlags == -0x10) // delete key with fontNormal (doesn't matter, happens because flags are set for entire row)
-                            //  original has LABEL_FLAGS_HAS_POPUP_HINT on < > in shift symbol (but there is no popup)
+                                && !(keyParams.mCode == -1 && keyParams.mLabelFlags - xmlParams.mLabelFlags == -0x10) // shift key with fontNormal (doesn't matter, happens because flags are set for entire row)
-                            // armenian, arabic, bangla,... and many with "symbols" original shift and delete have LABEL_FLAGS_FONT_NORMAL, mine not (but my period has)
+                                && !(keyParams.mCode == -5 && keyParams.mLabelFlags - xmlParams.mLabelFlags == -0x4010) // delete key with fontNormal|autoXScale (doesn't matter, happens because flags are set for entire row)
-                            // malayalam delete also has LABEL_FLAGS_AUTO_X_SCALE, mine not
+                                && !(keyParams.mCode == -1 && keyParams.mLabelFlags - xmlParams.mLabelFlags == -0x4010) // shift key with fontNormal|autoXScale (doesn't matter, happens because flags are set for entire row)
-                            // tamil & telugu my delete has LABEL_FLAGS_AUTO_X_SCALE, original not
+                                && !(keyParams.mLabelFlags - xmlParams.mLabelFlags == 0x10 && mParams.mId.mSubtype.keyboardLayoutSetName == "bengali_unijoy") // bangla (bd) doesn't have fontNormal, but for me it has -> that's fine, imo better
-                            // hindi / marathi: -5 has 10, mine has 0 -> delete and LABEL_FLAGS_FONT_NORMAL
+                            )
                             Log.w(TAG, "label flags different for ${keyParams.mLabel} / ${keyParams.mCode}: ${keyParams.mLabelFlags.toString(16)} vs ${xmlParams.mLabelFlags.toString(16)}")
                     }
                 }

app/src/main/java/org/dslul/openboard/inputmethod/keyboard/internal/keyboard_parser/KeyboardParser.kt

@@ -21,6 +21,7 @@ import org.dslul.openboard.inputmethod.latin.utils.InputTypeUtils
 import org.dslul.openboard.inputmethod.latin.utils.RunInLocale
 import org.dslul.openboard.inputmethod.latin.utils.ScriptUtils
 import org.dslul.openboard.inputmethod.latin.utils.sumOf
+import java.util.Locale
 
 /**
  * Abstract parser class that handles creation of keyboard from [KeyData] arranged in rows,
@@ -58,10 +59,14 @@ abstract class KeyboardParser(private val params: KeyboardParams, private val co
         if (!params.mId.mNumberRowEnabled && params.mId.mElementId == KeyboardId.ELEMENT_SYMBOLS) {
             // replace first symbols row with number row
             baseKeys[0] = params.mLocaleKeyTexts.getNumberRow()
-        } else if (!params.mId.mNumberRowEnabled && params.mId.isAlphabetKeyboard && params.mId.locale.language != "ko") {
+        } else if (!params.mId.mNumberRowEnabled && params.mId.isAlphabetKeyboard
+                && params.mId.locale.language != "ko"
+                && params.mId.locale.language != "th"
+                && params.mId.locale.language != "lo"
+            ) {
             // add number to the first 10 keys in first row
             // setting the correct moreKeys is handled in PopupSet
-            // not for korean layouts (add thai and lao to this), todo: should be decided in the layout, not in the parser
+            // not for korean/lao/thai layouts, todo: should be decided in the layout, not in the parser
             baseKeys.first().take(10).forEachIndexed { index, keyData -> keyData.popup.numberIndex = index }
             if (DebugFlags.DEBUG_ENABLED && baseKeys.first().size < 10) {
                 val message = "first row only has ${baseKeys.first().size} keys: ${baseKeys.first().map { it.label }}"
@@ -80,8 +85,9 @@ abstract class KeyboardParser(private val params: KeyboardParams, private val co
                 // add bottom row extra keys
                 // todo: question mark might be different -> get it from localeKeyTexts
                 //  also, maybe check device dimension int instead of getting this from resources, then using language labels is easier
+                //  and in shift symbols it should be inverted question/exclamation marks
                 it + context.getString(R.string.key_def_extra_bottom_right)
-                    .split(",").mapNotNull { if (it.isBlank()) null else it.trim().toTextKey() }
+                    .split(",").mapNotNull { if (it.isBlank()) null else it.trim().toTextKey(labelFlags = Key.LABEL_FLAGS_FONT_DEFAULT) }
             } else {
                 it
             }
@@ -186,7 +192,7 @@ abstract class KeyboardParser(private val params: KeyboardParams, private val co
 
     private fun splitFunctionalKeyDefs(def: String): List<String> {
         if (def.isBlank()) return emptyList()
-        return def.split(",").filter { infos.hasShiftKey || !it.startsWith("shift") }
+        return def.split(",").filter { infos.hasShiftKey || !it.trim().startsWith("shift") }
     }
 
     private fun getBottomRowAndAdjustBaseKeys(baseKeys: MutableList<List<KeyData>>): ArrayList<KeyParams> {
@@ -225,18 +231,18 @@ abstract class KeyboardParser(private val params: KeyboardParams, private val co
                         (adjustedKeys?.get(1)?.label ?: "<").rtlLabel(params),
                         params,
                         params.mDefaultRelativeKeyWidth,
-                        defaultLabelFlags,
+                        defaultLabelFlags or Key.LABEL_FLAGS_HAS_POPUP_HINT,
                         Key.BACKGROUND_TYPE_FUNCTIONAL,
-                        adjustedKeys?.get(1)?.popup?.toMoreKeys(params)
+                        adjustedKeys?.get(1)?.popup?.toMoreKeys(params) ?: arrayOf("!fixedColumnOrder!3", "‹", "≤", "«")
                     ))
                     bottomRow.add(keyParams)
                     bottomRow.add(KeyParams(
                         (adjustedKeys?.get(2)?.label ?: ">").rtlLabel(params),
                         params,
                         params.mDefaultRelativeKeyWidth,
-                        defaultLabelFlags,
+                        defaultLabelFlags or Key.LABEL_FLAGS_HAS_POPUP_HINT,
                         Key.BACKGROUND_TYPE_FUNCTIONAL,
-                        adjustedKeys?.get(2)?.popup?.toMoreKeys(params)
+                        adjustedKeys?.get(2)?.popup?.toMoreKeys(params) ?: arrayOf("!fixedColumnOrder!3", "›", "≥", "»")
                     ))
                 } else { // alphabet
                     if (params.mId.mLanguageSwitchKeyEnabled)
@@ -302,7 +308,9 @@ abstract class KeyboardParser(private val params: KeyboardParams, private val co
                 moreKeys?.let { getCommaMoreKeys() + it } ?: getCommaMoreKeys()
             )
             FunctionalKey.PERIOD -> KeyParams(
-                label ?: params.mLocaleKeyTexts.labelPeriod,
+                // special period moreKey only in alphabet layout, except for ar and fa
+                // todo: here is not the place to decide this, put it somewhere else (labelPeriod and labelPeriodSymbols?)
+                label ?: if (params.mId.isAlphabetKeyboard || params.mId.locale.language in listOf("ar", "fa")) params.mLocaleKeyTexts.labelPeriod else ".",
                 params,
                 width,
                 Key.LABEL_FLAGS_HAS_POPUP_HINT
@@ -351,7 +359,7 @@ abstract class KeyboardParser(private val params: KeyboardParams, private val co
                 if (params.mId.mElementId == KeyboardId.ELEMENT_ALPHABET_SHIFT_LOCK_SHIFTED || params.mId.mElementId == KeyboardId.ELEMENT_ALPHABET_SHIFT_LOCKED)
                     Key.BACKGROUND_TYPE_STICKY_ON
                 else Key.BACKGROUND_TYPE_STICKY_OFF,
-                arrayOf("!noPanelAutoMoreKey!", " |!code/key_capslock")
+                if (params.mId.isAlphabetKeyboard) arrayOf("!noPanelAutoMoreKey!", " |!code/key_capslock") else null // why the alphabe morekeys actually?
             )
             FunctionalKey.EMOJI -> KeyParams(
                 "!icon/emoji_normal_key|!code/key_emoji",
@@ -373,7 +381,7 @@ abstract class KeyboardParser(private val params: KeyboardParams, private val co
                 width,
                 Key.LABEL_FLAGS_AUTO_X_SCALE or Key.LABEL_FLAGS_FONT_NORMAL or Key.LABEL_FLAGS_HAS_POPUP_HINT or Key.LABEL_FLAGS_PRESERVE_CASE,
                 Key.BACKGROUND_TYPE_FUNCTIONAL,
-                arrayOf("!hasLabels!", ".net", ".org", ".gov", ".edu")
+                arrayOf(Key.MORE_KEYS_HAS_LABELS, ".net", ".org", ".gov", ".edu")
             )
             FunctionalKey.LANGUAGE_SWITCH -> KeyParams(
                 "!icon/language_switch_key|!code/key_language_switch",
@@ -477,8 +485,8 @@ abstract class KeyboardParser(private val params: KeyboardParams, private val co
             if (replacementText == iconName) { // i.e. we have the drawable
                 moreKeys.add(moreKey)
             } else {
-                moreKeys.add("!hasLabels!")
+                moreKeys.add(Key.MORE_KEYS_HAS_LABELS)
-                moreKeys.add(replacementText)
+                moreKeys.add("$replacementText|${iconPrefixRemoved.substringAfter("|")}")
             }
         }
         return moreKeys.toTypedArray()
@@ -497,7 +505,8 @@ abstract class KeyboardParser(private val params: KeyboardParams, private val co
         val ril = object : RunInLocale<String>() { // todo (later): simpler way of doing this in a single line?
             override fun job(res: Resources) = res.getString(id)
         }
-        return ril.runInLocale(context.resources, params.mId.locale)
+        val locale = if (params.mId.locale.toString().lowercase() == "hi_zz") Locale("en", "IN") else params.mId.locale // crappy workaround...
+        return ril.runInLocale(context.resources, locale)
     }
 
     private fun getToSymbolLabel() =
@@ -553,8 +562,11 @@ abstract class KeyboardParser(private val params: KeyboardParams, private val co
     private fun getPunctuationMoreKeys(): Array<String> {
         if (params.mId.mElementId == KeyboardId.ELEMENT_SYMBOLS || params.mId.mElementId == KeyboardId.ELEMENT_SYMBOLS_SHIFTED)
             return arrayOf("…")
-        val moreKeys = params.mLocaleKeyTexts.getMoreKeys("punctuation") ?:
+        val moreKeys = params.mLocaleKeyTexts.getMoreKeys("punctuation")!!
-            arrayOf("${Key.MORE_KEYS_AUTO_COLUMN_ORDER}8", "\\,", "?", "!", "#", ")", "(", "/", ";", "'", "@", ":", "-", "\"", "+", "\\%", "&")
+        if (params.mId.mSubtype.isRtlSubtype) {
+            for (i in moreKeys.indices)
+                moreKeys[i] = moreKeys[i].rtlLabel(params) // for parentheses
+        }
         if (context.resources.getInteger(R.integer.config_screen_metrics) >= 3 && moreKeys.contains("!") && moreKeys.contains("?")) {
             // we have a tablet, remove ! and ? keys and reduce number in autoColumnOrder
             // this makes use of removal of empty moreKeys in MoreKeySpec.insertAdditionalMoreKeys
@@ -627,7 +639,7 @@ abstract class KeyboardParser(private val params: KeyboardParams, private val co
                     R.array.touch_position_correction_data_default
                 else R.array.touch_position_correction_data_holo
             val hasZwnjKey = params.mId.locale.language in listOf("fa", "ne", "kn", "te") // determine from language, user might have custom layout
-            val hasShiftKey = name !in listOf("hindi_compact", "bengali", "arabic", "arabic_pc", "hebrew", "kannada", "malayalam", "marathi", "farsi", "tamil", "telugu")
+            val hasShiftKey = !params.mId.isAlphabetKeyboard || name !in listOf("hindi_compact", "bengali", "arabic", "arabic_pc", "hebrew", "kannada", "malayalam", "marathi", "farsi", "tamil", "telugu")
             return LayoutInfos(labelFlags, enableProximityCharsCorrection, allowRedundantMoreKeys, touchPositionCorrectionData, hasZwnjKey, hasShiftKey)
         }
     }
@@ -659,7 +671,7 @@ fun String.rtlLabel(params: KeyboardParams): String {
         "}" -> "}|{"
         "(" -> "(|)"
         ")" -> ")|("
-        "[" -> "{|]"
+        "[" -> "[|]"
         "]" -> "]|["
         "<" -> "<|>"
         ">" -> ">|<"

app/src/main/java/org/dslul/openboard/inputmethod/keyboard/internal/keyboard_parser/LocaleKeyTexts.kt

@@ -6,6 +6,7 @@ import org.dslul.openboard.inputmethod.keyboard.Key
 import org.dslul.openboard.inputmethod.keyboard.internal.KeyboardParams
 import org.dslul.openboard.inputmethod.keyboard.internal.keyboard_parser.floris.KeyData
 import org.dslul.openboard.inputmethod.keyboard.internal.keyboard_parser.floris.toTextKey
+import org.dslul.openboard.inputmethod.latin.common.splitOnFirstSpacesOnly
 import org.dslul.openboard.inputmethod.latin.common.splitOnWhitespace
 import org.dslul.openboard.inputmethod.latin.settings.Settings
 import java.io.InputStream
@@ -52,6 +53,8 @@ class LocaleKeyTexts(dataStream: InputStream?, locale: Locale) {
             moreKeys["!"] = arrayOf("¡")
         if ("?" !in moreKeys)
             moreKeys["?"] = arrayOf("¿")
+        if ("punctuation" !in moreKeys)
+            moreKeys["punctuation"] = arrayOf("${Key.MORE_KEYS_AUTO_COLUMN_ORDER}8", "\\,", "?", "!", "#", ")", "(", "/", ";", "'", "@", ":", "-", "\"", "+", "\\%", "&")
     }
 
     private fun readStream(stream: InputStream?, onlyMoreKeys: Boolean) {
@@ -68,7 +71,7 @@ class LocaleKeyTexts(dataStream: InputStream?, locale: Locale) {
                     "[number_row]" -> { mode = READER_MODE_NUMBER_ROW; return@forEachLine }
                 }
                 when (mode) {
-                    READER_MODE_MORE_KEYS -> addMoreKeys(line.splitOnWhitespace())
+                    READER_MODE_MORE_KEYS -> addMoreKeys(line)
                     READER_MODE_EXTRA_KEYS -> if (!onlyMoreKeys) addExtraKey(line.split(colonSpaceRegex, 2))
                     READER_MODE_LABELS -> if (!onlyMoreKeys) addLabel(line.split(colonSpaceRegex, 2))
                     READER_MODE_NUMBER_ROW -> setNumberRow(line.splitOnWhitespace(), onlyMoreKeys)
@@ -89,16 +92,22 @@ class LocaleKeyTexts(dataStream: InputStream?, locale: Locale) {
         readStream(dataStream, true)
     }
 
-    private fun addMoreKeys(split: List<String>) {
+    private fun addMoreKeys(line: String) {
+        val split = if (line.contains("|"))
+                // if a moreKey contains label/code separately, there are cases where space can be in there too
+                // normally this should work for all moreKeys, but if we split them on whitespace there is less chance for unnecessary issues
+                line.splitOnFirstSpacesOnly()
+            else line.splitOnWhitespace()
         if (split.size == 1) return
         val key = split.first()
         val existingMoreKeys = moreKeys[key]
         val newMoreKeys = if (existingMoreKeys == null)
                 Array(split.size - 1) { split[it + 1] }
             else mergeMoreKeys(existingMoreKeys, split.drop(1))
-        moreKeys[key] = if (key == "'" || key == "\"") // also do for parenthesis?
+        moreKeys[key] = when (key) {
-                addFixedColumnOrder(newMoreKeys)
+            "'", "\"", "«", "»", ")", "(" -> addFixedColumnOrder(newMoreKeys)
-            else newMoreKeys
+            else -> newMoreKeys
+        }
     }
 
     private fun addExtraKey(split: List<String>) {
@@ -266,17 +275,17 @@ private fun getCurrencyKey(locale: Locale): Pair<String, Array<String>> {
         return ruble
     if (locale.country == "LK" || locale.country == "BD")
         return genericCurrencyKey(getCurrency(locale))
-    if (locale.country == "IN" && locale.language == "ta")
+    if (locale.country != "IN" && locale.language == "ta")
         return genericCurrencyKey("௹")
     if (locale.country == "IN" || locale.language.matches("hi|kn|ml|mr|ta|te".toRegex()))
         return rupee
     if (locale.country == "GB")
         return pound
-    return genericCurrencyKey("$")
+    return dollar
 }
 
 private fun genericCurrencyKey(currency: String) = currency to genericCurrencyMoreKeys
-private val genericCurrencyMoreKeys = arrayOf("$", "¢", "£", "€", "¥", "₱")
+private val genericCurrencyMoreKeys = arrayOf("£", "€", "$", "¢", "¥", "₱")
 
 private fun getCurrency(locale: Locale): String {
     if (locale.country == "BD") return "৳"
@@ -298,12 +307,13 @@ private fun getCurrency(locale: Locale): String {
 }
 
 // needs at least 4 moreKeys for working shift-symbol keyboard
-private val euro = "€" to arrayOf("¢", "£", "$", "¥", "₱")
+private val euro = "€" to arrayOf("£", "¥", "$", "¢", "₱")
-private val dram = "֏" to arrayOf("€", "$", "₽", "¥", "£")
+private val dram = "֏" to arrayOf("€", "₽", "$", "£", "¥")
-private val rupee = "₹" to arrayOf("¢", "£", "€", "¥", "₱")
+private val rupee = "₹" to arrayOf("£", "€", "$", "¢", "¥", "₱")
-private val pound = "£" to arrayOf("¢", "$", "€", "¥", "₱")
+private val pound = "£" to arrayOf("€", "¥", "$", "¢", "₱")
 private val ruble = "₽" to arrayOf("€", "$", "£", "¥")
 private val lira = "₺" to arrayOf("€", "$", "£", "¥")
+private val dollar = "$" to arrayOf("£", "¢", "€", "¥", "₱")
 private val euroCountries = "AD|AT|BE|BG|HR|CY|CZ|DA|EE|FI|FR|DE|GR|HU|IE|IT|XK|LV|LT|LU|MT|MO|ME|NL|PL|PT|RO|SM|SK|SI|ES|VA".toRegex()
 private val euroLocales = "bg|ca|cs|da|de|el|en|es|et|eu|fi|fr|ga|gl|hr|hu|it|lb|lt|lv|mt|nl|pl|pt|ro|sk|sl|sq|sr|sv".toRegex()
 

app/src/main/java/org/dslul/openboard/inputmethod/latin/common/StringUtils.kt

@@ -53,6 +53,30 @@ fun hasLetterBeforeLastSpaceBeforeCursor(s: CharSequence): Boolean {
     return letter
 }
 
+/** split the string on the first of consecutive space only, further consecutive spaces are added to the next split */
+fun String.splitOnFirstSpacesOnly(): List<String> {
+    val out = mutableListOf<String>()
+    val sb = StringBuilder()
+    var previousSpace = false
+    for (c in this) {
+        if (c != ' ') {
+            sb.append(c)
+            previousSpace = false
+            continue
+        }
+        if (!previousSpace) {
+            out.add(sb.toString())
+            sb.clear()
+            previousSpace = true
+        } else {
+            sb.append(c)
+        }
+    }
+    if (sb.isNotBlank())
+        out.add(sb.toString())
+    return out
+}
+
 fun isEmoji(c: Int): Boolean = mightBeEmoji(c) && isEmoji(newSingleCodePointString(c))
 
 fun isEmoji(s: String): Boolean = mightBeEmoji(s) && s.matches(emoRegex)

app/src/main/java/org/dslul/openboard/inputmethod/latin/settings/PreferencesSettingsFragment.java

@@ -95,7 +95,7 @@ public final class PreferencesSettingsFragment extends SubScreenFragment {
         final Preference pref = findPreference(Settings.PREF_LOCALIZED_NUMBER_ROW);
         if (pref == null) return;
         // locales that have a number row defined (not good to have it hardcoded, but reading a bunch of files may be noticeably slow)
-        final String[] numberRowLocales = new String[] { "ar", "bn", "fa", "hi", "mr", "ne", "ur", "th" };
+        final String[] numberRowLocales = new String[] { "ar", "bn", "fa", "hi", "mr", "ne", "ur" };
         for (final InputMethodSubtype subtype : SubtypeSettingsKt.getEnabledSubtypes(getSharedPreferences(), true)) {
             if (ArraysKt.any(numberRowLocales, (l) -> l.equals(subtype.getLocale().substring(0, 2)))) {
                 pref.setVisible(true);

tools/morekeys_reader.py

@@ -272,42 +272,5 @@ def main():
 #    write_combined_lists(keys)
 
 
-# need to check strings:
-# latin, but only in symbol layout
-#  single_quotes, double_quotes (both used in morekeys of single/double quotes in symbol keyboard)
-#  single_angle_quotes, double_angle_quotes (same place as above -> merge into the same base ' or ")
-#  -> just treat them like morekeys_' and morekeys_"
-#  ... resolving those is really horrible, check different things and maybe include all if not too much?
-# latin, but for layout and not for moreKeys
-#  keyspec_nordic_row (+swiss and spanish) -> normal keys, what do? really specify a layout? or allow modifying?
-#  keyspec_q + w, y, x (eo only -> hmm, have a separate layout?)
-# not latin, but cyrillic (and maybe other non-latin)
-#  keyspec_east_slavic_row
-#  keylabel_to_alpha
-#  label_go_key and other keys (hi-rZZ and sr-rZZ -> why here? they should be in app strings, right?)
-# not in latin (so far)
-#  keyspec_symbols
-#  additional_morekeys_symbols
-#  keyspec_currency
-#  keylabel_to_symbol
-#  keyspec_comma
-#  keyhintlabel_period -> that's with the shifted key hint maybe
-#  keyhintlabel_tablet_period
-#  keyspec_period
-#  keyspec_tablet_period
-#  keyspec_symbols_question
-#  keyspec_symbols_semicolon
-#  keyspec_symbols_percent
-#  keyspec_tablet_comma
-#  keyhintlabel_tablet_comma
-#  keyspec_left_parenthesis + right
-#  keyspec_left_square_bracket + right
-#  keyspec_left_curly_bracket + right
-#  keyspec_less_than + greater
-#  keyspec_less_than_equal + greater
-#  keyspec_left_double_angle_quote + right
-#  keyspec_left_single_angle_quote + right
-
-
 if __name__ == "__main__":
     main()