add basic support for modifier keys, fixes #479

README.md

@@ -159,7 +159,10 @@ See make-dict-list tool [README](tools/make-dict-list/README.md).
 __Planned features and improvements:__
 * Customizable functional key layout
   * Will likely result in having the same functional key layout for alphabet and symbols layouts
-* Support for _alt_, _ctrl_, _meta_ and _fn_ (#479)
+* Improve support for modifier keys (_alt_, _ctrl_, _meta_ and _fn_), some ideas:
+  * keep modifier keys on with long press
+  * keep modifier keys on until the next key press
+  * use sliding input
 * Less complicated addition of new keyboard languages (e.g. #519)
 * Additional and customizable key swipe functionality
   * Some functionality will not be possible when using glide typing

app/src/main/java/helium314/keyboard/event/DeadKeyCombiner.kt

@@ -261,7 +261,7 @@ class DeadKeyCombiner : Combiner {
             var lastEvent: Event? = null
             do {
                 val codePoint = Character.codePointBefore(text, index)
-                lastEvent = Event.createHardwareKeypressEvent(codePoint, originalEvent.mKeyCode, lastEvent, false)
+                lastEvent = Event.createHardwareKeypressEvent(codePoint, originalEvent.mKeyCode, 0, lastEvent, false)
                 index -= Character.charCount(codePoint)
             } while (index > 0)
             // can't be null because

app/src/main/java/helium314/keyboard/event/Event.kt

@@ -39,6 +39,9 @@ class Event private constructor(
         // this to be equal to mCodePoint for convenience. If this is not a key, this must contain
         // NOT_A_KEY_CODE.
         val mKeyCode: Int,
+        // State of meta keys (currently ctrl, alt, fn, meta)
+        // same value as https://developer.android.com/reference/android/view/KeyEvent#getMetaState()
+        val mMetaState: Int,
         // Coordinates of the touch event, if relevant. If useful, we may want to replace this with
         // a MotionEvent or something in the future. This is only relevant when the keypress is from
         // a software keyboard obviously, unless there are touch-sensitive hardware keyboards in the
@@ -57,7 +60,7 @@ class Event private constructor(
     // Returns whether this is a function key like backspace, ctrl, settings... as opposed to keys
     // that result in input like letters or space.
     val isFunctionalKeyEvent: Boolean
-        get() = NOT_A_CODE_POINT == mCodePoint // This logic may need to be refined in the future
+        get() = NOT_A_CODE_POINT == mCodePoint || mMetaState != 0 // This logic may need to be refined in the future
 
     // Returns whether this event is for a dead character. @see {@link #FLAG_DEAD}
     val isDead: Boolean get() = 0 != FLAG_DEAD and mFlags
@@ -131,24 +134,22 @@ class Event private constructor(
         private const val FLAG_COMBINING = 0x8
 
         @JvmStatic
-        fun createSoftwareKeypressEvent(codePoint: Int, keyCode: Int,
+        fun createSoftwareKeypressEvent(codePoint: Int, keyCode: Int, metaState: Int, x: Int, y: Int, isKeyRepeat: Boolean): Event {
-                                        x: Int, y: Int, isKeyRepeat: Boolean): Event {
+            return Event(EVENT_TYPE_INPUT_KEYPRESS, null, codePoint, keyCode, metaState, x, y,
-            return Event(EVENT_TYPE_INPUT_KEYPRESS, null /* text */, codePoint, keyCode, x, y,
+                    null, if (isKeyRepeat) FLAG_REPEAT else FLAG_NONE, null)
-                    null /* suggestedWordInfo */, if (isKeyRepeat) FLAG_REPEAT else FLAG_NONE, null)
         }
 
-        fun createHardwareKeypressEvent(codePoint: Int, keyCode: Int,
+        fun createHardwareKeypressEvent(codePoint: Int, keyCode: Int, metaState: Int, next: Event?, isKeyRepeat: Boolean): Event {
-                                        next: Event?, isKeyRepeat: Boolean): Event {
+            return Event(EVENT_TYPE_INPUT_KEYPRESS, null, codePoint, keyCode, metaState,
-            return Event(EVENT_TYPE_INPUT_KEYPRESS, null /* text */, codePoint, keyCode,
                     Constants.EXTERNAL_KEYBOARD_COORDINATE, Constants.EXTERNAL_KEYBOARD_COORDINATE,
-                    null /* suggestedWordInfo */, if (isKeyRepeat) FLAG_REPEAT else FLAG_NONE, next)
+                    null, if (isKeyRepeat) FLAG_REPEAT else FLAG_NONE, next)
         }
 
         // This creates an input event for a dead character. @see {@link #FLAG_DEAD}
-        fun createDeadEvent(codePoint: Int, keyCode: Int, next: Event?): Event { // TODO: add an argument or something if we ever create a software layout with dead keys.
+        fun createDeadEvent(codePoint: Int, keyCode: Int, metaState: Int, next: Event?): Event { // TODO: add an argument or something if we ever create a software layout with dead keys.
-            return Event(EVENT_TYPE_INPUT_KEYPRESS, null /* text */, codePoint, keyCode,
+            return Event(EVENT_TYPE_INPUT_KEYPRESS, null, codePoint, keyCode, metaState,
                     Constants.EXTERNAL_KEYBOARD_COORDINATE, Constants.EXTERNAL_KEYBOARD_COORDINATE,
-                    null /* suggestedWordInfo */, FLAG_DEAD, next)
+                    null, FLAG_DEAD, next)
         }
 
         /**
@@ -160,9 +161,8 @@ class Event private constructor(
          */
         @JvmStatic
         fun createEventForCodePointFromUnknownSource(codePoint: Int): Event { // TODO: should we have a different type of event for this? After all, it's not a key press.
-            return Event(EVENT_TYPE_INPUT_KEYPRESS, null /* text */, codePoint, NOT_A_KEY_CODE,
+            return Event(EVENT_TYPE_INPUT_KEYPRESS, null, codePoint, NOT_A_KEY_CODE, 0,
-                    Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE,
+                    Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE, null, FLAG_NONE, null)
-                    null /* suggestedWordInfo */, FLAG_NONE, null /* next */)
         }
 
         /**
@@ -176,8 +176,8 @@ class Event private constructor(
         @JvmStatic
         fun createEventForCodePointFromAlreadyTypedText(codePoint: Int,
                                                         x: Int, y: Int): Event { // TODO: should we have a different type of event for this? After all, it's not a key press.
-            return Event(EVENT_TYPE_INPUT_KEYPRESS, null /* text */, codePoint, NOT_A_KEY_CODE,
+            return Event(EVENT_TYPE_INPUT_KEYPRESS, null, codePoint, NOT_A_KEY_CODE, 0,
-                    x, y, null /* suggestedWordInfo */, FLAG_NONE, null /* next */)
+                    x, y, null, FLAG_NONE, null)
         }
 
         /**
@@ -187,9 +187,9 @@ class Event private constructor(
         @JvmStatic
         fun createSuggestionPickedEvent(suggestedWordInfo: SuggestedWordInfo): Event {
             return Event(EVENT_TYPE_SUGGESTION_PICKED, suggestedWordInfo.mWord,
-                    NOT_A_CODE_POINT, NOT_A_KEY_CODE,
+                    NOT_A_CODE_POINT, NOT_A_KEY_CODE, 0,
                     Constants.SUGGESTION_STRIP_COORDINATE, Constants.SUGGESTION_STRIP_COORDINATE,
-                    suggestedWordInfo, FLAG_NONE, null /* next */)
+                    suggestedWordInfo, FLAG_NONE, null)
         }
 
         /**
@@ -203,9 +203,8 @@ class Event private constructor(
          */
         @JvmStatic
         fun createSoftwareTextEvent(text: CharSequence?, keyCode: Int, next: Event?): Event {
-            return Event(EVENT_TYPE_SOFTWARE_GENERATED_STRING, text, NOT_A_CODE_POINT, keyCode,
+            return Event(EVENT_TYPE_SOFTWARE_GENERATED_STRING, text, NOT_A_CODE_POINT, keyCode, 0,
-                Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE,
+                Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE,null, FLAG_NONE, next)
-                null /* suggestedWordInfo */, FLAG_NONE, next)
         }
 
         @JvmStatic
@@ -217,13 +216,10 @@ class Event private constructor(
          * @return an event for this suggestion pick.
          */
         @JvmStatic
-        fun createPunctuationSuggestionPickedEvent(
+        fun createPunctuationSuggestionPickedEvent(suggestedWordInfo: SuggestedWordInfo): Event {
-                suggestedWordInfo: SuggestedWordInfo): Event {
             val primaryCode = suggestedWordInfo.mWord[0].code
-            return Event(EVENT_TYPE_SUGGESTION_PICKED, suggestedWordInfo.mWord, primaryCode,
+            return Event(EVENT_TYPE_SUGGESTION_PICKED, suggestedWordInfo.mWord, primaryCode, NOT_A_KEY_CODE, 0,
-                    NOT_A_KEY_CODE, Constants.SUGGESTION_STRIP_COORDINATE,
+                Constants.SUGGESTION_STRIP_COORDINATE, Constants.SUGGESTION_STRIP_COORDINATE, suggestedWordInfo, FLAG_NONE,null)
-                    Constants.SUGGESTION_STRIP_COORDINATE, suggestedWordInfo, FLAG_NONE,
-                    null /* next */)
         }
 
         /**
@@ -234,7 +230,7 @@ class Event private constructor(
          */
         @JvmStatic
         fun createCursorMovedEvent(moveAmount: Int): Event {
-            return Event(EVENT_TYPE_CURSOR_MOVE, null, NOT_A_CODE_POINT, NOT_A_KEY_CODE,
+            return Event(EVENT_TYPE_CURSOR_MOVE, null, NOT_A_CODE_POINT, NOT_A_KEY_CODE, 0,
                     moveAmount, Constants.NOT_A_COORDINATE, null, FLAG_NONE, null)
         }
 
@@ -244,21 +240,18 @@ class Event private constructor(
          * @return an identical event marked as consumed.
          */
         fun createConsumedEvent(source: Event): Event { // A consumed event should not input any text at all, so we pass the empty string as text.
-            return Event(source.mEventType, source.mText, source.mCodePoint, source.mKeyCode,
+            return Event(source.mEventType, source.mText, source.mCodePoint, source.mKeyCode, source.mMetaState,
-                    source.mX, source.mY, source.mSuggestedWordInfo, source.mFlags or FLAG_CONSUMED,
+                    source.mX, source.mY, source.mSuggestedWordInfo, source.mFlags or FLAG_CONSUMED, source.mNextEvent)
-                    source.mNextEvent)
         }
 
         fun createCombiningEvent(source: Event): Event {
-            return Event(source.mEventType, source.mText, source.mCodePoint, source.mKeyCode,
+            return Event(source.mEventType, source.mText, source.mCodePoint, source.mKeyCode, source.mMetaState,
-                    source.mX, source.mY, source.mSuggestedWordInfo, source.mFlags or FLAG_COMBINING,
+                    source.mX, source.mY, source.mSuggestedWordInfo, source.mFlags or FLAG_COMBINING, source.mNextEvent)
-                    source.mNextEvent)
         }
 
         fun createNotHandledEvent(): Event {
-            return Event(EVENT_TYPE_NOT_HANDLED, null /* text */, NOT_A_CODE_POINT, NOT_A_KEY_CODE,
+            return Event(EVENT_TYPE_NOT_HANDLED, null, NOT_A_CODE_POINT, NOT_A_KEY_CODE, 0,
-                    Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE,
+                    Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE, null, FLAG_NONE, null)
-                    null /* suggestedWordInfo */, FLAG_NONE, null)
         }
     }
 

app/src/main/java/helium314/keyboard/event/HangulCombiner.kt

@@ -25,7 +25,7 @@ class HangulCombiner : Combiner {
                 return when {
                     history.size == 1 && composingWord.isEmpty() || history.isEmpty() && composingWord.length == 1 -> {
                         reset()
-                        Event.createHardwareKeypressEvent(0x20, Constants.CODE_SPACE, event, event.isKeyRepeat)
+                        Event.createHardwareKeypressEvent(0x20, Constants.CODE_SPACE, 0, event, event.isKeyRepeat)
                     }
                     history.isNotEmpty() -> {
                         history.removeAt(history.lastIndex)

app/src/main/java/helium314/keyboard/event/HangulEventDecoder.kt

@@ -13,7 +13,7 @@ object HangulEventDecoder {
     fun decodeHardwareKeyEvent(subtype: RichInputMethodSubtype, event: KeyEvent, defaultEvent: () -> Event): Event {
         val layout = LAYOUTS[subtype.keyboardLayoutSetName] ?: return defaultEvent()
         val codePoint = layout[event.keyCode]?.let { if (event.isShiftPressed) it.second else it.first } ?: return defaultEvent()
-        val hardwareEvent = Event.createHardwareKeypressEvent(codePoint, event.keyCode, null, event.repeatCount != 0)
+        val hardwareEvent = Event.createHardwareKeypressEvent(codePoint, event.keyCode, event.metaState, null, event.repeatCount != 0)
         return decodeSoftwareKeyEvent(hardwareEvent)
     }
 

app/src/main/java/helium314/keyboard/event/HardwareKeyboardEventDecoder.kt

@@ -29,30 +29,25 @@ class HardwareKeyboardEventDecoder(val mDeviceId: Int) : HardwareEventDecoder {
         // do not necessarily map to a unicode character. This represents a physical key, like
         // the key for 'A' or Space, but also Backspace or Ctrl or Caps Lock.
         val keyCode = keyEvent.keyCode
+        val metaState = keyEvent.metaState
         val isKeyRepeat = 0 != keyEvent.repeatCount
-        if (KeyEvent.KEYCODE_DEL == keyCode) {
+        return if (KeyEvent.KEYCODE_DEL == keyCode) {
-            return Event.createHardwareKeypressEvent(Event.NOT_A_CODE_POINT, KeyCode.DELETE, null /* next */, isKeyRepeat)
+            Event.createHardwareKeypressEvent(Event.NOT_A_CODE_POINT, KeyCode.DELETE, metaState, null, isKeyRepeat)
-        }
+        } else if (keyEvent.isPrintingKey || KeyEvent.KEYCODE_SPACE == keyCode || KeyEvent.KEYCODE_ENTER == keyCode) {
-        if (keyEvent.isPrintingKey || KeyEvent.KEYCODE_SPACE == keyCode || KeyEvent.KEYCODE_ENTER == keyCode) {
             if (0 != codePointAndFlags and KeyCharacterMap.COMBINING_ACCENT) { // A dead key.
-                return Event.createDeadEvent(
+                Event.createDeadEvent(codePointAndFlags and KeyCharacterMap.COMBINING_ACCENT_MASK, keyCode, metaState, null)
-                        codePointAndFlags and KeyCharacterMap.COMBINING_ACCENT_MASK, keyCode, null /* next */)
+            } else if (KeyEvent.KEYCODE_ENTER == keyCode) {
-            }
-            return if (KeyEvent.KEYCODE_ENTER == keyCode) {
                 // The Enter key. If the Shift key is not being pressed, this should send a
                 // CODE_ENTER to trigger the action if any, or a carriage return otherwise. If the
                 // Shift key is being pressed, this should send a CODE_SHIFT_ENTER and let
                 // Latin IME decide what to do with it.
                 if (keyEvent.isShiftPressed) {
-                    Event.createHardwareKeypressEvent(Event.NOT_A_CODE_POINT,
+                    Event.createHardwareKeypressEvent(Event.NOT_A_CODE_POINT, // todo: maybe remove, see also related comment in input logic
-                            KeyCode.SHIFT_ENTER, null /* next */, isKeyRepeat)
+                            KeyCode.SHIFT_ENTER, 0, null, isKeyRepeat)
-                } else Event.createHardwareKeypressEvent(Constants.CODE_ENTER, keyCode,
+                } else Event.createHardwareKeypressEvent(Constants.CODE_ENTER, keyCode, metaState, null, isKeyRepeat)
-                        null /* next */, isKeyRepeat)
+            } else Event.createHardwareKeypressEvent(codePointAndFlags, keyCode, metaState, null, isKeyRepeat)
-            } else Event.createHardwareKeypressEvent(codePointAndFlags, keyCode, null /* next */, isKeyRepeat)
             // If not Enter, then this is just a regular keypress event for a normal character
             // that can be committed right away, taking into account the current state.
-        }
+        } else Event.createNotHandledEvent()
-        return Event.createNotHandledEvent()
     }
-
 }

app/src/main/java/helium314/keyboard/keyboard/Key.java

@@ -514,6 +514,8 @@ public class Key implements Comparable<Key> {
 
     public final boolean isModifier() {
         return mCode == KeyCode.SHIFT || mCode == KeyCode.SYMBOL_ALPHA || mCode == KeyCode.ALPHA || mCode == KeyCode.SYMBOL;
+        // todo: if this is used, sliding input starts on those keys, but it's not yet implemented
+//                || mCode == KeyCode.CTRL || mCode == KeyCode.ALT || mCode == KeyCode.FN || mCode == KeyCode.META;
     }
 
     public final boolean isRepeatable() {

app/src/main/java/helium314/keyboard/keyboard/KeyboardActionListener.java

@@ -100,6 +100,7 @@ public interface KeyboardActionListener {
 
     void onMoveDeletePointer(int steps);
     void onUpWithDeletePointerActive();
+    void resetMetaState();
 
     KeyboardActionListener EMPTY_LISTENER = new Adapter();
 
@@ -144,5 +145,7 @@ public interface KeyboardActionListener {
         public void onMoveDeletePointer(int steps) {}
         @Override
         public void onUpWithDeletePointerActive() {}
+        @Override
+        public void resetMetaState() {}
     }
 }

app/src/main/java/helium314/keyboard/keyboard/KeyboardActionListenerImpl.kt

@@ -1,5 +1,6 @@
 package helium314.keyboard.keyboard
 
+import android.view.KeyEvent
 import helium314.keyboard.keyboard.internal.keyboard_parser.floris.KeyCode
 import helium314.keyboard.latin.LatinIME
 import helium314.keyboard.latin.RichInputMethodManager
@@ -13,18 +14,34 @@ class KeyboardActionListenerImpl(private val latinIME: LatinIME, private val inp
 
     private val keyboardSwitcher = KeyboardSwitcher.getInstance()
     private val settings = Settings.getInstance()
+    private var metaState = 0 // is this enough, or are there threading issues with the different PointerTrackers?
+
+    // todo: maybe keep meta state presses to KeyboardActionListenerImpl, and avoid calls to press/release key
+    private fun adjustMetaState(code: Int, remove: Boolean) {
+        val metaCode = when (code) {
+            KeyCode.CTRL -> KeyEvent.META_CTRL_ON
+            KeyCode.ALT -> KeyEvent.META_ALT_ON
+            KeyCode.FN -> KeyEvent.META_FUNCTION_ON
+            KeyCode.META -> KeyEvent.META_META_ON
+            else -> return
+        }
+        metaState = if (remove) metaState and metaCode.inv()
+            else metaState or metaCode
+    }
 
     override fun onPressKey(primaryCode: Int, repeatCount: Int, isSinglePointer: Boolean) {
+        adjustMetaState(primaryCode, false)
         keyboardSwitcher.onPressKey(primaryCode, isSinglePointer, latinIME.currentAutoCapsState, latinIME.currentRecapitalizeState)
         latinIME.hapticAndAudioFeedback(primaryCode, repeatCount)
     }
 
     override fun onReleaseKey(primaryCode: Int, withSliding: Boolean) {
+        adjustMetaState(primaryCode, true)
         keyboardSwitcher.onReleaseKey(primaryCode, withSliding, latinIME.currentAutoCapsState, latinIME.currentRecapitalizeState)
     }
 
     override fun onCodeInput(primaryCode: Int, x: Int, y: Int, isKeyRepeat: Boolean) =
-        latinIME.onCodeInput(primaryCode, x, y, isKeyRepeat)
+        latinIME.onCodeInput(primaryCode, metaState, x, y, isKeyRepeat)
 
     override fun onTextInput(text: String?) = latinIME.onTextInput(text)
 
@@ -74,6 +91,10 @@ class KeyboardActionListenerImpl(private val latinIME: LatinIME, private val inp
         onCodeInput(KeyCode.DELETE, Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE, false)
     }
 
+    override fun resetMetaState() {
+        metaState = 0
+    }
+
     private fun onLanguageSlide(steps: Int): Boolean {
         if (abs(steps) < 4) return false
         val subtypes = RichInputMethodManager.getInstance().getMyEnabledInputMethodSubtypeList(false)

app/src/main/java/helium314/keyboard/keyboard/PointerTracker.java

@@ -737,6 +737,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element,
         }
         if (!sInGesture && key != null && Character.isLetter(key.getCode())
                 && mBatchInputArbiter.mayStartBatchInput(this)) {
+            sListener.resetMetaState(); // avoid metaState getting stuck, doesn't work with gesture typing anyway
             sInGesture = true;
         }
         if (sInGesture) {
@@ -1117,6 +1118,10 @@ public final class PointerTracker implements PointerTrackerQueue.Element,
         if (popupKeysPanel == null) {
             return;
         }
+        if (code == KeyCode.CTRL || code == KeyCode.ALT || code == KeyCode.FN || code == KeyCode.META) {
+            // avoid metaState getting stuck
+            sListener.onReleaseKey(code, false);
+        }
         final int translatedX = popupKeysPanel.translateX(mLastX);
         final int translatedY = popupKeysPanel.translateY(mLastY);
         popupKeysPanel.onDownEvent(translatedX, translatedY, mPointerId, SystemClock.uptimeMillis());

app/src/main/java/helium314/keyboard/keyboard/internal/keyboard_parser/KeyboardParser.kt

@@ -298,7 +298,6 @@ class KeyboardParser(private val params: KeyboardParams, private val context: Co
             // this is ugly...
             if (label.length > 8 && label.startsWith("!string/")) {
                 val id = context.resources.getIdentifier(label.substringAfter("!string/"), "string", context.packageName)
-                Log.i("test", "id of $label: $id")
                 if (id != 0) copy(newLabel = getInLocale(id))
                 else this
             } else this

app/src/main/java/helium314/keyboard/keyboard/internal/keyboard_parser/floris/KeyCode.kt

@@ -5,6 +5,8 @@
  */
 package helium314.keyboard.keyboard.internal.keyboard_parser.floris
 
+import android.view.KeyEvent
+
 // taken from FlorisBoard and modified
 object KeyCode {
     object Spec {
@@ -134,6 +136,8 @@ object KeyCode {
     const val CLIPBOARD_COPY_ALL =        -10009
     const val PAGE_UP =                   -10010
     const val PAGE_DOWN =                 -10011
+    const val META =                      -10012
+    const val META_LOCK =                 -10013 // to be consistent with the CTRL/ALT(/FN LOCK codes, not sure whether this will be used
 
     /** to make sure a FlorisBoard code works when reading a JSON layout */
     fun Int.checkAndConvertCode(): Int = if (this > 0) this else when (this) {
@@ -142,7 +146,7 @@ object KeyCode {
         VOICE_INPUT, LANGUAGE_SWITCH, SETTINGS, DELETE, ALPHA, SYMBOL, EMOJI, CLIPBOARD,
         UNDO, REDO, ARROW_DOWN, ARROW_UP, ARROW_RIGHT, ARROW_LEFT, CLIPBOARD_COPY, CLIPBOARD_SELECT_ALL,
         CLIPBOARD_SELECT_WORD, TOGGLE_INCOGNITO_MODE, TOGGLE_AUTOCORRECT, MOVE_START_OF_LINE, MOVE_END_OF_LINE,
-        SHIFT, CAPS_LOCK, MULTIPLE_CODE_POINTS, UNSPECIFIED,
+        SHIFT, CAPS_LOCK, MULTIPLE_CODE_POINTS, UNSPECIFIED, CTRL, ALT, FN, META,
 
         // heliboard only
         SYMBOL_ALPHA, START_ONE_HANDED_MODE, STOP_ONE_HANDED_MODE, SWITCH_ONE_HANDED_MODE, SHIFT_ENTER,
@@ -156,4 +160,47 @@ object KeyCode {
 
         else -> throw IllegalStateException("key code $this not yet supported")
     }
+
+    // todo: add more keys, see near https://developer.android.com/reference/android/view/KeyEvent#KEYCODE_0
+    // maybe not toChar for conversion of some special keys?
+    /** convert a codePoint to a KeyEvent.KEYCODE_<xxx>, fallback to KeyEvent.KEYCODE_UNKNOWN */
+    fun Int.toKeyEventCode(): Int = when (this.toChar().uppercaseChar()) {
+        '0' -> KeyEvent.KEYCODE_0
+        '1' -> KeyEvent.KEYCODE_1
+        '2' -> KeyEvent.KEYCODE_2
+        '3' -> KeyEvent.KEYCODE_3
+        '4' -> KeyEvent.KEYCODE_4
+        '5' -> KeyEvent.KEYCODE_5
+        '6' -> KeyEvent.KEYCODE_6
+        '7' -> KeyEvent.KEYCODE_7
+        '8' -> KeyEvent.KEYCODE_8
+        '9' -> KeyEvent.KEYCODE_9
+        'A' -> KeyEvent.KEYCODE_A
+        'B' -> KeyEvent.KEYCODE_B
+        'C' -> KeyEvent.KEYCODE_C
+        'D' -> KeyEvent.KEYCODE_D
+        'E' -> KeyEvent.KEYCODE_E
+        'F' -> KeyEvent.KEYCODE_F
+        'G' -> KeyEvent.KEYCODE_G
+        'H' -> KeyEvent.KEYCODE_H
+        'I' -> KeyEvent.KEYCODE_I
+        'J' -> KeyEvent.KEYCODE_J
+        'K' -> KeyEvent.KEYCODE_K
+        'L' -> KeyEvent.KEYCODE_L
+        'M' -> KeyEvent.KEYCODE_M
+        'N' -> KeyEvent.KEYCODE_N
+        'O' -> KeyEvent.KEYCODE_O
+        'P' -> KeyEvent.KEYCODE_P
+        'Q' -> KeyEvent.KEYCODE_Q
+        'R' -> KeyEvent.KEYCODE_R
+        'S' -> KeyEvent.KEYCODE_S
+        'T' -> KeyEvent.KEYCODE_T
+        'U' -> KeyEvent.KEYCODE_U
+        'V' -> KeyEvent.KEYCODE_V
+        'W' -> KeyEvent.KEYCODE_W
+        'X' -> KeyEvent.KEYCODE_X
+        'Y' -> KeyEvent.KEYCODE_Y
+        'Z' -> KeyEvent.KEYCODE_Z
+        else -> KeyEvent.KEYCODE_UNKNOWN
+    }
 }

app/src/main/java/helium314/keyboard/latin/LatinIME.java

@@ -1075,6 +1075,7 @@ public class LatinIME extends InputMethodService implements
         mHandler.cancelUpdateSuggestionStrip();
         // Should do the following in onFinishInputInternal but until JB MR2 it's not called :(
         mInputLogic.finishInput();
+        mKeyboardActionListener.resetMetaState();
     }
 
     protected void deallocateMemory() {
@@ -1453,9 +1454,13 @@ public class LatinIME extends InputMethodService implements
         }
     }
 
-    // Implementation of {@link KeyboardActionListener}.
+    // Implementation of {@link SuggestionStripView.Listener}.
     @Override
     public void onCodeInput(final int codePoint, final int x, final int y, final boolean isKeyRepeat) {
+        onCodeInput(codePoint, 0, x, y, isKeyRepeat);
+    }
+
+    public void onCodeInput(final int codePoint, final int metaState, final int x, final int y, final boolean isKeyRepeat) {
         if (codePoint < 0) {
             switch (codePoint) {
                 case KeyCode.TOGGLE_AUTOCORRECT -> {mSettings.toggleAutoCorrect(); return; }
@@ -1471,7 +1476,7 @@ public class LatinIME extends InputMethodService implements
         // this transformation, it should be done already before calling onEvent.
         final int keyX = mainKeyboardView.getKeyX(x);
         final int keyY = mainKeyboardView.getKeyY(y);
-        final Event event = createSoftwareKeypressEvent(codePoint, keyX, keyY, isKeyRepeat);
+        final Event event = createSoftwareKeypressEvent(codePoint, metaState, keyX, keyY, isKeyRepeat);
         onEvent(event);
     }
 
@@ -1493,8 +1498,8 @@ public class LatinIME extends InputMethodService implements
     // squashed into the same variable, and this method should be removed.
     // public for testing, as we don't want to copy the same logic into test code
     @NonNull
-    public static Event createSoftwareKeypressEvent(final int keyCodeOrCodePoint, final int keyX,
+    public static Event createSoftwareKeypressEvent(final int keyCodeOrCodePoint, final int metaState,
-                                                    final int keyY, final boolean isKeyRepeat) {
+                                                    final int keyX, final int keyY, final boolean isKeyRepeat) {
         final int keyCode;
         final int codePoint;
         if (keyCodeOrCodePoint <= 0) {
@@ -1504,7 +1509,7 @@ public class LatinIME extends InputMethodService implements
             keyCode = Event.NOT_A_KEY_CODE;
             codePoint = keyCodeOrCodePoint;
         }
-        return Event.createSoftwareKeypressEvent(codePoint, keyCode, keyX, keyY, isKeyRepeat);
+        return Event.createSoftwareKeypressEvent(codePoint, keyCode, metaState, keyX, keyY, isKeyRepeat);
     }
 
     public void onTextInput(final String rawText) {

app/src/main/java/helium314/keyboard/latin/inputlogic/InputLogic.java

@@ -704,8 +704,9 @@ public final class InputLogic {
                 handleClipboardPaste();
                 break;
             case KeyCode.SHIFT_ENTER:
+                // todo: try using sendDownUpKeyEventWithMetaState() and remove the key code maybe
                 final Event tmpEvent = Event.createSoftwareKeypressEvent(Constants.CODE_ENTER,
-                        event.getMKeyCode(), event.getMX(), event.getMY(), event.isKeyRepeat());
+                        event.getMKeyCode(), 0, event.getMX(), event.getMY(), event.isKeyRepeat());
                 handleNonSpecialCharacterEvent(tmpEvent, inputTransaction, handler);
                 // Shift + Enter is treated as a functional key but it results in adding a new
                 // line, so that does affect the contents of the editor.
@@ -733,7 +734,7 @@ public final class InputLogic {
                 if (mConnection.hasSelection()) {
                     mConnection.copyText(true);
                     // fake delete keypress to remove the text
-                    final Event backspaceEvent = LatinIME.createSoftwareKeypressEvent(KeyCode.DELETE,
+                    final Event backspaceEvent = LatinIME.createSoftwareKeypressEvent(KeyCode.DELETE, 0,
                             event.getMX(), event.getMY(), event.isKeyRepeat());
                     handleBackspaceEvent(backspaceEvent, inputTransaction, currentKeyboardScript);
                     inputTransaction.setDidAffectContents();
@@ -783,9 +784,18 @@ public final class InputLogic {
             case KeyCode.START_ONE_HANDED_MODE:
             case KeyCode.STOP_ONE_HANDED_MODE:
             case KeyCode.SWITCH_ONE_HANDED_MODE:
+            case KeyCode.CTRL:
+            case KeyCode.ALT:
+            case KeyCode.FN:
+            case KeyCode.META:
                 break;
             default:
-                throw new RuntimeException("Unknown key code : " + event.getMKeyCode());
+                if (event.getMMetaState() != 0) {
+                    // need to convert codepoint to KeyEvent.KEYCODE_<xxx>
+                    int keyEventCode = KeyCode.INSTANCE.toKeyEventCode(event.getMCodePoint());
+                    sendDownUpKeyEventWithMetaState(keyEventCode, event.getMMetaState());
+                } else
+                    throw new RuntimeException("Unknown key code : " + event.getMKeyCode());
         }
     }
 

app/src/test/java/helium314/keyboard/latin/InputLogicTest.kt

@@ -591,7 +591,7 @@ class InputLogicTest {
 
     private fun functionalKeyPress(keyCode: Int) {
         require(keyCode < 0) { "not a functional key code: $keyCode" }
-        latinIME.onEvent(Event.createSoftwareKeypressEvent(Event.NOT_A_CODE_POINT, keyCode, Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE, false))
+        latinIME.onEvent(Event.createSoftwareKeypressEvent(Event.NOT_A_CODE_POINT, keyCode, 0, Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE, false))
         handleMessages()
         checkConnectionConsistency()
     }

layouts.md

@@ -22,10 +22,10 @@ If the layout has exactly 2 keys in the bottom row, these keys will replace comm
 * Allows more flexibility than the simple format, e.g. changing keys depending on input type, shift state or layout direction
 * You can use character layouts from [FlorisBoard](https://github.com/florisboard/florisboard/blob/master/CONTRIBUTING.md#adding-the-layout)
   * Support is not 100% there yet, notably `kana_selector` and `char_width_selector` do not work.
+* Lines _starting_ with `//` are ignored.
 * There is no need for specifying a `code`, it will be determined from the label automatically
   * You can still specify it, but it's only necessary if you want key label and code to be different (please avoid contributing layout with unnecessary codes to HeliBoard)
   * Note that not all _special codes_ (negative numbers) from FlorisBoard are supported
-* More details on the formal will be provided. For now you can check other layouts, often you just need to copy lines and change the labels.
 * Key classes: specified with `$`, usually you can omit them in HeliBoard 
   * `text_key`: normal key, default
   * `auto_text_key`: used in FlorisBoard for a key that changes text case when shift is enabled, HeliBoard does that anyway unless disabled with a _labelFlag_
@@ -48,6 +48,9 @@ If the layout has exactly 2 keys in the bottom row, these keys will replace comm
   * There are some more values, but they do nothing
 * `code`: code point that is entered when the key is pressed, determined from the label by default, not available for `multi_text_key`
   * There are special negative values available, e.g. the ones used by functional keys, see [KeyCode.kt](/app/src/main/java/helium314/keyboard/keyboard/internal/keyboard_parser/floris/KeyCode.kt). There are several not yet supported key codes in there, you can see in the function `checkAndConvertCode` which ones are working.
+  * Special notes for the modifier keys `CTRL`, `ALT`, `FN`, `META`
+    * Currently there is no special lock-treatment, so you need to hold the key and press another key at the same time (like on a hardware keyboard)
+    * this means you should avoid putting popups on modifier keys (or press the other key quickly)
 * `codePoints`: when multiple code points should be entered, only available for `multi_text_key`
 * `label`: text to display on the key, determined from code if empty
   * There are some special values, see the [label section](#labels)