Added tools:make-emoji-tool module

settings.gradle

@@ -1,3 +1,3 @@
 include ':app'
-include ':tools'
 include ':tools:make-keyboard-text'
+include ':tools:make-emoji-keys'

tools/make-emoji-keys/build.gradle

@@ -0,0 +1,40 @@
+apply plugin: 'java'
+apply plugin: 'kotlin'
+
+version 'unspecified'
+
+jar {
+    manifest {
+        attributes["Main-Class"] = 'com/majeur/inputmethod/tools/emoji/MakeEmojiKeys'
+    }
+    from {
+        configurations.runtimeClasspath.collect {
+            it.isDirectory() ? it : zipTree(it)
+        }
+    }
+    from("src/main/ressources")
+
+}
+
+task makeEmoji(type: JavaExec, dependsOn: ['jar']) {
+    main = '-jar'
+    args jar.archiveFile.get()
+    args '-res'
+    args project.rootProject.project('app').projectDir.path + File.separator + 'src' +
+            File.separator + 'main' + File.separator + 'res'
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
+}
+
+
+
+java {
+    sourceCompatibility = JavaVersion.VERSION_1_8
+    targetCompatibility = JavaVersion.VERSION_1_8
+}

tools/make-emoji-keys/src/main/kotlin/com/majeur/inputmethod/tools/emoji/AndroidEmojiSupportFileParser.kt

@@ -0,0 +1,43 @@
+package com.majeur.inputmethod.tools.emoji
+
+
+class AndroidEmojiSupportFileParser : TextFileParser<Map<Int, Int>>() {
+
+    private val map = mutableMapOf<Int, Int>()
+    private var currentApiLevel = 0
+
+    override fun getParseResult() = map
+
+    override fun parseLine(content: String) {
+        ifStartsWith(content,
+            API_LEVEL_MARK to ::parseApiLevel,
+            UNICODE_MARK to ::parseCodePoints)
+    }
+
+    private fun parseApiLevel(content: String) {
+        currentApiLevel = content
+                .substringBefore("#")
+                .trim()
+                .toInt()
+    }
+
+    private fun parseCodePoints(content: String) {
+        val codePointsHash = content
+                .substringBefore("#")
+                .trim()
+                .split(" ")
+                .map { it
+                        .trim()
+                        .removePrefix("U+")
+                        .toInt(radix = 16) }
+                .joinToString(separator = "")
+                .hashCode()
+        map[codePointsHash] = currentApiLevel
+    }
+
+    companion object {
+
+        private const val API_LEVEL_MARK = "@"
+        private const val UNICODE_MARK = "U"
+    }
+}

tools/make-emoji-keys/src/main/kotlin/com/majeur/inputmethod/tools/emoji/EmojiCategoriesResource.kt

@@ -0,0 +1,153 @@
+package com.majeur.inputmethod.tools.emoji
+
+import com.majeur.inputmethod.tools.emoji.model.EmojiData
+import com.majeur.inputmethod.tools.emoji.model.EmojiGroup
+import java.io.*
+import java.nio.charset.Charset
+import java.util.jar.JarFile
+
+class EmojiCategoriesResource(private val jarFile: JarFile) {
+
+    fun writeToAndroidRes(outDir: String?, emojiData: EmojiData, supportData: Map<Int, Int>) {
+        val template = JarUtils.getAndroidResTemplateResource(jarFile)
+        val resourceDir = template.substring(0, template.lastIndexOf('/'))
+        var ps: PrintStream? = null
+        var lnr: LineNumberReader? = null
+        try {
+            ps = if (outDir == null) {
+                System.out
+            } else {
+                val outDir = File(outDir, resourceDir)
+                val outputFile = File(outDir,
+                        ANDROID_RES_TEMPLATE.replace(".tmpl", ".xml"))
+                outDir.mkdirs()
+                println("Building android resource file into ${outputFile.absoluteFile}")
+                PrintStream(outputFile, Charset.forName("UTF-8"))
+            }
+            lnr = LineNumberReader(InputStreamReader(JarUtils.openResource(template), Charset.forName("UTF-8")))
+            inflateTemplate(lnr, ps!!, emojiData, supportData)
+        } catch (e: IOException) {
+            throw RuntimeException(e)
+        } finally {
+            JarUtils.close(lnr)
+            JarUtils.close(ps)
+        }
+    }
+
+    @Throws(IOException::class)
+    private fun inflateTemplate(reader: LineNumberReader, out: PrintStream,
+                                emojis: EmojiData, supportData: Map<Int, Int>) {
+        reader.lines().forEach {
+            when {
+                it.contains(MARK_UNICODE_VER) ->
+                    out.println(it.replace(MARK_UNICODE_VER, emojis.unicodeVersion))
+                it.contains(MARK_API_LEVEL) ->
+                    out.println(it.replace(MARK_API_LEVEL, supportData.values.maxOrNull().toString()))
+                it.contains(MARK_SMILEYS_AND_EMOTION) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.SMILEYS_AND_EMOTION)
+                it.contains(MARK_PEOPLE_AND_BODY) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.PEOPLE_AND_BODY)
+                it.contains(MARK_ANIMALS_AND_NATURE) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.ANIMALS_AND_NATURE)
+                it.contains(MARK_FOOD_AND_DRINK) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.FOOD_AND_DRINK)
+                it.contains(MARK_TRAVEL_AND_PLACES) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.TRAVEL_AND_PLACES)
+                it.contains(MARK_ACTIVITIES) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.ACTIVITIES)
+                it.contains(MARK_OBJECTS) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.OBJECTS)
+                it.contains(MARK_SYMBOLS) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.SYMBOLS)
+                it.contains(MARK_FLAGS) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.FLAGS)
+                it.contains(MARK_PEOPLE_AND_BODY_MORE) ->
+                    dumpEmojiSpecsVariant(out, emojis, supportData,EmojiGroup.PEOPLE_AND_BODY)
+                else -> out.println(it)
+            }
+        }
+    }
+
+    private fun dumpEmojiSpecs(out: PrintStream, emojiData: EmojiData, supportData: Map<Int, Int>,
+                               group: EmojiGroup) {
+        emojiData[group].forEach { emoji ->
+            val minApi = getMinApi(emoji.codes, supportData)
+            if (minApi < 0) {
+                // We have no clue of which android version supports this emoji,
+                // so we ignore it.
+                printCompatNotFound(emoji.codes)
+                return@forEach
+            }
+            val text = makeEmojiKey(emoji.codes, minApi)
+            out.println("        <item>$text</item>")
+        }
+    }
+
+    private fun dumpEmojiSpecsVariant(out: PrintStream, emojiData: EmojiData, supportData: Map<Int, Int>,
+                               group: EmojiGroup) {
+        emojiData[group].forEach { baseEmoji ->
+            val minApi = getMinApi(baseEmoji.codes, supportData)
+            if (minApi < 0) {
+                // Same thing, we already encountered it when dumping base emoji,
+                // ignoring this one silently.
+                return@forEach
+            }
+
+            val text = baseEmoji.variants.filter { emoji ->
+                if (getMinApi(emoji.codes, supportData) < 0) {
+                    // Again
+                    printCompatNotFound(emoji.codes)
+                    return@filter false
+                }
+                true
+            }.map { emoji ->
+                // Not very efficient, minApi is accessed twice,
+                // but hey, we are making tooling here
+                makeEmojiKey(emoji.codes, getMinApi(emoji.codes, supportData))
+            }.filter { key ->
+                key.isNotBlank()
+            }.joinToString(separator = ";")
+
+            if (text.isNotBlank()) out.println("        <item>$text</item>")
+            else out.println("        <item/>")
+        }
+    }
+
+    private fun makeEmojiKey(codes: IntArray, minApi: Int): String {
+        val cps = codes
+                .joinToString(separator = ",") {
+                    it.toString(radix = 16)
+                            .uppercase()
+                }
+        return if (minApi > 19) "$cps||$minApi" else cps
+    }
+
+    private fun getMinApi(codes: IntArray, supportData: Map<Int, Int>): Int {
+        val hash = codes
+                .joinToString(separator = "")
+                .hashCode()
+        return supportData[hash] ?: -1
+    }
+
+    private fun printCompatNotFound(codes: IntArray) {
+        val formattedCps = codes.joinToString(" ") { "U+" + it.toString(radix = 16).uppercase() }
+        println(" - No android compatibility found for emoji $formattedCps, ignoring...")
+    }
+
+    companion object {
+        private const val ANDROID_RES_TEMPLATE = "emoji-categories.tmpl"
+        private const val MARK_UNICODE_VER = "@UNICODE_VERSION@"
+        private const val MARK_API_LEVEL = "@ANDROID_API_LEVEL@"
+        private const val MARK_SMILEYS_AND_EMOTION = "@SMILEYS_AND_EMOTION@"
+        private const val MARK_PEOPLE_AND_BODY = "@PEOPLE_AND_BODY@"
+        private const val MARK_PEOPLE_AND_BODY_MORE = "@PEOPLE_AND_BODY MORE@"
+        private const val MARK_ANIMALS_AND_NATURE = "@ANIMALS_AND_NATURE@"
+        private const val MARK_FOOD_AND_DRINK = "@FOOD_AND_DRINKS@"
+        private const val MARK_TRAVEL_AND_PLACES = "@TRAVEL_AND_PLACES@"
+        private const val MARK_ACTIVITIES = "@ACTIVITIES@"
+        private const val MARK_OBJECTS = "@OBJECTS@"
+        private const val MARK_SYMBOLS = "@SYMBOLS@"
+        private const val MARK_FLAGS = "@FLAGS@"
+    }
+
+}

tools/make-emoji-keys/src/main/kotlin/com/majeur/inputmethod/tools/emoji/EmojiUCDTestFileParser.kt

@@ -0,0 +1,88 @@
+package com.majeur.inputmethod.tools.emoji
+
+import com.majeur.inputmethod.tools.emoji.model.EmojiData
+import com.majeur.inputmethod.tools.emoji.model.EmojiGroup
+
+class EmojiUCDTestFileParser: TextFileParser<EmojiData>() {
+
+    private var count = 0
+    private var emojiData = EmojiData()
+
+    private var currentGroup = EmojiGroup.SMILEYS_AND_EMOTION
+
+    override fun getParseResult() = emojiData
+
+    override fun parseLine(content: String) {
+        ifStartsWith(content,
+            "#" to ::parseComment,
+            "" to ::parseEmojiSpec
+        )
+    }
+
+    private fun parseComment(content: String) {
+        ifStartsWith(content,
+            PROP_DATE to { emojiData.dataDate = it},
+            PROP_UNICODE_VER to {
+                emojiData.unicodeVersion = it
+                println("Parsing emoji table from Unicode $it")
+            },
+            PROP_GROUP to ::parseGroup,
+            PROP_SUBGROUP to { },
+            "${currentGroup.rawName} subtotal:" to ::parseGroupSubtotal,
+            EOF to { println("Parsed a total of $count emojis") }
+        )
+    }
+
+    private fun parseGroup(content: String) {
+        currentGroup = EmojiGroup.get(content)
+    }
+
+    private fun parseGroupSubtotal(content: String) {
+        if (content.contains("w/o modifiers")) return
+        val expected = content.toInt()
+        val count = emojiData.emojiGroupCount(currentGroup)
+        println(" - $count/$expected emojis for group ${currentGroup.rawName}")
+    }
+
+    private fun parseEmojiSpec(content: String) {
+        if (content.isEmpty()) return
+
+        val codePoints = content
+                .substringBefore(';')
+                .trim()
+        val status = content
+                .substringAfter(';')
+                .substringBefore('#')
+                .trim()
+        val extras = content.substringAfter('#')
+
+        if (status != "fully-qualified") return
+
+        val rawVersion = EMOJI_VERSION_REGEX.find(extras)?.value ?: "O.0"
+        val version = rawVersion.toFloat()
+        val name = extras
+                .substringAfter(rawVersion)
+                .trim()
+
+        val cps = codePoints
+                .split(" ")
+                .map { it.toInt(radix = 16) }
+                .toIntArray()
+
+        emojiData.insertEmoji(currentGroup, cps, version, name)
+        count++
+    }
+
+    companion object {
+
+        private const val PROP_UNICODE_VER = "Version:"
+        private const val PROP_DATE = "Date:"
+        private const val PROP_GROUP = "group:"
+        private const val PROP_SUBGROUP = "subgroup:"
+        private const val EOF = "EOF"
+
+        private val EMOJI_VERSION_REGEX = "[0-9]*[.]?[0-9]+".toRegex()
+    }
+
+
+}

tools/make-emoji-keys/src/main/kotlin/com/majeur/inputmethod/tools/emoji/JarUtils.kt

@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.majeur.inputmethod.tools.emoji
+
+import java.io.Closeable
+import java.io.IOException
+import java.io.InputStream
+import java.io.UnsupportedEncodingException
+import java.net.URLDecoder
+import java.util.HashMap
+import java.util.jar.JarFile
+import kotlin.RuntimeException
+
+object JarUtils {
+
+    fun getJarFile(mainClass: Class<*>): JarFile {
+        val mainClassPath = "/${mainClass.name.replace('.', '/')}.class"
+        val resUrl = mainClass.getResource(mainClassPath)
+        if (resUrl?.protocol != "jar") {
+            throw RuntimeException("Should run as jar and not as " + resUrl?.protocol)
+        }
+        val path = resUrl.path
+        if (!path.startsWith("file:")) {
+            throw RuntimeException("Unknown jar path: $path")
+        }
+        val jarPath = path.substring("file:".length, path.indexOf('!'))
+        try {
+            return JarFile(URLDecoder.decode(jarPath, "UTF-8"))
+        } catch (e: UnsupportedEncodingException) {
+            throw RuntimeException(e)
+        } catch (e: IOException) {
+            throw RuntimeException(e)
+        }
+    }
+
+    fun openResource(name: String): InputStream {
+        return javaClass.getResourceAsStream("/$name")
+    }
+
+    fun getLatestEmojiTestResource(jar: JarFile) : String {
+        var latestUnicodeVersion = 0.0
+        var name = ""
+        jar.entries().iterator().forEach {
+            if (it.name.endsWith("emoji-test.txt")) {
+                val ver = it.name
+                        .removeSuffix("/emoji-test.txt")
+                        .substringAfterLast("/")
+                        .toDouble()
+                if (ver > latestUnicodeVersion) {
+                    latestUnicodeVersion = ver
+                    name = it.name
+                }
+            }
+        }
+        if (name.isEmpty())
+            throw RuntimeException("No emoji specs provided in resources")
+        return name
+    }
+
+    fun getAndroidResTemplateResource(jar: JarFile) : String {
+        jar.entries().iterator().forEach {
+            if (it.name.endsWith("emoji-categories.tmpl")) {
+                return it.name
+            }
+        }
+        throw RuntimeException("No template provided in resources")
+    }
+
+    fun close(stream: Closeable?) {
+        try {
+            stream?.close()
+        } catch (e: IOException) {
+        }
+    }
+
+    fun getEmojiSupportResource(jar: JarFile): String {
+        jar.entries().iterator().forEach {
+            if (it.name.endsWith("android-emoji-support.txt")) {
+                return it.name
+            }
+        }
+        throw RuntimeException("No emoji support file provided in resources")
+    }
+}

tools/make-emoji-keys/src/main/kotlin/com/majeur/inputmethod/tools/emoji/MakeEmojiKeys.kt

@@ -0,0 +1,56 @@
+package com.majeur.inputmethod.tools.emoji
+
+import java.util.*
+import kotlin.system.exitProcess
+
+class MakeEmojiKeys {
+
+    class Options(argsArray: Array<String>) {
+
+        private val OPTION_RES = "-res"
+
+        var resPath: String? = null
+
+        init {
+            val args = listOf(*argsArray).toMutableList()
+            var arg: String? = null
+            try {
+                while (args.isNotEmpty()) {
+                    arg = args.removeFirst()
+                    if (arg == OPTION_RES) {
+                        resPath = args.removeFirst()
+                    } else {
+                        usage("Unknown option: $arg")
+                    }
+                }
+            } catch (e: NoSuchElementException) {
+                usage("Option $arg needs argument")
+            }
+        }
+
+        fun usage(message: String?) {
+            message?.let { System.err.println(it) }
+            System.err.println("usage: make-emoji-keys $OPTION_RES <res_output_dir>")
+            exitProcess(1)
+        }
+    }
+
+    companion object {
+
+        @JvmStatic fun main(args: Array<String>) {
+            val options = Options(args)
+            val jar = JarUtils.getJarFile(Companion::class.java)
+
+            val parser = EmojiUCDTestFileParser()
+            parser.parse(JarUtils.getLatestEmojiTestResource(jar))
+            val emojis = parser.getParsedData()
+
+            val parser2 = AndroidEmojiSupportFileParser()
+            parser2.parse(JarUtils.getEmojiSupportResource(jar))
+            val supportData = parser2.getParsedData()
+
+            EmojiCategoriesResource(jar).writeToAndroidRes(options.resPath, emojis, supportData)
+        }
+
+    }
+}

tools/make-emoji-keys/src/main/kotlin/com/majeur/inputmethod/tools/emoji/TextFileParser.kt

@@ -0,0 +1,45 @@
+package com.majeur.inputmethod.tools.emoji
+
+import java.io.InputStreamReader
+import java.io.LineNumberReader
+import java.lang.IllegalStateException
+
+abstract class TextFileParser<T> {
+
+    private var parsed = false
+
+    fun parse(resource: String) {
+        if (parsed) throw IllegalStateException("parse() has already been called")
+        LineNumberReader(InputStreamReader(JarUtils.openResource(resource))).use { reader ->
+            reader.lines().forEach { content ->
+                parseLine(content)
+            }
+        }
+        parsed = true
+    }
+
+    fun getParsedData(): T {
+        if (!parsed) throw IllegalStateException("parse() must be called before calling getParsedData()")
+        return getParseResult()
+    }
+
+    protected fun ifStartsWith(content: String, vararg pairs: Pair<String, (String) -> Unit>) : Boolean {
+        pairs.forEach { pair ->
+            if (ifStartsWith(content, pair.first, pair.second)) return true
+        }
+        return false
+    }
+
+    protected fun ifStartsWith(content: String, prefix: String, call: (String) -> Unit) : Boolean {
+        if (content.startsWith(prefix)) {
+            call.invoke(content.removePrefix(prefix).trim())
+            return true
+        }
+        return false
+    }
+
+    protected abstract fun getParseResult(): T
+
+    protected abstract fun parseLine(content: String)
+
+}

tools/make-emoji-keys/src/main/kotlin/com/majeur/inputmethod/tools/emoji/model/EmojiData.kt

@@ -0,0 +1,170 @@
+package com.majeur.inputmethod.tools.emoji.model
+
+class EmojiData {
+
+    var unicodeVersion = ""
+    var dataDate = ""
+
+    private var emojiGroups = mutableMapOf<EmojiGroup, MutableList<EmojiSpec>>()
+
+    operator fun get(group: EmojiGroup) = emojiGroups.getValue(group)
+
+    fun emojiCount(group: EmojiGroup): Int {
+        var acc = 0
+        emojiGroups.values.forEach { acc += it.size }
+        return acc
+    }
+
+    fun emojiGroupCount(group: EmojiGroup) = emojiGroups[group]?.size ?: 0
+
+    fun insertEmoji(group: EmojiGroup, codes: IntArray, unicodeVer: Float, name: String): EmojiSpec {
+        return EmojiSpec(codes, unicodeVer, name).also { emoji ->
+            val baseEmoji = findBaseEmoji(group, emoji)
+            if (baseEmoji != null && onEmojiVariantInserted(group, baseEmoji, emoji)) {
+                baseEmoji.variants.add(emoji)
+            } else if (onEmojiInserted(group, emoji)) {
+                emojiGroups.getOrPut(group) { mutableListOf() }.add(emoji)
+            }
+        }
+    }
+
+    private fun onEmojiInserted(group: EmojiGroup, emoji: EmojiSpec): Boolean {
+        // Unicode RGI does not include letter symbols but Android supports them, so we inject them manually.
+        if (emoji.codes contentEquals RAW_CPS_KEYCAP_HASH) {
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_A), 2.0f, "regional indicator symbol letter a")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_B), 2.0f, "regional indicator symbol letter b")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_C), 2.0f, "regional indicator symbol letter c")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_D), 2.0f, "regional indicator symbol letter d")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_E), 2.0f, "regional indicator symbol letter e")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_F), 2.0f, "regional indicator symbol letter f")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_G), 2.0f, "regional indicator symbol letter g")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_H), 2.0f, "regional indicator symbol letter h")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_I), 2.0f, "regional indicator symbol letter i")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_J), 2.0f, "regional indicator symbol letter j")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_K), 2.0f, "regional indicator symbol letter k")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_L), 2.0f, "regional indicator symbol letter l")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_M), 2.0f, "regional indicator symbol letter m")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_N), 2.0f, "regional indicator symbol letter n")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_O), 2.0f, "regional indicator symbol letter o")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_P), 2.0f, "regional indicator symbol letter p")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_Q), 2.0f, "regional indicator symbol letter q")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_R), 2.0f, "regional indicator symbol letter r")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_S), 2.0f, "regional indicator symbol letter s")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_T), 2.0f, "regional indicator symbol letter t")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_U), 2.0f, "regional indicator symbol letter u")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_V), 2.0f, "regional indicator symbol letter v")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_W), 2.0f, "regional indicator symbol letter w")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_X), 2.0f, "regional indicator symbol letter x")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_Y), 2.0f, "regional indicator symbol letter y")
+            insertEmoji(group, intArrayOf(CP_REGIONAL_INDICATOR_SYMBOL_LETTER_Z), 2.0f, "regional indicator symbol letter z")
+        }
+
+        if (hasMultipleSkinModifiers(emoji.codes)) {
+            // For now Openboard implementation is not robust enough to handle such complicated sequences.
+            // Emoji palettes get polluted with too much emoji variations, so we'll ignore them.
+            return false
+        }
+
+        return true
+    }
+
+    private fun hasMultipleSkinModifiers(codes: IntArray): Boolean {
+        var count = 0
+        codes.forEach {
+            when (it) {
+                CP_LIGHT_SKIN_TONE, CP_MEDIUM_LIGHT_SKIN_TONE, CP_MEDIUM_SKIN_TONE,
+                CP_MEDIUM_DARK_SKIN_TONE, CP_DARK_SKIN_TONE ->
+                    count += 1
+            }
+        }
+        return count > 1
+    }
+
+    private fun onEmojiVariantInserted(group: EmojiGroup, baseSpec: EmojiSpec, emojiSpec: EmojiSpec): Boolean {
+        return true
+    }
+
+    private fun findBaseEmoji(group: EmojiGroup, emoji: EmojiSpec): EmojiSpec? {
+        val (baseCodePoints, componentCode) = withoutComponentCodes(emoji.codes)
+
+        // No component codes found, this emoji is a standalone one
+        if (componentCode == CP_NUL) return null
+
+        // Second try for emojis with U+FE0F suffix
+        val baseCodePoints2 = baseCodePoints + CP_VARIANT_SELECTOR
+
+        // Third try for emojis with U+FE0F prefix before an eventual ZWJ
+        val baseCodePoints3 = emoji.codes.toMutableList()
+                .apply { set(emoji.codes.indexOf(componentCode), CP_VARIANT_SELECTOR) }.toIntArray()
+
+        val base = emojiGroups[group]?.firstOrNull { it.codes contentEquals  baseCodePoints }
+                ?: emojiGroups[group]?.firstOrNull { it.codes contentEquals baseCodePoints2 }
+                ?: emojiGroups[group]?.firstOrNull { it.codes contentEquals baseCodePoints3 }
+
+        // We keep track the component modifier of this emoji
+        if (base != null) emoji.component = componentCode
+
+        return base
+    }
+
+    private fun withoutComponentCodes(codes: IntArray) : Pair<IntArray, Int> {
+        codes.forEach { code ->
+            when (code) {
+                CP_LIGHT_SKIN_TONE, CP_MEDIUM_LIGHT_SKIN_TONE, CP_MEDIUM_SKIN_TONE,
+                CP_MEDIUM_DARK_SKIN_TONE, CP_DARK_SKIN_TONE ->
+                    return codes.asList().minus(code).toIntArray() to code
+            }
+        }
+        return codes to CP_NUL
+    }
+
+    companion object {
+
+        private val RAW_CPS_KEYCAP_HASH = intArrayOf(0x0023, 0xFE0F, 0x20E3)
+
+        const val CP_NUL = 0x0000
+
+        private const val CP_ZWJ = 0x200D
+        private const val CP_FEMALE_SIGN = 0x2640
+        private const val CP_MALE_SIGN = 0x2642
+        private const val CP_LIGHT_SKIN_TONE = 0x1F3FB
+        private const val CP_MEDIUM_LIGHT_SKIN_TONE = 0x1F3FC
+        private const val CP_MEDIUM_SKIN_TONE = 0x1F3FD
+        private const val CP_MEDIUM_DARK_SKIN_TONE = 0x1F3FE
+        private const val CP_DARK_SKIN_TONE = 0x1F3FF
+        private const val CP_RED_HAIR = 0x1F9B0
+        private const val CP_CURLY_HAIR = 0x1F9B1
+        private const val CP_WHITE_HAIR = 0x1F9B3
+        private const val CP_BARLD = 0x1F9B2
+        private const val CP_VARIANT_SELECTOR = 0xFE0F
+
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_A = 0x1F1E6
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_B = 0x1F1E7
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_C = 0x1F1E8
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_D = 0x1F1E9
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_E = 0x1F1EA
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_F = 0x1F1EB
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_G = 0x1F1EC
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_H = 0x1F1ED
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_I = 0x1F1EE
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_J = 0x1F1EF
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_K = 0x1F1F0
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_L = 0x1F1F1
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_M = 0x1F1F2
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_N = 0x1F1F3
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_O = 0x1F1F4
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_P = 0x1F1F5
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_Q = 0x1F1F6
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_R = 0x1F1F7
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_S = 0x1F1F8
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_T = 0x1F1F9
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_U = 0x1F1FA
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_V = 0x1F1FB
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_W = 0x1F1FC
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_X = 0x1F1FD
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_Y = 0x1F1FE
+        private const val CP_REGIONAL_INDICATOR_SYMBOL_LETTER_Z = 0x1F1FF
+    }
+
+
+}

tools/make-emoji-keys/src/main/kotlin/com/majeur/inputmethod/tools/emoji/model/EmojiGroup.kt

@@ -0,0 +1,19 @@
+package com.majeur.inputmethod.tools.emoji.model
+
+enum class EmojiGroup(val rawName: String) {
+    SMILEYS_AND_EMOTION("Smileys & Emotion"),
+    PEOPLE_AND_BODY("People & Body"),
+    COMPONENT("Component"),
+    ANIMALS_AND_NATURE("Animals & Nature"),
+    FOOD_AND_DRINK("Food & Drink"),
+    TRAVEL_AND_PLACES("Travel & Places"),
+    ACTIVITIES("Activities"),
+    OBJECTS("Objects"),
+    SYMBOLS("Symbols"),
+    FLAGS("Flags");
+
+    companion object {
+        fun get(rawName: String) = values().first { it.rawName == rawName }
+    }
+
+}

tools/make-emoji-keys/src/main/kotlin/com/majeur/inputmethod/tools/emoji/model/EmojiSpec.kt

@@ -0,0 +1,21 @@
+package com.majeur.inputmethod.tools.emoji.model
+
+import com.majeur.inputmethod.tools.emoji.model.EmojiData.Companion.CP_NUL
+
+data class EmojiSpec(val codes: IntArray, val unicodeVer: Float, val name: String) {
+
+    var component = CP_NUL
+
+    val variants by lazy { mutableListOf<EmojiSpec>() }
+
+    override fun toString() = name
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (javaClass != other?.javaClass) return false
+        other as EmojiSpec
+        return codes contentEquals other.codes
+    }
+
+    override fun hashCode() = codes.contentHashCode()
+}

tools/make-emoji-keys/src/main/ressources/values/emoji-categories.tmpl

@@ -0,0 +1,127 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+
+    !!!!! DO NOT EDIT THIS FILE !!!!!
+
+    This file is generated by tools/make-emoji-keys. The base template file is
+    tools/make-emoji-keys/src/main/resources/values/emoji-categories.tmpl
+
+    This file must be updated when any a new release of unicode comes out. Base data
+    can be found at https://unicode.org/Public/emoji/. Table must be defined in
+    tools/make-emoji-keys/src/main/resources/emoji/UNICODE_VERSION/emoji-test.txt.
+
+    To update this file, please run the following commands.
+      $ gradle :tools:make-emoji-keys:makeEmoji
+
+    The updated source file will be generated to the following path (this file).
+      app/src/main/res/values/emoji-categories.xml
+
+    Unicode version @UNICODE_VERSION@.
+    Platform compatibility up to API level @ANDROID_API_LEVEL@.
+
+ -->
+
+<resources>
+
+    <array
+        name="emoji_smileys_emotion"
+        format="string">
+        <!-- @SMILEYS_AND_EMOTION@ -->
+    </array>
+
+    <array
+        name="emoji_people_body"
+        format="string">
+        <!-- @PEOPLE_AND_BODY@ -->
+    </array>
+
+    <array
+        name="emoji_people_body_more"
+        format="string">
+        <!-- @PEOPLE_AND_BODY MORE@ -->
+    </array>
+
+    <array
+        name="emoji_animals_nature"
+        format="string">
+        <!-- @ANIMALS_AND_NATURE@ -->
+    </array>
+
+    <array
+        name="emoji_food_drink"
+        format="string">
+        <!-- @FOOD_AND_DRINKS@ -->
+    </array>
+
+    <array
+        name="emoji_travel_places"
+        format="string">
+        <!-- @TRAVEL_AND_PLACES@ -->
+        </array>
+
+    <array
+        name="emoji_activities"
+        format="string">
+        <!-- @ACTIVITIES@ -->
+        </array>
+
+    <array
+        name="emoji_objects"
+        format="string">
+        <!-- @OBJECTS@ -->
+    </array>
+
+    <array
+        name="emoji_symbols"
+        format="string">
+        <!-- @SYMBOLS@ -->
+        </array>
+
+    <array
+        name="emoji_flags"
+        format="string">
+        <!-- @FLAGS@ -->
+    </array>
+
+    <!-- Dummy codeArrays for recents emoji keyboard.
+             Do not remove these keys, because they are used as a template. -->
+    <array
+        name="emoji_recents"
+        format="string">
+        <!-- These code point should be aligned with {@link RecentsKeyboard#TEMPLATE_KEY_CODE_*. -->
+        <item>30</item>
+        <item>31</item>
+    </array>
+
+    <array
+        name="emoji_emoticons"
+        format="string">
+        <item>:-)</item>
+        <item>;-)</item>
+        <item>:-(</item>
+        <item>:-!</item>
+        <item>:-$</item>
+        <item>B-)</item>
+        <item>=-O</item>
+        <item>:-P</item>
+        <item>:O</item>
+        <item>:-*</item>
+        <item>:-D</item>
+        <item>:\'(</item>
+        <item>:-\\</item>
+        <item>O:-)</item>
+        <item>:-[</item>
+        <item>(╯°</item>
+        <item>□°）</item>
+        <item>╯︵</item>
+        <item>┻━┻</item>
+        <item>¯\\_</item>
+        <item>(ツ)</item>
+        <item>_/¯</item>
+        <item>┬─┬</item>
+        <item>︵ /(</item>
+        <item>.□.\\</item>
+    </array>
+
+</resources>
+