Add URL detection (#157)

* detect all as one word if it contains one of `.`, `:`, `_`, `@` (and no whitespace)
* resume word suggestions after entering a non-space after one of the characters above

app/src/main/java/org/dslul/openboard/inputmethod/latin/RichInputConnection.java

@@ -397,9 +397,14 @@ public final class RichInputConnection implements PrivateCommandPerformer {
         return Character.codePointBefore(mCommittedTextBeforeComposingText, length);
     }
 
+    public int getCharBeforeBeforeCursor() {
+        final int length = mCommittedTextBeforeComposingText.length();
+        if (length < 2) return Constants.NOT_A_CODE;
+        return mCommittedTextBeforeComposingText.charAt(length - 2);
+    }
+
     public CharSequence getTextBeforeCursor(final int n, final int flags) {
-        final int cachedLength =
+        final int cachedLength = mCommittedTextBeforeComposingText.length() + mComposingText.length();
-                mCommittedTextBeforeComposingText.length() + mComposingText.length();
         // If we have enough characters to satisfy the request, or if we have all characters in
         // the text field, then we can return the cached version right away.
         // However, if we don't have an expected cursor position, then we should always
@@ -650,7 +655,6 @@ public final class RichInputConnection implements PrivateCommandPerformer {
         if (DEBUG_PREVIOUS_TEXT) checkConsistencyForDebug();
     }
 
-    @SuppressWarnings("unused")
     @NonNull
     public NgramContext getNgramContextFromNthPreviousWord(
             final SpacingAndPunctuations spacingAndPunctuations, final int n) {
@@ -671,14 +675,12 @@ public final class RichInputConnection implements PrivateCommandPerformer {
             if (internal.length() > checkLength) {
                 internal.delete(0, internal.length() - checkLength);
                 if (!(reference.equals(internal.toString()))) {
-                    final String context =
+                    final String context = "Expected text = " + internal + "\nActual text = " + reference;
-                            "Expected text = " + internal + "\nActual text = " + reference;
                     ((LatinIME)mParent).debugDumpStateAndCrashWithException(context);
                 }
             }
         }
-        return NgramContextUtils.getNgramContextFromNthPreviousWord(
+        return NgramContextUtils.getNgramContextFromNthPreviousWord(prev, spacingAndPunctuations, n);
-                prev, spacingAndPunctuations, n);
     }
 
     private static boolean isPartOfCompositionForScript(final int codePoint,
@@ -738,10 +740,28 @@ public final class RichInputConnection implements PrivateCommandPerformer {
         }
 
         // Going backward, find the first breaking point (separator)
+        // todo: break if there are 2 consecutive sometimesWordConnectors (more complicated once again, great...)
         int startIndexInBefore = before.length();
+        int endIndexInAfter = -1;
         while (startIndexInBefore > 0) {
             final int codePoint = Character.codePointBefore(before, startIndexInBefore);
             if (!isPartOfCompositionForScript(codePoint, spacingAndPunctuations, scriptId)) {
+                if (Character.isWhitespace(codePoint) || !spacingAndPunctuations.mCurrentLanguageHasSpaces)
+                    break;
+                // continue to the next whitespace and see whether this contains a sometimesWordConnector
+                for (int i = startIndexInBefore - 1; i >= 0; i--) {
+                    final char c = before.charAt(i);
+                    if (spacingAndPunctuations.isSometimesWordConnector(c)) {
+                        // if yes -> whitespace is the index
+                        startIndexInBefore = Math.max(StringUtils.charIndexOfLastWhitespace(before), 0);;
+                        final int firstSpaceAfter = StringUtils.charIndexOfFirstWhitespace(after);
+                        endIndexInAfter = firstSpaceAfter == -1 ? (after.length() - 1) : firstSpaceAfter -1;
+                        break;
+                    } else if (Character.isWhitespace(c)) {
+                        // if no, just break normally
+                        break;
+                    }
+                }
                 break;
             }
             --startIndexInBefore;
@@ -751,17 +771,42 @@ public final class RichInputConnection implements PrivateCommandPerformer {
         }
 
         // Find last word separator after the cursor
-        int endIndexInAfter = -1;
+        if (endIndexInAfter == -1) {
-        while (++endIndexInAfter < after.length()) {
+            while (++endIndexInAfter < after.length()) {
-            final int codePoint = Character.codePointAt(after, endIndexInAfter);
+                final int codePoint = Character.codePointAt(after, endIndexInAfter);
-            if (!isPartOfCompositionForScript(codePoint, spacingAndPunctuations, scriptId)) {
+                if (!isPartOfCompositionForScript(codePoint, spacingAndPunctuations, scriptId)) {
-                break;
+                    if (Character.isWhitespace(codePoint) || !spacingAndPunctuations.mCurrentLanguageHasSpaces)
-            }
+                        break;
-            if (Character.isSupplementaryCodePoint(codePoint)) {
+                    // continue to the next whitespace and see whether this contains a sometimesWordConnector
-                ++endIndexInAfter;
+                    for (int i = endIndexInAfter; i < after.length(); i++) {
+                        final char c = after.charAt(i);
+                        if (spacingAndPunctuations.isSometimesWordConnector(c)) {
+                            // if yes -> whitespace is next to the index
+                            startIndexInBefore = Math.max(StringUtils.charIndexOfLastWhitespace(before), 0);;
+                            final int firstSpaceAfter = StringUtils.charIndexOfFirstWhitespace(after);
+                            endIndexInAfter = firstSpaceAfter == -1 ? (after.length() - 1) : firstSpaceAfter - 1;
+                            break;
+                        } else if (Character.isWhitespace(c)) {
+                            // if no, just break normally
+                            break;
+                        }
+                    }
+                    break;
+                }
+                if (Character.isSupplementaryCodePoint(codePoint)) {
+                    ++endIndexInAfter;
+                }
             }
         }
 
+        // we don't want the end characters to be word separators
+        while (endIndexInAfter > 0 && spacingAndPunctuations.isWordSeparator(after.charAt(endIndexInAfter - 1))) {
+            --endIndexInAfter;
+        }
+        while (startIndexInBefore < before.length() && spacingAndPunctuations.isWordSeparator(before.charAt(startIndexInBefore))) {
+            ++startIndexInBefore;
+        }
+
         final boolean hasUrlSpans =
                 SpannableStringUtils.hasUrlSpans(before, startIndexInBefore, before.length())
                 || SpannableStringUtils.hasUrlSpans(after, 0, endIndexInAfter);
@@ -954,6 +999,18 @@ public final class RichInputConnection implements PrivateCommandPerformer {
         return mCommittedTextBeforeComposingText.lastIndexOf(" ") < mCommittedTextBeforeComposingText.lastIndexOf("@");
     }
 
+    public CharSequence textBeforeCursorUntilLastWhitespace() {
+        int afterLastSpace = 0;
+        for (int i = mCommittedTextBeforeComposingText.length() - 1; i >= 0; i--) {
+            final char c = mCommittedTextBeforeComposingText.charAt(i);
+            if (Character.isWhitespace(c)) {
+                afterLastSpace = i + 1;
+                break;
+            }
+        }
+        return mCommittedTextBeforeComposingText.subSequence(afterLastSpace, mCommittedTextBeforeComposingText.length());
+    }
+
     /**
      * Looks at the text just before the cursor to find out if we are inside a double quote.
      *

app/src/main/java/org/dslul/openboard/inputmethod/latin/common/StringUtils.java

@@ -738,4 +738,24 @@ public final class StringUtils {
         }
         return true;
     }
+
+    public static int charIndexOfFirstWhitespace(final CharSequence s) {
+        for (int i = 0; i < s.length() - 1; i++) {
+            final char c = s.charAt(i);
+            if (Character.isWhitespace(c)) {
+                return i + 1;
+            }
+        }
+        return -1;
+    }
+
+    public static int charIndexOfLastWhitespace(final CharSequence s) {
+        for (int i = s.length() - 1; i >= 0; i--) {
+            final char c = s.charAt(i);
+            if (Character.isWhitespace(c)) {
+                return i + 1;
+            }
+        }
+        return -1;
+    }
 }

app/src/main/java/org/dslul/openboard/inputmethod/latin/inputlogic/InputLogic.java

@@ -446,6 +446,9 @@ public final class InputLogic {
                 && (event.getMCodePoint() >= 0x1100 || Character.isWhitespace(event.getMCodePoint()))) {
             mWordComposer.setHangul(true);
             final Event hangulDecodedEvent = HangulEventDecoder.decodeSoftwareKeyEvent(event);
+            // todo: here hangul combiner does already consume the event, and appends typed codepoint
+            //  to the current word instead of considering the cursor position
+            //  position is actually not visible to the combiner, how to fix?
             processedEvent = mWordComposer.processEvent(hangulDecodedEvent);
         } else {
             mWordComposer.setHangul(false);
@@ -666,6 +669,11 @@ public final class InputLogic {
                 if (mSuggestedWords.isPrediction()) {
                     inputTransaction.setRequiresUpdateSuggestions();
                 }
+                // undo phantom space if it's because after punctuation
+                // users who want to start a sentence with a lowercase letter may not like it
+                if (mSpaceState == SpaceState.PHANTOM
+                        && inputTransaction.getMSettingsValues().isUsuallyFollowedBySpace(mConnection.getCodePointBeforeCursor()))
+                    mSpaceState = SpaceState.NONE;
                 break;
             case Constants.CODE_CAPSLOCK:
                 // Note: Changing keyboard to shift lock state is handled in
@@ -818,8 +826,20 @@ public final class InputLogic {
             final LatinIME.UIHandler handler) {
         final int codePoint = event.getMCodePoint();
         mSpaceState = SpaceState.NONE;
-        if (inputTransaction.getMSettingsValues().isWordSeparator(codePoint)
+        final SettingsValues sv = inputTransaction.getMSettingsValues();
-                || Character.getType(codePoint) == Character.OTHER_SYMBOL) {
+        // don't treat separators as for handling URLs and similar
+        //  otherwise it would work too, but whenever a separator is entered, the word is not selected
+        //  until the next character is entered, and the word is added to history
+        //  -> the changing selection would be confusing, and adding to history is usually bad
+        if (Character.getType(codePoint) == Character.OTHER_SYMBOL
+                || (sv.isWordSeparator(codePoint)
+                    && (!sv.mUrlDetectionEnabled
+                        || Character.isWhitespace(codePoint)
+                        || !sv.mSpacingAndPunctuations.containsSometimesWordConnector(mWordComposer.getTypedWord())
+                    )
+                )
+        ) {
+            Log.i("test1", "separator");
             handleSeparatorEvent(event, inputTransaction, handler);
         } else {
             if (SpaceState.PHANTOM == inputTransaction.getMSpaceState()) {
@@ -827,14 +847,15 @@ public final class InputLogic {
                     // If we are in the middle of a recorrection, we need to commit the recorrection
                     // first so that we can insert the character at the current cursor position.
                     // We also need to unlearn the original word that is now being corrected.
-                    unlearnWord(mWordComposer.getTypedWord(), inputTransaction.getMSettingsValues(), Constants.EVENT_BACKSPACE);
+                    unlearnWord(mWordComposer.getTypedWord(), sv, Constants.EVENT_BACKSPACE);
                     resetEntireInputState(mConnection.getExpectedSelectionStart(),
                             mConnection.getExpectedSelectionEnd(), true /* clearSuggestionStrip */);
                 } else {
-                    commitTyped(inputTransaction.getMSettingsValues(), LastComposedWord.NOT_A_SEPARATOR);
+                    commitTyped(sv, LastComposedWord.NOT_A_SEPARATOR);
                 }
             }
-            handleNonSeparatorEvent(event, inputTransaction.getMSettingsValues(), inputTransaction);
+            Log.i("test1", "nonseparator");
+            handleNonSeparatorEvent(event, sv, inputTransaction);
         }
     }
 
@@ -853,6 +874,18 @@ public final class InputLogic {
         // not the same.
         boolean isComposingWord = mWordComposer.isComposingWord();
 
+        // if we continue directly after a sometimesWordConnector, restart suggestions for the whole word
+        // (only with URL detection enabled)
+        if (settingsValues.mUrlDetectionEnabled && !isComposingWord && SpaceState.NONE == inputTransaction.getMSpaceState()
+                && settingsValues.mSpacingAndPunctuations.isSometimesWordConnector(mConnection.getCodePointBeforeCursor())
+                // but not if there are two consecutive sometimesWordConnectors (e.g. "...bla")
+                && !settingsValues.mSpacingAndPunctuations.isSometimesWordConnector(mConnection.getCharBeforeBeforeCursor())
+        ) {
+            final CharSequence text = mConnection.textBeforeCursorUntilLastWhitespace();
+            final TextRange range = new TextRange(text, 0, text.length(), text.length(), false);
+            isComposingWord = true;
+            restartSuggestions(range, mConnection.mExpectedSelStart);
+        }
         // TODO: remove isWordConnector() and use isUsuallyFollowedBySpace() instead.
         // See onStartBatchInput() to see how to do it.
         if (SpaceState.PHANTOM == inputTransaction.getMSpaceState()
@@ -1493,10 +1526,24 @@ public final class InputLogic {
                 mWordComposer.wasAutoCapitalized() && !mWordComposer.isMostlyCaps();
         final int timeStampInSeconds = (int)TimeUnit.MILLISECONDS.toSeconds(
                 System.currentTimeMillis());
-        mDictionaryFacilitator.addToUserHistory(suggestion, wasAutoCapitalized,
+        mDictionaryFacilitator.addToUserHistory(stripWordSeparatorsFromEnd(suggestion, settingsValues), wasAutoCapitalized,
                 ngramContext, timeStampInSeconds, settingsValues.mBlockPotentiallyOffensive);
     }
 
+    // strip word separators from end (may be necessary for urls, e.g. when the user has typed
+    //  "go to example.com, and" -> we don't want the ",")
+    private String stripWordSeparatorsFromEnd(final String word, final SettingsValues settingsValues) {
+        final String result;
+        if (settingsValues.mSpacingAndPunctuations.isWordSeparator(word.codePointBefore(word.length()))) {
+            int endIndex = word.length() - 1;
+            while (settingsValues.mSpacingAndPunctuations.isWordSeparator(word.codePointBefore(endIndex)))
+                --endIndex;
+            result = word.substring(0, endIndex);
+        } else
+            result = word;
+        return result;
+    }
+
     public void performUpdateSuggestionStripSync(final SettingsValues settingsValues, final int inputStyle) {
         long startTimeMillis = 0;
         if (DebugFlags.DEBUG_ENABLED) {
@@ -1609,6 +1656,10 @@ public final class InputLogic {
             mConnection.finishComposingText();
             return;
         }
+        restartSuggestions(range, expectedCursorPosition);
+    }
+
+    private void restartSuggestions(final TextRange range, final int expectedCursorPosition) {
         final int numberOfCharsInWordBeforeCursor = range.getNumberOfCharsInWordBeforeCursor();
         if (numberOfCharsInWordBeforeCursor > expectedCursorPosition) return;
         final ArrayList<SuggestedWordInfo> suggestions = new ArrayList<>();

app/src/main/java/org/dslul/openboard/inputmethod/latin/settings/Settings.java

@@ -130,6 +130,7 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang
     public static final String PREF_SELECTED_INPUT_STYLE = "pref_selected_input_style";
     public static final String PREF_USE_SYSTEM_LOCALES = "pref_use_system_locales";
     public static final String PREF_SHOW_ALL_MORE_KEYS = "pref_show_all_more_keys";
+    public static final String PREF_URL_DETECTION = "pref_url_detection";
 
     public static final String PREF_DONT_SHOW_MISSING_DICTIONARY_DIALOG = "pref_dont_show_missing_dict_dialog";
 

app/src/main/java/org/dslul/openboard/inputmethod/latin/settings/SettingsValues.java

@@ -59,7 +59,6 @@ public class SettingsValues {
 
     // From resources:
     public final SpacingAndPunctuations mSpacingAndPunctuations;
-    public final int mDelayInMillisecondsToUpdateOldSuggestions;
     public final long mDoubleSpacePeriodTimeout;
     // From configuration:
     public final Locale mLocale;
@@ -111,6 +110,7 @@ public class SettingsValues {
     public final boolean mUseContactsDictionary;
     public final boolean mCustomNavBarColor;
     public final float mKeyboardHeightScale;
+    public final boolean mUrlDetectionEnabled;
 
     // From the input box
     @NonNull
@@ -138,8 +138,6 @@ public class SettingsValues {
                           @NonNull final InputAttributes inputAttributes) {
         mLocale = res.getConfiguration().locale;
         // Get the resources
-        mDelayInMillisecondsToUpdateOldSuggestions =
-                res.getInteger(R.integer.config_delay_in_milliseconds_to_update_old_suggestions);
         mSpacingAndPunctuations = new SpacingAndPunctuations(res);
 
         // Store the input attributes
@@ -233,6 +231,7 @@ public class SettingsValues {
                 mBlockPotentiallyOffensive,
                 prefs.getBoolean(Settings.PREF_GESTURE_SPACE_AWARE, false)
         );
+        mUrlDetectionEnabled = prefs.getBoolean(Settings.PREF_URL_DETECTION, false);
     }
 
     public boolean isApplicationSpecifiedCompletionsOn() {
@@ -350,8 +349,6 @@ public class SettingsValues {
         final StringBuilder sb = new StringBuilder("Current settings :");
         sb.append("\n   mSpacingAndPunctuations = ");
         sb.append("" + mSpacingAndPunctuations.dump());
-        sb.append("\n   mDelayInMillisecondsToUpdateOldSuggestions = ");
-        sb.append("" + mDelayInMillisecondsToUpdateOldSuggestions);
         sb.append("\n   mAutoCap = ");
         sb.append("" + mAutoCap);
         sb.append("\n   mVibrateOn = ");

app/src/main/java/org/dslul/openboard/inputmethod/latin/settings/SpacingAndPunctuations.java

@@ -33,6 +33,7 @@ public final class SpacingAndPunctuations {
     private final int[] mSortedSymbolsFollowedBySpace;
     private final int[] mSortedSymbolsClusteringTogether;
     private final int[] mSortedWordConnectors;
+    private final int[] mSortedSometimesWordConnectors; // maybe rename... they are some sort of glue for words containing separators
     public final int[] mSortedWordSeparators;
     public final PunctuationSuggestions mSuggestPuncList;
     private final int mSentenceSeparator;
@@ -45,25 +46,21 @@ public final class SpacingAndPunctuations {
 
     public SpacingAndPunctuations(final Resources res) {
         // To be able to binary search the code point. See {@link #isUsuallyPrecededBySpace(int)}.
-        mSortedSymbolsPrecededBySpace = StringUtils.toSortedCodePointArray(
+        mSortedSymbolsPrecededBySpace = StringUtils.toSortedCodePointArray(res.getString(R.string.symbols_preceded_by_space));
-                res.getString(R.string.symbols_preceded_by_space));
         // To be able to binary search the code point. See {@link #isUsuallyFollowedBySpace(int)}.
-        mSortedSymbolsFollowedBySpace = StringUtils.toSortedCodePointArray(
+        mSortedSymbolsFollowedBySpace = StringUtils.toSortedCodePointArray(res.getString(R.string.symbols_followed_by_space));
-                res.getString(R.string.symbols_followed_by_space));
+        mSortedSymbolsClusteringTogether = StringUtils.toSortedCodePointArray(res.getString(R.string.symbols_clustering_together));
-        mSortedSymbolsClusteringTogether = StringUtils.toSortedCodePointArray(
-                res.getString(R.string.symbols_clustering_together));
         // To be able to binary search the code point. See {@link #isWordConnector(int)}.
-        mSortedWordConnectors = StringUtils.toSortedCodePointArray(
+        mSortedWordConnectors = StringUtils.toSortedCodePointArray(res.getString(R.string.symbols_word_connectors));
-                res.getString(R.string.symbols_word_connectors));
+        mSortedWordSeparators = StringUtils.toSortedCodePointArray(res.getString(R.string.symbols_word_separators));
-        mSortedWordSeparators = StringUtils.toSortedCodePointArray(
+        mSortedSentenceTerminators = StringUtils.toSortedCodePointArray(res.getString(R.string.symbols_sentence_terminators));
-                res.getString(R.string.symbols_word_separators));
-        mSortedSentenceTerminators = StringUtils.toSortedCodePointArray(
-                res.getString(R.string.symbols_sentence_terminators));
         mSentenceSeparator = res.getInteger(R.integer.sentence_separator);
         mAbbreviationMarker = res.getInteger(R.integer.abbreviation_marker);
         mSentenceSeparatorAndSpace = new String(new int[] {
                 mSentenceSeparator, Constants.CODE_SPACE }, 0, 2);
         mCurrentLanguageHasSpaces = res.getBoolean(R.bool.current_language_has_spaces);
+        // make it empty if language doesn't have spaces, to avoid weird glitches
+        mSortedSometimesWordConnectors = mCurrentLanguageHasSpaces ? StringUtils.toSortedCodePointArray(res.getString(R.string.symbols_sometimes_word_connectors)) : new int[0];
         final Locale locale = res.getConfiguration().locale;
         // Heuristic: we use American Typography rules because it's the most common rules for all
         // English variants. German rules (not "German typography") also have small gotchas.
@@ -81,6 +78,7 @@ public final class SpacingAndPunctuations {
         mSortedSymbolsFollowedBySpace = model.mSortedSymbolsFollowedBySpace;
         mSortedSymbolsClusteringTogether = model.mSortedSymbolsClusteringTogether;
         mSortedWordConnectors = model.mSortedWordConnectors;
+        mSortedSometimesWordConnectors = model.mSortedSometimesWordConnectors;
         mSortedWordSeparators = overrideSortedWordSeparators;
         mSortedSentenceTerminators = model.mSortedSentenceTerminators;
         mSuggestPuncList = model.mSuggestPuncList;
@@ -100,6 +98,19 @@ public final class SpacingAndPunctuations {
         return Arrays.binarySearch(mSortedWordConnectors, code) >= 0;
     }
 
+    public boolean isSometimesWordConnector(final int code) {
+        return Arrays.binarySearch(mSortedSometimesWordConnectors, code) >= 0;
+    }
+
+    public boolean containsSometimesWordConnector(final CharSequence word) {
+        // todo: this only works if all mSortedSometimesWordConnectors are simple chars
+        for (int i = 0; i < word.length(); i++) {
+            if (isSometimesWordConnector(word.charAt(i)))
+                return true;
+        }
+        return false;
+    }
+
     public boolean isWordCodePoint(final int code) {
         return Character.isLetter(code) || isWordConnector(code);
     }

app/src/main/res/values/config-common.xml

@@ -103,7 +103,6 @@
     <integer name="config_suggestions_count_in_strip">3</integer>
     <fraction name="config_center_suggestion_percentile">36%</fraction>
     <integer name="config_delay_in_milliseconds_to_update_suggestions">100</integer>
-    <integer name="config_delay_in_milliseconds_to_update_old_suggestions">300</integer>
 
     <!-- Common more suggestions configuraion. -->
     <dimen name="config_more_suggestions_key_horizontal_padding">12dp</dimen>

app/src/main/res/values/donottranslate-config-spacing-and-punctuations.xml

@@ -33,6 +33,12 @@
     <string name="symbols_word_separators">"&#x0009;&#x0020;&#x000A;&#x00A0;"()[]{}*&amp;&lt;&gt;+=|.,;:!?/_\"„“</string>
     <!-- Word connectors -->
     <string name="symbols_word_connectors">\'-</string>
+    <!-- Symbols that may act as word connectors, e.g. in URLs or mail addresses
+         / is not included, because a URL typically contains . or : before /, and including it might be unwanted in some cases
+         @ is not a word separator, but including it here allows continuing suggestions when typing a period of a mail address
+         todo: keep _ or not?
+     -->
+    <string name="symbols_sometimes_word_connectors">.:_@</string>
     <!-- The sentence separator code point, for capitalization and auto-insertion -->
     <!-- U+002E: "." FULL STOP   ; 2Eh = 46d -->
     <integer name="sentence_separator">46</integer>

app/src/main/res/values/strings.xml

@@ -197,6 +197,10 @@
     <string name="show_all_more_keys_title">Show all keys in popup</string>
     <!-- Description for "show_all_more_keys" option. -->
     <string name="show_all_more_keys_summary">When using a latin keyboard, more characters are available on long-pressing a key</string>
+    <!-- Preferences item for enabling URL detection -->
+    <string name="url_detection_title">URL detection</string>
+    <!-- Description for "url_detection_title" option. -->
+    <string name="url_detection_summary">Try to detect URLs and similar as a single word</string>
     <!-- Preferences item for disabling word learning -->
     <string name="prefs_force_incognito_mode">Force incognito mode</string>
     <!-- Description for "prefs_force_incognito_mode" option. -->

app/src/main/res/xml/prefs_screen_advanced.xml

@@ -94,6 +94,12 @@
             android:summary="@string/show_all_more_keys_summary"
             android:defaultValue="false" />
 
+        <SwitchPreferenceCompat
+            android:key="pref_url_detection"
+            android:title="@string/url_detection_title"
+            android:summary="@string/url_detection_summary"
+            android:defaultValue="false" />
+
         <Preference
             android:key="load_gesture_library"
             android:title="@string/load_gesture_library"

app/src/test/java/org/dslul/openboard/inputmethod/latin/InputLogicTest.kt

@@ -38,6 +38,7 @@ import kotlin.math.min
     ShadowInputMethodService::class,
     ShadowKeyboardSwitcher::class,
     ShadowHandler::class,
+    ShadowFacilitator2::class,
 ])
 class InputLogicTest {
     private lateinit var latinIME: LatinIME
@@ -134,6 +135,14 @@ class InputLogicTest {
         assertEquals(4, cursor)
     }
 
+    @Test fun separatorUnselectsWord() {
+        reset()
+        setText("hello")
+        assertEquals("hello", composingText)
+        input('.'.code)
+        assertEquals("", composingText)
+    }
+
     // todo: try the same if there is text afterwards (not touching)
     @Test fun autospace() {
         reset()
@@ -142,7 +151,6 @@ class InputLogicTest {
         input('a'.code)
         assertEquals("hello.a", textBeforeCursor)
         DeviceProtectedUtils.getSharedPreferences(latinIME).edit { putBoolean(Settings.PREF_AUTOSPACE_AFTER_PUNCTUATION, true) }
-        assert(settingsValues.mAutospaceAfterPunctuationEnabled)
         setText("hello")
         input('.'.code)
         input('a'.code)
@@ -158,7 +166,6 @@ class InputLogicTest {
         assertEquals("hello.a", textBeforeCursor)
         assertEquals("hello.a there", text)
         DeviceProtectedUtils.getSharedPreferences(latinIME).edit { putBoolean(Settings.PREF_AUTOSPACE_AFTER_PUNCTUATION, true) }
-        assert(settingsValues.mAutospaceAfterPunctuationEnabled)
         setText("hello there")
         setCursorPosition(5) // after hello
         input('.'.code)
@@ -167,6 +174,62 @@ class InputLogicTest {
         assertEquals("hello. a there", text)
     }
 
+    @Test fun urlDetectionThings() {
+        reset()
+        DeviceProtectedUtils.getSharedPreferences(latinIME).edit { putBoolean(Settings.PREF_URL_DETECTION, true) }
+        input('.'.code)
+        input('.'.code)
+        input('.'.code)
+        input('h'.code)
+        assertEquals("...h", text)
+        assertEquals("h", composingText)
+        reset()
+        DeviceProtectedUtils.getSharedPreferences(latinIME).edit { putBoolean(Settings.PREF_URL_DETECTION, true) }
+        input("bla")
+        input('.'.code)
+        input('.'.code)
+        assertEquals("bla..", text)
+        assertEquals("", composingText)
+        reset()
+        DeviceProtectedUtils.getSharedPreferences(latinIME).edit { putBoolean(Settings.PREF_URL_DETECTION, true) }
+        input("bla")
+        input('.'.code)
+        input('c'.code)
+        assertEquals("bla.c", text)
+        assertEquals("bla.c", composingText)
+        reset()
+        DeviceProtectedUtils.getSharedPreferences(latinIME).edit { putBoolean(Settings.PREF_URL_DETECTION, true) }
+        DeviceProtectedUtils.getSharedPreferences(latinIME).edit { putBoolean(Settings.PREF_AUTOSPACE_AFTER_PUNCTUATION, true) }
+        input("bla")
+        input('.'.code)
+        functionalKeyPress(Constants.CODE_SHIFT) // should remove the phantom space (in addition to normal effect)
+        input('c'.code)
+        assertEquals("bla.c", text)
+        assertEquals("bla.c", composingText)
+    }
+
+    @Test fun stripSeparatorsBeforeAddingToHistoryWithURLDetection() {
+        reset()
+        DeviceProtectedUtils.getSharedPreferences(latinIME).edit { putBoolean(Settings.PREF_URL_DETECTION, true) }
+        setText("example.co")
+        input('m'.code)
+        input('.'.code)
+        assertEquals("example.com.", composingText)
+        input(' '.code)
+        assertEquals("example.com", ShadowFacilitator2.lastAddedWord)
+    }
+
+    @Test fun dontSelectConsecutiveSeparatorsWithURLDetection() {
+        reset()
+        DeviceProtectedUtils.getSharedPreferences(latinIME).edit { putBoolean(Settings.PREF_URL_DETECTION, true) }
+        setText("bl")
+        input('a'.code)
+        input('.'.code)
+        input('.'.code)
+        assertEquals("", composingText)
+        assertEquals("bla..", text)
+    }
+
     // ------- helper functions ---------
 
     // should be called before every test, so the same state is guaranteed
@@ -185,11 +248,7 @@ class InputLogicTest {
 
         currentInputType = InputType.TYPE_CLASS_TEXT
 
-        // todo: does setText("") work?
+        setText("")
-        //  plus restarting = true maybe?
-        //  that may be the better method for setting a new text field
-        connection.setSelection(0, 0) // resets cache
-        inputLogic.restartSuggestionsOnWordTouchedByCursor(settingsValues, currentScript)
     }
 
     private fun input(codePoint: Int) {
@@ -218,7 +277,7 @@ class InputLogicTest {
         val oldBefore = textBeforeCursor
         val oldAfter = textAfterCursor
 
-        latinIME.onTextInput(text)
+        latinIME.onTextInput(insert)
         handleMessages()
 
         assertEquals(oldBefore + insert, textBeforeCursor)
@@ -233,13 +292,6 @@ class InputLogicTest {
         false
     ).mWord
 
-    private fun getUnderlinedWord(): String {
-        val word = getText().substring(inputLogic.composingStart, inputLogic.composingStart + inputLogic.composingLength)
-        assertEquals(word, composingText)
-        assertEquals(word, connectionComposingText) // no, this will fail as it returns only text until the cursor
-        return word
-    }
-
     private fun setCursorPosition(start: Int, end: Int = start, weirdTextField: Boolean = false) {
         val ei = EditorInfo()
         ei.inputType = currentInputType
@@ -263,7 +315,6 @@ class InputLogicTest {
         handleMessages()
 
         if (weirdTextField) {
-            // todo: when to handle messages from update selection?
             latinIME.mHandler.onStartInput(ei, true) // essentially does nothing
             latinIME.mHandler.onStartInputView(ei, true) // does the thing
             handleMessages()
@@ -283,8 +334,6 @@ class InputLogicTest {
     }
 
     // just sets the text and starts input so connection it set up correctly
-    // todo: update selection to automatically set composing span?
-    //  here it's -1, -1 in the end, but it probably shouldn't be (and isn't in tests)
     private fun setText(newText: String) {
         text = newText
         selectionStart = newText.length
@@ -324,7 +373,9 @@ class InputLogicTest {
             messages.removeFirst()
         }
         while (delayedMessages.isNotEmpty()) {
-            latinIME.mHandler.handleMessage(delayedMessages.first())
+            val msg = delayedMessages.first()
+            if (msg.what != 2) // MSG_UPDATE_SUGGESTION_STRIP, we want to ignore it because it's irrelevant and has a 500 ms timeout
+                latinIME.mHandler.handleMessage(delayedMessages.first())
             delayedMessages.removeFirst()
             // delayed messages may post further messages, handle before next delayed message
             while (messages.isNotEmpty()) {
@@ -378,7 +429,6 @@ private val ic = object : InputConnection {
     // this REPLACES currently composing text (even if at a different position)
     // moves the cursor: positive means relative to composing text start, negative means relative to start
     override fun setComposingText(newText: CharSequence, cursor: Int): Boolean {
-        println("set composing text $newText, $cursor")
         // first remove the composing text if any
         if (composingStart != -1 && composingEnd != -1)
             text = textBeforeComposingText + text.substring(composingEnd)
@@ -538,3 +588,16 @@ class ShadowKeyboardSwitcher {
     // only affects view
     fun getCurrentKeyboardScriptId() = currentScript
 }
+
+@Implements(DictionaryFacilitatorImpl::class)
+class ShadowFacilitator2 {
+    @Implementation
+    fun addToUserHistory(suggestion: String, wasAutoCapitalized: Boolean,
+                         ngramContext: NgramContext, timeStampInSeconds: Long,
+                         blockPotentiallyOffensive: Boolean) {
+        lastAddedWord = suggestion
+    }
+    companion object {
+        var lastAddedWord = ""
+    }
+}

app/src/test/java/org/dslul/openboard/inputmethod/latin/SuggestTest.kt

@@ -297,8 +297,8 @@ fun suggestion(word: String, score: Int, locale: Locale) =
 
 @Implements(DictionaryFacilitatorImpl::class)
 class ShadowFacilitator {
-     @Implementation
+    @Implementation
-     fun getCurrentLocale() = currentTypingLocale
+    fun getCurrentLocale() = currentTypingLocale
     @Implementation
     fun hasAtLeastOneInitializedMainDictionary() = true // otherwise no autocorrect
 }