Add warning for BIDI characters in page renders and in diffs (#17562)

Fix #17514

Given the comments I've adjusted this somewhat. The numbers of characters detected are increased and include things like the use of U+300 to make à instead of à and non-breaking spaces.

There is a button which can be used to escape the content to show it.

Signed-off-by: Andrew Thornton <art27@cantab.net>
Co-authored-by: Gwyneth Morgan <gwymor@tilde.club>
Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

web_src/js/features/common-global.js

@@ -297,8 +297,20 @@ export function initGlobalButtons() {
   });
 
   $('.hide-panel.button').on('click', function (event) {
-    $($(this).data('panel')).hide();
+    // a `.hide-panel.button` can hide a panel, by `data-panel="selector"` or `data-panel-closest="selector"`
     event.preventDefault();
+    let sel = $(this).attr('data-panel');
+    if (sel) {
+      $(sel).hide();
+      return;
+    }
+    sel = $(this).attr('data-panel-closest');
+    if (sel) {
+      $(this).closest(sel).hide();
+      return;
+    }
+    // should never happen, otherwise there is a bug in code
+    alert('Nothing to hide');
   });
 
   $('.show-modal.button').on('click', function () {

web_src/js/features/repo-legacy.js

@@ -10,6 +10,7 @@ import {
   initRepoIssueWipToggle, initRepoPullRequestMerge, initRepoPullRequestUpdate,
   updateIssuesMeta,
 } from './repo-issue.js';
+import {initUnicodeEscapeButton} from './repo-unicode-escape.js';
 import {svg} from '../svg.js';
 import {htmlEscape} from 'escape-goat';
 import {initRepoBranchTagDropdown} from '../components/RepoBranchTagDropdown.js';
@@ -533,6 +534,8 @@ export function initRepository() {
       easyMDE.codemirror.refresh();
     });
   }
+
+  initUnicodeEscapeButton();
 }
 
 function initRepoIssueCommentEdit() {

web_src/js/features/repo-unicode-escape.js

@@ -0,0 +1,28 @@
+export function initUnicodeEscapeButton() {
+  $(document).on('click', 'a.escape-button', (e) => {
+    e.preventDefault();
+    $(e.target).parents('.file-content, .non-diff-file-content').find('.file-code, .file-view').addClass('unicode-escaped');
+    $(e.target).hide();
+    $(e.target).siblings('a.unescape-button').show();
+  });
+  $(document).on('click', 'a.unescape-button', (e) => {
+    e.preventDefault();
+    $(e.target).parents('.file-content, .non-diff-file-content').find('.file-code, .file-view').removeClass('unicode-escaped');
+    $(e.target).hide();
+    $(e.target).siblings('a.escape-button').show();
+  });
+  $(document).on('click', 'a.toggle-escape-button', (e) => {
+    e.preventDefault();
+    const fileContent = $(e.target).parents('.file-content, .non-diff-file-content');
+    const fileView = fileContent.find('.file-code, .file-view');
+    if (fileView.hasClass('unicode-escaped')) {
+      fileView.removeClass('unicode-escaped');
+      fileContent.find('a.unescape-button').hide();
+      fileContent.find('a.escape-button').show();
+    } else {
+      fileView.addClass('unicode-escaped');
+      fileContent.find('a.unescape-button').show();
+      fileContent.find('a.escape-button').hide();
+    }
+  });
+}

web_src/less/_base.less

@@ -668,6 +668,12 @@ a.ui.card:hover,
   color: var(--color-text-dark);
 }
 
+.ui.error.message .header,
+.ui.warning.message .header {
+  color: inherit;
+  filter: saturate(2);
+}
+
 .dont-break-out {
   overflow-wrap: break-word;
   word-wrap: break-word;
@@ -1569,6 +1575,10 @@ a.ui.label:hover {
   }
 }
 
+.lines-escape {
+  width: 0;
+}
+
 .lines-code {
   background-color: var(--color-code-bg);
   padding-left: 5px;

web_src/less/_repository.less

@@ -76,6 +76,24 @@
     }
   }
 
+  .unicode-escaped .escaped-code-point {
+    &[data-escaped]::before {
+      visibility: visible;
+      content: attr(data-escaped);
+      font-family: var(--fonts-monospace);
+      color: var(--color-red);
+    }
+
+    .char {
+      display: none;
+    }
+  }
+
+  .broken-code-point {
+    font-family: var(--fonts-monospace);
+    color: blue;
+  }
+
   .metas {
     .menu {
       overflow-x: auto;
@@ -3020,6 +3038,26 @@ td.blob-excerpt {
   padding-left: 8px;
 }
 
+.ui.message.unicode-escape-prompt {
+  margin-bottom: 0;
+  border-radius: 0;
+  display: flex;
+  flex-direction: column;
+}
+
+.wiki-content-sidebar .ui.message.unicode-escape-prompt,
+.wiki-content-footer .ui.message.unicode-escape-prompt {
+  p {
+    display: none;
+  }
+}
+
+/* fomantic's last-child selector does not work with hidden last child */
+.ui.buttons .unescape-button {
+  border-top-right-radius: .28571429rem;
+  border-bottom-right-radius: .28571429rem;
+}
+
 .webhook-info {
   padding: 7px 12px;
   margin: 10px 0;
@@ -3110,6 +3148,7 @@ td.blob-excerpt {
 .code-diff-unified .del-code,
 .code-diff-unified .del-code td,
 .code-diff-split .del-code .lines-num-old,
+.code-diff-split .del-code .lines-escape-old,
 .code-diff-split .del-code .lines-type-marker-old,
 .code-diff-split .del-code .lines-code-old {
   background: var(--color-diff-removed-row-bg);
@@ -3120,9 +3159,11 @@ td.blob-excerpt {
 .code-diff-unified .add-code td,
 .code-diff-split .add-code .lines-num-new,
 .code-diff-split .add-code .lines-type-marker-new,
+.code-diff-split .add-code .lines-escape-new,
 .code-diff-split .add-code .lines-code-new,
 .code-diff-split .del-code .add-code.lines-num-new,
 .code-diff-split .del-code .add-code.lines-type-marker-new,
+.code-diff-split .del-code .add-code.lines-escape-new,
 .code-diff-split .del-code .add-code.lines-code-new {
   background: var(--color-diff-added-row-bg);
   border-color: var(--color-diff-added-row-border);
@@ -3131,7 +3172,9 @@ td.blob-excerpt {
 .code-diff-split .del-code .lines-num-new,
 .code-diff-split .del-code .lines-type-marker-new,
 .code-diff-split .del-code .lines-code-new,
+.code-diff-split .del-code .lines-escape-new,
 .code-diff-split .add-code .lines-num-old,
+.code-diff-split .add-code .lines-escape-old,
 .code-diff-split .add-code .lines-type-marker-old,
 .code-diff-split .add-code .lines-code-old {
   background: var(--color-diff-inactive);

web_src/less/_review.less

@@ -16,6 +16,17 @@
   }
 }
 
+.lines-escape a.toggle-escape-button::before {
+  visibility: visible;
+  content: '⚠️';
+  font-family: var(--fonts-emoji);
+  color: var(--color-red);
+}
+
+.repository .diff-file-box .code-diff td.lines-escape {
+  padding-left: 0 !important;
+}
+
 .diff-file-box .lines-code:hover .ui.button.add-code-comment {
   opacity: 1;
 }