From 72c892e014b49de0bc91828e97928f589be83691 Mon Sep 17 00:00:00 2001
From: Mayfield <3902405+mayfieldiv@users.noreply.github.com>
Date: Thu, 4 Nov 2021 11:12:39 -0400
Subject: [PATCH] ui: render markdown in release notes (#22754)

* convert release notes from markdown to html

* fall back to previous behavior if utf8 decoding or markdown parsing throws

* make simple markdown parser to avoid needing a library

* add unit test

* move markdown parser to common. add unit test

use `markdown-it-py` instead of `markdown` dependency for test comparison since it's already in Pipfile.lock

* test (almost) all release notes and add some extra html encoding

* update lock

Co-authored-by: Willem Melching <willem.melching@gmail.com>
old-commit-hash: 1aebe6ff6e7097cef973f73a990c270d1f6d682b
---
 Pipfile                        |  4 +--
 Pipfile.lock                   |  4 +--
 common/markdown.py             | 48 ++++++++++++++++++++++++++++++++++
 common/tests/test_markdown.py  | 26 ++++++++++++++++++
 selfdrive/test/test_updated.py |  2 +-
 selfdrive/updated.py           |  9 ++++---
 6 files changed, 85 insertions(+), 8 deletions(-)
 create mode 100755 common/markdown.py
 create mode 100755 common/tests/test_markdown.py

diff --git a/Pipfile b/Pipfile
index 04ca0935d5..75cdb29c3d 100644
--- a/Pipfile
+++ b/Pipfile
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75cf958eb51e024e9a7a8a2f5c74d392be9d103bc5d35848d6b4c2e71a0d8580
-size 2026
+oid sha256:a8c79f0c17747345aaa88d127cd7253ce72cab155af167a89d825f99454353fc
+size 2047
diff --git a/Pipfile.lock b/Pipfile.lock
index c8a14b9bd0..57f65b84c0 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a292a1cd9ece673c2ad294801ff742c38584280844a303a5a0bfa2982f58672c
-size 264815
+oid sha256:0964a435b0151e3ea435d32213877612e3706b7651bd89233e341f792b7a6751
+size 264794
diff --git a/common/markdown.py b/common/markdown.py
new file mode 100755
index 0000000000..30c5bc2c09
--- /dev/null
+++ b/common/markdown.py
@@ -0,0 +1,48 @@
+from typing import List
+
+HTML_REPLACEMENTS = [
+  (r'&', r'&amp;'),
+  (r'"', r'&quot;'),
+]
+
+
+def parse_markdown(text: str, tab_length: int = 2) -> str:
+  lines = text.split("\n")
+  output: List[str] = []
+  list_level = 0
+
+  def end_outstanding_lists(level: int, end_level: int) -> int:
+    while level > end_level:
+      level -= 1
+      output.append("</ul>")
+      if level > 0:
+        output.append("</li>")
+    return end_level
+
+  for i, line in enumerate(lines):
+    if i + 1 < len(lines) and lines[i + 1].startswith("==="):  # heading
+      output.append(f"<h1>{line}</h1>")
+    elif line.startswith("==="):
+      pass
+    elif line.lstrip().startswith("* "):  # list
+      line_level = 1 + line.count(" " * tab_length, 0, line.index("*"))
+      if list_level >= line_level:
+        list_level = end_outstanding_lists(list_level, line_level)
+      else:
+        list_level += 1
+        if list_level > 1:
+          output[-1] = output[-1].replace("</li>", "")
+        output.append("<ul>")
+      output.append(f"<li>{line.replace('*', '', 1).lstrip()}</li>")
+    else:
+      list_level = end_outstanding_lists(list_level, 0)
+      if len(line) > 0:
+        output.append(line)
+
+  end_outstanding_lists(list_level, 0)
+  output_str = "\n".join(output) + "\n"
+
+  for (fr, to) in HTML_REPLACEMENTS:
+    output_str = output_str.replace(fr, to)
+
+  return output_str
diff --git a/common/tests/test_markdown.py b/common/tests/test_markdown.py
new file mode 100755
index 0000000000..9ac67f7e50
--- /dev/null
+++ b/common/tests/test_markdown.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+from markdown_it import MarkdownIt
+import os
+import unittest
+
+from common.basedir import BASEDIR
+from common.markdown import parse_markdown
+
+
+class TestMarkdown(unittest.TestCase):
+  # validate that our simple markdown parser produces the same output as `markdown_it` from pip
+  def test_current_release_notes(self):
+    self.maxDiff = None
+
+    with open(os.path.join(BASEDIR, "RELEASES.md")) as f:
+      for r in f.read().split("\n\n"):
+
+        # No hyperlink support is ok
+        if '[' in r:
+          continue
+
+        self.assertEqual(MarkdownIt().render(r), parse_markdown(r))
+
+
+if __name__ == "__main__":
+  unittest.main()
diff --git a/selfdrive/test/test_updated.py b/selfdrive/test/test_updated.py
index d30bc3c6ed..4c0ed2fddf 100755
--- a/selfdrive/test/test_updated.py
+++ b/selfdrive/test/test_updated.py
@@ -53,7 +53,7 @@ class TestUpdated(unittest.TestCase):
       f"cd {self.basedir} && scons -j{os.cpu_count()} cereal/ common/"
     ])
 
-    self.params = Params(db=os.path.join(self.basedir, "persist/params"))
+    self.params = Params(os.path.join(self.basedir, "persist/params"))
     self.params.clear_all()
     os.sync()
 
diff --git a/selfdrive/updated.py b/selfdrive/updated.py
index a74c8c799d..78760bea44 100755
--- a/selfdrive/updated.py
+++ b/selfdrive/updated.py
@@ -35,6 +35,7 @@ from pathlib import Path
 from typing import List, Tuple, Optional
 
 from common.basedir import BASEDIR
+from common.markdown import parse_markdown
 from common.params import Params
 from selfdrive.hardware import EON, TICI, HARDWARE
 from selfdrive.swaglog import cloudlog
@@ -113,9 +114,11 @@ def set_params(new_version: bool, failed_count: int, exception: Optional[str]) -
   if new_version:
     try:
       with open(os.path.join(FINALIZED, "RELEASES.md"), "rb") as f:
-        r = f.read()
-      r = r[:r.find(b'\n\n')]  # Slice latest release notes
-      params.put("ReleaseNotes", r + b"\n")
+        r = f.read().split(b'\n\n', 1)[0]  # Slice latest release notes
+      try:
+        params.put("ReleaseNotes", parse_markdown(r.decode("utf-8")))
+      except Exception:
+        params.put("ReleaseNotes", r + b"\n")
     except Exception:
       params.put("ReleaseNotes", "")
     params.put_bool("UpdateAvailable", True)