sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

multilang: parameterize unit tests (#30842)

Browse Source 
* init

* fix indents

* remove import

* safer

* TemporaryDirectory

* much cleaner

---------

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
old-commit-hash: 9d7f618bc5
chrysler-long2
royjr 1 year ago committed by GitHub
parent 4ca48060bc
commit f82d7f453f
1 changed files with 64 additions and 88 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 152
      selfdrive/ui/tests/test_translations.py

152
selfdrive/ui/tests/test_translations.py
Unescape View File

@ -2,33 +2,26 @@
import json import json
import os import os
import re import re
import shutil
import unittest import unittest
import shutil
import tempfile
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
import string import string
import requests import requests
from parameterized import parameterized_class
from openpilot.selfdrive.ui.update_translations import TRANSLATIONS_DIR, LANGUAGES_FILE, update_translations from openpilot.selfdrive.ui.update_translations import TRANSLATIONS_DIR, LANGUAGES_FILE, update_translations
TMP_TRANSLATIONS_DIR = os.path.join(TRANSLATIONS_DIR, "tmp") with open(LANGUAGES_FILE, "r") as f:
translation_files = json.load(f)
UNFINISHED_TRANSLATION_TAG = "<translation type=\"unfinished\"" # non-empty translations can be marked unfinished UNFINISHED_TRANSLATION_TAG = "<translation type=\"unfinished\"" # non-empty translations can be marked unfinished
LOCATION_TAG = "<location " LOCATION_TAG = "<location "
FORMAT_ARG = re.compile("%[0-9]+") FORMAT_ARG = re.compile("%[0-9]+")
@parameterized_class(("name", "file"), translation_files.items())
class TestTranslations(unittest.TestCase): class TestTranslations(unittest.TestCase):
@classmethod
def setUpClass(cls):
with open(LANGUAGES_FILE, "r") as f:
cls.translation_files = json.load(f)
# Set up temp directory
shutil.copytree(TRANSLATIONS_DIR, TMP_TRANSLATIONS_DIR, dirs_exist_ok=True)
@classmethod
def tearDownClass(cls):
shutil.rmtree(TMP_TRANSLATIONS_DIR, ignore_errors=True)
@staticmethod @staticmethod
def _read_translation_file(path, file): def _read_translation_file(path, file):
tr_file = os.path.join(path, f"{file}.ts") tr_file = os.path.join(path, f"{file}.ts")
@ -36,39 +29,29 @@ class TestTranslations(unittest.TestCase):
return f.read() return f.read()
def test_missing_translation_files(self): def test_missing_translation_files(self):
for name, file in self.translation_files.items(): self.assertTrue(os.path.exists(os.path.join(TRANSLATIONS_DIR, f"{self.file}.ts")),
with self.subTest(name=name, file=file): f"{self.name} has no XML translation file, run selfdrive/ui/update_translations.py")
self.assertTrue(os.path.exists(os.path.join(TRANSLATIONS_DIR, f"{file}.ts")),
f"{name} has no XML translation file, run selfdrive/ui/update_translations.py")
def test_translations_updated(self): def test_translations_updated(self):
update_translations(plural_only=["main_en"], translations_dir=TMP_TRANSLATIONS_DIR) with tempfile.TemporaryDirectory() as tmpdir:
shutil.copytree(TRANSLATIONS_DIR, tmpdir, dirs_exist_ok=True)
for name, file in self.translation_files.items(): update_translations(plural_only=["main_en"], translations_dir=tmpdir)
with self.subTest(name=name, file=file):
# caught by test_missing_translation_files
if not os.path.exists(os.path.join(TRANSLATIONS_DIR, f"{file}.ts")):
self.skipTest(f"{name} missing translation file")
cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file) cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
new_translations = self._read_translation_file(TMP_TRANSLATIONS_DIR, file) new_translations = self._read_translation_file(tmpdir, self.file)
self.assertEqual(cur_translations, new_translations, self.assertEqual(cur_translations, new_translations,
f"{file} ({name}) XML translation file out of date. Run selfdrive/ui/update_translations.py to update the translation files") f"{self.file} ({self.name}) XML translation file out of date. Run selfdrive/ui/update_translations.py to update the translation files")
@unittest.skip("Only test unfinished translations before going to release") @unittest.skip("Only test unfinished translations before going to release")
def test_unfinished_translations(self): def test_unfinished_translations(self):
for name, file in self.translation_files.items(): cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
with self.subTest(name=name, file=file): self.assertTrue(UNFINISHED_TRANSLATION_TAG not in cur_translations,
cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file) f"{self.file} ({self.name}) translation file has unfinished translations. Finish translations or mark them as completed in Qt Linguist")
self.assertTrue(UNFINISHED_TRANSLATION_TAG not in cur_translations,
f"{file} ({name}) translation file has unfinished translations. Finish translations or mark them as completed in Qt Linguist")
def test_vanished_translations(self): def test_vanished_translations(self):
for name, file in self.translation_files.items(): cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
with self.subTest(name=name, file=file): self.assertTrue("<translation type=\"vanished\">" not in cur_translations,
cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file) f"{self.file} ({self.name}) translation file has obsolete translations. Run selfdrive/ui/update_translations.py --vanish to remove them")
self.assertTrue("<translation type=\"vanished\">" not in cur_translations,
f"{file} ({name}) translation file has obsolete translations. Run selfdrive/ui/update_translations.py --vanish to remove them")
def test_finished_translations(self): def test_finished_translations(self):
""" """
@ -80,75 +63,68 @@ class TestTranslations(unittest.TestCase):
- that translation is not empty - that translation is not empty
- that translation format arguments are consistent - that translation format arguments are consistent
""" """
for name, file in self.translation_files.items(): tr_xml = ET.parse(os.path.join(TRANSLATIONS_DIR, f"{self.file}.ts"))
with self.subTest(name=name, file=file):
tr_xml = ET.parse(os.path.join(TRANSLATIONS_DIR, f"{file}.ts"))
for context in tr_xml.getroot(): for context in tr_xml.getroot():
for message in context.iterfind("message"): for message in context.iterfind("message"):
translation = message.find("translation") translation = message.find("translation")
source_text = message.find("source").text source_text = message.find("source").text
# Do not test unfinished translations # Do not test unfinished translations
if translation.get("type") == "unfinished": if translation.get("type") == "unfinished":
continue continue
if message.get("numerus") == "yes": if message.get("numerus") == "yes":
numerusform = [t.text for t in translation.findall("numerusform")] numerusform = [t.text for t in translation.findall("numerusform")]
for nf in numerusform: for nf in numerusform:
self.assertIsNotNone(nf, f"Ensure all plural translation forms are completed: {source_text}") self.assertIsNotNone(nf, f"Ensure all plural translation forms are completed: {source_text}")
self.assertIn("%n", nf, "Ensure numerus argument (%n) exists in translation.") self.assertIn("%n", nf, "Ensure numerus argument (%n) exists in translation.")
self.assertIsNone(FORMAT_ARG.search(nf), "Plural translations must use %n, not %1, %2, etc.: {}".format(numerusform)) self.assertIsNone(FORMAT_ARG.search(nf), "Plural translations must use %n, not %1, %2, etc.: {}".format(numerusform))
else: else:
self.assertIsNotNone(translation.text, f"Ensure translation is completed: {source_text}") self.assertIsNotNone(translation.text, f"Ensure translation is completed: {source_text}")
source_args = FORMAT_ARG.findall(source_text) source_args = FORMAT_ARG.findall(source_text)
translation_args = FORMAT_ARG.findall(translation.text) translation_args = FORMAT_ARG.findall(translation.text)
self.assertEqual(sorted(source_args), sorted(translation_args), self.assertEqual(sorted(source_args), sorted(translation_args),
f"Ensure format arguments are consistent: `{source_text}` vs. `{translation.text}`") f"Ensure format arguments are consistent: `{source_text}` vs. `{translation.text}`")
def test_no_locations(self): def test_no_locations(self):
for name, file in self.translation_files.items(): for line in self._read_translation_file(TRANSLATIONS_DIR, self.file).splitlines():
with self.subTest(name=name, file=file): self.assertFalse(line.strip().startswith(LOCATION_TAG),
for line in self._read_translation_file(TRANSLATIONS_DIR, file).splitlines(): f"Line contains location tag: {line.strip()}, remove all line numbers.")
self.assertFalse(line.strip().startswith(LOCATION_TAG),
f"Line contains location tag: {line.strip()}, remove all line numbers.")
def test_entities_error(self): def test_entities_error(self):
for name, file in self.translation_files.items(): cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
with self.subTest(name=name, file=file): matches = re.findall(r'@(\w+);', cur_translations)
cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file) self.assertEqual(len(matches), 0, f"The string(s) {matches} were found with '@' instead of '&'")
matches = re.findall(r'@(\w+);', cur_translations)
self.assertEqual(len(matches), 0, f"The string(s) {matches} were found with '@' instead of '&'")
def test_bad_language(self): def test_bad_language(self):
IGNORED_WORDS = {'pédale'} IGNORED_WORDS = {'pédale'}
for name, file in self.translation_files.items(): match = re.search(r'_([a-zA-Z]{2,3})', self.file)
match = re.search(r'_([a-zA-Z]{2,3})', file) assert match, f"{self.name} - could not parse language"
assert match, f"{name} - could not parse language"
response = requests.get(f"https://raw.githubusercontent.com/LDNOOBW/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/master/{match.group(1)}") response = requests.get(f"https://raw.githubusercontent.com/LDNOOBW/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/master/{match.group(1)}")
response.raise_for_status() response.raise_for_status()
banned_words = {line.strip() for line in response.text.splitlines()} banned_words = {line.strip() for line in response.text.splitlines()}
for context in ET.parse(os.path.join(TRANSLATIONS_DIR, f"{file}.ts")).getroot(): for context in ET.parse(os.path.join(TRANSLATIONS_DIR, f"{self.file}.ts")).getroot():
for message in context.iterfind("message"): for message in context.iterfind("message"):
translation = message.find("translation") translation = message.find("translation")
if translation.get("type") == "unfinished": if translation.get("type") == "unfinished":
continue continue
translation_text = " ".join([t.text for t in translation.findall("numerusform")]) if message.get("numerus") == "yes" else translation.text translation_text = " ".join([t.text for t in translation.findall("numerusform")]) if message.get("numerus") == "yes" else translation.text
if not translation_text: if not translation_text:
continue continue
words = set(translation_text.translate(str.maketrans('', '', string.punctuation + '%n')).lower().split()) words = set(translation_text.translate(str.maketrans('', '', string.punctuation + '%n')).lower().split())
bad_words_found = words & (banned_words - IGNORED_WORDS) bad_words_found = words & (banned_words - IGNORED_WORDS)
assert not bad_words_found, f"Bad language found in {name}: '{translation_text}'. Bad word(s): {', '.join(bad_words_found)}" assert not bad_words_found, f"Bad language found in {self.name}: '{translation_text}'. Bad word(s): {', '.join(bad_words_found)}"
if __name__ == "__main__": if __name__ == "__main__":

Write Preview
Loading…
Cancel
Save