sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

multilang: parameterize unit tests (#30842)

Browse Source 
* init

* fix indents

* remove import

* safer

* TemporaryDirectory

* much cleaner

---------

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
old-commit-hash: 9d7f618bc5
chrysler-long2
royjr 1 year ago committed by GitHub
parent 4ca48060bc
commit f82d7f453f
1 changed files with 64 additions and 88 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 152
      selfdrive/ui/tests/test_translations.py

152
selfdrive/ui/tests/test_translations.py
Unescape View File

@ -2,33 +2,26 @@
import json
import os
import re
import shutil
import unittest
import shutil
import tempfile
import xml.etree.ElementTree as ET
import string
import requests
from parameterized import parameterized_class
from openpilot.selfdrive.ui.update_translations import TRANSLATIONS_DIR, LANGUAGES_FILE, update_translations
TMP_TRANSLATIONS_DIR = os.path.join(TRANSLATIONS_DIR, "tmp")
with open(LANGUAGES_FILE, "r") as f:
translation_files = json.load(f)
UNFINISHED_TRANSLATION_TAG = "<translation type=\"unfinished\"" # non-empty translations can be marked unfinished
LOCATION_TAG = "<location "
FORMAT_ARG = re.compile("%[0-9]+")
@parameterized_class(("name", "file"), translation_files.items())
class TestTranslations(unittest.TestCase):
@classmethod
def setUpClass(cls):
with open(LANGUAGES_FILE, "r") as f:
cls.translation_files = json.load(f)
# Set up temp directory
shutil.copytree(TRANSLATIONS_DIR, TMP_TRANSLATIONS_DIR, dirs_exist_ok=True)
@classmethod
def tearDownClass(cls):
shutil.rmtree(TMP_TRANSLATIONS_DIR, ignore_errors=True)
@staticmethod
def _read_translation_file(path, file):
tr_file = os.path.join(path, f"{file}.ts")
@ -36,39 +29,29 @@ class TestTranslations(unittest.TestCase):
return f.read()
def test_missing_translation_files(self):
for name, file in self.translation_files.items():
with self.subTest(name=name, file=file):
self.assertTrue(os.path.exists(os.path.join(TRANSLATIONS_DIR, f"{file}.ts")),
f"{name} has no XML translation file, run selfdrive/ui/update_translations.py")
self.assertTrue(os.path.exists(os.path.join(TRANSLATIONS_DIR, f"{self.file}.ts")),
f"{self.name} has no XML translation file, run selfdrive/ui/update_translations.py")
def test_translations_updated(self):
update_translations(plural_only=["main_en"], translations_dir=TMP_TRANSLATIONS_DIR)
for name, file in self.translation_files.items():
with self.subTest(name=name, file=file):
# caught by test_missing_translation_files
if not os.path.exists(os.path.join(TRANSLATIONS_DIR, f"{file}.ts")):
self.skipTest(f"{name} missing translation file")
with tempfile.TemporaryDirectory() as tmpdir:
shutil.copytree(TRANSLATIONS_DIR, tmpdir, dirs_exist_ok=True)
update_translations(plural_only=["main_en"], translations_dir=tmpdir)
cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file)
new_translations = self._read_translation_file(TMP_TRANSLATIONS_DIR, file)
self.assertEqual(cur_translations, new_translations,
f"{file} ({name}) XML translation file out of date. Run selfdrive/ui/update_translations.py to update the translation files")
cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
new_translations = self._read_translation_file(tmpdir, self.file)
self.assertEqual(cur_translations, new_translations,
f"{self.file} ({self.name}) XML translation file out of date. Run selfdrive/ui/update_translations.py to update the translation files")
@unittest.skip("Only test unfinished translations before going to release")
def test_unfinished_translations(self):
for name, file in self.translation_files.items():
with self.subTest(name=name, file=file):
cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file)
self.assertTrue(UNFINISHED_TRANSLATION_TAG not in cur_translations,
f"{file} ({name}) translation file has unfinished translations. Finish translations or mark them as completed in Qt Linguist")
cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
self.assertTrue(UNFINISHED_TRANSLATION_TAG not in cur_translations,
f"{self.file} ({self.name}) translation file has unfinished translations. Finish translations or mark them as completed in Qt Linguist")
def test_vanished_translations(self):
for name, file in self.translation_files.items():
with self.subTest(name=name, file=file):
cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file)
self.assertTrue("<translation type=\"vanished\">" not in cur_translations,
f"{file} ({name}) translation file has obsolete translations. Run selfdrive/ui/update_translations.py --vanish to remove them")
cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
self.assertTrue("<translation type=\"vanished\">" not in cur_translations,
f"{self.file} ({self.name}) translation file has obsolete translations. Run selfdrive/ui/update_translations.py --vanish to remove them")
def test_finished_translations(self):
"""
@ -80,75 +63,68 @@ class TestTranslations(unittest.TestCase):
- that translation is not empty
- that translation format arguments are consistent
"""
for name, file in self.translation_files.items():
with self.subTest(name=name, file=file):
tr_xml = ET.parse(os.path.join(TRANSLATIONS_DIR, f"{file}.ts"))
tr_xml = ET.parse(os.path.join(TRANSLATIONS_DIR, f"{self.file}.ts"))
for context in tr_xml.getroot():
for message in context.iterfind("message"):
translation = message.find("translation")
source_text = message.find("source").text
for context in tr_xml.getroot():
for message in context.iterfind("message"):
translation = message.find("translation")
source_text = message.find("source").text
# Do not test unfinished translations
if translation.get("type") == "unfinished":
continue
# Do not test unfinished translations
if translation.get("type") == "unfinished":
continue
if message.get("numerus") == "yes":
numerusform = [t.text for t in translation.findall("numerusform")]
if message.get("numerus") == "yes":
numerusform = [t.text for t in translation.findall("numerusform")]
for nf in numerusform:
self.assertIsNotNone(nf, f"Ensure all plural translation forms are completed: {source_text}")
self.assertIn("%n", nf, "Ensure numerus argument (%n) exists in translation.")
self.assertIsNone(FORMAT_ARG.search(nf), "Plural translations must use %n, not %1, %2, etc.: {}".format(numerusform))
for nf in numerusform:
self.assertIsNotNone(nf, f"Ensure all plural translation forms are completed: {source_text}")
self.assertIn("%n", nf, "Ensure numerus argument (%n) exists in translation.")
self.assertIsNone(FORMAT_ARG.search(nf), "Plural translations must use %n, not %1, %2, etc.: {}".format(numerusform))
else:
self.assertIsNotNone(translation.text, f"Ensure translation is completed: {source_text}")
else:
self.assertIsNotNone(translation.text, f"Ensure translation is completed: {source_text}")
source_args = FORMAT_ARG.findall(source_text)
translation_args = FORMAT_ARG.findall(translation.text)
self.assertEqual(sorted(source_args), sorted(translation_args),
f"Ensure format arguments are consistent: `{source_text}` vs. `{translation.text}`")
source_args = FORMAT_ARG.findall(source_text)
translation_args = FORMAT_ARG.findall(translation.text)
self.assertEqual(sorted(source_args), sorted(translation_args),
f"Ensure format arguments are consistent: `{source_text}` vs. `{translation.text}`")
def test_no_locations(self):
for name, file in self.translation_files.items():
with self.subTest(name=name, file=file):
for line in self._read_translation_file(TRANSLATIONS_DIR, file).splitlines():
self.assertFalse(line.strip().startswith(LOCATION_TAG),
f"Line contains location tag: {line.strip()}, remove all line numbers.")
for line in self._read_translation_file(TRANSLATIONS_DIR, self.file).splitlines():
self.assertFalse(line.strip().startswith(LOCATION_TAG),
f"Line contains location tag: {line.strip()}, remove all line numbers.")
def test_entities_error(self):
for name, file in self.translation_files.items():
with self.subTest(name=name, file=file):
cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file)
matches = re.findall(r'@(\w+);', cur_translations)
self.assertEqual(len(matches), 0, f"The string(s) {matches} were found with '@' instead of '&'")
cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
matches = re.findall(r'@(\w+);', cur_translations)
self.assertEqual(len(matches), 0, f"The string(s) {matches} were found with '@' instead of '&'")
def test_bad_language(self):
IGNORED_WORDS = {'pédale'}
for name, file in self.translation_files.items():
match = re.search(r'_([a-zA-Z]{2,3})', file)
assert match, f"{name} - could not parse language"
match = re.search(r'_([a-zA-Z]{2,3})', self.file)
assert match, f"{self.name} - could not parse language"
response = requests.get(f"https://raw.githubusercontent.com/LDNOOBW/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/master/{match.group(1)}")
response.raise_for_status()
response = requests.get(f"https://raw.githubusercontent.com/LDNOOBW/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/master/{match.group(1)}")
response.raise_for_status()
banned_words = {line.strip() for line in response.text.splitlines()}
banned_words = {line.strip() for line in response.text.splitlines()}
for context in ET.parse(os.path.join(TRANSLATIONS_DIR, f"{file}.ts")).getroot():
for message in context.iterfind("message"):
translation = message.find("translation")
if translation.get("type") == "unfinished":
continue
for context in ET.parse(os.path.join(TRANSLATIONS_DIR, f"{self.file}.ts")).getroot():
for message in context.iterfind("message"):
translation = message.find("translation")
if translation.get("type") == "unfinished":
continue
translation_text = " ".join([t.text for t in translation.findall("numerusform")]) if message.get("numerus") == "yes" else translation.text
translation_text = " ".join([t.text for t in translation.findall("numerusform")]) if message.get("numerus") == "yes" else translation.text
if not translation_text:
continue
if not translation_text:
continue
words = set(translation_text.translate(str.maketrans('', '', string.punctuation + '%n')).lower().split())
bad_words_found = words & (banned_words - IGNORED_WORDS)
assert not bad_words_found, f"Bad language found in {name}: '{translation_text}'. Bad word(s): {', '.join(bad_words_found)}"
words = set(translation_text.translate(str.maketrans('', '', string.punctuation + '%n')).lower().split())
bad_words_found = words & (banned_words - IGNORED_WORDS)
assert not bad_words_found, f"Bad language found in {self.name}: '{translation_text}'. Bad word(s): {', '.join(bad_words_found)}"
if __name__ == "__main__":

Write Preview
Loading…
Cancel
Save