multilanguage: compile QM in scons (#25217)

* All in scons

* delete all this

* delete the qm files

* No need to check QM files in test_translations.py anymore

* readme

* add lupdate to third party

* fix

* one line

* update files_common

* readme imp

* add j flag

* add to path

* duplicate scons!

* update readme

* fix path

fix path

fix path

* no path
old-commit-hash: c528decd17

.gitignore

@@ -32,6 +32,7 @@ a.out
 *.class
 *.pyxbldc
 *.vcd
+*.qm
 config.json
 clcache
 compile_commands.json

release/files_common

@@ -303,7 +303,8 @@ selfdrive/ui/soundd/*.cc
 selfdrive/ui/soundd/*.h
 selfdrive/ui/soundd/soundd
 selfdrive/ui/soundd/.gitignore
-selfdrive/ui/translations/*
+selfdrive/ui/translations/*.ts
+selfdrive/ui/translations/languages.json
 
 selfdrive/ui/qt/*.cc
 selfdrive/ui/qt/*.h
@@ -421,6 +422,8 @@ third_party/acados/x86_64/**
 third_party/acados/larch64/**
 third_party/acados/include/**
 
+third_party/qt5/larch64/bin/**
+
 scripts/update_now.sh
 scripts/stop_updater.sh
 

selfdrive/ui/SConscript

@@ -63,6 +63,13 @@ if GetOption('test'):
   qt_env.Program('tests/test_translations', [asset_obj, 'tests/test_runner.cc', 'tests/test_translations.cc'] + qt_src, LIBS=qt_libs)
 
 
+# build translation files
+translation_sources = Glob("#selfdrive/ui/translations/*.ts", strings=True)
+translation_targets = [src.replace(".ts", ".qm") for src in translation_sources]
+lrelease = 'third_party/qt5/larch64/bin/lrelease' if arch == 'larch64' else 'lrelease'
+qt_env.Command(translation_targets, translation_sources, f"{lrelease} $SOURCES")
+
+
 # setup and factory resetter
 if GetOption('extras'):
   qt_env.Program("qt/setup/reset", ["qt/setup/reset.cc"], LIBS=qt_libs)

selfdrive/ui/tests/test_translations.py

@@ -23,8 +23,8 @@ class TestTranslations(unittest.TestCase):
     shutil.rmtree(TMP_TRANSLATIONS_DIR, ignore_errors=True)
 
   @staticmethod
-  def _read_translation_file(path, file, file_ext):
-    tr_file = os.path.join(path, f"{file}.{file_ext}")
+  def _read_translation_file(path, file):
+    tr_file = os.path.join(path, f"{file}.ts")
     with open(tr_file, "rb") as f:
       # fix relative path depth
       return f.read().replace(b"filename=\"../../", b"filename=\"../")
@@ -35,9 +35,8 @@ class TestTranslations(unittest.TestCase):
         if not len(file):
           self.skipTest(f"{name} translation has no defined file")
 
-        if not (os.path.exists(os.path.join(TRANSLATIONS_DIR, f"{file}.ts")) and
-                os.path.exists(os.path.join(TRANSLATIONS_DIR, f"{file}.qm"))):
-          self.fail(f"{name} is missing translation files, run selfdrive/ui/update_translations.py")
+        self.assertTrue(os.path.exists(os.path.join(TRANSLATIONS_DIR, f"{file}.ts")),
+                        f"{name} has no XML translation file, run selfdrive/ui/update_translations.py")
 
   def test_translations_updated(self):
     update_translations(translations_dir=TMP_TRANSLATIONS_DIR)
@@ -47,18 +46,14 @@ class TestTranslations(unittest.TestCase):
         if not len(file):
           self.skipTest(f"{name} translation has no defined file")
 
-        for file_ext in ["ts", "qm"]:
-          with self.subTest(file_ext=file_ext):
+        # caught by test_missing_translation_files
+        if not os.path.exists(os.path.join(TRANSLATIONS_DIR, f"{file}.ts")):
+          self.skipTest(f"{name} missing translation file")
 
-            # caught by test_missing_translation_files
-            if not os.path.exists(os.path.join(TRANSLATIONS_DIR, f"{file}.{file_ext}")):
-              self.skipTest(f"{name} missing translation file")
-
-            cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file, file_ext)
-            new_translations = self._read_translation_file(TMP_TRANSLATIONS_DIR, file, file_ext)
-
-            self.assertEqual(cur_translations, new_translations,
-                             f"{file} ({name}) {file_ext.upper()} translation file out of date. Run selfdrive/ui/update_translations.py to update the translation files")
+        cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file)
+        new_translations = self._read_translation_file(TMP_TRANSLATIONS_DIR, file)
+        self.assertEqual(cur_translations, new_translations,
+                         f"{file} ({name}) XML translation file out of date. Run selfdrive/ui/update_translations.py to update the translation files")
 
   @unittest.skip("Only test unfinished translations before going to release")
   def test_unfinished_translations(self):
@@ -67,7 +62,7 @@ class TestTranslations(unittest.TestCase):
         if not len(file):
           raise self.skipTest(f"{name} translation has no defined file")
 
-        cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file, "ts")
+        cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file)
         self.assertTrue(b"<translation type=\"unfinished\">" not in cur_translations,
                         f"{file} ({name}) translation file has unfinished translations. Finish translations or mark them as completed in Qt Linguist")
 

selfdrive/ui/translations/README.md

@@ -9,7 +9,7 @@ Before getting started, make sure you have set up the openpilot Ubuntu developme
 openpilot provides a few tools to help contributors manage their translations and to ensure quality. To get started:
 
 1. Add your new language to [languages.json](/selfdrive/ui/translations/languages.json) with the appropriate [language code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) and the localized language name (Simplified Chinese is `中文（繁體）`).
-2. Generate the translation file (`*.ts`):
+2. Generate the XML translation file (`*.ts`):
    ```shell
    selfdrive/ui/update_translations.py
    ```
@@ -17,22 +17,22 @@ openpilot provides a few tools to help contributors manage their translations an
    ```shell
    linguist selfdrive/ui/translations/your_language_file.ts
    ```
-4. Save your file and generate the compiled QM file used by the Qt UI:
+4. View your finished translations by compiling and starting the UI, then find it in the language selector:
    ```shell
-   selfdrive/ui/update_translations.py --release
+   scons -j$(nproc) selfdrive/ui && selfdrive/ui/ui
    ```
 
 ### Improving an Existing Language
 
-Follow the steps above, omitting steps 1. and 2. Any time you edit translations you'll want to make sure to compile them.
+Follow step 3. above, you can review existing translations and add missing ones. Once you're done, just open a pull request to openpilot.
 
 ### Updating the UI
 
-Any time you edit source code in the UI, you need to update and compile the translations to ensure the line numbers and contexts are up to date (last step above).
+Any time you edit source code in the UI, you need to update the translations to ensure the line numbers and contexts are up to date (first step above).
 
 ### Testing
 
-openpilot has a few unit tests to make sure all translations are up to date and that all strings are wrapped in a translation marker.
+openpilot has a few unit tests to make sure all translations are up to date and that all strings are wrapped in a translation marker. They are run in CI, but you can also run them locally.
 
 Tests translation files up to date:
 

selfdrive/ui/translations/main_ja.qm

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8cb8a9918fefe9bb2d047fd4b81063747857d5740f682ee98d8b95ba68a095ac
-size 20202

selfdrive/ui/translations/main_ko.qm

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fb7e26c65d5dee24c17532f9810a00e6a278f6f8f851bdd06418e4ed13fed213
-size 19439

selfdrive/ui/translations/main_zh-CHS.qm

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ddfc47a6fec5375c88f63275d02846cb7f4bf5431346cad895038a8705724ba3
-size 17931

selfdrive/ui/translations/main_zh-CHT.qm

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:846dcae2af3d8071cdcfd1a20a65e06b7e7105cb128bae8b8244f2a29fe59dc4
-size 17969

selfdrive/ui/update_translations.py

@@ -26,9 +26,6 @@ def update_translations(vanish=False, translations_dir=TRANSLATIONS_DIR):
     ret = os.system(args)
     assert ret == 0
 
-    ret = os.system(f"lrelease {tr_file}")
-    assert ret == 0
-
 
 if __name__ == "__main__":
   parser = argparse.ArgumentParser(description="Update translation files for UI",

third_party/qt5/larch64/bin/lrelease

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56ddfc4ead01eaf43cba41e2095ec4f52309c53a60ba9e351d8dbd900151228f
+size 546824

third_party/qt5/larch64/bin/lupdate

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7af679aa708031f6d19baabbd48e92d23462c6ff287caccdcb5c6324168d985d
+size 1095744