openpilot v0.4.0.1 tweaks

apk/external/.gitignore

@@ -0,0 +1,2 @@
+src/*
+out/*

apk/external/patcher.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python2.7
+
+import os
+import sys
+import glob
+import shutil
+import urllib2
+import hashlib
+import subprocess
+
+
+EXTERNAL_PATH = os.path.dirname(os.path.abspath(__file__))
+
+if os.path.exists("/init.qcom.rc"):
+  # android
+  APKPATCH = os.path.join(EXTERNAL_PATH, 'tools/apkpatch_android')
+  SIGNAPK = os.path.join(EXTERNAL_PATH, 'tools/signapk_android')
+else:
+  APKPATCH = os.path.join(EXTERNAL_PATH, 'tools/apkpatch')
+  SIGNAPK = os.path.join(EXTERNAL_PATH, 'tools/signapk')
+
+APKS = {
+  'com.waze': {
+    'src': 'https://apkcache.s3.amazonaws.com/com.waze_1021278.apk',
+    'src_sha256': 'f00957e93e2389f9e30502ac54994b98ac769314b0963c263d4e8baa625ab0c2',
+    'patch': 'com.waze.apkpatch',
+    'out_sha256': '9ec8b0ea3c78c666342865b1bfb66e368a3f5c911df2ad12835206ec8b19f444'
+  },
+  'com.spotify.music': {
+    'src': 'https://apkcache.s3.amazonaws.com/com.spotify.music_24382006.apk',
+    'src_sha256': '0610fea68ee7ba5f8e4e0732ad429d729dd6cbb8bc21222c4c99db6cb09fbff4',
+    'patch': 'com.spotify.music.apkpatch',
+    'out_sha256': '5a3d6f478c7e40403a98ccc8906d7e0ae12b06543b41f5df52149dd09c647c11'
+  },
+}
+
+def sha256_path(path):
+  with open(path, 'rb') as f:
+    return hashlib.sha256(f.read()).hexdigest()
+
+def remove(path):
+  try:
+    os.remove(path)
+  except OSError:
+    pass
+
+def process(download, patch):
+  # clean up any junk apks
+  for out_apk in glob.glob(os.path.join(EXTERNAL_PATH, 'out/*.apk')):
+    app = os.path.basename(out_apk)[:-4]
+    if app not in APKS:
+      print "remove junk", out_apk
+      remove(out_apk)
+
+  complete = True
+  for k,v in APKS.iteritems():
+    apk_path = os.path.join(EXTERNAL_PATH, 'out', k+'.apk')
+    print "checking", apk_path
+    if os.path.exists(apk_path) and sha256_path(apk_path) == v['out_sha256']:
+      # nothing to do
+      continue
+
+    complete = False
+
+    remove(apk_path)
+
+    src_path = os.path.join(EXTERNAL_PATH, 'src', v['src_sha256'])
+    if not os.path.exists(src_path) or sha256_path(src_path) != v['src_sha256']:
+      if not download:
+        continue
+
+      print "downloading", v['src'], "to", src_path
+      # download it
+      resp = urllib2.urlopen(v['src'])
+      data = resp.read()
+      with open(src_path, 'wb') as src_f:
+        src_f.write(data)
+
+      if sha256_path(src_path) != v['src_sha256']:
+        print "download was corrupted..."
+        continue
+
+    if not patch:
+      continue
+
+    # ignoring lots of TOCTTOU here...
+
+    apk_temp = "/tmp/"+k+".patched"
+    remove(apk_temp)
+    apk_temp2 = "/tmp/"+k+".signed"
+    remove(apk_temp2)
+
+    try:
+      print "patching", v['patch']
+      subprocess.check_call([APKPATCH, 'apply', src_path, apk_temp, os.path.join(EXTERNAL_PATH, v['patch'])])
+      print "signing", apk_temp
+      subprocess.check_call([SIGNAPK,
+        os.path.join(EXTERNAL_PATH, 'tools/certificate.pem'), os.path.join(EXTERNAL_PATH, 'tools/key.pk8'),
+        apk_temp, apk_temp2])
+
+      out_sha256 = sha256_path(apk_temp2) if os.path.exists(apk_temp2) else None
+
+      if out_sha256 == v['out_sha256']:
+        print "done", apk_path
+        shutil.move(apk_temp2, apk_path)
+      else:
+        print "patch was corrupted", apk_temp2, out_sha256
+    finally:
+      remove(apk_temp)
+      remove(apk_temp2)
+
+  return complete
+
+if __name__ == "__main__":
+  ret = True
+  if len(sys.argv) == 2 and sys.argv[1] == "download":
+    ret = process(True, False)
+  elif len(sys.argv) == 2 and sys.argv[1] == "patch":
+    ret = process(False, True)
+  else:
+    ret = process(True, True)
+  sys.exit(0 if ret else 1)

apk/external/tools/apkpatch_android

@@ -0,0 +1,7 @@
+#!/system/bin/sh
+
+DIR="$(cd "$(dirname "$0")" && pwd)"
+
+export LD_LIBRARY_PATH=/system/lib64
+export CLASSPATH="$DIR"/ApkPatch.android.jar
+exec app_process "$DIR" ApkPatch "$@"

apk/external/tools/certificate.pem

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICtTCCAh4CCQDm79UqF+Dc5zANBgkqhkiG9w0BAQUFADCBnjELMAkGA1UEBhMC
+SUQxEzARBgNVBAgTCkphd2EgQmFyYXQxEDAOBgNVBAcTB0JhbmR1bmcxEjAQBgNV
+BAoTCUxvbmRhdGlnYTETMBEGA1UECxMKQW5kcm9pZERldjEaMBgGA1UEAxMRTG9y
+ZW5zaXVzIFcuIEwuIFQxIzAhBgkqhkiG9w0BCQEWFGxvcmVuekBsb25kYXRpZ2Eu
+bmV0MB4XDTEwMDUwNTA5MjEzOFoXDTEzMDEyODA5MjEzOFowgZ4xCzAJBgNVBAYT
+AklEMRMwEQYDVQQIEwpKYXdhIEJhcmF0MRAwDgYDVQQHEwdCYW5kdW5nMRIwEAYD
+VQQKEwlMb25kYXRpZ2ExEzARBgNVBAsTCkFuZHJvaWREZXYxGjAYBgNVBAMTEUxv
+cmVuc2l1cyBXLiBMLiBUMSMwIQYJKoZIhvcNAQkBFhRsb3JlbnpAbG9uZGF0aWdh
+Lm5ldDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAy2oWtbdVXMHGiS6cA3qi
+3VfZt5Vz9jTlux+TEcGx5h18ZKwclyo+z2B0L/p5bYdnrTdFEiD7IxvX+h3lu0JV
+B9rdXZdyrzXNOw5YFrsn2k7hKvB8KEBaga1gZEwodlc6N14H3FbZdZkIA9V716Pu
+e5CWBZ2VqU03lUJmKnpH8c8CAwEAATANBgkqhkiG9w0BAQUFAAOBgQBpNgXh8dw9
+uMjZxzLUXovV5ptHd61jAcZlQlffqPsz6/2QNfIShVdGH9jkm0IudfKkbvvOKive
+a77t9c4sDh2Sat2L/rx6BfTuS1+y9wFr1Ee8Rrr7wGHhRkx2qqGrXGVWqXn8aE3E
+P6e7BTPF0ibS+tG8cdDPEisqGFxw36nTNQ==
+-----END CERTIFICATE-----

apk/external/tools/signapk_android

@@ -0,0 +1,7 @@
+#!/system/bin/sh
+
+DIR="$(cd "$(dirname "$0")" && pwd)"
+
+export LD_LIBRARY_PATH=/system/lib64
+export CLASSPATH="$DIR"/signapk.android.jar
+exec app_process "$DIR" com.android.signapk.SignApk "$@"

launch_openpilot.sh

@@ -8,12 +8,6 @@ function launch {
      exec "${BASH_SOURCE[0]}"
   fi
 
-  # check if NEOS update is required
-  while [ "$(cat /VERSION)" -lt 4 ] && [ ! -e /data/media/0/noupdate ]; do
-    curl -o /tmp/updater https://neos.comma.ai/updater && chmod +x /tmp/updater && /tmp/updater
-    sleep 10
-  done
-
   # no cpu rationing for now
   echo 0-3 > /dev/cpuset/background/cpus
   echo 0-3 > /dev/cpuset/system-background/cpus
@@ -26,6 +20,12 @@ function launch {
   until ping -W 1 -c 1 8.8.8.8; do sleep 1; done
   kill $spin_pid
 
+  # check if NEOS update is required
+  while [ "$(cat /VERSION)" -lt 4 ] && [ ! -e /data/media/0/noupdate ]; do
+    curl -o /tmp/updater https://neos.comma.ai/updater && chmod +x /tmp/updater && /tmp/updater
+    sleep 10
+  done
+
   export PYTHONPATH="$PWD"
 
   # start manager

selfdrive/debug/get_fingerprint.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+# simple script to get a vehicle fingerprint.
+# keep this script running for few seconds: some messages are published every few seconds
+
+# Instructions:
+# - connect to a Panda
+# - run selfdrive/boardd/boardd
+# - launching this script
+# - since some messages are published at low frequency, keep this script running for few seconds, 
+#   until all messages are received at least once
+
+import zmq
+import selfdrive.messaging as messaging
+from selfdrive.services import service_list
+
+context = zmq.Context()
+logcan = messaging.sub_sock(context, service_list['can'].port)
+msgs = {}
+while True:
+  lc = messaging.recv_sock(logcan, True)
+  for c in lc.can:
+    if c.src == 0:
+      msgs[c.address] = len(c.dat)
+
+  fingerprint = ', '.join("%d: %d" % v for v in sorted(msgs.items()))
+
+  print "number of messages:", len(msgs)
+  print "fingerprint", fingerprint

selfdrive/manager.py

@@ -1,4 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
+
 import os
 import sys
 import time
@@ -40,6 +41,7 @@ if __name__ == "__main__":
 
     os._exit(os.wait()[1])
 
+import glob
 import shutil
 import hashlib
 import importlib
@@ -54,6 +56,7 @@ os.environ['BASEDIR'] = BASEDIR
 import usb1
 import zmq
 from setproctitle import setproctitle
+from smbus2 import SMBus
 
 from common.params import Params
 from common.realtime import sec_since_boot
@@ -68,6 +71,8 @@ import selfdrive.crash as crash
 
 from selfdrive.loggerd.config import ROOT
 
+EON = os.path.exists("/EON")
+
 # comment out anything you don't want to run
 managed_processes = {
   "uploader": "selfdrive.loggerd.uploader",
@@ -272,10 +277,6 @@ def system(cmd):
       output=e.output[-1024:],
       returncode=e.returncode)
 
-EON = os.path.exists("/EON")
-if EON:
-  from smbus2 import SMBus
-
 def setup_eon_fan():
   if not EON:
     return
@@ -530,34 +531,50 @@ def install_apk(path):
   return ret == 0
 
 def update_apks():
+  # patch apks
+  if os.getenv("PREPAREONLY"):
+    # assume we have internet, download too
+    patched = subprocess.call([os.path.join(BASEDIR, "apk/external/patcher.py")])
+  else:
+    patched = subprocess.call([os.path.join(BASEDIR, "apk/external/patcher.py"), "patch"])
+  cloudlog.info("patcher: %r" % (patched,))
+
   # install apks
   installed = get_installed_apks()
-  for app in os.listdir(os.path.join(BASEDIR, "apk/")):
+
-    if ".apk" in app:
+  install_apks = (glob.glob(os.path.join(BASEDIR, "apk/*.apk"))
-      app = app.split(".apk")[0]
+                  + glob.glob(os.path.join(BASEDIR, "apk/external/out/*.apk")))
-      if app not in installed:
+  for apk in install_apks:
-        installed[app] = None
+    app = os.path.basename(apk)[:-4]
+    if app not in installed:
+      installed[app] = None
+
   cloudlog.info("installed apks %s" % (str(installed), ))
 
   for app in installed.iterkeys():
-    apk_path = os.path.join(BASEDIR, "apk/"+app+".apk")
-    if os.path.isfile(apk_path):
-      h1 = hashlib.sha1(open(apk_path).read()).hexdigest()
-      h2 = None
-      if installed[app] is not None:
-        h2 = hashlib.sha1(open(installed[app]).read()).hexdigest()
-        cloudlog.info("comparing version of %s  %s vs %s" % (app, h1, h2))
-
-      if h2 is None or h1 != h2:
-        cloudlog.info("installing %s" % app)
     
+    apk_path = os.path.join(BASEDIR, "apk/"+app+".apk")
+    if not os.path.exists(apk_path):
+      apk_path = os.path.join(BASEDIR, "apk/external/out/"+app+".apk")
+    if not os.path.exists(apk_path):
+      continue
+
+    h1 = hashlib.sha1(open(apk_path).read()).hexdigest()
+    h2 = None
+    if installed[app] is not None:
+      h2 = hashlib.sha1(open(installed[app]).read()).hexdigest()
+      cloudlog.info("comparing version of %s  %s vs %s" % (app, h1, h2))
+
+    if h2 is None or h1 != h2:
+      cloudlog.info("installing %s" % app)
+
+      success = install_apk(apk_path)
+      if not success:
+        cloudlog.info("needing to uninstall %s" % app)
+        system("pm uninstall %s" % app)
         success = install_apk(apk_path)
-        if not success:
-          cloudlog.info("needing to uninstall %s" % app)
-          system("pm uninstall %s" % app)
-          success = install_apk(apk_path)
 
-        assert success
+      assert success
 
 def manager_update():
   update_apks()

selfdrive/updated.py

@@ -6,6 +6,9 @@ import os
 import time
 import subprocess
 
+from common.basedir import BASEDIR
+from selfdrive.swaglog import cloudlog
+
 def main(gctx=None):
   while True:
     # try network
@@ -14,12 +17,16 @@ def main(gctx=None):
       time.sleep(60)
       continue
 
-    # try fetch
     r = subprocess.call(["nice", "-n", "19", "git", "fetch", "--depth=1"])
+    cloudlog.info("git fetch: %r", r)
     if r:
       time.sleep(60)
       continue
 
+    # download apks
+    r = subprocess.call(["nice", "-n", "19", os.path.join(BASEDIR, "apk/external/patcher.py"), "download"])
+    cloudlog.info("patcher download: %r", r)
+
     time.sleep(60*60*3)
 
 if __name__ == "__main__":