manager runs on Mac, and other openpilot for PC fixes (#1037)

* use the openpilot/persist directory on PC

* manager runs on mac

* sim runs w/o carla

* fix params location in test

* that rmtree can fail and it's okay

* refactor params clear functionality

* set PARAMS_PATH

old-commit-hash: c42e2ecc50

common/android.py

@@ -92,6 +92,8 @@ def parse_service_call_bytes(ret):
 
 def get_network_type(NetworkType):
   wifi_check = parse_service_call_string(service_call(["connectivity", "2"]))
+  if wifi_check is None:
+    return NetworkType.none
   if 'WIFI' in wifi_check:
     return NetworkType.wifi
   else:

common/api/__init__.py

@@ -1,13 +1,13 @@
 import jwt
 import requests
 from datetime import datetime, timedelta
-
+from common.basedir import PERSIST
 from selfdrive.version import version
 
 class Api():
   def __init__(self, dongle_id):
     self.dongle_id = dongle_id
-    with open('/persist/comma/id_rsa') as f:
+    with open(PERSIST+'/comma/id_rsa') as f:
       self.private_key = f.read()
 
   def get(self, *args, **kwargs):

common/basedir.py

@@ -1,4 +1,11 @@
 import os
 BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))
 
+from common.android import ANDROID
+if ANDROID:
+  PERSIST = "/persist"
+  PARAMS = "/data/params"
+else:
+  PERSIST = os.path.join(BASEDIR, "persist")
+  PARAMS = os.path.join(BASEDIR, "persist", "params")
 

common/ffi_wrapper.py

@@ -2,8 +2,14 @@ import os
 import sys
 import fcntl
 import hashlib
+import platform
 from cffi import FFI
 
+def suffix():
+  if platform.system() == "Darwin":
+    return ".dylib"
+  else:
+    return ".so"
 
 def ffi_wrap(name, c_code, c_header, tmpdir="/tmp/ccache", cflags="", libraries=None):
   if libraries is None:

common/params.py

@@ -29,7 +29,7 @@ import fcntl
 import tempfile
 import threading
 from enum import Enum
-
+from common.basedir import PARAMS
 
 def mkdirs_exists_ok(path):
   try:
@@ -320,7 +320,7 @@ def write_db(params_path, key, value):
     lock.release()
 
 class Params():
-  def __init__(self, db='/data/params'):
+  def __init__(self, db=PARAMS):
     self.db = db
 
     # create the database if it doesn't exist...
@@ -328,6 +328,11 @@ class Params():
       with self.transaction(write=True):
         pass
 
+  def clear_all(self):
+    shutil.rmtree(self.db, ignore_errors=True)
+    with self.transaction(write=True):
+      pass
+
   def transaction(self, write=False):
     if write:
       return DBWriter(self.db)

selfdrive/athena/athenad.py

@@ -19,6 +19,7 @@ from selfdrive.loggerd.config import ROOT
 
 import cereal.messaging as messaging
 from common import android
+from common.basedir import PERSIST
 from common.api import Api
 from common.params import Params
 from cereal.services import service_list
@@ -188,10 +189,10 @@ def startLocalProxy(global_end_event, remote_ws_uri, local_port):
 
 @dispatcher.add_method
 def getPublicKey():
-  if not os.path.isfile('/persist/comma/id_rsa.pub'):
+  if not os.path.isfile(PERSIST+'/comma/id_rsa.pub'):
     return None
 
-  with open('/persist/comma/id_rsa.pub', 'r') as f:
+  with open(PERSIST+'/comma/id_rsa.pub', 'r') as f:
     return f.read()
 
 @dispatcher.add_method

selfdrive/controls/lib/cluster/fastcluster_py.py

@@ -2,9 +2,10 @@ import os
 import numpy as np
 
 from cffi import FFI
+from common.ffi_wrapper import suffix
 
 cluster_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)))
-cluster_fn = os.path.join(cluster_dir, "libfastcluster.so")
+cluster_fn = os.path.join(cluster_dir, "libfastcluster"+suffix())
 
 ffi = FFI()
 ffi.cdef("""

selfdrive/controls/lib/lateral_mpc/libmpc_py.py

@@ -1,9 +1,10 @@
 import os
 
 from cffi import FFI
+from common.ffi_wrapper import suffix
 
 mpc_dir = os.path.dirname(os.path.abspath(__file__))
-libmpc_fn = os.path.join(mpc_dir, "libmpc.so")
+libmpc_fn = os.path.join(mpc_dir, "libmpc"+suffix())
 
 ffi = FFI()
 ffi.cdef("""

selfdrive/controls/lib/longitudinal_mpc/libmpc_py.py

@@ -1,11 +1,12 @@
 import os
 
 from cffi import FFI
+from common.ffi_wrapper import suffix
 
 mpc_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)))
 
 def _get_libmpc(mpc_id):
-    libmpc_fn = os.path.join(mpc_dir, "libmpc%d.so" % mpc_id)
+    libmpc_fn = os.path.join(mpc_dir, "libmpc%d%s" % (mpc_id, suffix()))
 
     ffi = FFI()
     ffi.cdef("""

selfdrive/manager.py

@@ -9,7 +9,7 @@ import shutil
 import subprocess
 import datetime
 
-from common.basedir import BASEDIR
+from common.basedir import BASEDIR, PARAMS
 from common.android import ANDROID
 sys.path.append(os.path.join(BASEDIR, "pyextra"))
 os.environ['BASEDIR'] = BASEDIR
@@ -22,6 +22,8 @@ try:
   os.mkdir("/dev/shm")
 except FileExistsError:
   pass
+except PermissionError:
+  print("WARNING: failed to make /dev/shm")
 
 if ANDROID:
   os.chmod("/dev/shm", 0o777)
@@ -365,6 +367,7 @@ def manager_init(should_register=True):
     pass
 
   # ensure shared libraries are readable by apks
+  if ANDROID:
     os.chmod(BASEDIR, 0o755)
     os.chmod(os.path.join(BASEDIR, "cereal"), 0o755)
     os.chmod(os.path.join(BASEDIR, "cereal", "libmessaging_shared.so"), 0o755)
@@ -454,6 +457,8 @@ def uninstall():
   android.reboot(reason="recovery")
 
 def main():
+  os.environ['PARAMS_PATH'] = PARAMS
+
   # the flippening!
   os.system('LD_LIBRARY_PATH="" content insert --uri content://settings/system --bind name:s:user_rotation --bind value:i:1')
 

selfdrive/registration.py

@@ -8,6 +8,7 @@ from common.android import get_imei, get_serial, get_subscriber_info
 from common.api import api_get
 from common.params import Params
 from common.file_helpers import mkdirs_exists_ok
+from common.basedir import PERSIST
 
 def register():
   params = Params()
@@ -22,24 +23,24 @@ def register():
   # create a key for auth
   # your private key is kept on your device persist partition and never sent to our servers
   # do not erase your persist partition
-  if not os.path.isfile("/persist/comma/id_rsa.pub"):
+  if not os.path.isfile(PERSIST+"/comma/id_rsa.pub"):
     cloudlog.warning("generating your personal RSA key")
-    mkdirs_exists_ok("/persist/comma")
-    assert os.system("openssl genrsa -out /persist/comma/id_rsa.tmp 2048") == 0
-    assert os.system("openssl rsa -in /persist/comma/id_rsa.tmp -pubout -out /persist/comma/id_rsa.tmp.pub") == 0
-    os.rename("/persist/comma/id_rsa.tmp", "/persist/comma/id_rsa")
-    os.rename("/persist/comma/id_rsa.tmp.pub", "/persist/comma/id_rsa.pub")
+    mkdirs_exists_ok(PERSIST+"/comma")
+    assert os.system("openssl genrsa -out "+PERSIST+"/comma/id_rsa.tmp 2048") == 0
+    assert os.system("openssl rsa -in "+PERSIST+"/comma/id_rsa.tmp -pubout -out "+PERSIST+"/comma/id_rsa.tmp.pub") == 0
+    os.rename(PERSIST+"/comma/id_rsa.tmp", PERSIST+"/comma/id_rsa")
+    os.rename(PERSIST+"/comma/id_rsa.tmp.pub", PERSIST+"/comma/id_rsa.pub")
 
   # make key readable by app users (ai.comma.plus.offroad)
-  os.chmod('/persist/comma/', 0o755)
-  os.chmod('/persist/comma/id_rsa', 0o744)
+  os.chmod(PERSIST+'/comma/', 0o755)
+  os.chmod(PERSIST+'/comma/id_rsa', 0o744)
 
   dongle_id, access_token = params.get("DongleId", encoding='utf8'), params.get("AccessToken", encoding='utf8')
-  public_key = open("/persist/comma/id_rsa.pub").read()
+  public_key = open(PERSIST+"/comma/id_rsa.pub").read()
 
   # create registration token
   # in the future, this key will make JWTs directly
-  private_key = open("/persist/comma/id_rsa").read()
+  private_key = open(PERSIST+"/comma/id_rsa").read()
 
   # late import
   import jwt

selfdrive/test/longitudinal_maneuvers/test_longitudinal.py

@@ -4,7 +4,6 @@ os.environ['OLD_CAN'] = '1'
 os.environ['NOCRASH'] = '1'
 
 import unittest
-import shutil
 import matplotlib
 matplotlib.use('svg')
 
@@ -328,8 +327,8 @@ class LongitudinalControl(unittest.TestCase):
 
     setup_output()
 
-    shutil.rmtree('/data/params', ignore_errors=True)
     params = Params()
+    params.clear_all()
     params.put("Passive", "1" if os.getenv("PASSIVE") else "0")
     params.put("OpenpilotEnabledToggle", "1")
     params.put("CommunityFeaturesToggle", "1")

selfdrive/test/process_replay/process_replay.py

@@ -3,7 +3,6 @@ import os
 import sys
 import threading
 import importlib
-import shutil
 
 if "CI" in os.environ:
   tqdm = lambda x: x
@@ -260,8 +259,8 @@ def replay_process(cfg, lr):
   all_msgs = sorted(lr, key=lambda msg: msg.logMonoTime)
   pub_msgs = [msg for msg in all_msgs if msg.which() in list(cfg.pub_sub.keys())]
 
-  shutil.rmtree('/data/params', ignore_errors=True)
   params = Params()
+  params.clear_all()
   params.manager_start()
   params.put("OpenpilotEnabledToggle", "1")
   params.put("Passive", "0")

selfdrive/test/test_car_models.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-import shutil
 import time
 import os
 import sys
@@ -487,8 +486,8 @@ if __name__ == "__main__":
     elif "UNLOGGER_PATH" not in os.environ:
       continue
 
-    shutil.rmtree('/data/params')
     params = Params()
+    params.clear_all()
     params.manager_start()
     params.put("OpenpilotEnabledToggle", "1")
     params.put("CommunityFeaturesToggle", "1")

tools/sim/controller.py

@@ -5,7 +5,6 @@ import math
 import atexit
 import numpy as np
 import threading
-import carla
 import random
 import cereal.messaging as messaging
 from common.params import Params
@@ -62,15 +61,16 @@ def health_function():
     rk.keep_time()
 
 def fake_driver_monitoring():
-  pm = messaging.PubMaster(['driverMonitoring'])
+  pm = messaging.PubMaster(['driverState'])
   while 1:
     dat = messaging.new_message()
-    dat.init('driverMonitoring')
-    dat.driverMonitoring.faceProb = 1.0
-    pm.send('driverMonitoring', dat)
+    dat.init('driverState')
+    dat.driverState.faceProb = 1.0
+    pm.send('driverState', dat)
     time.sleep(0.1)
 
 def go():
+  import carla
   client = carla.Client("127.0.0.1", 2000)
   client.set_timeout(5.0)
   world = client.load_world('Town03')
@@ -123,8 +123,6 @@ def go():
     print("done")
   atexit.register(destroy)
 
-  threading.Thread(target=health_function).start()
-  threading.Thread(target=fake_driver_monitoring).start()
 
   # can loop
   sendcan = messaging.sub_sock('sendcan')
@@ -151,5 +149,16 @@ if __name__ == "__main__":
   params.put("HasAcceptedTerms", terms_version)
   params.put("CompletedTrainingVersion", training_version)
 
+  threading.Thread(target=health_function).start()
+  threading.Thread(target=fake_driver_monitoring).start()
+
+  # no carla, still run
+  try:
+    import carla
+  except ImportError:
+    print("WARNING: NO CARLA")
+    while 1:
+      time.sleep(1)
+    
   go()