micd: scale sound volume with ambient noise level (#26399)

* test changing sound volume

* test changing sound volume

* create system/hardware/pc/hardware.h

* implement Hardware::set_volume using pactl

* soundd: use Hardware::set_volume

* add sounddevice dependency

* sounddevice example

* simple micd

* cleanup

* remove this

* fix process config

* add to release files

* hardware: get sound input device

* no more offroad

* debug

* calculate volume from all measurements since last update

* use microphone noise level to update sound volume

* fix scale

* mute microphone during alerts

* log raw noise level

* hardware: reduce tici min volume

* improve scale

* add package

* clear measurements on muted

* change default to min volume and respond quicker

* fixes

Co-authored-by: Shane Smiskol <shane@smiskol.com>

* logarithmic scaling

* fix

* respond quicker

* fixes

* tweak scaling

* specify default device

* Revert "hardware: get sound input device"

This reverts commit 50f594f7a3bab005023482bc793147a8c8dae5d7.

* tuning

* forgot to update submaster

* tuning

* don't mute microphone, and clip measurement

* remove submaster

* fixes

* tuning

* implement Hardware::set_volume using pactl

* Revert "test changing sound volume"

This reverts commit 4bbd870746ec86d1c9871a6175def96cf7f751a6.

* draft

* draft

* calculate sound pressure level in dB

* fix setting

* faster filter

* start at initial value

* don't run command in background

* pactl: use default sink

* use sound pressure db

* tuning

* bump up max volume threshold

* update filter slower

* fix divide by zero

* bump cereal

Co-authored-by: Shane Smiskol <shane@smiskol.com>
old-commit-hash: 108ff15f5d

cereal

@@ -1 +1 @@
-Subproject commit 19a0c46b71150a8dabc5644eb24f261feee45b9c
+Subproject commit dbc9846ac9c9e735ee2f4a281ce079cfff7ea285

poetry.lock

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:616bfb7e1761adc9f7a27571d1bde023727147971f5426207ee09fab592f97bc
+oid sha256:88495068650500a25401ed83cf3f2c4eb2216a1c3f8c2dcda0fa33bfd801b85f
-size 561251
+size 560742

pyproject.toml

@@ -49,6 +49,7 @@ sentry-sdk = "^1.6.0"
 setproctitle = "^1.2.3"
 six = "^1.16.0"
 smbus2 = "^0.4.2"
+sounddevice = "^0.4.5"
 sympy = "^1.10.1"
 timezonefinder = "^6.0.1"
 tqdm = "^4.64.0"

release/files_common

@@ -71,6 +71,7 @@ selfdrive/rtshield.py
 selfdrive/statsd.py
 
 system/logmessaged.py
+system/micd.py
 system/swaglog.py
 system/version.py
 

selfdrive/manager/process_config.py

@@ -24,6 +24,7 @@ procs = [
   NativeProcess("logcatd", "system/logcatd", ["./logcatd"]),
   NativeProcess("proclogd", "system/proclogd", ["./proclogd"]),
   PythonProcess("logmessaged", "system.logmessaged", offroad=True),
+  PythonProcess("micd", "system.micd"),
   PythonProcess("timezoned", "system.timezoned", enabled=not PC, offroad=True),
 
   DaemonProcess("manage_athenad", "selfdrive.athena.manage_athenad", "AthenadPid"),

selfdrive/ui/soundd/sound.cc

@@ -12,7 +12,7 @@
 // TODO: detect when we can't play sounds
 // TODO: detect when we can't display the UI
 
-Sound::Sound(QObject *parent) : sm({"carState", "controlsState", "deviceState"}) {
+Sound::Sound(QObject *parent) : sm({"controlsState", "deviceState", "microphone"}) {
   qInfo() << "default audio device: " << QAudioDeviceInfo::defaultOutputDevice().deviceName();
 
   for (auto &[alert, fn, loops] : sound_list) {
@@ -47,8 +47,8 @@ void Sound::update() {
   }
 
   // scale volume with speed
-  if (sm.updated("carState")) {
+  if (sm.updated("microphone")) {
-    float volume = util::map_val(sm["carState"].getCarState().getVEgo(), 11.f, 20.f, 0.f, 1.f);
+    float volume = util::map_val(sm["microphone"].getMicrophone().getFilteredSoundPressureDb(), 58.f, 77.f, 0.f, 1.f);
     volume = QAudio::convertVolume(volume, QAudio::LogarithmicVolumeScale, QAudio::LinearVolumeScale);
     Hardware::set_volume(volume);
   }

system/hardware/tici/hardware.h

@@ -10,7 +10,7 @@
 class HardwareTici : public HardwareNone {
 public:
   static constexpr float MAX_VOLUME = 0.9;
-  static constexpr float MIN_VOLUME = 0.2;
+  static constexpr float MIN_VOLUME = 0.1;
   static bool TICI() { return true; }
   static bool AGNOS() { return true; }
   static std::string get_os_version() {

system/micd.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+import sounddevice as sd
+import numpy as np
+
+from cereal import messaging
+from common.filter_simple import FirstOrderFilter
+from common.realtime import Ratekeeper
+from system.swaglog import cloudlog
+
+RATE = 10
+DT_MIC = 1. / RATE
+REFERENCE_SPL = 2 * 10 ** -5  # newtons/m^2
+
+
+class Mic:
+  def __init__(self, pm):
+    self.pm = pm
+    self.rk = Ratekeeper(RATE)
+
+    self.measurements = np.empty(0)
+    self.spl_filter = FirstOrderFilter(0, 4, DT_MIC, initialized=False)
+
+  def update(self):
+    # self.measurements contains amplitudes from -1 to 1 which we use to
+    # calculate an uncalibrated sound pressure level
+    if len(self.measurements) > 0:
+      # https://www.engineeringtoolbox.com/sound-pressure-d_711.html
+      sound_pressure = np.sqrt(np.mean(self.measurements ** 2))  # RMS of amplitudes
+      sound_pressure_level = 20 * np.log10(sound_pressure / REFERENCE_SPL) if sound_pressure > 0 else 0  # dB
+      self.spl_filter.update(sound_pressure_level)
+    else:
+      sound_pressure = 0
+      sound_pressure_level = 0
+
+    self.measurements = np.empty(0)
+
+    msg = messaging.new_message('microphone')
+    msg.microphone.soundPressure = float(sound_pressure)
+    msg.microphone.soundPressureDb = float(sound_pressure_level)
+    msg.microphone.filteredSoundPressureDb = float(self.spl_filter.x)
+
+    self.pm.send('microphone', msg)
+    self.rk.keep_time()
+
+  def callback(self, indata, frames, time, status):
+    self.measurements = np.concatenate((self.measurements, indata[:, 0]))
+
+  def micd_thread(self, device=None):
+    if device is None:
+      device = "sysdefault"
+
+    with sd.InputStream(device=device, channels=1, samplerate=44100, callback=self.callback) as stream:
+      cloudlog.info(f"micd stream started: {stream.samplerate=} {stream.channels=} {stream.dtype=} {stream.device=}")
+      while True:
+        self.update()
+
+
+def main(pm=None, sm=None):
+  if pm is None:
+    pm = messaging.PubMaster(['microphone'])
+
+  mic = Mic(pm)
+  mic.micd_thread()
+
+
+if __name__ == "__main__":
+  main()

tools/ubuntu_setup.sh

@@ -56,6 +56,7 @@ function install_ubuntu_common_requirements() {
     libomp-dev \
     libopencv-dev \
     libpng16-16 \
+    libportaudio2 \
     libssl-dev \
     libsqlite3-dev \
     libusb-1.0-0-dev \