micd: scale sound volume with ambient noise level (#26399)

* test changing sound volume

* test changing sound volume

* create system/hardware/pc/hardware.h

* implement Hardware::set_volume using pactl

* soundd: use Hardware::set_volume

* add sounddevice dependency

* sounddevice example

* simple micd

* cleanup

* remove this

* fix process config

* add to release files

* hardware: get sound input device

* no more offroad

* debug

* calculate volume from all measurements since last update

* use microphone noise level to update sound volume

* fix scale

* mute microphone during alerts

* log raw noise level

* hardware: reduce tici min volume

* improve scale

* add package

* clear measurements on muted

* change default to min volume and respond quicker

* fixes

Co-authored-by: Shane Smiskol <shane@smiskol.com>

* logarithmic scaling

* fix

* respond quicker

* fixes

* tweak scaling

* specify default device

* Revert "hardware: get sound input device"

This reverts commit 50f594f7a3bab005023482bc793147a8c8dae5d7.

* tuning

* forgot to update submaster

* tuning

* don't mute microphone, and clip measurement

* remove submaster

* fixes

* tuning

* implement Hardware::set_volume using pactl

* Revert "test changing sound volume"

This reverts commit 4bbd870746.

* draft

* draft

* calculate sound pressure level in dB

* fix setting

* faster filter

* start at initial value

* don't run command in background

* pactl: use default sink

* use sound pressure db

* tuning

* bump up max volume threshold

* update filter slower

* fix divide by zero

* bump cereal

Co-authored-by: Shane Smiskol <shane@smiskol.com>

cereal

@@ -1 +1 @@
-Subproject commit 19a0c46b71150a8dabc5644eb24f261feee45b9c
+Subproject commit dbc9846ac9c9e735ee2f4a281ce079cfff7ea285

poetry.lock

@@ -3696,6 +3696,20 @@ category = "dev"
 optional = false
 python-versions = "*"
 
+[[package]]
+name = "sounddevice"
+version = "0.4.5"
+description = "Play and Record Sound with Python"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+CFFI = ">=1.0"
+
+[package.extras]
+numpy = ["NumPy"]
+
 [[package]]
 name = "soupsieve"
 version = "2.3.2.post1"
@@ -4386,7 +4400,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "1.1"
 python-versions = "~3.8"
-content-hash = "d2854112975a9d83a9540175b2d430487e40e0292d48a1ba6c591db60a08c136"
+content-hash = "a41b1d669ec4f0a94abe12af8a9a7d709b2bb56e9c83aa87801cdf5d9af074f8"
 
 [metadata.files]
 adal = [
@@ -5392,7 +5406,6 @@ gevent = [
     {file = "gevent-22.10.1-cp310-cp310-win_amd64.whl", hash = "sha256:d2ea4ce36c09355379bc038be2bd50118f97d2eb6381b7096de4d05aa4c3e241"},
     {file = "gevent-22.10.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e73c9f71aa2a6795ecbec9b57282b002375e863e283558feb87b62840c8c1ac"},
     {file = "gevent-22.10.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5bc3758f0dc95007c1780d28a9fd2150416a79c50f308f62a674d78a845ea1b9"},
-    {file = "gevent-22.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03c10ca0beeab0c6be516030471ea630447ddd1f649d3335e5b162097cd4130a"},
     {file = "gevent-22.10.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fe2c0ff095171c49f78f1d4e6dc89fa58253783c7b6dccab9f1d76e2ee391f10"},
     {file = "gevent-22.10.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d18fcc324f39a3b21795022eb47c7752d6e4f4ed89d8cca41f1cc604553265b3"},
     {file = "gevent-22.10.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06ea39c70ce166c4a1d4386c7fae96cb8d84ad799527b3378406051104d15443"},
@@ -6563,11 +6576,6 @@ pillow-avif-plugin = [
     {file = "pillow_avif_plugin-1.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:017e5e52cb4320414e8ce3e2089eae2cb87c22c73ff6012b17ae326fc5753b20"},
     {file = "pillow_avif_plugin-1.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2a57136d4866de5dc80cfb24d66655955fbdd87acf1d11d88c8dc2ab41023e46"},
     {file = "pillow_avif_plugin-1.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:f339511d0fccb69e3a5e3af39f8fe6700b0a07279015006ea56f8f49e7fecff4"},
-    {file = "pillow_avif_plugin-1.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:05e821ecd90bb0b8d2dc7610625372cc47de9cb893d09662528bad572f669d1c"},
-    {file = "pillow_avif_plugin-1.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d33886a5f9796fe9a8a3bc25ccfdeba7db119adb50b7004f1928a14b07d0213a"},
-    {file = "pillow_avif_plugin-1.2.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:75b7ed186c2f740dd26e556f6a966c59a170b70263e429a2c81920fe444da8a7"},
-    {file = "pillow_avif_plugin-1.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a11aef6b79078b8dad25c928e5871c146ab94424472851d5bf539ba62abde9ac"},
-    {file = "pillow_avif_plugin-1.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:10696c536d68a14cefea3b98edb8d5a7ae29e8e07458f1d59c5d1cd780a8bf2a"},
     {file = "pillow_avif_plugin-1.2.2-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:1a7291d6a5fb7336e72685a31d193e0b3a6bee9986c9ac4d8bd4b68dbe6d4f7f"},
     {file = "pillow_avif_plugin-1.2.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:14b9c5dbf237e7dc12f69819ea181a457b3bd4f59f8cd71d028d3635fd3bcab4"},
     {file = "pillow_avif_plugin-1.2.2-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:799cbfbeee831332d280c80df9ce16b5c3b1224c318264e97e89df8da32e870e"},
@@ -6956,13 +6964,11 @@ pyprof2calltree = [
 ]
 pyproj = [
     {file = "pyproj-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f343725566267a296b09ee7e591894f1fdc90f84f8ad5ec476aeb53bd4479c07"},
-    {file = "pyproj-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5816807ca0bdc7256558770c6206a6783a3f02bcf844f94ee245f197bb5f7285"},
     {file = "pyproj-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7e609903572a56cca758bbaee5c1663c3e829ddce5eec4f368e68277e37022b"},
     {file = "pyproj-3.4.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4fd425ee8b6781c249c7adb7daa2e6c41ce573afabe4f380f5eecd913b56a3be"},
     {file = "pyproj-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:954b068136518b3174d0a99448056e97af62b63392a95c420894f7de2229dae6"},
     {file = "pyproj-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:4a23d84c5ffc383c7d9f0bde3a06fc1f6697b1b96725597f8f01e7b4bef0a2b5"},
     {file = "pyproj-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1f9c100fd0fd80edbc7e4daa303600a8cbef6f0de43d005617acb38276b88dc0"},
-    {file = "pyproj-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aa5171f700f174777a9e9ed8f4655583243967c0f9cf2c90e3f54e54ff740134"},
     {file = "pyproj-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a496d9057b2128db9d733e66b206f2d5954bbae6b800d412f562d780561478c"},
     {file = "pyproj-3.4.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52e54796e2d9554a5eb8f11df4748af1fbbc47f76aa234d6faf09216a84554c5"},
     {file = "pyproj-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a454a7c4423faa2a14e939d08ef293ee347fa529c9df79022b0585a6e1d8310c"},
@@ -6973,7 +6979,6 @@ pyproj = [
     {file = "pyproj-3.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f80adda8c54b84271a93829477a01aa57bc178c834362e9f74e1de1b5033c74c"},
     {file = "pyproj-3.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:221d8939685e0c43ee594c9f04b6a73a10e8e1cc0e85f28be0b4eb2f1bc8777d"},
     {file = "pyproj-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d94afed99f31673d3d19fe750283621e193e2a53ca9e0443bf9d092c3905833b"},
-    {file = "pyproj-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0fff9c3a991508f16027be27d153f6c5583d03799443639d13c681e60f49e2d7"},
     {file = "pyproj-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b85acf09e5a9e35cd9ee72989793adb7089b4e611be02a43d3d0bda50ad116b"},
     {file = "pyproj-3.4.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:45554f47d1a12a84b0620e4abc08a2a1b5d9f273a4759eaef75e74788ec7162a"},
     {file = "pyproj-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12f62c20656ac9b6076ebb213e9a635d52f4f01fef95310121d337e62e910cb6"},
@@ -7515,6 +7520,13 @@ sortedcontainers = [
     {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"},
     {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"},
 ]
+sounddevice = [
+    {file = "sounddevice-0.4.5-py3-none-any.whl", hash = "sha256:5cea4afd9412e731f50ae09a54d68b10628a604cfd56b42a976c54d424c6c39d"},
+    {file = "sounddevice-0.4.5-py3-none-macosx_10_6_x86_64.macosx_10_6_universal2.whl", hash = "sha256:0875173595a8bd5a66b5a03a3d958e7b89c3b956b8befbe4491a24a3ce7784c0"},
+    {file = "sounddevice-0.4.5-py3-none-win32.whl", hash = "sha256:442adf53850916374a58f902200aaf9412227378181264e60c966da64be47d41"},
+    {file = "sounddevice-0.4.5-py3-none-win_amd64.whl", hash = "sha256:d3216c5d3d678c3301058e9aac7000879e255140c524c9ef98730091b67ea676"},
+    {file = "sounddevice-0.4.5.tar.gz", hash = "sha256:2fe0d41299e4f3037dad2acede4eff0666b34a1fa3da5335e47120373964bef5"},
+]
 soupsieve = [
     {file = "soupsieve-2.3.2.post1-py3-none-any.whl", hash = "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759"},
     {file = "soupsieve-2.3.2.post1.tar.gz", hash = "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"},

pyproject.toml

@@ -49,6 +49,7 @@ sentry-sdk = "^1.6.0"
 setproctitle = "^1.2.3"
 six = "^1.16.0"
 smbus2 = "^0.4.2"
+sounddevice = "^0.4.5"
 sympy = "^1.10.1"
 timezonefinder = "^6.0.1"
 tqdm = "^4.64.0"

release/files_common

@@ -71,6 +71,7 @@ selfdrive/rtshield.py
 selfdrive/statsd.py
 
 system/logmessaged.py
+system/micd.py
 system/swaglog.py
 system/version.py
 

selfdrive/manager/process_config.py

@@ -24,6 +24,7 @@ procs = [
   NativeProcess("logcatd", "system/logcatd", ["./logcatd"]),
   NativeProcess("proclogd", "system/proclogd", ["./proclogd"]),
   PythonProcess("logmessaged", "system.logmessaged", offroad=True),
+  PythonProcess("micd", "system.micd"),
   PythonProcess("timezoned", "system.timezoned", enabled=not PC, offroad=True),
 
   DaemonProcess("manage_athenad", "selfdrive.athena.manage_athenad", "AthenadPid"),

selfdrive/ui/soundd/sound.cc

@@ -12,7 +12,7 @@
 // TODO: detect when we can't play sounds
 // TODO: detect when we can't display the UI
 
-Sound::Sound(QObject *parent) : sm({"carState", "controlsState", "deviceState"}) {
+Sound::Sound(QObject *parent) : sm({"controlsState", "deviceState", "microphone"}) {
   qInfo() << "default audio device: " << QAudioDeviceInfo::defaultOutputDevice().deviceName();
 
   for (auto &[alert, fn, loops] : sound_list) {
@@ -47,8 +47,8 @@ void Sound::update() {
   }
 
   // scale volume with speed
-  if (sm.updated("carState")) {
+  if (sm.updated("microphone")) {
-    float volume = util::map_val(sm["carState"].getCarState().getVEgo(), 11.f, 20.f, 0.f, 1.f);
+    float volume = util::map_val(sm["microphone"].getMicrophone().getFilteredSoundPressureDb(), 58.f, 77.f, 0.f, 1.f);
     volume = QAudio::convertVolume(volume, QAudio::LogarithmicVolumeScale, QAudio::LinearVolumeScale);
     Hardware::set_volume(volume);
   }

system/hardware/tici/hardware.h

@@ -10,7 +10,7 @@
 class HardwareTici : public HardwareNone {
 public:
   static constexpr float MAX_VOLUME = 0.9;
-  static constexpr float MIN_VOLUME = 0.2;
+  static constexpr float MIN_VOLUME = 0.1;
   static bool TICI() { return true; }
   static bool AGNOS() { return true; }
   static std::string get_os_version() {

system/micd.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+import sounddevice as sd
+import numpy as np
+
+from cereal import messaging
+from common.filter_simple import FirstOrderFilter
+from common.realtime import Ratekeeper
+from system.swaglog import cloudlog
+
+RATE = 10
+DT_MIC = 1. / RATE
+REFERENCE_SPL = 2 * 10 ** -5  # newtons/m^2
+
+
+class Mic:
+  def __init__(self, pm):
+    self.pm = pm
+    self.rk = Ratekeeper(RATE)
+
+    self.measurements = np.empty(0)
+    self.spl_filter = FirstOrderFilter(0, 4, DT_MIC, initialized=False)
+
+  def update(self):
+    # self.measurements contains amplitudes from -1 to 1 which we use to
+    # calculate an uncalibrated sound pressure level
+    if len(self.measurements) > 0:
+      # https://www.engineeringtoolbox.com/sound-pressure-d_711.html
+      sound_pressure = np.sqrt(np.mean(self.measurements ** 2))  # RMS of amplitudes
+      sound_pressure_level = 20 * np.log10(sound_pressure / REFERENCE_SPL) if sound_pressure > 0 else 0  # dB
+      self.spl_filter.update(sound_pressure_level)
+    else:
+      sound_pressure = 0
+      sound_pressure_level = 0
+
+    self.measurements = np.empty(0)
+
+    msg = messaging.new_message('microphone')
+    msg.microphone.soundPressure = float(sound_pressure)
+    msg.microphone.soundPressureDb = float(sound_pressure_level)
+    msg.microphone.filteredSoundPressureDb = float(self.spl_filter.x)
+
+    self.pm.send('microphone', msg)
+    self.rk.keep_time()
+
+  def callback(self, indata, frames, time, status):
+    self.measurements = np.concatenate((self.measurements, indata[:, 0]))
+
+  def micd_thread(self, device=None):
+    if device is None:
+      device = "sysdefault"
+
+    with sd.InputStream(device=device, channels=1, samplerate=44100, callback=self.callback) as stream:
+      cloudlog.info(f"micd stream started: {stream.samplerate=} {stream.channels=} {stream.dtype=} {stream.device=}")
+      while True:
+        self.update()
+
+
+def main(pm=None, sm=None):
+  if pm is None:
+    pm = messaging.PubMaster(['microphone'])
+
+  mic = Mic(pm)
+  mic.micd_thread()
+
+
+if __name__ == "__main__":
+  main()

tools/ubuntu_setup.sh

@@ -56,6 +56,7 @@ function install_ubuntu_common_requirements() {
     libomp-dev \
     libopencv-dev \
     libpng16-16 \
+    libportaudio2 \
     libssl-dev \
     libsqlite3-dev \
     libusb-1.0-0-dev \