From 9bcf83f493cf5e179bb23904d6f70ef855839a6d Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Wed, 30 Nov 2022 21:56:03 -0800 Subject: [PATCH] micd: scale sound volume with ambient noise level (#26399) * test changing sound volume * test changing sound volume * create system/hardware/pc/hardware.h * implement Hardware::set_volume using pactl * soundd: use Hardware::set_volume * add sounddevice dependency * sounddevice example * simple micd * cleanup * remove this * fix process config * add to release files * hardware: get sound input device * no more offroad * debug * calculate volume from all measurements since last update * use microphone noise level to update sound volume * fix scale * mute microphone during alerts * log raw noise level * hardware: reduce tici min volume * improve scale * add package * clear measurements on muted * change default to min volume and respond quicker * fixes Co-authored-by: Shane Smiskol * logarithmic scaling * fix * respond quicker * fixes * tweak scaling * specify default device * Revert "hardware: get sound input device" This reverts commit 50f594f7a3bab005023482bc793147a8c8dae5d7. * tuning * forgot to update submaster * tuning * don't mute microphone, and clip measurement * remove submaster * fixes * tuning * implement Hardware::set_volume using pactl * Revert "test changing sound volume" This reverts commit 4bbd870746ec86d1c9871a6175def96cf7f751a6. * draft * draft * calculate sound pressure level in dB * fix setting * faster filter * start at initial value * don't run command in background * pactl: use default sink * use sound pressure db * tuning * bump up max volume threshold * update filter slower * fix divide by zero * bump cereal Co-authored-by: Shane Smiskol old-commit-hash: 108ff15f5dc16f79a36a2d33397b36dba42d70cf --- cereal | 2 +- poetry.lock | 4 +- pyproject.toml | 1 + release/files_common | 1 + selfdrive/manager/process_config.py | 1 + selfdrive/ui/soundd/sound.cc | 6 +-- system/hardware/tici/hardware.h | 2 +- system/micd.py | 67 +++++++++++++++++++++++++++++ tools/ubuntu_setup.sh | 1 + 9 files changed, 78 insertions(+), 7 deletions(-) create mode 100755 system/micd.py diff --git a/cereal b/cereal index 19a0c46b71..dbc9846ac9 160000 --- a/cereal +++ b/cereal @@ -1 +1 @@ -Subproject commit 19a0c46b71150a8dabc5644eb24f261feee45b9c +Subproject commit dbc9846ac9c9e735ee2f4a281ce079cfff7ea285 diff --git a/poetry.lock b/poetry.lock index 6482e97660..7be21e9653 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:616bfb7e1761adc9f7a27571d1bde023727147971f5426207ee09fab592f97bc -size 561251 +oid sha256:88495068650500a25401ed83cf3f2c4eb2216a1c3f8c2dcda0fa33bfd801b85f +size 560742 diff --git a/pyproject.toml b/pyproject.toml index 7e76b9cdfc..c890b78875 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ sentry-sdk = "^1.6.0" setproctitle = "^1.2.3" six = "^1.16.0" smbus2 = "^0.4.2" +sounddevice = "^0.4.5" sympy = "^1.10.1" timezonefinder = "^6.0.1" tqdm = "^4.64.0" diff --git a/release/files_common b/release/files_common index 2400c3c0dc..aa6c0ac55c 100644 --- a/release/files_common +++ b/release/files_common @@ -71,6 +71,7 @@ selfdrive/rtshield.py selfdrive/statsd.py system/logmessaged.py +system/micd.py system/swaglog.py system/version.py diff --git a/selfdrive/manager/process_config.py b/selfdrive/manager/process_config.py index dbccb8d4a9..50c19610ed 100644 --- a/selfdrive/manager/process_config.py +++ b/selfdrive/manager/process_config.py @@ -24,6 +24,7 @@ procs = [ NativeProcess("logcatd", "system/logcatd", ["./logcatd"]), NativeProcess("proclogd", "system/proclogd", ["./proclogd"]), PythonProcess("logmessaged", "system.logmessaged", offroad=True), + PythonProcess("micd", "system.micd"), PythonProcess("timezoned", "system.timezoned", enabled=not PC, offroad=True), DaemonProcess("manage_athenad", "selfdrive.athena.manage_athenad", "AthenadPid"), diff --git a/selfdrive/ui/soundd/sound.cc b/selfdrive/ui/soundd/sound.cc index 841bea3b8b..3deb6ceca0 100644 --- a/selfdrive/ui/soundd/sound.cc +++ b/selfdrive/ui/soundd/sound.cc @@ -12,7 +12,7 @@ // TODO: detect when we can't play sounds // TODO: detect when we can't display the UI -Sound::Sound(QObject *parent) : sm({"carState", "controlsState", "deviceState"}) { +Sound::Sound(QObject *parent) : sm({"controlsState", "deviceState", "microphone"}) { qInfo() << "default audio device: " << QAudioDeviceInfo::defaultOutputDevice().deviceName(); for (auto &[alert, fn, loops] : sound_list) { @@ -47,8 +47,8 @@ void Sound::update() { } // scale volume with speed - if (sm.updated("carState")) { - float volume = util::map_val(sm["carState"].getCarState().getVEgo(), 11.f, 20.f, 0.f, 1.f); + if (sm.updated("microphone")) { + float volume = util::map_val(sm["microphone"].getMicrophone().getFilteredSoundPressureDb(), 58.f, 77.f, 0.f, 1.f); volume = QAudio::convertVolume(volume, QAudio::LogarithmicVolumeScale, QAudio::LinearVolumeScale); Hardware::set_volume(volume); } diff --git a/system/hardware/tici/hardware.h b/system/hardware/tici/hardware.h index 02becb76e4..d388f9c48a 100644 --- a/system/hardware/tici/hardware.h +++ b/system/hardware/tici/hardware.h @@ -10,7 +10,7 @@ class HardwareTici : public HardwareNone { public: static constexpr float MAX_VOLUME = 0.9; - static constexpr float MIN_VOLUME = 0.2; + static constexpr float MIN_VOLUME = 0.1; static bool TICI() { return true; } static bool AGNOS() { return true; } static std::string get_os_version() { diff --git a/system/micd.py b/system/micd.py new file mode 100755 index 0000000000..d2a5a2849f --- /dev/null +++ b/system/micd.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +import sounddevice as sd +import numpy as np + +from cereal import messaging +from common.filter_simple import FirstOrderFilter +from common.realtime import Ratekeeper +from system.swaglog import cloudlog + +RATE = 10 +DT_MIC = 1. / RATE +REFERENCE_SPL = 2 * 10 ** -5 # newtons/m^2 + + +class Mic: + def __init__(self, pm): + self.pm = pm + self.rk = Ratekeeper(RATE) + + self.measurements = np.empty(0) + self.spl_filter = FirstOrderFilter(0, 4, DT_MIC, initialized=False) + + def update(self): + # self.measurements contains amplitudes from -1 to 1 which we use to + # calculate an uncalibrated sound pressure level + if len(self.measurements) > 0: + # https://www.engineeringtoolbox.com/sound-pressure-d_711.html + sound_pressure = np.sqrt(np.mean(self.measurements ** 2)) # RMS of amplitudes + sound_pressure_level = 20 * np.log10(sound_pressure / REFERENCE_SPL) if sound_pressure > 0 else 0 # dB + self.spl_filter.update(sound_pressure_level) + else: + sound_pressure = 0 + sound_pressure_level = 0 + + self.measurements = np.empty(0) + + msg = messaging.new_message('microphone') + msg.microphone.soundPressure = float(sound_pressure) + msg.microphone.soundPressureDb = float(sound_pressure_level) + msg.microphone.filteredSoundPressureDb = float(self.spl_filter.x) + + self.pm.send('microphone', msg) + self.rk.keep_time() + + def callback(self, indata, frames, time, status): + self.measurements = np.concatenate((self.measurements, indata[:, 0])) + + def micd_thread(self, device=None): + if device is None: + device = "sysdefault" + + with sd.InputStream(device=device, channels=1, samplerate=44100, callback=self.callback) as stream: + cloudlog.info(f"micd stream started: {stream.samplerate=} {stream.channels=} {stream.dtype=} {stream.device=}") + while True: + self.update() + + +def main(pm=None, sm=None): + if pm is None: + pm = messaging.PubMaster(['microphone']) + + mic = Mic(pm) + mic.micd_thread() + + +if __name__ == "__main__": + main() diff --git a/tools/ubuntu_setup.sh b/tools/ubuntu_setup.sh index 403437bfb3..89d33a6127 100755 --- a/tools/ubuntu_setup.sh +++ b/tools/ubuntu_setup.sh @@ -56,6 +56,7 @@ function install_ubuntu_common_requirements() { libomp-dev \ libopencv-dev \ libpng16-16 \ + libportaudio2 \ libssl-dev \ libsqlite3-dev \ libusb-1.0-0-dev \