sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

QT UI: sounds (#2078)

Browse Source 
* move android into own dir

* fix name

* maybe this works? qt ui doesn't work on mac

* fix that

* pc sound works

* fix pc build

* lowercase

* that needs to be real_arch

* split into classes

* fix typo in lib

* Fix cycle alerts

* Add qt multimedia libs to install scripts

* Add ui/android folder

* Fix android build

* Raise exception if sound init fails

* add missing return

Co-authored-by: Willem Melching <willem.melching@gmail.com>
Co-authored-by: Comma Device <device@comma.ai>
pull/2097/head
Adeeb Shihadeh 5 years ago committed by GitHub
parent 9eca642c76
commit acd1bde496
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 178 additions and 100 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      Dockerfile.openpilot_base
  2. 2
      SConstruct
  3. 3
      release/files_common
  4. 71
      selfdrive/debug/cycle_alerts.py
  5. 9
      selfdrive/ui/SConscript
  6. 46
      selfdrive/ui/android/sl_sound.cc
  7. 26
      selfdrive/ui/android/sl_sound.hpp
  8. 8
      selfdrive/ui/android/ui.cc
  9. 29
      selfdrive/ui/qt/qt_sound.cc
  10. 16
      selfdrive/ui/qt/qt_sound.hpp
  11. 10
      selfdrive/ui/qt/window.cc
  12. 2
      selfdrive/ui/qt/window.hpp
  13. 38
      selfdrive/ui/sound.hpp
  14. 6
      selfdrive/ui/ui.cc
  15. 2
      selfdrive/ui/ui.hpp
  16. 1
      tools/ubuntu_setup.sh

1
Dockerfile.openpilot_base
Unescape View File

@ -38,6 +38,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
python-dev \
python-pip \
qt5-default \
qtmultimedia5-dev \
sudo \
wget \
&& rm -rf /var/lib/apt/lists/*

2
SConstruct
Unescape View File

@ -190,6 +190,7 @@ if arch in ["x86_64", "Darwin", "larch64"]:
QT_BASE + "include/QtGui",
QT_BASE + "include/QtCore",
QT_BASE + "include/QtDBus",
QT_BASE + "include/QtMultimedia",
]
qt_env["LINKFLAGS"] += ["-F" + QT_BASE + "lib"]
else:
@ -199,6 +200,7 @@ if arch in ["x86_64", "Darwin", "larch64"]:
f"/usr/include/{real_arch}-linux-gnu/qt5/QtGui",
f"/usr/include/{real_arch}-linux-gnu/qt5/QtCore",
f"/usr/include/{real_arch}-linux-gnu/qt5/QtDBus",
f"/usr/include/{real_arch}-linux-gnu/qt5/QtMultimedia",
]
qt_env.Tool('qt')

3
release/files_common
Unescape View File

@ -350,6 +350,9 @@ selfdrive/ui/text/text.c
selfdrive/ui/qt/*.cc
selfdrive/ui/qt/*.hpp
selfdrive/ui/android/*.cc
selfdrive/ui/android/*.hpp
selfdrive/camerad/SConscript
selfdrive/camerad/main.cc
selfdrive/camerad/bufs.h

71
selfdrive/debug/cycle_alerts.py
Unescape View File

@ -7,52 +7,63 @@ import argparse
import time
import cereal.messaging as messaging
from selfdrive.controls.lib.events import EVENTS, Alert
from selfdrive.car.honda.interface import CarInterface
from selfdrive.controls.lib.events import ET, EVENTS, Alert, Events
from selfdrive.controls.lib.alertmanager import AlertManager
def now_millis(): return time.time() * 1000
ALERTS = [a for _, et in EVENTS.items() for _, a in et.items() if isinstance(a, Alert)]
def cycle_alerts(duration=200, is_metric=False):
alerts = list(EVENTS.keys())
print(alerts)
#from cereal import car
#ALERTS = [a for a in ALERTS if a.audible_alert == car.CarControl.HUDControl.AudibleAlert.chimeWarningRepeat]
default_alerts = sorted(ALERTS, key=lambda alert: (alert.alert_size, len(alert.alert_text_2)))
def cycle_alerts(duration_millis, alerts=None):
if alerts is None:
alerts = default_alerts
CP = CarInterface.get_params("HONDA CIVIC 2016 TOURING")
sm = messaging.SubMaster(['thermal', 'health', 'frame', 'model', 'liveCalibration',
'dMonitoringState', 'plan', 'pathPlan', 'liveLocationKalman'])
controls_state = messaging.pub_sock('controlsState')
thermal = messaging.pub_sock('thermal')
idx, last_alert_millis = 0, 0
alert = alerts[0]
events = Events()
AM = AlertManager()
frame = 0
while 1:
if (now_millis() - last_alert_millis) > duration_millis:
alert = alerts[idx]
if frame % duration == 0:
idx = (idx + 1) % len(alerts)
last_alert_millis = now_millis()
print('sending {}'.format(str(alert)))
events.clear()
events.add(alerts[idx])
current_alert_types = [ET.PERMANENT, ET.USER_DISABLE, ET.IMMEDIATE_DISABLE,
ET.SOFT_DISABLE, ET.PRE_ENABLE, ET.NO_ENTRY,
ET.ENABLE, ET.WARNING]
a = events.create_alerts(current_alert_types, [CP, sm, is_metric])
AM.add_many(frame, a)
AM.process_alerts(frame)
dat = messaging.new_message()
dat.init('controlsState')
dat.controlsState.alertType = alert.alert_type
dat.controlsState.alertText1 = alert.alert_text_1
dat.controlsState.alertText2 = alert.alert_text_2
dat.controlsState.alertSize = alert.alert_size
#dat.controlsState.alertStatus = alert.alert_status
dat.controlsState.alertSound = alert.audible_alert
dat.controlsState.alertText1 = AM.alert_text_1
dat.controlsState.alertText2 = AM.alert_text_2
dat.controlsState.alertSize = AM.alert_size
dat.controlsState.alertStatus = AM.alert_status
dat.controlsState.alertBlinkingRate = AM.alert_rate
dat.controlsState.alertType = AM.alert_type
dat.controlsState.alertSound = AM.audible_alert
controls_state.send(dat.to_bytes())
dat = messaging.new_message()
dat.init('thermal')
dat.thermal.started = True
thermal.send(dat.to_bytes())
frame += 1
time.sleep(0.01)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--duration', type=int, default=1000)
parser.add_argument('--alert-types', nargs='+')
args = parser.parse_args()
alerts = None
if args.alert_types:
alerts = [next(a for a in ALERTS if a.alert_type==alert_type) for alert_type in args.alert_types]
cycle_alerts(args.duration, alerts=alerts)
cycle_alerts()

9
selfdrive/ui/SConscript
Unescape View File

@ -5,11 +5,10 @@ libs = [common, 'zmq', 'czmq', 'capnp', 'kj', 'm', cereal, messaging, gpucommon,
if qt_env is None:
src += ['sound.cc']
libs += ['EGL', 'GLESv3', 'gnustl_shared', 'log', 'utils', 'gui', 'hardware', 'ui', 'CB', 'gsl', 'adreno_utils', 'OpenSLES', 'cutils', 'uuid', 'OpenCL']
linkflags = ['-Wl,-rpath=/system/lib64,-rpath=/system/comma/usr/lib']
src = ["android_ui.cc"] + src
src += ["android/ui.cc", "android/sl_sound.cc"]
env.Program('_ui', src,
LINKFLAGS=linkflags,
LIBS=libs)
@ -18,14 +17,14 @@ else:
qt_libs = ["pthread"]
if arch == "Darwin":
qt_env["FRAMEWORKS"] += ["QtWidgets", "QtGui", "QtCore", "QtDBus"]
qt_env["FRAMEWORKS"] += ["QtWidgets", "QtGui", "QtCore", "QtDBus", "QtMultimedia"]
else:
qt_libs += ["Qt5Widgets", "Qt5Gui", "Qt5Core", "Qt5DBus"]
qt_libs += ["Qt5Widgets", "Qt5Gui", "Qt5Core", "Qt5DBus", "Qt5Multimedia"]
if arch == "larch64":
qt_libs += ["GLESv2"]
else:
qt_libs += ["GL"]
qt_src = ["qt/ui.cc", "qt/window.cc", "qt/settings.cc"] + src
qt_src = ["qt/ui.cc", "qt/window.cc", "qt/settings.cc", "qt/qt_sound.cc"] + src
qt_env.Program("_ui", qt_src, LIBS=qt_libs + libs)

46
selfdrive/ui/sound.cc → selfdrive/ui/android/sl_sound.cc
Unescape View File

@ -1,35 +1,31 @@
#include "sound.hpp"
#include <math.h>
#include <stdlib.h>
#include <atomic>
#include "common/swaglog.h"
#include "common/timing.h"
#include "android/sl_sound.hpp"
#define LogOnError(func, msg) \
if ((func) != SL_RESULT_SUCCESS) { LOGW(msg); }
#define ReturnOnError(func, msg) \
if ((func) != SL_RESULT_SUCCESS) { LOGW(msg); return false; }
static std::map<AudibleAlert, std::pair<const char *, int>> sound_map {
{AudibleAlert::CHIME_DISENGAGE, {"../assets/sounds/disengaged.wav", 0}},
{AudibleAlert::CHIME_ENGAGE, {"../assets/sounds/engaged.wav", 0}},
{AudibleAlert::CHIME_WARNING1, {"../assets/sounds/warning_1.wav", 0}},
{AudibleAlert::CHIME_WARNING2, {"../assets/sounds/warning_2.wav", 0}},
{AudibleAlert::CHIME_WARNING2_REPEAT, {"../assets/sounds/warning_2.wav", 3}},
{AudibleAlert::CHIME_WARNING_REPEAT, {"../assets/sounds/warning_repeat.wav", 3}},
{AudibleAlert::CHIME_ERROR, {"../assets/sounds/error.wav", 0}},
{AudibleAlert::CHIME_PROMPT, {"../assets/sounds/error.wav", 0}}};
struct Sound::Player {
struct SLSound::Player {
SLObjectItf player;
SLPlayItf playItf;
// slplay_callback runs on a background thread,use atomic to ensure thread safe.
std::atomic<int> repeat;
};
bool Sound::init(int volume) {
SLSound::SLSound() {
if (!init()){
throw std::runtime_error("Failed to initialize sound");
}
}
bool SLSound::init() {
SLEngineOption engineOptions[] = {{SL_ENGINEOPTION_THREADSAFE, SL_BOOLEAN_TRUE}};
const SLInterfaceID ids[1] = {SL_IID_VOLUME};
const SLboolean req[1] = {SL_BOOLEAN_FALSE};
@ -54,15 +50,13 @@ bool Sound::init(int volume) {
ReturnOnError((*player)->GetInterface(player, SL_IID_PLAY, &playItf), "Failed to get player interface");
ReturnOnError((*playItf)->SetPlayState(playItf, SL_PLAYSTATE_PAUSED), "Failed to initialize playstate to SL_PLAYSTATE_PAUSED");
player_[kv.first] = new Sound::Player{player, playItf};
player_[kv.first] = new SLSound::Player{player, playItf};
}
setVolume(volume);
return true;
}
void SLAPIENTRY slplay_callback(SLPlayItf playItf, void *context, SLuint32 event) {
Sound::Player *s = reinterpret_cast<Sound::Player *>(context);
SLSound::Player *s = reinterpret_cast<SLSound::Player *>(context);
if (event == SL_PLAYEVENT_HEADATEND && s->repeat > 1) {
--s->repeat;
(*playItf)->SetPlayState(playItf, SL_PLAYSTATE_STOPPED);
@ -71,7 +65,7 @@ void SLAPIENTRY slplay_callback(SLPlayItf playItf, void *context, SLuint32 event
}
}
bool Sound::play(AudibleAlert alert) {
bool SLSound::play(AudibleAlert alert) {
if (currentSound_ != AudibleAlert::NONE) {
stop();
}
@ -93,7 +87,7 @@ bool Sound::play(AudibleAlert alert) {
return true;
}
void Sound::stop() {
void SLSound::stop() {
if (currentSound_ != AudibleAlert::NONE) {
auto player = player_.at(currentSound_);
player->repeat = 0;
@ -102,7 +96,7 @@ void Sound::stop() {
}
}
void Sound::setVolume(int volume) {
void SLSound::setVolume(int volume) {
if (last_volume_ == volume) return;
double current_time = nanos_since_boot();
@ -115,11 +109,15 @@ void Sound::setVolume(int volume) {
}
}
Sound::~Sound() {
SLSound::~SLSound() {
for (auto &kv : player_) {
(*(kv.second->player))->Destroy(kv.second->player);
delete kv.second;
}
if (outputMix_) (*outputMix_)->Destroy(outputMix_);
if (engine_) (*engine_)->Destroy(engine_);
if (outputMix_) {
(*outputMix_)->Destroy(outputMix_);
}
if (engine_) {
(*engine_)->Destroy(engine_);
}
}

26
selfdrive/ui/android/sl_sound.hpp
Unescape View File

@ -0,0 +1,26 @@
#pragma once
#include <SLES/OpenSLES.h>
#include <SLES/OpenSLES_Android.h>
#include "sound.hpp"
class SLSound : public Sound {
public:
SLSound();
~SLSound();
bool play(AudibleAlert alert);
void stop();
void setVolume(int volume);
private:
bool init();
SLObjectItf engine_ = nullptr;
SLObjectItf outputMix_ = nullptr;
int last_volume_ = 0;
double last_set_volume_time_ = 0.;
AudibleAlert currentSound_ = AudibleAlert::NONE;
struct Player;
std::map<AudibleAlert, Player *> player_;
friend void SLAPIENTRY slplay_callback(SLPlayItf playItf, void *context, SLuint32 event);
};

8
selfdrive/ui/android_ui.cc → selfdrive/ui/android/ui.cc
Unescape View File

@ -12,6 +12,7 @@
#include "ui.hpp"
#include "paint.hpp"
#include "android/sl_sound.hpp"
// Includes for light sensor
#include <cutils/properties.h>
@ -169,10 +170,13 @@ int main(int argc, char* argv[]) {
setpriority(PRIO_PROCESS, 0, -14);
signal(SIGINT, (sighandler_t)set_do_exit);
SLSound sound;
UIState uistate = {};
UIState *s = &uistate;
ui_init(s);
s->sound = &sound;
set_awake(s, true);
enable_event_processing(true);
@ -201,7 +205,7 @@ int main(int argc, char* argv[]) {
const int MIN_VOLUME = LEON ? 12 : 9;
const int MAX_VOLUME = LEON ? 15 : 12;
assert(s->sound.init(MIN_VOLUME));
s->sound->setVolume(MIN_VOLUME);
while (!do_exit) {
if (!s->started || !s->vision_connected) {
@ -238,7 +242,7 @@ int main(int argc, char* argv[]) {
}
// up one notch every 5 m/s
s->sound.setVolume(fmin(MAX_VOLUME, MIN_VOLUME + s->scene.controls_state.getVEgo() / 5));
s->sound->setVolume(fmin(MAX_VOLUME, MIN_VOLUME + s->scene.controls_state.getVEgo() / 5));
// set brightness
float clipped_brightness = fmin(512, (s->light_sensor*brightness_m) + brightness_b);

29
selfdrive/ui/qt/qt_sound.cc
Unescape View File

@ -0,0 +1,29 @@
#include <QUrl>
#include "qt/qt_sound.hpp"
QtSound::QtSound() {
for (auto &kv : sound_map) {
auto path = QUrl::fromLocalFile(kv.second.first);
sounds[kv.first].setSource(path);
}
}
bool QtSound::play(AudibleAlert alert) {
sounds[alert].setLoopCount(sound_map[alert].second);
sounds[alert].play();
return true;
}
void QtSound::stop() {
for (auto &kv : sounds) {
kv.second.stop();
}
}
void QtSound::setVolume(int volume) {
// TODO: implement this
}
QtSound::~QtSound() {
}

16
selfdrive/ui/qt/qt_sound.hpp
Unescape View File

@ -0,0 +1,16 @@
#pragma once
#include <QSoundEffect>
#include "sound.hpp"
class QtSound : public Sound {
public:
QtSound();
~QtSound();
bool play(AudibleAlert alert);
void stop();
void setVolume(int volume);
private:
std::map<AudibleAlert, QSoundEffect> sounds;
};

10
selfdrive/ui/qt/window.cc
Unescape View File

@ -13,7 +13,6 @@
#include "settings.hpp"
#include "paint.hpp"
#include "sound.hpp"
volatile sig_atomic_t do_exit = 0;
@ -52,6 +51,7 @@ void MainWindow::closeSettings(){
GLWindow::GLWindow(QWidget *parent) : QOpenGLWidget(parent) {
timer = new QTimer(this);
QObject::connect(timer, SIGNAL(timeout()), this, SLOT(timerUpdate()));
}
GLWindow::~GLWindow() {
@ -68,6 +68,7 @@ void GLWindow::initializeGL() {
ui_state = new UIState();
ui_init(ui_state);
ui_state->sound = &sound;
ui_state->fb_w = vwp_w;
ui_state->fb_h = vwp_h;
@ -103,13 +104,6 @@ void GLWindow::mousePressEvent(QMouseEvent *e) {
}
/* HACKS */
bool Sound::init(int volume) { return true; }
bool Sound::play(AudibleAlert alert) { printf("play sound: %d\n", (int)alert); return true; }
void Sound::stop() {}
void Sound::setVolume(int volume) {}
Sound::~Sound() {}
GLuint visionimg_to_gl(const VisionImg *img, EGLImageKHR *pkhr, void **pph) {
unsigned int texture;
glGenTextures(1, &texture);

2
selfdrive/ui/qt/window.hpp
Unescape View File

@ -7,6 +7,7 @@
#include <QOpenGLFunctions>
#include <QStackedLayout>
#include "qt/qt_sound.hpp"
#include "ui/ui.hpp"
class MainWindow : public QWidget
@ -45,6 +46,7 @@ protected:
private:
QTimer * timer;
UIState * ui_state;
QtSound sound;
public slots:
void timerUpdate();

38
selfdrive/ui/sound.hpp
Unescape View File

@ -2,31 +2,23 @@
#include <map>
#include "cereal/gen/cpp/log.capnp.h"
#if defined(QCOM)
#include <SLES/OpenSLES.h>
#include <SLES/OpenSLES_Android.h>
#endif
typedef cereal::CarControl::HUDControl::AudibleAlert AudibleAlert;
static std::map<AudibleAlert, std::pair<const char *, int>> sound_map {
// AudibleAlert, (file path, loop count)
{AudibleAlert::CHIME_DISENGAGE, {"../assets/sounds/disengaged.wav", 0}},
{AudibleAlert::CHIME_ENGAGE, {"../assets/sounds/engaged.wav", 0}},
{AudibleAlert::CHIME_WARNING1, {"../assets/sounds/warning_1.wav", 0}},
{AudibleAlert::CHIME_WARNING2, {"../assets/sounds/warning_2.wav", 0}},
{AudibleAlert::CHIME_WARNING2_REPEAT, {"../assets/sounds/warning_2.wav", 3}},
{AudibleAlert::CHIME_WARNING_REPEAT, {"../assets/sounds/warning_repeat.wav", 3}},
{AudibleAlert::CHIME_ERROR, {"../assets/sounds/error.wav", 0}},
{AudibleAlert::CHIME_PROMPT, {"../assets/sounds/error.wav", 0}}
};
class Sound {
public:
Sound() = default;
bool init(int volume);
bool play(AudibleAlert alert);
void stop();
void setVolume(int volume);
~Sound();
#if defined(QCOM)
private:
SLObjectItf engine_ = nullptr;
SLObjectItf outputMix_ = nullptr;
int last_volume_ = 0;
double last_set_volume_time_ = 0.;
AudibleAlert currentSound_ = AudibleAlert::NONE;
struct Player;
std::map<AudibleAlert, Player *> player_;
friend void SLAPIENTRY slplay_callback(SLPlayItf playItf, void *context, SLuint32 event);
#endif
virtual bool play(AudibleAlert alert) = 0;
virtual void stop() = 0;
virtual void setVolume(int volume) = 0;
};

6
selfdrive/ui/ui.cc
Unescape View File

@ -130,9 +130,9 @@ void update_sockets(UIState *s) {
auto alert_sound = scene.controls_state.getAlertSound();
if (scene.alert_type.compare(scene.controls_state.getAlertType()) != 0) {
if (alert_sound == AudibleAlert::NONE) {
s->sound.stop();
s->sound->stop();
} else {
s->sound.play(alert_sound);
s->sound->play(alert_sound);
}
}
scene.alert_text1 = scene.controls_state.getAlertText1();
@ -254,7 +254,7 @@ void ui_update(UIState *s) {
} else {
// car is started, but controls is lagging or died
if (s->scene.alert_text2 != "Controls Unresponsive") {
s->sound.play(AudibleAlert::CHIME_WARNING_REPEAT);
s->sound->play(AudibleAlert::CHIME_WARNING_REPEAT);
LOGE("Controls unresponsive");
}

2
selfdrive/ui/ui.hpp
Unescape View File

@ -174,7 +174,7 @@ typedef struct UIState {
SubMaster *sm;
Sound sound;
Sound *sound;
UIStatus status;
UIScene scene;
cereal::UiLayoutState::App active_app;

1
tools/ubuntu_setup.sh
Unescape View File

@ -43,6 +43,7 @@ sudo apt-get update && sudo apt-get install -y \
python-dev \
python-pip \
qt5-default \
qtmultimedia5-dev \
screen \
sudo \
vim \

Write Preview
Loading…
Cancel
Save