1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
// Copyright (c) 2012-2017 VideoStitch SAS
// Copyright (c) 2018 stitchEm
#include "audioplayer.hpp"
#include "libvideostitch/logging.hpp"
#include <QAudioOutput>
#include <QAudioDeviceInfo>
#include <iostream>
#define ERROR(tag) VideoStitch::Logger::error(tag)
#define WARNING(tag) VideoStitch::Logger::warning(tag)
#define INFO(tag) VideoStitch::Logger::info(tag)
#define VERBOSE(tag) VideoStitch::Logger::verbose(tag)
#define DEBUG(tag) VideoStitch::Logger::debug(tag)
namespace {
static const std::string tag = "AudioPlayer";
VideoStitch::Audio::SamplingDepth fmt2depth(QAudioFormat fmt) {
// Manage only interleaved data
switch (fmt.sampleType()) {
case QAudioFormat::SampleType::SignedInt:
if (fmt.sampleSize() == 16) {
return VideoStitch::Audio::SamplingDepth::INT16;
} else if (fmt.sampleSize() == 24) {
return VideoStitch::Audio::SamplingDepth::INT24;
} else if (fmt.sampleSize() == 32) {
return VideoStitch::Audio::SamplingDepth::INT32;
}
break;
case QAudioFormat::SampleType::UnSignedInt:
if (fmt.sampleSize() == 8) {
return VideoStitch::Audio::SamplingDepth::UINT8;
}
break;
case QAudioFormat::SampleType::Float:
if (fmt.sampleSize() == 32) {
return VideoStitch::Audio::SamplingDepth::FLT;
} else if (fmt.sampleSize() == 64) {
return VideoStitch::Audio::SamplingDepth::DBL;
}
break;
case QAudioFormat::SampleType::Unknown:
return VideoStitch::Audio::SamplingDepth::SD_NONE;
}
return VideoStitch::Audio::SamplingDepth::SD_NONE;
}
VideoStitch::Audio::ChannelLayout fmt2layout(QAudioFormat fmt) {
return VideoStitch::Audio::getAChannelLayoutFromNbChannels(fmt.channelCount());
}
} // namespace
AudioPlayer::AudioPlayer(QObject* parent)
: Output("playback"),
QObject(parent),
VideoStitch::Output::AudioWriter(VideoStitch::Audio::getSamplingRateFromInt(
QAudioDeviceInfo::defaultOutputDevice().preferredFormat().sampleRate()),
fmt2depth(QAudioDeviceInfo::defaultOutputDevice().preferredFormat()),
fmt2layout(QAudioDeviceInfo::defaultOutputDevice().preferredFormat())),
currentTimestamp(-1),
fmt(QAudioDeviceInfo::defaultOutputDevice().preferredFormat()),
audioOutput(nullptr),
info(QAudioDeviceInfo::defaultOutputDevice()),
volume(1.0) {
audioOutput = new QAudioOutput(info, fmt);
connect(audioOutput, &QAudioOutput::stateChanged, this, &AudioPlayer::handleStateChanged);
dev = audioOutput->start();
delay = (mtime_t)fmt.durationForBytes(audioOutput->bufferSize());
worker = new std::thread(run, this);
}
AudioPlayer::~AudioPlayer() {
// QAudioOutput (CoreAudioOutput) on Mac often crashes when deleted on a background thread
audioOutput->deleteLater();
{
std::unique_lock<std::mutex> lk(mu);
exit = true;
}
worker->join();
delete worker;
}
void AudioPlayer::pushAudio(VideoStitch::Audio::Samples& audioSamples) {
if (dev != nullptr && audioSamples.getNbOfSamples() > 0) {
if (audioOutput->state() != QAudio::SuspendedState) {
std::unique_lock<std::mutex> lk(mu);
while (!audioQueue.empty() && audioQueue.front().getTimestamp() > audioSamples.getTimestamp()) {
// must have seeked just before
audioQueue.pop();
}
audioQueue.push(audioSamples.clone());
}
}
}
std::string AudioPlayer::getName() const { return "playback"; }
void AudioPlayer::render(std::shared_ptr<VideoStitch::Core::PanoOpenGLSurface>, mtime_t ts) { currentTimestamp = ts; }
void AudioPlayer::renderCubemap(std::shared_ptr<VideoStitch::Core::CubemapOpenGLSurface>, mtime_t ts) {
currentTimestamp = ts;
}
void AudioPlayer::renderEquiangularCubemap(std::shared_ptr<VideoStitch::Core::CubemapOpenGLSurface>, mtime_t ts) {
currentTimestamp = ts;
}
// play the audio as fast as possible
void AudioPlayer::run(AudioPlayer* that) {
for (;;) {
VideoStitch::Audio::Samples as;
{
std::unique_lock<std::mutex> lock(that->mu);
if (that->exit) {
return;
}
if (that->audioQueue.empty() ||
size_t(that->audioOutput->bytesFree() / that->fmt.channelCount() / (that->fmt.sampleSize() / 8)) <
that->audioQueue.front().getNbOfSamples()) {
continue;
}
if (that->audioQueue.front().getTimestamp() - that->delay > that->currentTimestamp) {
continue; // audio in advance
}
as = std::move(that->audioQueue.front());
that->audioQueue.pop();
}
qint64 toWrite = that->fmt.bytesForFrames((qint32)as.getNbOfSamples());
qint64 written = 0;
while (toWrite > 0) {
QByteArray b((const char*)as.getSamples()[0] + written, toWrite);
qint64 actuallyWritten = that->dev->write(b);
written += actuallyWritten;
toWrite -= actuallyWritten;
if (actuallyWritten == 0) {
// cannot write in the queue
break;
}
}
}
}
void AudioPlayer::handleStateChanged(QAudio::State) {
if (audioOutput->error() != QAudio::NoError) {
logError(audioOutput->error());
}
}
void AudioPlayer::logError(QAudio::Error err) {
switch (err) {
case QAudio::OpenError:
ERROR(tag) << "An error opening the audio device, cannot play audio." << std::endl;
break;
case QAudio::IOError:
ERROR(tag) << "An error occurred during read/write of audio device, cannot play audio." << std::endl;
break;
case QAudio::UnderrunError: {
static auto underrunLogLevel = VideoStitch::Logger::Warning;
VideoStitch::Logger::get(underrunLogLevel, tag)
<< "Audio data is not being fed to the audio device at a fast enough rate, cannot play audio." << std::endl;
underrunLogLevel = VideoStitch::Logger::Verbose;
} break;
case QAudio::FatalError:
default:
ERROR(tag) << "A non-recoverable error has occurred, the audio device is not usable at this time." << std::endl;
break;
}
}
void AudioPlayer::onActivatePlayBack(bool b) {
std::lock_guard<std::mutex> lk(mu);
if (b) {
audioOutput->resume();
} else {
audioOutput->suspend();
}
}