Implement audio speed using libavfilter.

This commit is contained in:
John Preston 2023-03-08 13:31:58 +04:00
parent 0880a83c2c
commit 9c74c04738
17 changed files with 871 additions and 669 deletions

View file

@ -321,6 +321,12 @@ FramePointer MakeFramePointer() {
return FramePointer(av_frame_alloc()); return FramePointer(av_frame_alloc());
} }
FramePointer DuplicateFramePointer(AVFrame *frame) {
return frame
? FramePointer(av_frame_clone(frame))
: FramePointer();
}
bool FrameHasData(AVFrame *frame) { bool FrameHasData(AVFrame *frame) {
return (frame && frame->data[0] != nullptr); return (frame && frame->data[0] != nullptr);
} }

View file

@ -141,6 +141,7 @@ struct FrameDeleter {
}; };
using FramePointer = std::unique_ptr<AVFrame, FrameDeleter>; using FramePointer = std::unique_ptr<AVFrame, FrameDeleter>;
[[nodiscard]] FramePointer MakeFramePointer(); [[nodiscard]] FramePointer MakeFramePointer();
[[nodiscard]] FramePointer DuplicateFramePointer(AVFrame *frame);
[[nodiscard]] bool FrameHasData(AVFrame *frame); [[nodiscard]] bool FrameHasData(AVFrame *frame);
void ClearFrameMemory(AVFrame *frame); void ClearFrameMemory(AVFrame *frame);

View file

@ -45,15 +45,6 @@ ALCcontext *AudioContext = nullptr;
auto VolumeMultiplierAll = 1.; auto VolumeMultiplierAll = 1.;
auto VolumeMultiplierSong = 1.; auto VolumeMultiplierSong = 1.;
// Value for AL_PITCH_SHIFTER_COARSE_TUNE effect, 0.5 <= speed <= 2.
int CoarseTuneForSpeed(float64 speed) {
Expects(speed >= 0.5 && speed <= 2.);
constexpr auto kTuneSteps = 12;
const auto tuneRatio = std::log(speed) / std::log(2.);
return -int(base::SafeRound(kTuneSteps * tuneRatio));
}
} // namespace } // namespace
namespace Media { namespace Media {
@ -205,7 +196,6 @@ void Start(not_null<Instance*> instance) {
auto loglevel = getenv("ALSOFT_LOGLEVEL"); auto loglevel = getenv("ALSOFT_LOGLEVEL");
LOG(("OpenAL Logging Level: %1").arg(loglevel ? loglevel : "(not set)")); LOG(("OpenAL Logging Level: %1").arg(loglevel ? loglevel : "(not set)"));
OpenAL::LoadEFXExtension();
EnumeratePlaybackDevices(); EnumeratePlaybackDevices();
EnumerateCaptureDevices(); EnumerateCaptureDevices();
@ -244,9 +234,9 @@ bool AttachToDevice() {
return false; return false;
} }
if (auto m = Player::mixer()) { if (const auto m = Player::mixer()) {
m->reattachTracks(); m->reattachTracks();
m->faderOnTimer(); m->scheduleFaderCallback();
} }
crl::on_main([] { crl::on_main([] {
@ -282,16 +272,7 @@ void StopDetachIfNotUsedSafe() {
} }
bool SupportsSpeedControl() { bool SupportsSpeedControl() {
return OpenAL::HasEFXExtension() return true;
&& (alGetEnumValue("AL_AUXILIARY_SEND_FILTER") != 0)
&& (alGetEnumValue("AL_DIRECT_FILTER") != 0)
&& (alGetEnumValue("AL_EFFECT_TYPE") != 0)
&& (alGetEnumValue("AL_EFFECT_PITCH_SHIFTER") != 0)
&& (alGetEnumValue("AL_FILTER_TYPE") != 0)
&& (alGetEnumValue("AL_FILTER_LOWPASS") != 0)
&& (alGetEnumValue("AL_LOWPASS_GAIN") != 0)
&& (alGetEnumValue("AL_PITCH_SHIFTER_COARSE_TUNE") != 0)
&& (alGetEnumValue("AL_EFFECTSLOT_EFFECT") != 0);
} }
} // namespace Audio } // namespace Audio
@ -300,7 +281,7 @@ namespace Player {
namespace { namespace {
constexpr auto kVolumeRound = 10000; constexpr auto kVolumeRound = 10000;
constexpr auto kPreloadSamples = 2LL * kDefaultFrequency; // preload next part if less than 2 seconds remains constexpr auto kPreloadSeconds = 2LL; // preload next part if less than 2 seconds remains
constexpr auto kFadeDuration = crl::time(500); constexpr auto kFadeDuration = crl::time(500);
constexpr auto kCheckPlaybackPositionTimeout = crl::time(100); // 100ms per check audio position constexpr auto kCheckPlaybackPositionTimeout = crl::time(100); // 100ms per check audio position
constexpr auto kCheckPlaybackPositionDelta = 2400LL; // update position called each 2400 samples constexpr auto kCheckPlaybackPositionDelta = 2400LL; // update position called each 2400 samples
@ -348,43 +329,6 @@ void Mixer::Track::createStream(AudioMsgId::Type type) {
alGetEnumValue("AL_REMIX_UNMATCHED_SOFT")); alGetEnumValue("AL_REMIX_UNMATCHED_SOFT"));
} }
alGenBuffers(3, stream.buffers); alGenBuffers(3, stream.buffers);
if (speedEffect) {
applySourceSpeedEffect();
} else {
removeSourceSpeedEffect();
}
}
void Mixer::Track::removeSourceSpeedEffect() {
if (!Audio::SupportsSpeedControl()) {
return;
}
alSource3i(stream.source, alGetEnumValue("AL_AUXILIARY_SEND_FILTER"), alGetEnumValue("AL_EFFECTSLOT_NULL"), 0, 0);
alSourcei(stream.source, alGetEnumValue("AL_DIRECT_FILTER"), alGetEnumValue("AL_FILTER_NULL"));
alSourcef(stream.source, AL_PITCH, 1.f);
}
void Mixer::Track::applySourceSpeedEffect() {
if (!Audio::SupportsSpeedControl()) {
return;
}
Expects(speedEffect != nullptr);
if (!speedEffect->effect || !OpenAL::alIsEffect(speedEffect->effect)) {
OpenAL::alGenAuxiliaryEffectSlots(1, &speedEffect->effectSlot);
OpenAL::alGenEffects(1, &speedEffect->effect);
OpenAL::alGenFilters(1, &speedEffect->filter);
OpenAL::alEffecti(speedEffect->effect, alGetEnumValue("AL_EFFECT_TYPE"), alGetEnumValue("AL_EFFECT_PITCH_SHIFTER"));
OpenAL::alFilteri(speedEffect->filter, alGetEnumValue("AL_FILTER_TYPE"), alGetEnumValue("AL_FILTER_LOWPASS"));
OpenAL::alFilterf(speedEffect->filter, alGetEnumValue("AL_LOWPASS_GAIN"), 0.f);
}
OpenAL::alEffecti(speedEffect->effect, alGetEnumValue("AL_PITCH_SHIFTER_COARSE_TUNE"), speedEffect->coarseTune);
OpenAL::alAuxiliaryEffectSloti(speedEffect->effectSlot, alGetEnumValue("AL_EFFECTSLOT_EFFECT"), speedEffect->effect);
alSourcef(stream.source, AL_PITCH, speedEffect->speed);
alSource3i(stream.source, alGetEnumValue("AL_AUXILIARY_SEND_FILTER"), speedEffect->effectSlot, 0, 0);
alSourcei(stream.source, alGetEnumValue("AL_DIRECT_FILTER"), speedEffect->filter);
} }
void Mixer::Track::destroyStream() { void Mixer::Track::destroyStream() {
@ -396,45 +340,32 @@ void Mixer::Track::destroyStream() {
for (auto i = 0; i != 3; ++i) { for (auto i = 0; i != 3; ++i) {
stream.buffers[i] = 0; stream.buffers[i] = 0;
} }
resetSpeedEffect();
}
void Mixer::Track::resetSpeedEffect() {
if (!Audio::SupportsSpeedControl()) {
return;
}
if (!speedEffect) {
return;
} else if (speedEffect->effect && OpenAL::alIsEffect(speedEffect->effect)) {
if (isStreamCreated()) {
removeSourceSpeedEffect();
}
if (Player::mixer()) {
// Don't destroy effect slot immediately.
// See https://github.com/kcat/openal-soft/issues/486
Player::mixer()->scheduleEffectDestruction(*speedEffect);
}
}
speedEffect->effect = speedEffect->effectSlot = speedEffect->filter = 0;
} }
void Mixer::Track::reattach(AudioMsgId::Type type) { void Mixer::Track::reattach(AudioMsgId::Type type) {
if (isStreamCreated() if (isStreamCreated()
|| (!samplesCount[0] && !state.id.externalPlayId())) { || (!withSpeed.samples[0] && !state.id.externalPlayId())) {
return; return;
} }
createStream(type); createStream(type);
for (auto i = 0; i != kBuffersCount; ++i) { for (auto i = 0; i != kBuffersCount; ++i) {
if (!samplesCount[i]) { if (!withSpeed.samples[i]) {
break; break;
} }
alBufferData(stream.buffers[i], format, bufferSamples[i].constData(), bufferSamples[i].size(), frequency); alBufferData(
stream.buffers[i],
format,
withSpeed.buffered[i].constData(),
withSpeed.buffered[i].size(),
state.frequency);
alSourceQueueBuffers(stream.source, 1, stream.buffers + i); alSourceQueueBuffers(stream.source, 1, stream.buffers + i);
} }
alSourcei(stream.source, AL_SAMPLE_OFFSET, qMax(state.position - bufferedPosition, 0LL)); alSourcei(
stream.source,
AL_SAMPLE_OFFSET,
qMax(withSpeed.position - withSpeed.bufferedPosition, 0LL));
if (!IsStopped(state.state) if (!IsStopped(state.state)
&& (state.state != State::PausedAtEnd) && (state.state != State::PausedAtEnd)
&& !state.waitingForData) { && !state.waitingForData) {
@ -461,18 +392,12 @@ void Mixer::Track::clear() {
state = TrackState(); state = TrackState();
file = Core::FileLocation(); file = Core::FileLocation();
data = QByteArray(); data = QByteArray();
bufferedPosition = 0; format = 0;
bufferedLength = 0;
loading = false; loading = false;
loaded = false; loaded = false;
fadeStartPosition = 0; waitingForBuffer = false;
withSpeed = WithSpeed();
format = 0; speed = 1.;
frequency = kDefaultFrequency;
for (int i = 0; i != kBuffersCount; ++i) {
samplesCount[i] = 0;
bufferSamples[i] = QByteArray();
}
setExternalData(nullptr); setExternalData(nullptr);
lastUpdateWhen = 0; lastUpdateWhen = 0;
@ -482,17 +407,9 @@ void Mixer::Track::clear() {
void Mixer::Track::started() { void Mixer::Track::started() {
resetStream(); resetStream();
bufferedPosition = 0;
bufferedLength = 0;
loaded = false;
fadeStartPosition = 0;
format = 0; format = 0;
frequency = kDefaultFrequency; loaded = false;
for (auto i = 0; i != kBuffersCount; ++i) { withSpeed = WithSpeed();
samplesCount[i] = 0;
bufferSamples[i] = QByteArray();
}
} }
bool Mixer::Track::isStreamCreated() const { bool Mixer::Track::isStreamCreated() const {
@ -507,7 +424,7 @@ void Mixer::Track::ensureStreamCreated(AudioMsgId::Type type) {
int Mixer::Track::getNotQueuedBufferIndex() { int Mixer::Track::getNotQueuedBufferIndex() {
// See if there are no free buffers right now. // See if there are no free buffers right now.
while (samplesCount[kBuffersCount - 1] != 0) { while (withSpeed.samples[kBuffersCount - 1] != 0) {
// Try to unqueue some buffer. // Try to unqueue some buffer.
ALint processed = 0; ALint processed = 0;
alGetSourcei(stream.source, AL_BUFFERS_PROCESSED, &processed); alGetSourcei(stream.source, AL_BUFFERS_PROCESSED, &processed);
@ -523,17 +440,17 @@ int Mixer::Track::getNotQueuedBufferIndex() {
bool found = false; bool found = false;
for (auto i = 0; i != kBuffersCount; ++i) { for (auto i = 0; i != kBuffersCount; ++i) {
if (stream.buffers[i] == buffer) { if (stream.buffers[i] == buffer) {
auto samplesInBuffer = samplesCount[i]; const auto samplesInBuffer = withSpeed.samples[i];
bufferedPosition += samplesInBuffer; withSpeed.bufferedPosition += samplesInBuffer;
bufferedLength -= samplesInBuffer; withSpeed.bufferedLength -= samplesInBuffer;
for (auto j = i + 1; j != kBuffersCount; ++j) { for (auto j = i + 1; j != kBuffersCount; ++j) {
samplesCount[j - 1] = samplesCount[j]; withSpeed.samples[j - 1] = withSpeed.samples[j];
stream.buffers[j - 1] = stream.buffers[j]; stream.buffers[j - 1] = stream.buffers[j];
bufferSamples[j - 1] = bufferSamples[j]; withSpeed.buffered[j - 1] = withSpeed.buffered[j];
} }
samplesCount[kBuffersCount - 1] = 0; withSpeed.samples[kBuffersCount - 1] = 0;
stream.buffers[kBuffersCount - 1] = buffer; stream.buffers[kBuffersCount - 1] = buffer;
bufferSamples[kBuffersCount - 1] = QByteArray(); withSpeed.buffered[kBuffersCount - 1] = QByteArray();
found = true; found = true;
break; break;
} }
@ -545,7 +462,7 @@ int Mixer::Track::getNotQueuedBufferIndex() {
} }
for (auto i = 0; i != kBuffersCount; ++i) { for (auto i = 0; i != kBuffersCount; ++i) {
if (!samplesCount[i]) { if (!withSpeed.samples[i]) {
return i; return i;
} }
} }
@ -554,28 +471,33 @@ int Mixer::Track::getNotQueuedBufferIndex() {
void Mixer::Track::setExternalData( void Mixer::Track::setExternalData(
std::unique_ptr<ExternalSoundData> data) { std::unique_ptr<ExternalSoundData> data) {
changeSpeedEffect(data ? data->speed : 1.); nextSpeed = speed = data ? data->speed : 1.;
externalData = std::move(data); externalData = std::move(data);
} }
void Mixer::Track::changeSpeedEffect(float64 speed) { void Mixer::Track::updateStatePosition() {
if (!Audio::SupportsSpeedControl()) { state.position = SpeedIndependentPosition(withSpeed.position, speed);
return; }
}
if (speed != 1.) { void Mixer::Track::updateWithSpeedPosition() {
if (!speedEffect) { withSpeed.position = SpeedDependentPosition(state.position, speed);
speedEffect = std::make_unique<SpeedEffect>(); withSpeed.fineTunedPosition = withSpeed.position;
} }
speedEffect->speed = speed;
speedEffect->coarseTune = CoarseTuneForSpeed(speed); int64 Mixer::Track::SpeedIndependentPosition(
if (isStreamCreated()) { int64 position,
applySourceSpeedEffect(); float64 speed) {
} Expects(speed < 2.5);
} else if (speedEffect) {
resetSpeedEffect(); return int64(base::SafeRound(position * speed));
speedEffect = nullptr; }
}
int64 Mixer::Track::SpeedDependentPosition(
int64 position,
float64 speed) {
Expects(speed >= 0.5);
return int64(base::SafeRound(position / speed));
} }
void Mixer::Track::resetStream() { void Mixer::Track::resetStream() {
@ -589,24 +511,26 @@ Mixer::Track::~Track() = default;
Mixer::Mixer(not_null<Audio::Instance*> instance) Mixer::Mixer(not_null<Audio::Instance*> instance)
: _instance(instance) : _instance(instance)
, _effectsDestructionTimer([=] { destroyStaleEffectsSafe(); })
, _volumeVideo(kVolumeRound) , _volumeVideo(kVolumeRound)
, _volumeSong(kVolumeRound) , _volumeSong(kVolumeRound)
, _fader(new Fader(&_faderThread)) , _fader(new Fader(&_faderThread))
, _loader(new Loaders(&_loaderThread)) { , _loader(new Loaders(&_loaderThread)) {
connect(this, SIGNAL(faderOnTimer()), _fader, SLOT(onTimer()), Qt::QueuedConnection);
connect(this, SIGNAL(suppressSong()), _fader, SLOT(onSuppressSong())); connect(this, SIGNAL(suppressSong()), _fader, SLOT(onSuppressSong()));
connect(this, SIGNAL(unsuppressSong()), _fader, SLOT(onUnsuppressSong())); connect(this, SIGNAL(unsuppressSong()), _fader, SLOT(onUnsuppressSong()));
connect(this, SIGNAL(suppressAll(qint64)), _fader, SLOT(onSuppressAll(qint64))); connect(this, SIGNAL(suppressAll(qint64)), _fader, SLOT(onSuppressAll(qint64)));
Core::App().settings().songVolumeChanges( Core::App().settings().songVolumeChanges(
) | rpl::start_with_next([=] { ) | rpl::start_with_next([=] {
QMetaObject::invokeMethod(_fader, "onSongVolumeChanged"); InvokeQueued(_fader, [fader = _fader] {
fader->songVolumeChanged();
});
}, _lifetime); }, _lifetime);
Core::App().settings().videoVolumeChanges( Core::App().settings().videoVolumeChanges(
) | rpl::start_with_next([=] { ) | rpl::start_with_next([=] {
QMetaObject::invokeMethod(_fader, "onVideoVolumeChanged"); InvokeQueued(_fader, [fader = _fader] {
fader->videoVolumeChanged();
});
}, _lifetime); }, _lifetime);
connect(this, SIGNAL(loaderOnStart(const AudioMsgId&, qint64)), _loader, SLOT(onStart(const AudioMsgId&, qint64))); connect(this, SIGNAL(loaderOnStart(const AudioMsgId&, qint64)), _loader, SLOT(onStart(const AudioMsgId&, qint64)));
@ -645,6 +569,12 @@ Mixer::~Mixer() {
_loaderThread.wait(); _loaderThread.wait();
} }
void Mixer::scheduleFaderCallback() {
InvokeQueued(_fader, [fader = _fader] {
fader->onTimer();
});
}
void Mixer::onUpdated(const AudioMsgId &audio) { void Mixer::onUpdated(const AudioMsgId &audio) {
if (audio.externalPlayId()) { if (audio.externalPlayId()) {
externalSoundProgress(audio); externalSoundProgress(audio);
@ -656,60 +586,6 @@ void Mixer::onUpdated(const AudioMsgId &audio) {
}); });
} }
// Thread: Any. Must be locked: AudioMutex.
void Mixer::scheduleEffectDestruction(const SpeedEffect &effect) {
_effectsForDestruction.emplace_back(
crl::now() + kEffectDestructionDelay,
effect);
scheduleEffectsDestruction();
}
// Thread: Any. Must be locked: AudioMutex.
void Mixer::scheduleEffectsDestruction() {
if (_effectsForDestruction.empty()) {
return;
}
InvokeQueued(this, [=] {
if (!_effectsDestructionTimer.isActive()) {
_effectsDestructionTimer.callOnce(kEffectDestructionDelay + 1);
}
});
}
// Thread: Main. Locks: AudioMutex.
void Mixer::destroyStaleEffectsSafe() {
QMutexLocker lock(&AudioMutex);
destroyStaleEffects();
}
// Thread: Main. Must be locked: AudioMutex.
void Mixer::destroyStaleEffects() {
const auto now = crl::now();
const auto checkAndDestroy = [&](
const std::pair<crl::time, SpeedEffect> &pair) {
const auto &[when, effect] = pair;
if (when && when > now) {
return false;
}
OpenAL::alDeleteEffects(1, &effect.effect);
OpenAL::alDeleteAuxiliaryEffectSlots(1, &effect.effectSlot);
OpenAL::alDeleteFilters(1, &effect.filter);
return true;
};
_effectsForDestruction.erase(
ranges::remove_if(_effectsForDestruction, checkAndDestroy),
end(_effectsForDestruction));
scheduleEffectsDestruction();
}
// Thread: Main. Must be locked: AudioMutex.
void Mixer::destroyEffectsOnClose() {
for (auto &[when, effect] : _effectsForDestruction) {
when = 0;
}
destroyStaleEffects();
}
void Mixer::onError(const AudioMsgId &audio) { void Mixer::onError(const AudioMsgId &audio) {
stoppedOnError(audio); stoppedOnError(audio);
@ -789,7 +665,7 @@ void Mixer::resetFadeStartPosition(AudioMsgId::Type type, int positionInBuffered
} else if ((alState == AL_STOPPED) } else if ((alState == AL_STOPPED)
&& (alSampleOffset == 0) && (alSampleOffset == 0)
&& !internal::CheckAudioDeviceConnected()) { && !internal::CheckAudioDeviceConnected()) {
track->fadeStartPosition = track->state.position; track->withSpeed.fadeStartPosition = track->withSpeed.position;
return; return;
} }
@ -798,17 +674,19 @@ void Mixer::resetFadeStartPosition(AudioMsgId::Type type, int positionInBuffered
&& (!IsStopped(track->state.state) && (!IsStopped(track->state.state)
|| IsStoppedAtEnd(track->state.state))); || IsStoppedAtEnd(track->state.state)));
positionInBuffered = stoppedAtEnd positionInBuffered = stoppedAtEnd
? track->bufferedLength ? track->withSpeed.bufferedLength
: alSampleOffset; : alSampleOffset;
} else { } else {
positionInBuffered = 0; positionInBuffered = 0;
} }
} }
auto fullPosition = track->samplesCount[0] const auto withSpeedPosition = track->withSpeed.samples[0]
? (track->bufferedPosition + positionInBuffered) ? (track->withSpeed.bufferedPosition + positionInBuffered)
: track->state.position; : track->withSpeed.position;
track->state.position = fullPosition; track->withSpeed.fineTunedPosition = withSpeedPosition;
track->fadeStartPosition = fullPosition; track->withSpeed.position = withSpeedPosition;
track->withSpeed.fadeStartPosition = withSpeedPosition;
track->updateStatePosition();
} }
bool Mixer::fadedStop(AudioMsgId::Type type, bool *fadedStart) { bool Mixer::fadedStop(AudioMsgId::Type type, bool *fadedStart) {
@ -862,7 +740,7 @@ void Mixer::play(
} }
if (current->state.id) { if (current->state.id) {
loaderOnCancel(current->state.id); loaderOnCancel(current->state.id);
faderOnTimer(); scheduleFaderCallback();
} }
if (type != AudioMsgId::Type::Video) { if (type != AudioMsgId::Type::Video) {
auto foundCurrent = currentIndex(type); auto foundCurrent = currentIndex(type);
@ -887,6 +765,7 @@ void Mixer::play(
current->setExternalData(std::move(externalData)); current->setExternalData(std::move(externalData));
current->state.position = (positionMs * current->state.frequency) current->state.position = (positionMs * current->state.frequency)
/ 1000LL; / 1000LL;
current->updateWithSpeedPosition();
current->state.state = current->externalData current->state.state = current->externalData
? State::Paused ? State::Paused
: fadedStart : fadedStart
@ -916,7 +795,15 @@ void Mixer::setSpeedFromExternal(const AudioMsgId &audioId, float64 speed) {
QMutexLocker lock(&AudioMutex); QMutexLocker lock(&AudioMutex);
const auto track = trackForType(audioId.type()); const auto track = trackForType(audioId.type());
if (track->state.id == audioId) { if (track->state.id == audioId) {
track->changeSpeedEffect(speed); track->nextSpeed = speed;
if (track->speed != track->nextSpeed
&& !IsStoppedOrStopping(track->state.state)) {
track->loading = true;
track->loaded = false;
InvokeQueued(_loader, [loader = _loader, id = audioId] {
loader->onLoad(id);
});
}
} }
} }
@ -932,6 +819,10 @@ Streaming::TimePoint Mixer::getExternalSyncTimePoint(
if (track && track->state.id == audio && track->lastUpdateWhen > 0) { if (track && track->state.id == audio && track->lastUpdateWhen > 0) {
result.trackTime = track->lastUpdatePosition; result.trackTime = track->lastUpdatePosition;
result.worldTime = track->lastUpdateWhen; result.worldTime = track->lastUpdateWhen;
LOG(("Sync: Track Time %1, World Time: %2, Speed: %3"
).arg(result.trackTime / 1000.
).arg(result.worldTime / 1000.
).arg(track->speed));
} }
return result; return result;
} }
@ -957,9 +848,11 @@ void Mixer::externalSoundProgress(const AudioMsgId &audio) {
QMutexLocker lock(&AudioMutex); QMutexLocker lock(&AudioMutex);
const auto current = trackForType(type); const auto current = trackForType(type);
if (current && current->state.length && current->state.frequency) { if (current && current->state.length && current->state.frequency) {
if (current->state.id == audio && current->state.state == State::Playing) { if (current->state.id == audio
&& current->state.state == State::Playing) {
current->lastUpdateWhen = crl::now(); current->lastUpdateWhen = crl::now();
current->lastUpdatePosition = (current->state.position * 1000ULL) / current->state.frequency; current->lastUpdatePosition = (current->state.position * 1000LL)
/ current->state.frequency;
} }
} }
} }
@ -1014,7 +907,7 @@ void Mixer::pause(const AudioMsgId &audio, bool fast) {
} }
} }
faderOnTimer(); scheduleFaderCallback();
track->lastUpdateWhen = 0; track->lastUpdateWhen = 0;
track->lastUpdatePosition = 0; track->lastUpdatePosition = 0;
@ -1061,7 +954,10 @@ void Mixer::resume(const AudioMsgId &audio, bool fast) {
if (!checkCurrentALError(type)) return; if (!checkCurrentALError(type)) return;
if (state == AL_STOPPED) { if (state == AL_STOPPED) {
alSourcei(track->stream.source, AL_SAMPLE_OFFSET, qMax(track->state.position - track->bufferedPosition, 0LL)); alSourcei(
track->stream.source,
AL_SAMPLE_OFFSET,
qMax(track->withSpeed.position - track->withSpeed.bufferedPosition, 0LL));
if (!checkCurrentALError(type)) return; if (!checkCurrentALError(type)) return;
} }
alSourcePlay(track->stream.source); alSourcePlay(track->stream.source);
@ -1073,78 +969,10 @@ void Mixer::resume(const AudioMsgId &audio, bool fast) {
} }
} break; } break;
} }
faderOnTimer(); scheduleFaderCallback();
} }
if (current) updated(current); if (current) updated(current);
} }
//
// Right now all the music is played in the streaming player.
//
//void Mixer::seek(AudioMsgId::Type type, crl::time positionMs) {
// QMutexLocker lock(&AudioMutex);
//
// const auto current = trackForType(type);
// const auto audio = current->state.id;
//
// Audio::AttachToDevice();
// const auto streamCreated = current->isStreamCreated();
// const auto position = (positionMs * current->frequency) / 1000LL;
// const auto fastSeek = [&] {
// const auto loadedStart = current->bufferedPosition;
// const auto loadedLength = current->bufferedLength;
// const auto skipBack = (current->loaded ? 0 : kDefaultFrequency);
// const auto availableEnd = loadedStart + loadedLength - skipBack;
// if (position < loadedStart) {
// return false;
// } else if (position >= availableEnd) {
// return false;
// } else if (!streamCreated) {
// return false;
// } else if (IsStoppedOrStopping(current->state.state)) {
// return false;
// }
// return true;
// }();
// if (fastSeek) {
// alSourcei(current->stream.source, AL_SAMPLE_OFFSET, position - current->bufferedPosition);
// if (!checkCurrentALError(type)) return;
//
// alSourcef(current->stream.source, AL_GAIN, ComputeVolume(type));
// if (!checkCurrentALError(type)) return;
//
// resetFadeStartPosition(type, position - current->bufferedPosition);
// } else {
// setStoppedState(current);
// }
// switch (current->state.state) {
// case State::Pausing:
// case State::Paused:
// case State::PausedAtEnd: {
// if (current->state.state == State::PausedAtEnd) {
// current->state.state = State::Paused;
// }
// lock.unlock();
// return resume(audio, true);
// } break;
// case State::Starting:
// case State::Resuming:
// case State::Playing: {
// current->state.state = State::Pausing;
// resetFadeStartPosition(type);
// if (type == AudioMsgId::Type::Voice) {
// emit unsuppressSong();
// }
// } break;
// case State::Stopping:
// case State::Stopped:
// case State::StoppedAtEnd:
// case State::StoppedAtError:
// case State::StoppedAtStart: {
// lock.unlock();
// } return play(audio, positionMs);
// }
// emit faderOnTimer();
//}
void Mixer::stop(const AudioMsgId &audio) { void Mixer::stop(const AudioMsgId &audio) {
AudioMsgId current; AudioMsgId current;
@ -1239,6 +1067,8 @@ TrackState Mixer::currentState(AudioMsgId::Type type) {
void Mixer::setStoppedState(Track *current, State state) { void Mixer::setStoppedState(Track *current, State state) {
current->state.state = state; current->state.state = state;
current->state.position = 0; current->state.position = 0;
current->withSpeed.position = 0;
current->withSpeed.fineTunedPosition = 0;
if (current->isStreamCreated()) { if (current->isStreamCreated()) {
alSourceStop(current->stream.source); alSourceStop(current->stream.source);
alSourcef(current->stream.source, AL_GAIN, 1); alSourcef(current->stream.source, AL_GAIN, 1);
@ -1255,8 +1085,6 @@ void Mixer::prepareToCloseDevice() {
trackForType(AudioMsgId::Type::Song, i)->detach(); trackForType(AudioMsgId::Type::Song, i)->detach();
} }
_videoTrack.detach(); _videoTrack.detach();
destroyEffectsOnClose();
} }
// Thread: Main. Must be locked: AudioMutex. // Thread: Main. Must be locked: AudioMutex.
@ -1427,12 +1255,13 @@ int32 Fader::updateOnePlayback(Mixer::Track *track, bool &hasPlaying, bool &hasF
&& (!IsStopped(track->state.state) && (!IsStopped(track->state.state)
|| IsStoppedAtEnd(track->state.state))); || IsStoppedAtEnd(track->state.state)));
const auto positionInBuffered = stoppedAtEnd const auto positionInBuffered = stoppedAtEnd
? track->bufferedLength ? track->withSpeed.bufferedLength
: alSampleOffset; : alSampleOffset;
const auto waitingForDataOld = track->state.waitingForData; const auto waitingForDataOld = track->state.waitingForData;
track->state.waitingForData = stoppedAtEnd track->state.waitingForData = stoppedAtEnd
&& (track->state.state != State::Stopping); && (track->state.state != State::Stopping);
const auto fullPosition = track->bufferedPosition + positionInBuffered; const auto withSpeedPosition = track->withSpeed.bufferedPosition
+ positionInBuffered;
auto playing = (track->state.state == State::Playing); auto playing = (track->state.state == State::Playing);
auto fading = IsFading(track->state.state); auto fading = IsFading(track->state.state);
@ -1451,7 +1280,8 @@ int32 Fader::updateOnePlayback(Mixer::Track *track, bool &hasPlaying, bool &hasF
emitSignals |= EmitStopped; emitSignals |= EmitStopped;
} }
} else if (fading && alState == AL_PLAYING) { } else if (fading && alState == AL_PLAYING) {
auto fadingForSamplesCount = (fullPosition - track->fadeStartPosition); const auto fadingForSamplesCount = withSpeedPosition
- track->withSpeed.fadeStartPosition;
if (crl::time(1000) * fadingForSamplesCount >= kFadeDuration * track->state.frequency) { if (crl::time(1000) * fadingForSamplesCount >= kFadeDuration * track->state.frequency) {
fading = false; fading = false;
alSourcef(track->stream.source, AL_GAIN, 1. * volumeMultiplier); alSourcef(track->stream.source, AL_GAIN, 1. * volumeMultiplier);
@ -1488,20 +1318,22 @@ int32 Fader::updateOnePlayback(Mixer::Track *track, bool &hasPlaying, bool &hasF
if (errorHappened()) return EmitError; if (errorHappened()) return EmitError;
} }
} }
if (alState == AL_PLAYING && fullPosition >= track->state.position + kCheckPlaybackPositionDelta) { track->withSpeed.fineTunedPosition = withSpeedPosition;
track->state.position = fullPosition; if (alState == AL_PLAYING && withSpeedPosition >= track->withSpeed.position + kCheckPlaybackPositionDelta) {
track->withSpeed.position = withSpeedPosition;
track->updateStatePosition();
emitSignals |= EmitPositionUpdated; emitSignals |= EmitPositionUpdated;
} else if (track->state.waitingForData && !waitingForDataOld) { } else if (track->state.waitingForData && !waitingForDataOld) {
if (fullPosition > track->state.position) { if (withSpeedPosition > track->withSpeed.position) {
track->state.position = fullPosition; track->withSpeed.position = withSpeedPosition;
} }
// When stopped because of insufficient data while streaming, // When stopped because of insufficient data while streaming,
// inform the player about the last position we were at. // inform the player about the last position we were at.
emitSignals |= EmitPositionUpdated; emitSignals |= EmitPositionUpdated;
} }
if (playing || track->state.state == State::Starting || track->state.state == State::Resuming) { if (playing || track->state.state == State::Starting || track->state.state == State::Resuming) {
if (!track->loaded && !track->loading) { if ((!track->loaded && !track->loading) || track->waitingForBuffer) {
auto needPreload = (track->state.position + kPreloadSamples > track->bufferedPosition + track->bufferedLength); auto needPreload = (track->withSpeed.position + kPreloadSeconds * track->state.frequency > track->withSpeed.bufferedPosition + track->withSpeed.bufferedLength);
if (needPreload) { if (needPreload) {
track->loading = true; track->loading = true;
emitSignals |= EmitNeedToPreload; emitSignals |= EmitNeedToPreload;
@ -1549,12 +1381,12 @@ void Fader::onSuppressAll(qint64 duration) {
onTimer(); onTimer();
} }
void Fader::onSongVolumeChanged() { void Fader::songVolumeChanged() {
_volumeChangedSong = true; _volumeChangedSong = true;
onTimer(); onTimer();
} }
void Fader::onVideoVolumeChanged() { void Fader::videoVolumeChanged() {
_volumeChangedVideo = true; _volumeChangedVideo = true;
onTimer(); onTimer();
} }
@ -1615,13 +1447,11 @@ public:
: AbstractFFMpegLoader(file, data, bytes::vector()) { : AbstractFFMpegLoader(file, data, bytes::vector()) {
} }
bool open(crl::time positionMs) override { bool open(crl::time positionMs, float64 speed = 1.) override {
if (!AbstractFFMpegLoader::open(positionMs)) { if (!AbstractFFMpegLoader::open(positionMs, speed)) {
return false; return false;
} }
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
for (int32 i = 0, l = fmtContext->nb_streams; i < l; ++i) { for (int32 i = 0, l = fmtContext->nb_streams; i < l; ++i) {
const auto stream = fmtContext->streams[i]; const auto stream = fmtContext->streams[i];
if (stream->disposition & AV_DISPOSITION_ATTACHED_PIC) { if (stream->disposition & AV_DISPOSITION_ATTACHED_PIC) {
@ -1644,11 +1474,10 @@ public:
} }
} }
} else if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { } else if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
DEBUG_LOG(("Audio Read Error: Found video stream in file '%1', data size '%2', error %3, %4") DEBUG_LOG(("Audio Read Error: Found video stream in file '%1', data size '%2', stream %3.")
.arg(_file.name()) .arg(_file.name())
.arg(_data.size()) .arg(_data.size())
.arg(i) .arg(i));
.arg(av_make_error_string(err, sizeof(err), streamId)));
return false; return false;
} }
} }
@ -1729,8 +1558,8 @@ Ui::PreparedFileInformation PrepareForSending(
auto result = Ui::PreparedFileInformation::Song(); auto result = Ui::PreparedFileInformation::Song();
FFMpegAttributesReader reader(Core::FileLocation(fname), data); FFMpegAttributesReader reader(Core::FileLocation(fname), data);
const auto positionMs = crl::time(0); const auto positionMs = crl::time(0);
if (reader.open(positionMs) && reader.samplesCount() > 0) { if (reader.open(positionMs) && reader.duration() > 0) {
result.duration = reader.samplesCount() / reader.samplesFrequency(); result.duration = reader.duration() / 1000;
result.title = reader.title(); result.title = reader.title();
result.performer = reader.performer(); result.performer = reader.performer();
result.cover = reader.cover(); result.cover = reader.cover();
@ -1745,17 +1574,18 @@ public:
FFMpegWaveformCounter(const Core::FileLocation &file, const QByteArray &data) : FFMpegLoader(file, data, bytes::vector()) { FFMpegWaveformCounter(const Core::FileLocation &file, const QByteArray &data) : FFMpegLoader(file, data, bytes::vector()) {
} }
bool open(crl::time positionMs) override { bool open(crl::time positionMs, float64 speed = 1.) override {
if (!FFMpegLoader::open(positionMs)) { if (!FFMpegLoader::open(positionMs, speed)) {
return false; return false;
} }
QByteArray buffer; QByteArray buffer;
buffer.reserve(kWaveformCounterBufferSize); buffer.reserve(kWaveformCounterBufferSize);
int64 countbytes = sampleSize() * samplesCount(); const auto samplesCount = samplesFrequency() * duration() / 1000;
int64 countbytes = sampleSize() * samplesCount;
int64 processed = 0; int64 processed = 0;
int64 sumbytes = 0; int64 sumbytes = 0;
if (samplesCount() < Media::Player::kWaveformSamplesCount) { if (samplesCount < Media::Player::kWaveformSamplesCount) {
return false; return false;
} }
@ -1775,16 +1605,16 @@ public:
}; };
while (processed < countbytes) { while (processed < countbytes) {
const auto result = readMore(); const auto result = readMore();
const auto sampleBytes = v::is<bytes::const_span>(result) Assert(result != ReadError::Wait); // Not a child loader.
? v::get<bytes::const_span>(result) if (result == ReadError::Retry) {
: bytes::const_span(); continue;
if (result == ReadError::Other } else if (result == ReadError::Other
|| result == ReadError::EndOfFile) { || result == ReadError::EndOfFile) {
break; break;
} else if (sampleBytes.empty()) {
continue;
} }
Assert(v::is<bytes::const_span>(result));
const auto sampleBytes = v::get<bytes::const_span>(result);
Assert(!sampleBytes.empty());
if (fmt == AL_FORMAT_MONO8 || fmt == AL_FORMAT_STEREO8) { if (fmt == AL_FORMAT_MONO8 || fmt == AL_FORMAT_STEREO8) {
Media::Audio::IterateSamples<uchar>(sampleBytes, callback); Media::Audio::IterateSamples<uchar>(sampleBytes, callback);
} else if (fmt == AL_FORMAT_MONO16 || fmt == AL_FORMAT_STEREO16) { } else if (fmt == AL_FORMAT_MONO16 || fmt == AL_FORMAT_STEREO16) {

View file

@ -35,6 +35,9 @@ namespace Audio {
class Instance; class Instance;
inline constexpr auto kSpeedMin = 0.5;
inline constexpr auto kSpeedMax = 2.5;
// Thread: Main. // Thread: Main.
void Start(not_null<Instance*> instance); void Start(not_null<Instance*> instance);
void Finish(not_null<Instance*> instance); void Finish(not_null<Instance*> instance);
@ -180,6 +183,8 @@ public:
void setVideoVolume(float64 volume); void setVideoVolume(float64 volume);
float64 getVideoVolume() const; float64 getVideoVolume() const;
void scheduleFaderCallback();
~Mixer(); ~Mixer();
private Q_SLOTS: private Q_SLOTS:
@ -194,21 +199,11 @@ Q_SIGNALS:
void loaderOnStart(const AudioMsgId &audio, qint64 positionMs); void loaderOnStart(const AudioMsgId &audio, qint64 positionMs);
void loaderOnCancel(const AudioMsgId &audio); void loaderOnCancel(const AudioMsgId &audio);
void faderOnTimer();
void suppressSong(); void suppressSong();
void unsuppressSong(); void unsuppressSong();
void suppressAll(qint64 duration); void suppressAll(qint64 duration);
private: private:
struct SpeedEffect {
uint32 effect = 0;
uint32 effectSlot = 0;
uint32 filter = 0;
int coarseTune = 0;
float64 speed = 1.;
};
class Track { class Track {
public: public:
static constexpr int kBuffersCount = 3; static constexpr int kBuffersCount = 3;
@ -229,7 +224,16 @@ private:
// Thread: Main. Must be locked: AudioMutex. // Thread: Main. Must be locked: AudioMutex.
void setExternalData(std::unique_ptr<ExternalSoundData> data); void setExternalData(std::unique_ptr<ExternalSoundData> data);
void changeSpeedEffect(float64 speed);
void updateStatePosition();
void updateWithSpeedPosition();
[[nodiscard]] static int64 SpeedIndependentPosition(
int64 position,
float64 speed);
[[nodiscard]] static int64 SpeedDependentPosition(
int64 position,
float64 speed);
~Track(); ~Track();
@ -237,25 +241,35 @@ private:
Core::FileLocation file; Core::FileLocation file;
QByteArray data; QByteArray data;
int64 bufferedPosition = 0;
int64 bufferedLength = 0; int format = 0;
bool loading = false; bool loading = false;
bool loaded = false; bool loaded = false;
int64 fadeStartPosition = 0; bool waitingForBuffer = false;
int32 format = 0; // Speed dependent values.
int32 frequency = kDefaultFrequency; float64 speed = 1.;
int samplesCount[kBuffersCount] = { 0 }; float64 nextSpeed = 1.;
QByteArray bufferSamples[kBuffersCount]; struct WithSpeed {
int64 fineTunedPosition = 0;
int64 position = 0;
int64 length = 0;
int64 bufferedPosition = 0;
int64 bufferedLength = 0;
int64 fadeStartPosition = 0;
int samples[kBuffersCount] = { 0 };
QByteArray buffered[kBuffersCount];
};
WithSpeed withSpeed;
struct Stream { struct Stream {
uint32 source = 0; uint32 source = 0;
uint32 buffers[kBuffersCount] = { 0 }; uint32 buffers[kBuffersCount] = { 0 };
}; };
Stream stream; Stream stream;
std::unique_ptr<ExternalSoundData> externalData; std::unique_ptr<ExternalSoundData> externalData;
std::unique_ptr<SpeedEffect> speedEffect;
crl::time lastUpdateWhen = 0; crl::time lastUpdateWhen = 0;
crl::time lastUpdatePosition = 0; crl::time lastUpdatePosition = 0;
@ -263,9 +277,6 @@ private:
void createStream(AudioMsgId::Type type); void createStream(AudioMsgId::Type type);
void destroyStream(); void destroyStream();
void resetStream(); void resetStream();
void resetSpeedEffect();
void applySourceSpeedEffect();
void removeSourceSpeedEffect();
}; };
@ -283,17 +294,6 @@ private:
int *currentIndex(AudioMsgId::Type type); int *currentIndex(AudioMsgId::Type type);
const int *currentIndex(AudioMsgId::Type type) const; const int *currentIndex(AudioMsgId::Type type) const;
// Thread: Any. Must be locked: AudioMutex.
void scheduleEffectDestruction(const SpeedEffect &effect);
void scheduleEffectsDestruction();
// Thread: Main. Must be locked: AudioMutex.
void destroyStaleEffects();
void destroyEffectsOnClose();
// Thread: Main. Locks: AudioMutex.
void destroyStaleEffectsSafe();
const not_null<Audio::Instance*> _instance; const not_null<Audio::Instance*> _instance;
int _audioCurrent = 0; int _audioCurrent = 0;
@ -304,9 +304,6 @@ private:
Track _videoTrack; Track _videoTrack;
std::vector<std::pair<crl::time, SpeedEffect>> _effectsForDestruction;
base::Timer _effectsDestructionTimer;
QAtomicInt _volumeVideo; QAtomicInt _volumeVideo;
QAtomicInt _volumeSong; QAtomicInt _volumeSong;
@ -329,6 +326,9 @@ class Fader : public QObject {
public: public:
Fader(QThread *thread); Fader(QThread *thread);
void songVolumeChanged();
void videoVolumeChanged();
Q_SIGNALS: Q_SIGNALS:
void error(const AudioMsgId &audio); void error(const AudioMsgId &audio);
void playPositionUpdated(const AudioMsgId &audio); void playPositionUpdated(const AudioMsgId &audio);
@ -342,8 +342,6 @@ public Q_SLOTS:
void onSuppressSong(); void onSuppressSong();
void onUnsuppressSong(); void onUnsuppressSong();
void onSuppressAll(qint64 duration); void onSuppressAll(qint64 duration);
void onSongVolumeChanged();
void onVideoVolumeChanged();
private: private:
enum { enum {

View file

@ -11,7 +11,18 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include "ffmpeg/ffmpeg_utility.h" #include "ffmpeg/ffmpeg_utility.h"
#include "base/bytes.h" #include "base/bytes.h"
extern "C" {
#include <libavfilter/buffersink.h>
#include <libavfilter/buffersrc.h>
} // extern "C"
namespace Media { namespace Media {
namespace {
using FFmpeg::AvErrorWrap;
using FFmpeg::LogError;
} // namespace
#if !DA_FFMPEG_NEW_CHANNEL_LAYOUT #if !DA_FFMPEG_NEW_CHANNEL_LAYOUT
uint64_t AbstractFFMpegLoader::ComputeChannelLayout( uint64_t AbstractFFMpegLoader::ComputeChannelLayout(
@ -30,14 +41,11 @@ int64 AbstractFFMpegLoader::Mul(int64 value, AVRational rational) {
return value * rational.num / rational.den; return value * rational.num / rational.den;
} }
bool AbstractFFMpegLoader::open(crl::time positionMs) { bool AbstractFFMpegLoader::open(crl::time positionMs, float64 speed) {
if (!AudioPlayerLoader::openFile()) { if (!AudioPlayerLoader::openFile()) {
return false; return false;
} }
int res = 0;
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
ioBuffer = (uchar *)av_malloc(FFmpeg::kAVBlockSize); ioBuffer = (uchar *)av_malloc(FFmpeg::kAVBlockSize);
if (!_data.isEmpty()) { if (!_data.isEmpty()) {
ioContext = avio_alloc_context(ioBuffer, FFmpeg::kAVBlockSize, 0, reinterpret_cast<void *>(this), &AbstractFFMpegLoader::ReadData, 0, &AbstractFFMpegLoader::SeekData); ioContext = avio_alloc_context(ioBuffer, FFmpeg::kAVBlockSize, 0, reinterpret_cast<void *>(this), &AbstractFFMpegLoader::ReadData, 0, &AbstractFFMpegLoader::SeekData);
@ -48,27 +56,26 @@ bool AbstractFFMpegLoader::open(crl::time positionMs) {
} }
fmtContext = avformat_alloc_context(); fmtContext = avformat_alloc_context();
if (!fmtContext) { if (!fmtContext) {
DEBUG_LOG(("Audio Read Error: Unable to avformat_alloc_context for file '%1', data size '%2'").arg(_file.name()).arg(_data.size())); LogError(u"avformat_alloc_context"_q);
return false; return false;
} }
fmtContext->pb = ioContext; fmtContext->pb = ioContext;
if ((res = avformat_open_input(&fmtContext, 0, 0, 0)) < 0) { if (AvErrorWrap error = avformat_open_input(&fmtContext, 0, 0, 0)) {
ioBuffer = 0; ioBuffer = nullptr;
LogError(u"avformat_open_input"_q, error);
DEBUG_LOG(("Audio Read Error: Unable to avformat_open_input for file '%1', data size '%2', error %3, %4").arg(_file.name()).arg(_data.size()).arg(res).arg(av_make_error_string(err, sizeof(err), res)));
return false; return false;
} }
_opened = true; _opened = true;
if ((res = avformat_find_stream_info(fmtContext, 0)) < 0) { if (AvErrorWrap error = avformat_find_stream_info(fmtContext, 0)) {
DEBUG_LOG(("Audio Read Error: Unable to avformat_find_stream_info for file '%1', data size '%2', error %3, %4").arg(_file.name()).arg(_data.size()).arg(res).arg(av_make_error_string(err, sizeof(err), res))); LogError(u"avformat_find_stream_info"_q, error);
return false; return false;
} }
streamId = av_find_best_stream(fmtContext, AVMEDIA_TYPE_AUDIO, -1, -1, &codec, 0); streamId = av_find_best_stream(fmtContext, AVMEDIA_TYPE_AUDIO, -1, -1, &codec, 0);
if (streamId < 0) { if (streamId < 0) {
LOG(("Audio Error: Unable to av_find_best_stream for file '%1', data size '%2', error %3, %4").arg(_file.name()).arg(_data.size()).arg(streamId).arg(av_make_error_string(err, sizeof(err), streamId))); FFmpeg::LogError(u"av_find_best_stream"_q, AvErrorWrap(streamId));
return false; return false;
} }
@ -76,15 +83,11 @@ bool AbstractFFMpegLoader::open(crl::time positionMs) {
const auto params = stream->codecpar; const auto params = stream->codecpar;
_samplesFrequency = params->sample_rate; _samplesFrequency = params->sample_rate;
if (stream->duration != AV_NOPTS_VALUE) { if (stream->duration != AV_NOPTS_VALUE) {
_samplesCount = Mul( _duration = Mul(stream->duration * 1000, stream->time_base);
stream->duration * _samplesFrequency,
stream->time_base);
} else { } else {
_samplesCount = Mul( _duration = Mul(fmtContext->duration * 1000, { 1, AV_TIME_BASE });
fmtContext->duration * _samplesFrequency,
{ 1, AV_TIME_BASE });
} }
_startedAtSample = (positionMs * _samplesFrequency) / 1000LL;
return true; return true;
} }
@ -203,10 +206,42 @@ AbstractAudioFFMpegLoader::AbstractAudioFFMpegLoader(
, _frame(FFmpeg::MakeFramePointer()) { , _frame(FFmpeg::MakeFramePointer()) {
} }
void AbstractAudioFFMpegLoader::dropFramesTill(int64 samples) {
const auto isAfter = [&](const EnqueuedFrame &frame) {
return frame.position > samples;
};
const auto from = begin(_framesQueued);
const auto after = ranges::find_if(_framesQueued, isAfter);
if (from == after) {
return;
}
const auto till = after - 1;
const auto erasing = till - from;
if (erasing > 0) {
if (_framesQueuedIndex >= 0) {
Assert(_framesQueuedIndex >= erasing);
_framesQueuedIndex -= erasing;
}
_framesQueued.erase(from, till);
if (_framesQueued.empty()) {
_framesQueuedIndex = -1;
}
}
}
int64 AbstractAudioFFMpegLoader::startReadingQueuedFrames(float64 newSpeed) {
changeSpeedFilter(newSpeed);
if (_framesQueued.empty()) {
_framesQueuedIndex = -1;
return -1;
}
_framesQueuedIndex = 0;
return _framesQueued.front().position;
}
bool AbstractAudioFFMpegLoader::initUsingContext( bool AbstractAudioFFMpegLoader::initUsingContext(
not_null<AVCodecContext*> context, not_null<AVCodecContext*> context,
int64 initialCount, float64 speed) {
int initialFrequency) {
_swrSrcSampleFormat = context->sample_fmt; _swrSrcSampleFormat = context->sample_fmt;
#if DA_FFMPEG_NEW_CHANNEL_LAYOUT #if DA_FFMPEG_NEW_CHANNEL_LAYOUT
const AVChannelLayout mono = AV_CHANNEL_LAYOUT_MONO; const AVChannelLayout mono = AV_CHANNEL_LAYOUT_MONO;
@ -275,15 +310,8 @@ bool AbstractAudioFFMpegLoader::initUsingContext(
} }
} }
if (_swrDstRate == initialFrequency) { createSpeedFilter(speed);
_outputSamplesCount = initialCount;
} else {
_outputSamplesCount = av_rescale_rnd(
initialCount,
_swrDstRate,
initialFrequency,
AV_ROUND_UP);
}
return true; return true;
} }
@ -297,44 +325,98 @@ auto AbstractAudioFFMpegLoader::replaceFrameAndRead(
auto AbstractAudioFFMpegLoader::readFromReadyContext( auto AbstractAudioFFMpegLoader::readFromReadyContext(
not_null<AVCodecContext*> context) not_null<AVCodecContext*> context)
-> ReadResult { -> ReadResult {
const auto res = avcodec_receive_frame(context, _frame.get()); if (_filterGraph) {
if (res >= 0) { AvErrorWrap error = av_buffersink_get_frame(
_filterSink,
_filteredFrame.get());
if (!error) {
if (!_filteredFrame->nb_samples) {
return ReadError::Retry;
}
return bytes::const_span(
reinterpret_cast<const bytes::type*>(
_filteredFrame->extended_data[0]),
_filteredFrame->nb_samples * _outputSampleSize);
} else if (error.code() == AVERROR_EOF) {
return ReadError::EndOfFile;
} else if (error.code() != AVERROR(EAGAIN)) {
LogError(u"av_buffersink_get_frame"_q, error);
return ReadError::Other;
}
}
using Enqueued = not_null<const EnqueuedFrame*>;
const auto queueResult = fillFrameFromQueued();
if (queueResult == ReadError::RetryNotQueued) {
return ReadError::RetryNotQueued;
} else if (const auto enqueued = std::get_if<Enqueued>(&queueResult)) {
const auto raw = (*enqueued)->frame.get();
Assert(frameHasDesiredFormat(raw));
return readOrBufferForFilter(raw, (*enqueued)->samples);
}
const auto queueError = v::get<ReadError>(queueResult);
AvErrorWrap error = (queueError == ReadError::EndOfFile)
? AVERROR_EOF
: avcodec_receive_frame(context, _frame.get());
if (!error) {
return readFromReadyFrame(); return readFromReadyFrame();
} }
if (res == AVERROR_EOF) { if (error.code() == AVERROR_EOF) {
return ReadError::EndOfFile; enqueueFramesFinished();
} else if (res != AVERROR(EAGAIN)) { if (!_filterGraph) {
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; return ReadError::EndOfFile;
LOG(("Audio Error: " }
"Unable to avcodec_receive_frame() file '%1', data size '%2', " AvErrorWrap error = av_buffersrc_add_frame(_filterSrc, nullptr);
"error %3, %4" if (!error) {
).arg(_file.name() return ReadError::Retry;
).arg(_data.size() }
).arg(res LogError(u"av_buffersrc_add_frame"_q, error);
).arg(av_make_error_string(err, sizeof(err), res) return ReadError::Other;
)); } else if (error.code() != AVERROR(EAGAIN)) {
LogError(u"avcodec_receive_frame"_q, error);
return ReadError::Other; return ReadError::Other;
} }
return ReadError::Wait; return ReadError::Wait;
} }
bool AbstractAudioFFMpegLoader::frameHasDesiredFormat() const { auto AbstractAudioFFMpegLoader::fillFrameFromQueued()
-> std::variant<not_null<const EnqueuedFrame*>, ReadError> {
if (_framesQueuedIndex == _framesQueued.size()) {
_framesQueuedIndex = -1;
return ReadError::RetryNotQueued;
} else if (_framesQueuedIndex < 0) {
return ReadError::Wait;
}
const auto &queued = _framesQueued[_framesQueuedIndex];
++_framesQueuedIndex;
if (!queued.frame) {
return ReadError::EndOfFile;
}
LOG(("Returning At %1 Data: %2"
).arg(queued.position
).arg(quintptr(queued.frame->extended_data[0])));
return &queued;
}
bool AbstractAudioFFMpegLoader::frameHasDesiredFormat(
not_null<AVFrame*> frame) const {
const auto sameChannelLayout = [&] { const auto sameChannelLayout = [&] {
#if DA_FFMPEG_NEW_CHANNEL_LAYOUT #if DA_FFMPEG_NEW_CHANNEL_LAYOUT
return !av_channel_layout_compare( return !av_channel_layout_compare(
&_frame->ch_layout, &frame->ch_layout,
&_swrDstChannelLayout); &_swrDstChannelLayout);
#else // DA_FFMPEG_NEW_CHANNEL_LAYOUT #else // DA_FFMPEG_NEW_CHANNEL_LAYOUT
const auto frameChannelLayout = ComputeChannelLayout( const auto frameChannelLayout = ComputeChannelLayout(
_frame->channel_layout, frame->channel_layout,
_frame->channels); frame->channels);
return (frameChannelLayout == _swrDstChannelLayout); return (frameChannelLayout == _swrDstChannelLayout);
#endif // DA_FFMPEG_NEW_CHANNEL_LAYOUT #endif // DA_FFMPEG_NEW_CHANNEL_LAYOUT
}; };
return true return true
&& (_frame->format == _swrDstSampleFormat) && (frame->format == _swrDstSampleFormat)
&& (_frame->sample_rate == _swrDstRate) && (frame->sample_rate == _swrDstRate)
&& sameChannelLayout(); && sameChannelLayout();
} }
@ -392,9 +474,9 @@ bool AbstractAudioFFMpegLoader::initResampleForFrame() {
} }
bool AbstractAudioFFMpegLoader::initResampleUsingFormat() { bool AbstractAudioFFMpegLoader::initResampleUsingFormat() {
int res = 0; AvErrorWrap error = 0;
#if DA_FFMPEG_NEW_CHANNEL_LAYOUT #if DA_FFMPEG_NEW_CHANNEL_LAYOUT
res = swr_alloc_set_opts2( error = swr_alloc_set_opts2(
&_swrContext, &_swrContext,
&_swrDstChannelLayout, &_swrDstChannelLayout,
_swrDstSampleFormat, _swrDstSampleFormat,
@ -416,38 +498,25 @@ bool AbstractAudioFFMpegLoader::initResampleUsingFormat() {
0, 0,
nullptr); nullptr);
#endif // DA_FFMPEG_NEW_CHANNEL_LAYOUT #endif // DA_FFMPEG_NEW_CHANNEL_LAYOUT
if (res < 0 || !_swrContext) { if (error || !_swrContext) {
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; LogError(u"swr_alloc_set_opts2"_q, error);
LOG(("Audio Error: "
"Unable to swr_alloc_set_opts2 for file '%1', data size '%2', "
"error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(res
).arg(av_make_error_string(err, sizeof(err), res)
));
return false; return false;
} else if ((res = swr_init(_swrContext)) < 0) { } else if (AvErrorWrap error = swr_init(_swrContext)) {
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; LogError(u"swr_init"_q, error);
LOG(("Audio Error: "
"Unable to swr_init for file '%1', data size '%2', "
"error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(res
).arg(av_make_error_string(err, sizeof(err), res)
));
return false; return false;
} }
if (_swrDstData) { _resampledFrame = nullptr;
av_freep(&_swrDstData[0]); _resampledFrameCapacity = 0;
_swrDstDataCapacity = -1;
}
return true; return true;
} }
bool AbstractAudioFFMpegLoader::ensureResampleSpaceAvailable(int samples) { bool AbstractAudioFFMpegLoader::ensureResampleSpaceAvailable(int samples) {
if (_swrDstData != nullptr && _swrDstDataCapacity >= samples) { const auto enlarge = (_resampledFrameCapacity < samples);
if (!_resampledFrame) {
_resampledFrame = FFmpeg::MakeFramePointer();
} else if (enlarge || !av_frame_is_writable(_resampledFrame.get())) {
av_frame_unref(_resampledFrame.get());
} else {
return true; return true;
} }
const auto allocate = std::max(samples, int(av_rescale_rnd( const auto allocate = std::max(samples, int(av_rescale_rnd(
@ -455,46 +524,166 @@ bool AbstractAudioFFMpegLoader::ensureResampleSpaceAvailable(int samples) {
_swrDstRate, _swrDstRate,
_swrSrcRate, _swrSrcRate,
AV_ROUND_UP))); AV_ROUND_UP)));
_resampledFrame->sample_rate = _swrDstRate;
if (_swrDstData) { _resampledFrame->format = _swrDstSampleFormat;
av_freep(&_swrDstData[0]); av_channel_layout_copy(
} &_resampledFrame->ch_layout,
const auto res = _swrDstData &_swrDstChannelLayout);
? av_samples_alloc( _resampledFrame->nb_samples = allocate;
_swrDstData, if (AvErrorWrap error = av_frame_get_buffer(_resampledFrame.get(), 0)) {
nullptr, LogError(u"av_frame_get_buffer"_q, error);
_outputChannels,
allocate,
_swrDstSampleFormat,
0)
: av_samples_alloc_array_and_samples(
&_swrDstData,
nullptr,
_outputChannels,
allocate,
_swrDstSampleFormat,
0);
if (res < 0) {
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
LOG(("Audio Error: "
"Unable to av_samples_alloc for file '%1', data size '%2', "
"error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(res
).arg(av_make_error_string(err, sizeof(err), res)
));
return false; return false;
} }
_swrDstDataCapacity = allocate; _resampledFrameCapacity = allocate;
return true; return true;
} }
auto AbstractAudioFFMpegLoader::readFromReadyFrame() -> ReadResult { bool AbstractAudioFFMpegLoader::changeSpeedFilter(float64 speed) {
if (frameHasDesiredFormat()) { speed = std::clamp(speed, Audio::kSpeedMin, Audio::kSpeedMax);
return bytes::const_span( if (_filterSpeed == speed) {
reinterpret_cast<const bytes::type*>(_frame->extended_data[0]), return false;
_frame->nb_samples * _outputSampleSize); }
avfilter_graph_free(&_filterGraph);
const auto guard = gsl::finally([&] {
if (!_filterGraph) {
_filteredFrame = nullptr;
_filterSpeed = 1.;
}
});
createSpeedFilter(speed);
return true;
}
void AbstractAudioFFMpegLoader::createSpeedFilter(float64 speed) {
Expects(!_filterGraph);
if (speed == 1.) {
return;
}
const auto abuffer = avfilter_get_by_name("abuffer");
const auto abuffersink = avfilter_get_by_name("abuffersink");
const auto atempo = avfilter_get_by_name("atempo");
if (!abuffer || !abuffersink || !atempo) {
LOG(("FFmpeg Error: Could not find abuffer / abuffersink /atempo."));
return;
}
auto graph = avfilter_graph_alloc();
if (!graph) {
LOG(("FFmpeg Error: Unable to create filter graph."));
return;
}
const auto guard = gsl::finally([&] {
avfilter_graph_free(&graph);
});
_filterSrc = avfilter_graph_alloc_filter(graph, abuffer, "src");
_atempo = avfilter_graph_alloc_filter(graph, atempo, "atempo");
_filterSink = avfilter_graph_alloc_filter(graph, abuffersink, "sink");
if (!_filterSrc || !atempo || !_filterSink) {
LOG(("FFmpeg Error: "
"Could not allocate abuffer / abuffersink /atempo."));
return;
}
char layout[64] = { 0 };
av_channel_layout_describe(
&_swrDstChannelLayout,
layout,
sizeof(layout));
av_opt_set(
_filterSrc,
"channel_layout",
layout,
AV_OPT_SEARCH_CHILDREN);
av_opt_set_sample_fmt(
_filterSrc,
"sample_fmt",
_swrDstSampleFormat,
AV_OPT_SEARCH_CHILDREN);
av_opt_set_q(
_filterSrc,
"time_base",
AVRational{ 1, _swrDstRate },
AV_OPT_SEARCH_CHILDREN);
av_opt_set_int(
_filterSrc,
"sample_rate",
_swrDstRate,
AV_OPT_SEARCH_CHILDREN);
av_opt_set_double(
_atempo,
"tempo",
speed,
AV_OPT_SEARCH_CHILDREN);
AvErrorWrap error = 0;
if ((error = avfilter_init_str(_filterSrc, nullptr))) {
LogError(u"avfilter_init_str(src)"_q, error);
return;
} else if ((error = avfilter_init_str(_atempo, nullptr))) {
LogError(u"avfilter_init_str(atempo)"_q, error);
avfilter_graph_free(&graph);
return;
} else if ((error = avfilter_init_str(_filterSink, nullptr))) {
LogError(u"avfilter_init_str(sink)"_q, error);
avfilter_graph_free(&graph);
return;
} else if ((error = avfilter_link(_filterSrc, 0, _atempo, 0))) {
LogError(u"avfilter_link(src->atempo)"_q, error);
avfilter_graph_free(&graph);
return;
} else if ((error = avfilter_link(_atempo, 0, _filterSink, 0))) {
LogError(u"avfilter_link(atempo->sink)"_q, error);
avfilter_graph_free(&graph);
return;
} else if ((error = avfilter_graph_config(graph, nullptr))) {
LogError("avfilter_link(atempo->sink)"_q, error);
avfilter_graph_free(&graph);
return;
}
_filterGraph = base::take(graph);
_filteredFrame = FFmpeg::MakeFramePointer();
_filterSpeed = speed;
}
void AbstractAudioFFMpegLoader::enqueueNormalFrame(
not_null<AVFrame*> frame,
int64 samples) {
if (_framesQueuedIndex >= 0) {
return;
}
if (!samples) {
samples = frame->nb_samples;
}
_framesQueued.push_back({
.position = startedAtSample() + _framesQueuedSamples,
.samples = samples,
.frame = FFmpeg::DuplicateFramePointer(frame),
});
LOG(("Added At %1 Data: %2"
).arg(_framesQueued.back().position
).arg(quintptr(_framesQueued.back().frame->extended_data[0])));
_framesQueuedSamples += samples;
}
void AbstractAudioFFMpegLoader::enqueueFramesFinished() {
if (_framesQueuedIndex >= 0) {
return;
}
_framesQueued.push_back({
.position = startedAtSample() + _framesQueuedSamples,
});
}
auto AbstractAudioFFMpegLoader::readFromReadyFrame()
-> ReadResult {
const auto raw = _frame.get();
if (frameHasDesiredFormat(raw)) {
if (!raw->nb_samples) {
return ReadError::Retry;
}
return readOrBufferForFilter(raw, raw->nb_samples);
} else if (!initResampleForFrame()) { } else if (!initResampleForFrame()) {
return ReadError::Other; return ReadError::Other;
} }
@ -509,57 +698,68 @@ auto AbstractAudioFFMpegLoader::readFromReadyFrame() -> ReadResult {
} }
const auto samples = swr_convert( const auto samples = swr_convert(
_swrContext, _swrContext,
_swrDstData, (uint8_t**)_resampledFrame->extended_data,
maxSamples, maxSamples,
(const uint8_t * *)_frame->extended_data, (const uint8_t **)_frame->extended_data,
_frame->nb_samples); _frame->nb_samples);
if (samples < 0) { if (AvErrorWrap error = samples) {
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; LogError(u"swr_convert"_q, error);
LOG(("Audio Error: " return ReadError::Other;
"Unable to swr_convert for file '%1', data size '%2', " } else if (!samples) {
"error %3, %4" return ReadError::Retry;
).arg(_file.name() }
).arg(_data.size() return readOrBufferForFilter(_resampledFrame.get(), samples);
).arg(samples }
).arg(av_make_error_string(err, sizeof(err), samples)
)); auto AbstractAudioFFMpegLoader::readOrBufferForFilter(
not_null<AVFrame*> frame,
int64 samplesOverride)
-> ReadResult {
enqueueNormalFrame(frame, samplesOverride);
const auto was = frame->nb_samples;
frame->nb_samples = samplesOverride;
const auto guard = gsl::finally([&] {
frame->nb_samples = was;
});
if (!_filterGraph) {
return bytes::const_span(
reinterpret_cast<const bytes::type*>(frame->extended_data[0]),
frame->nb_samples * _outputSampleSize);
}
AvErrorWrap error = av_buffersrc_add_frame_flags(
_filterSrc,
frame,
AV_BUFFERSRC_FLAG_KEEP_REF);
if (error) {
LogError(u"av_buffersrc_add_frame_flags"_q, error);
return ReadError::Other; return ReadError::Other;
} }
return bytes::const_span( return ReadError::Retry;
reinterpret_cast<const bytes::type*>(_swrDstData[0]),
samples * _outputSampleSize);
} }
AbstractAudioFFMpegLoader::~AbstractAudioFFMpegLoader() { AbstractAudioFFMpegLoader::~AbstractAudioFFMpegLoader() {
if (_filterGraph) {
avfilter_graph_free(&_filterGraph);
}
if (_swrContext) { if (_swrContext) {
swr_free(&_swrContext); swr_free(&_swrContext);
} }
if (_swrDstData) {
if (_swrDstData[0]) {
av_freep(&_swrDstData[0]);
}
av_freep(&_swrDstData);
}
} }
FFMpegLoader::FFMpegLoader( FFMpegLoader::FFMpegLoader(
const Core::FileLocation & file, const Core::FileLocation &file,
const QByteArray & data, const QByteArray &data,
bytes::vector && buffer) bytes::vector &&buffer)
: AbstractAudioFFMpegLoader(file, data, std::move(buffer)) { : AbstractAudioFFMpegLoader(file, data, std::move(buffer)) {
} }
bool FFMpegLoader::open(crl::time positionMs) { bool FFMpegLoader::open(crl::time positionMs, float64 speed) {
if (!AbstractFFMpegLoader::open(positionMs)) { return AbstractFFMpegLoader::open(positionMs)
return false; && openCodecContext()
} && initUsingContext(_codecContext, speed)
if (!openCodecContext()) { && seekTo(positionMs);
return false;
}
if (!initUsingContext(_codecContext, _samplesCount, _samplesFrequency)) {
return false;
}
return seekTo(positionMs);
} }
bool FFMpegLoader::openCodecContext() { bool FFMpegLoader::openCodecContext() {
@ -577,31 +777,18 @@ bool FFMpegLoader::openCodecContext() {
} }
const auto stream = fmtContext->streams[streamId]; const auto stream = fmtContext->streams[streamId];
if ((res = avcodec_parameters_to_context( AvErrorWrap error = avcodec_parameters_to_context(
_codecContext, _codecContext,
stream->codecpar)) < 0) { stream->codecpar);
LOG(("Audio Error: " if (error) {
"Unable to avcodec_parameters_to_context for file '%1', " LogError(u"avcodec_parameters_to_context"_q, error);
"data size '%2', error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(res
).arg(av_make_error_string(err, sizeof(err), res)
));
return false; return false;
} }
_codecContext->pkt_timebase = stream->time_base; _codecContext->pkt_timebase = stream->time_base;
av_opt_set_int(_codecContext, "refcounted_frames", 1, 0); av_opt_set_int(_codecContext, "refcounted_frames", 1, 0);
if ((res = avcodec_open2(_codecContext, codec, 0)) < 0) { if (AvErrorWrap error = avcodec_open2(_codecContext, codec, 0)) {
LOG(("Audio Error: " LogError(u"avcodec_open2"_q, error);
"Unable to avcodec_open2 for file '%1', data size '%2', "
"error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(res
).arg(av_make_error_string(err, sizeof(err), res)
));
return false; return false;
} }
return true; return true;
@ -630,48 +817,34 @@ FFMpegLoader::ReadResult FFMpegLoader::readMore() {
return readResult; return readResult;
} }
auto res = 0; if (AvErrorWrap error = av_read_frame(fmtContext, &_packet)) {
if ((res = av_read_frame(fmtContext, &_packet)) < 0) { if (error.code() != AVERROR_EOF) {
if (res != AVERROR_EOF) { LogError(u"av_read_frame"_q, error);
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
LOG(("Audio Error: "
"Unable to av_read_frame() file '%1', data size '%2', "
"error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(res
).arg(av_make_error_string(err, sizeof(err), res)
));
return ReadError::Other; return ReadError::Other;
} }
avcodec_send_packet(_codecContext, nullptr); // drain error = avcodec_send_packet(_codecContext, nullptr); // drain
return bytes::const_span(); if (!error) {
return ReadError::Retry;
}
LogError(u"avcodec_send_packet"_q, error);
return ReadError::Other;
} }
if (_packet.stream_index == streamId) { if (_packet.stream_index == streamId) {
res = avcodec_send_packet(_codecContext, &_packet); AvErrorWrap error = avcodec_send_packet(_codecContext, &_packet);
if (res < 0) { if (error) {
av_packet_unref(&_packet); av_packet_unref(&_packet);
LogError(u"avcodec_send_packet"_q, error);
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
LOG(("Audio Error: "
"Unable to avcodec_send_packet() file '%1', data size '%2', "
"error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(res
).arg(av_make_error_string(err, sizeof(err), res)
));
// There is a sample voice message where skipping such packet // There is a sample voice message where skipping such packet
// results in a crash (read_access to nullptr) in swr_convert(). // results in a crash (read_access to nullptr) in swr_convert().
//if (res == AVERROR_INVALIDDATA) { //if (error.code() == AVERROR_INVALIDDATA) {
// return ReadResult::NotYet; // try to skip bad packet // return ReadResult::Retry; // try to skip bad packet
//} //}
return ReadError::Other; return ReadError::Other;
} }
} }
av_packet_unref(&_packet); av_packet_unref(&_packet);
return bytes::const_span(); return ReadError::Retry;
} }
FFMpegLoader::~FFMpegLoader() { FFMpegLoader::~FFMpegLoader() {

View file

@ -16,6 +16,7 @@ extern "C" {
#include <libavformat/avformat.h> #include <libavformat/avformat.h>
#include <libavutil/opt.h> #include <libavutil/opt.h>
#include <libswresample/swresample.h> #include <libswresample/swresample.h>
#include <libavfilter/avfilter.h>
} // extern "C" } // extern "C"
#include <al.h> #include <al.h>
@ -35,10 +36,14 @@ public:
: AudioPlayerLoader(file, data, std::move(buffer)) { : AudioPlayerLoader(file, data, std::move(buffer)) {
} }
bool open(crl::time positionMs) override; bool open(crl::time positionMs, float64 speed = 1.) override;
int64 samplesCount() override { crl::time duration() override {
return _samplesCount; return _duration;
}
void overrideDuration(int64 startedAtSample, crl::time duration) {
_startedAtSample = startedAtSample;
_duration = duration;
} }
int samplesFrequency() override { int samplesFrequency() override {
@ -51,13 +56,18 @@ public:
int channels); int channels);
#endif // !DA_FFMPEG_NEW_CHANNEL_LAYOUT #endif // !DA_FFMPEG_NEW_CHANNEL_LAYOUT
[[nodiscard]] int64 startedAtSample() const {
return _startedAtSample;
}
~AbstractFFMpegLoader(); ~AbstractFFMpegLoader();
protected: protected:
static int64 Mul(int64 value, AVRational rational); static int64 Mul(int64 value, AVRational rational);
int _samplesFrequency = Media::Player::kDefaultFrequency; int _samplesFrequency = Media::Player::kDefaultFrequency;
int64 _samplesCount = 0; int64 _startedAtSample = 0;
crl::time _duration = 0;
uchar *ioBuffer = nullptr; uchar *ioBuffer = nullptr;
AVIOContext *ioContext = nullptr; AVIOContext *ioContext = nullptr;
@ -88,9 +98,8 @@ public:
const QByteArray &data, const QByteArray &data,
bytes::vector &&buffer); bytes::vector &&buffer);
int64 samplesCount() override { void dropFramesTill(int64 samples) override;
return _outputSamplesCount; int64 startReadingQueuedFrames(float64 newSpeed) override;
}
int samplesFrequency() override { int samplesFrequency() override {
return _swrDstRate; return _swrDstRate;
@ -107,10 +116,7 @@ public:
~AbstractAudioFFMpegLoader(); ~AbstractAudioFFMpegLoader();
protected: protected:
bool initUsingContext( bool initUsingContext(not_null<AVCodecContext*> context, float64 speed);
not_null<AVCodecContext *> context,
int64 initialCount,
int initialFrequency);
[[nodiscard]] ReadResult readFromReadyContext( [[nodiscard]] ReadResult readFromReadyContext(
not_null<AVCodecContext*> context); not_null<AVCodecContext*> context);
@ -119,17 +125,42 @@ protected:
[[nodiscard]] ReadResult replaceFrameAndRead(FFmpeg::FramePointer frame); [[nodiscard]] ReadResult replaceFrameAndRead(FFmpeg::FramePointer frame);
private: private:
struct EnqueuedFrame {
int64 position = 0;
int64 samples = 0;
FFmpeg::FramePointer frame;
};
[[nodiscard]] ReadResult readFromReadyFrame(); [[nodiscard]] ReadResult readFromReadyFrame();
bool frameHasDesiredFormat() const; [[nodiscard]] ReadResult readOrBufferForFilter(
not_null<AVFrame*> frame,
int64 samplesOverride);
bool frameHasDesiredFormat(not_null<AVFrame*> frame) const;
bool initResampleForFrame(); bool initResampleForFrame();
bool initResampleUsingFormat(); bool initResampleUsingFormat();
bool ensureResampleSpaceAvailable(int samples); bool ensureResampleSpaceAvailable(int samples);
bool changeSpeedFilter(float64 speed);
void createSpeedFilter(float64 speed);
void enqueueNormalFrame(
not_null<AVFrame*> frame,
int64 samples = 0);
void enqueueFramesFinished();
[[nodiscard]] auto fillFrameFromQueued()
-> std::variant<not_null<const EnqueuedFrame*>, ReadError>;
FFmpeg::FramePointer _frame; FFmpeg::FramePointer _frame;
FFmpeg::FramePointer _resampledFrame;
FFmpeg::FramePointer _filteredFrame;
int _resampledFrameCapacity = 0;
int64 _framesQueuedSamples = 0;
std::deque<EnqueuedFrame> _framesQueued;
int _framesQueuedIndex = -1;
int _outputFormat = AL_FORMAT_STEREO16; int _outputFormat = AL_FORMAT_STEREO16;
int _outputChannels = 2; int _outputChannels = 2;
int _outputSampleSize = 2 * sizeof(uint16); int _outputSampleSize = 2 * sizeof(uint16);
int64 _outputSamplesCount = 0;
SwrContext *_swrContext = nullptr; SwrContext *_swrContext = nullptr;
@ -146,8 +177,12 @@ private:
uint64_t _swrSrcChannelLayout = 0; uint64_t _swrSrcChannelLayout = 0;
uint64_t _swrDstChannelLayout = AV_CH_LAYOUT_STEREO; uint64_t _swrDstChannelLayout = AV_CH_LAYOUT_STEREO;
#endif // DA_FFMPEG_NEW_CHANNEL_LAYOUT #endif // DA_FFMPEG_NEW_CHANNEL_LAYOUT
uint8_t **_swrDstData = nullptr;
int _swrDstDataCapacity = 0; AVFilterGraph *_filterGraph = nullptr;
float64 _filterSpeed = 1.;
AVFilterContext *_filterSrc = nullptr;
AVFilterContext *_atempo = nullptr;
AVFilterContext *_filterSink = nullptr;
}; };
@ -158,7 +193,7 @@ public:
const QByteArray &data, const QByteArray &data,
bytes::vector &&buffer); bytes::vector &&buffer);
bool open(crl::time positionMs) override; bool open(crl::time positionMs, float64 speed = 1.) override;
ReadResult readMore() override; ReadResult readMore() override;

View file

@ -52,6 +52,18 @@ bool AudioPlayerLoader::holdsSavedDecodedSamples() const {
return _holdsSavedSamples; return _holdsSavedSamples;
} }
void AudioPlayerLoader::dropDecodedSamples() {
_savedSamples = {};
_holdsSavedSamples = false;
}
int AudioPlayerLoader::bytesPerBuffer() {
if (!_bytesPerBuffer) {
_bytesPerBuffer = samplesFrequency() * sampleSize();
}
return _bytesPerBuffer;
}
bool AudioPlayerLoader::openFile() { bool AudioPlayerLoader::openFile() {
if (_data.isEmpty() && _bytes.empty()) { if (_data.isEmpty() && _bytes.empty()) {
if (_f.isOpen()) _f.close(); if (_f.isOpen()) _f.close();

View file

@ -21,17 +21,29 @@ public:
bytes::vector &&buffer); bytes::vector &&buffer);
virtual ~AudioPlayerLoader(); virtual ~AudioPlayerLoader();
virtual bool check(const Core::FileLocation &file, const QByteArray &data); virtual bool check(
const Core::FileLocation &file,
const QByteArray &data);
virtual bool open(crl::time positionMs) = 0; virtual bool open(crl::time positionMs, float64 speed = 1.) = 0;
virtual int64 samplesCount() = 0; virtual crl::time duration() = 0;
virtual int samplesFrequency() = 0; virtual int samplesFrequency() = 0;
virtual int sampleSize() = 0; virtual int sampleSize() = 0;
virtual int format() = 0; virtual int format() = 0;
virtual void dropFramesTill(int64 samples) {
}
[[nodiscard]] virtual int64 startReadingQueuedFrames(float64 newSpeed) {
Unexpected(
"startReadingQueuedFrames() on not AbstractAudioFFMpegLoader");
}
[[nodiscard]] int bytesPerBuffer();
enum class ReadError { enum class ReadError {
Other, Other,
NotYet, Retry,
RetryNotQueued,
Wait, Wait,
EndOfFile, EndOfFile,
}; };
@ -51,6 +63,7 @@ public:
void saveDecodedSamples(not_null<QByteArray*> samples); void saveDecodedSamples(not_null<QByteArray*> samples);
void takeSavedDecodedSamples(not_null<QByteArray*> samples); void takeSavedDecodedSamples(not_null<QByteArray*> samples);
bool holdsSavedDecodedSamples() const; bool holdsSavedDecodedSamples() const;
void dropDecodedSamples();
protected: protected:
Core::FileLocation _file; Core::FileLocation _file;
@ -67,6 +80,8 @@ private:
QByteArray _savedSamples; QByteArray _savedSamples;
bool _holdsSavedSamples = false; bool _holdsSavedSamples = false;
int _bytesPerBuffer = 0;
}; };
} // namespace Media } // namespace Media

View file

@ -13,11 +13,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
namespace Media { namespace Media {
namespace Player { namespace Player {
namespace {
constexpr auto kPlaybackBufferSize = 256 * 1024;
} // namespace
Loaders::Loaders(QThread *thread) Loaders::Loaders(QThread *thread)
: _fromExternalNotify([=] { videoSoundAdded(); }) { : _fromExternalNotify([=] { videoSoundAdded(); }) {
@ -151,17 +146,29 @@ void Loaders::onLoad(const AudioMsgId &audio) {
} }
void Loaders::loadData(AudioMsgId audio, crl::time positionMs) { void Loaders::loadData(AudioMsgId audio, crl::time positionMs) {
auto err = SetupNoErrorStarted;
auto type = audio.type(); auto type = audio.type();
auto l = setupLoader(audio, err, positionMs); auto setup = setupLoader(audio, positionMs);
const auto l = setup.loader;
if (!l) { if (!l) {
if (err == SetupErrorAtStart) { if (setup.errorAtStart) {
emitError(type); emitError(type);
} }
return; return;
} }
auto started = (err == SetupNoErrorStarted); const auto sampleSize = l->sampleSize();
const auto speedChanged = (setup.newSpeed != setup.oldSpeed);
auto updatedWithSpeed = speedChanged
? rebufferOnSpeedChange(setup)
: std::optional<Mixer::Track::WithSpeed>();
if (!speedChanged && setup.oldSpeed > 0.) {
const auto normalPosition = Mixer::Track::SpeedIndependentPosition(
setup.position,
setup.oldSpeed);
l->dropFramesTill(normalPosition);
}
const auto started = setup.justStarted;
auto finished = false; auto finished = false;
auto waiting = false; auto waiting = false;
auto errAtStart = started; auto errAtStart = started;
@ -170,10 +177,15 @@ void Loaders::loadData(AudioMsgId audio, crl::time positionMs) {
auto accumulatedCount = 0; auto accumulatedCount = 0;
if (l->holdsSavedDecodedSamples()) { if (l->holdsSavedDecodedSamples()) {
l->takeSavedDecodedSamples(&accumulated); l->takeSavedDecodedSamples(&accumulated);
accumulatedCount = accumulated.size() / l->sampleSize(); accumulatedCount = accumulated.size() / sampleSize;
} }
while (accumulated.size() < kPlaybackBufferSize) { const auto accumulateTill = l->bytesPerBuffer();
while (accumulated.size() < accumulateTill) {
using Error = AudioPlayerLoader::ReadError;
const auto result = l->readMore(); const auto result = l->readMore();
if (result == Error::Retry) {
continue;
}
const auto sampleBytes = v::is<bytes::const_span>(result) const auto sampleBytes = v::is<bytes::const_span>(result)
? v::get<bytes::const_span>(result) ? v::get<bytes::const_span>(result)
: bytes::const_span(); : bytes::const_span();
@ -181,10 +193,8 @@ void Loaders::loadData(AudioMsgId audio, crl::time positionMs) {
accumulated.append( accumulated.append(
reinterpret_cast<const char*>(sampleBytes.data()), reinterpret_cast<const char*>(sampleBytes.data()),
sampleBytes.size()); sampleBytes.size());
accumulatedCount += sampleBytes.size() / l->sampleSize(); accumulatedCount += sampleBytes.size() / sampleSize;
} } else if (result == Error::Other) {
using Error = AudioPlayerLoader::ReadError;
if (result == Error::Other) {
if (errAtStart) { if (errAtStart) {
{ {
QMutexLocker lock(internal::audioPlayerMutex()); QMutexLocker lock(internal::audioPlayerMutex());
@ -201,7 +211,7 @@ void Loaders::loadData(AudioMsgId audio, crl::time positionMs) {
finished = true; finished = true;
break; break;
} else if (result == Error::Wait) { } else if (result == Error::Wait) {
waiting = (accumulated.size() < kPlaybackBufferSize) waiting = (accumulated.size() < accumulateTill)
&& (accumulated.isEmpty() || !l->forceToBuffer()); && (accumulated.isEmpty() || !l->forceToBuffer());
if (waiting) { if (waiting) {
l->saveDecodedSamples(&accumulated); l->saveDecodedSamples(&accumulated);
@ -225,10 +235,11 @@ void Loaders::loadData(AudioMsgId audio, crl::time positionMs) {
return; return;
} }
if (started || !accumulated.isEmpty()) { if (started || !accumulated.isEmpty() || updatedWithSpeed) {
Audio::AttachToDevice(); Audio::AttachToDevice();
} }
if (started) { if (started) {
Assert(!updatedWithSpeed);
track->started(); track->started();
if (!internal::audioCheckError()) { if (!internal::audioCheckError()) {
setStoppedState(track, State::StoppedAtStart); setStoppedState(track, State::StoppedAtStart);
@ -237,12 +248,20 @@ void Loaders::loadData(AudioMsgId audio, crl::time positionMs) {
} }
track->format = l->format(); track->format = l->format();
track->frequency = l->samplesFrequency(); track->state.frequency = l->samplesFrequency();
const auto position = (positionMs * track->frequency) / 1000LL; track->state.position = (positionMs * track->state.frequency)
track->bufferedPosition = position; / 1000LL;
track->state.position = position; track->updateWithSpeedPosition();
track->fadeStartPosition = position; track->withSpeed.bufferedPosition = track->withSpeed.position;
track->withSpeed.fadeStartPosition = track->withSpeed.position;
} else if (updatedWithSpeed) {
auto old = Mixer::Track();
old.stream = base::take(track->stream);
old.withSpeed = std::exchange(track->withSpeed, *updatedWithSpeed);
track->speed = setup.newSpeed;
track->reattach(type);
old.detach();
} }
if (!accumulated.isEmpty()) { if (!accumulated.isEmpty()) {
track->ensureStreamCreated(type); track->ensureStreamCreated(type);
@ -256,21 +275,23 @@ void Loaders::loadData(AudioMsgId audio, crl::time positionMs) {
} }
if (bufferIndex < 0) { // No free buffers, wait. if (bufferIndex < 0) { // No free buffers, wait.
track->waitingForBuffer = true;
l->saveDecodedSamples(&accumulated); l->saveDecodedSamples(&accumulated);
return; return;
} else if (l->forceToBuffer()) { } else if (l->forceToBuffer()) {
l->setForceToBuffer(false); l->setForceToBuffer(false);
} }
track->waitingForBuffer = false;
track->bufferSamples[bufferIndex] = accumulated; track->withSpeed.buffered[bufferIndex] = accumulated;
track->samplesCount[bufferIndex] = accumulatedCount; track->withSpeed.samples[bufferIndex] = accumulatedCount;
track->bufferedLength += accumulatedCount; track->withSpeed.bufferedLength += accumulatedCount;
alBufferData( alBufferData(
track->stream.buffers[bufferIndex], track->stream.buffers[bufferIndex],
track->format, track->format,
accumulated.constData(), accumulated.constData(),
accumulated.size(), accumulated.size(),
track->frequency); track->state.frequency);
alSourceQueueBuffers( alSourceQueueBuffers(
track->stream.source, track->stream.source,
@ -292,8 +313,11 @@ void Loaders::loadData(AudioMsgId audio, crl::time positionMs) {
if (finished) { if (finished) {
track->loaded = true; track->loaded = true;
track->state.length = track->bufferedPosition + track->bufferedLength; track->withSpeed.length = track->withSpeed.bufferedPosition
clear(type); + track->withSpeed.bufferedLength;
track->state.length = Mixer::Track::SpeedIndependentPosition(
track->withSpeed.length,
track->speed);
} }
track->loading = false; track->loading = false;
@ -323,7 +347,10 @@ void Loaders::loadData(AudioMsgId audio, crl::time positionMs) {
} }
if (state == AL_STOPPED) { if (state == AL_STOPPED) {
alSourcei(track->stream.source, AL_SAMPLE_OFFSET, qMax(track->state.position - track->bufferedPosition, 0LL)); alSourcei(
track->stream.source,
AL_SAMPLE_OFFSET,
qMax(track->withSpeed.position - track->withSpeed.bufferedPosition, 0LL));
if (!internal::audioCheckError()) { if (!internal::audioCheckError()) {
setStoppedState(track, State::StoppedAtError); setStoppedState(track, State::StoppedAtError);
emitError(type); emitError(type);
@ -340,20 +367,19 @@ void Loaders::loadData(AudioMsgId audio, crl::time positionMs) {
needToCheck(); needToCheck();
} }
AudioPlayerLoader *Loaders::setupLoader( Loaders::SetupLoaderResult Loaders::setupLoader(
const AudioMsgId &audio, const AudioMsgId &audio,
SetupError &err,
crl::time positionMs) { crl::time positionMs) {
err = SetupErrorAtStart;
QMutexLocker lock(internal::audioPlayerMutex()); QMutexLocker lock(internal::audioPlayerMutex());
if (!mixer()) return nullptr; if (!mixer()) {
return {};
}
auto track = mixer()->trackForType(audio.type()); auto track = mixer()->trackForType(audio.type());
if (!track || track->state.id != audio || !track->loading) { if (!track || track->state.id != audio || !track->loading) {
error(audio); error(audio);
LOG(("Audio Error: trying to load part of audio, that is not current at the moment")); LOG(("Audio Error: trying to load part of audio, that is not current at the moment"));
err = SetupErrorNotPlaying; return {};
return nullptr;
} }
bool isGoodId = false; bool isGoodId = false;
@ -369,6 +395,8 @@ AudioPlayerLoader *Loaders::setupLoader(
l = nullptr; l = nullptr;
} }
auto SpeedDependentPosition = Mixer::Track::SpeedDependentPosition;
auto SpeedIndependentPosition = Mixer::Track::SpeedIndependentPosition;
if (!l) { if (!l) {
std::unique_ptr<AudioPlayerLoader> *loader = nullptr; std::unique_ptr<AudioPlayerLoader> *loader = nullptr;
switch (audio.type()) { switch (audio.type()) {
@ -383,7 +411,7 @@ AudioPlayerLoader *Loaders::setupLoader(
track->state.state = State::StoppedAtError; track->state.state = State::StoppedAtError;
error(audio); error(audio);
LOG(("Audio Error: video sound data not ready")); LOG(("Audio Error: video sound data not ready"));
return nullptr; return {};
} }
*loader = std::make_unique<ChildFFMpegLoader>( *loader = std::make_unique<ChildFFMpegLoader>(
std::move(track->externalData)); std::move(track->externalData));
@ -395,24 +423,120 @@ AudioPlayerLoader *Loaders::setupLoader(
} }
l = loader->get(); l = loader->get();
if (!l->open(positionMs)) { track->speed = track->nextSpeed;
if (!l->open(positionMs, track->speed)) {
track->state.state = State::StoppedAtStart; track->state.state = State::StoppedAtStart;
return nullptr; return { .errorAtStart = true };
} }
auto length = l->samplesCount(); const auto duration = l->duration();
if (length <= 0) { if (duration <= 0) {
track->state.state = State::StoppedAtStart; track->state.state = State::StoppedAtStart;
return nullptr; return { .errorAtStart = true };
} }
track->state.length = length;
track->state.frequency = l->samplesFrequency(); track->state.frequency = l->samplesFrequency();
err = SetupNoErrorStarted; track->state.length = (duration * track->state.frequency) / 1000;
track->withSpeed.length = SpeedDependentPosition(
track->state.length,
track->speed);
return { .loader = l, .justStarted = true };
} else if (track->nextSpeed != track->speed) {
return {
.loader = l,
.oldSpeed = track->speed,
.newSpeed = track->nextSpeed,
.fadeStartPosition = track->withSpeed.fadeStartPosition,
.position = track->withSpeed.fineTunedPosition,
.normalLength = track->state.length,
.frequency = track->state.frequency,
};
} else if (track->loaded) { } else if (track->loaded) {
err = SetupErrorLoadedFull;
LOG(("Audio Error: trying to load part of audio, that is already loaded to the end")); LOG(("Audio Error: trying to load part of audio, that is already loaded to the end"));
return nullptr; return {};
} }
return l; return {
.loader = l,
.oldSpeed = track->speed,
.newSpeed = track->nextSpeed,
.position = track->withSpeed.fineTunedPosition,
.frequency = track->state.frequency,
};
}
Mixer::Track::WithSpeed Loaders::rebufferOnSpeedChange(
const SetupLoaderResult &setup) {
Expects(setup.oldSpeed > 0. && setup.newSpeed > 0.);
Expects(setup.loader != nullptr);
const auto speed = setup.newSpeed;
const auto change = setup.oldSpeed / speed;
const auto normalPosition = Mixer::Track::SpeedIndependentPosition(
setup.position,
setup.oldSpeed);
const auto newPosition = int64(base::SafeRound(setup.position * change));
auto result = Mixer::Track::WithSpeed{
.fineTunedPosition = newPosition,
.position = newPosition,
.length = Mixer::Track::SpeedDependentPosition(
setup.normalLength,
speed),
.fadeStartPosition = int64(
base::SafeRound(setup.fadeStartPosition * change)),
};
const auto l = setup.loader;
l->dropFramesTill(normalPosition);
const auto normalFrom = l->startReadingQueuedFrames(speed);
if (normalFrom < 0) {
result.bufferedPosition = newPosition;
return result;
}
result.bufferedPosition = Mixer::Track::SpeedDependentPosition(
normalFrom,
speed);
for (auto i = 0; i != Mixer::Track::kBuffersCount; ++i) {
auto finished = false;
auto accumulated = QByteArray();
auto accumulatedCount = int64();
const auto sampleSize = l->sampleSize();
const auto accumulateTill = l->bytesPerBuffer();
while (accumulated.size() < accumulateTill) {
const auto result = l->readMore();
const auto sampleBytes = v::is<bytes::const_span>(result)
? v::get<bytes::const_span>(result)
: bytes::const_span();
if (!sampleBytes.empty()) {
accumulated.append(
reinterpret_cast<const char*>(sampleBytes.data()),
sampleBytes.size());
accumulatedCount += sampleBytes.size() / sampleSize;
continue;
} else if (result == AudioPlayerLoader::ReadError::Retry) {
continue;
}
Assert(result == AudioPlayerLoader::ReadError::RetryNotQueued);
finished = true;
break;
}
if (!accumulated.isEmpty()) {
result.samples[i] = accumulatedCount;
result.bufferedLength += accumulatedCount;
result.buffered[i] = accumulated;
}
if (finished) {
break;
}
}
const auto limit = result.bufferedPosition + result.bufferedLength;
if (newPosition > limit) {
result.fineTunedPosition = limit;
result.position = limit;
}
if (limit > result.length) {
result.length = limit;
}
return result;
} }
Mixer::Track *Loaders::checkLoader(AudioMsgId::Type type) { Mixer::Track *Loaders::checkLoader(AudioMsgId::Type type) {

View file

@ -37,7 +37,31 @@ public Q_SLOTS:
void onCancel(const AudioMsgId &audio); void onCancel(const AudioMsgId &audio);
private: private:
struct SetupLoaderResult {
AudioPlayerLoader *loader = nullptr;
float64 oldSpeed = 0.;
float64 newSpeed = 0.;
int64 fadeStartPosition = 0;
int64 position = 0;
int64 normalLength = 0;
int frequency = 0;
bool errorAtStart = false;
bool justStarted = false;
};
void videoSoundAdded(); void videoSoundAdded();
[[nodiscard]] Mixer::Track::WithSpeed rebufferOnSpeedChange(
const SetupLoaderResult &setup);
void emitError(AudioMsgId::Type type);
AudioMsgId clear(AudioMsgId::Type type);
void setStoppedState(Mixer::Track *m, State state = State::Stopped);
void loadData(AudioMsgId audio, crl::time positionMs = 0);
[[nodiscard]] SetupLoaderResult setupLoader(
const AudioMsgId &audio,
crl::time positionMs);
Mixer::Track *checkLoader(AudioMsgId::Type type);
AudioMsgId _audio, _song, _video; AudioMsgId _audio, _song, _video;
std::unique_ptr<AudioPlayerLoader> _audioLoader; std::unique_ptr<AudioPlayerLoader> _audioLoader;
@ -51,23 +75,6 @@ private:
base::flat_set<AudioMsgId> _fromExternalForceToBuffer; base::flat_set<AudioMsgId> _fromExternalForceToBuffer;
SingleQueuedInvokation _fromExternalNotify; SingleQueuedInvokation _fromExternalNotify;
void emitError(AudioMsgId::Type type);
AudioMsgId clear(AudioMsgId::Type type);
void setStoppedState(Mixer::Track *m, State state = State::Stopped);
enum SetupError {
SetupErrorAtStart = 0,
SetupErrorNotPlaying = 1,
SetupErrorLoadedFull = 2,
SetupNoErrorStarted = 3,
};
void loadData(AudioMsgId audio, crl::time positionMs = 0);
AudioPlayerLoader *setupLoader(
const AudioMsgId &audio,
SetupError &err,
crl::time positionMs);
Mixer::Track *checkLoader(AudioMsgId::Type type);
}; };
} // namespace Player } // namespace Player

View file

@ -59,7 +59,8 @@ void Track::fillFromData(bytes::vector &&data) {
} }
auto format = loader.format(); auto format = loader.format();
_peakEachPosition = _peakDurationMs ? ((loader.samplesFrequency() * _peakDurationMs) / 1000) : 0; _peakEachPosition = _peakDurationMs ? ((loader.samplesFrequency() * _peakDurationMs) / 1000) : 0;
auto peaksCount = _peakEachPosition ? (loader.samplesCount() / _peakEachPosition) : 0; const auto samplesCount = (loader.duration() * loader.samplesFrequency()) / 1000;
const auto peaksCount = _peakEachPosition ? (samplesCount / _peakEachPosition) : 0;
_peaks.reserve(peaksCount); _peaks.reserve(peaksCount);
auto peakValue = uint16(0); auto peakValue = uint16(0);
auto peakSamples = 0; auto peakSamples = 0;
@ -79,32 +80,33 @@ void Track::fillFromData(bytes::vector &&data) {
do { do {
using Error = AudioPlayerLoader::ReadError; using Error = AudioPlayerLoader::ReadError;
const auto result = loader.readMore(); const auto result = loader.readMore();
const auto sampleBytes = v::is<bytes::const_span>(result) Assert(result != Error::Wait && result != Error::RetryNotQueued);
? v::get<bytes::const_span>(result)
: bytes::const_span(); if (result == Error::Retry) {
if (!sampleBytes.empty()) { continue;
_samplesCount += sampleBytes.size() / loader.sampleSize(); } else if (result == Error::EndOfFile) {
_samples.insert(_samples.end(), sampleBytes.data(), sampleBytes.data() + sampleBytes.size());
if (peaksCount) {
if (format == AL_FORMAT_MONO8 || format == AL_FORMAT_STEREO8) {
Media::Audio::IterateSamples<uchar>(sampleBytes, peakCallback);
} else if (format == AL_FORMAT_MONO16 || format == AL_FORMAT_STEREO16) {
Media::Audio::IterateSamples<int16>(sampleBytes, peakCallback);
}
}
} else if (result == Error::Other
|| result == Error::NotYet
|| result == Error::Wait) {
_failed = true;
}
if (!v::is<bytes::const_span>(result)) {
break; break;
} else if (result == Error::Other || result == Error::Wait) {
_failed = true;
break;
}
Assert(v::is<bytes::const_span>(result));
const auto sampleBytes = v::get<bytes::const_span>(result);
Assert(!sampleBytes.empty());
_samplesCount += sampleBytes.size() / loader.sampleSize();
_samples.insert(_samples.end(), sampleBytes.data(), sampleBytes.data() + sampleBytes.size());
if (peaksCount) {
if (format == AL_FORMAT_MONO8 || format == AL_FORMAT_STEREO8) {
Media::Audio::IterateSamples<uchar>(sampleBytes, peakCallback);
} else if (format == AL_FORMAT_MONO16 || format == AL_FORMAT_STEREO16) {
Media::Audio::IterateSamples<int16>(sampleBytes, peakCallback);
}
} }
} while (true); } while (true);
_alFormat = loader.format(); _alFormat = loader.format();
_sampleRate = loader.samplesFrequency(); _sampleRate = loader.samplesFrequency();
_lengthMs = (loader.samplesCount() * crl::time(1000)) / _sampleRate; _lengthMs = loader.duration();
} }
void Track::fillFromFile(const Core::FileLocation &location) { void Track::fillFromFile(const Core::FileLocation &location) {

View file

@ -11,6 +11,11 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include "core/file_location.h" #include "core/file_location.h"
namespace Media { namespace Media {
namespace {
using FFmpeg::AvErrorWrap;
} // namespace
ChildFFMpegLoader::ChildFFMpegLoader( ChildFFMpegLoader::ChildFFMpegLoader(
std::unique_ptr<ExternalSoundData> &&data) std::unique_ptr<ExternalSoundData> &&data)
@ -22,14 +27,13 @@ ChildFFMpegLoader::ChildFFMpegLoader(
Expects(_parentData->codec != nullptr); Expects(_parentData->codec != nullptr);
} }
bool ChildFFMpegLoader::open(crl::time positionMs) { bool ChildFFMpegLoader::open(crl::time positionMs, float64 speed) {
return initUsingContext( const auto sample = (positionMs * samplesFrequency()) / 1000LL;
_parentData->codec.get(), overrideDuration(sample, _parentData->duration);
_parentData->length, return initUsingContext(_parentData->codec.get(), speed);
_parentData->frequency);
} }
AudioPlayerLoader::ReadResult ChildFFMpegLoader::readFromInitialFrame() { auto ChildFFMpegLoader::readFromInitialFrame() -> ReadResult {
if (!_parentData->frame) { if (!_parentData->frame) {
return ReadError::Wait; return ReadError::Wait;
} }
@ -58,28 +62,22 @@ auto ChildFFMpegLoader::readMore() -> ReadResult {
_eofReached = packet.empty(); _eofReached = packet.empty();
if (_eofReached) { if (_eofReached) {
avcodec_send_packet(_parentData->codec.get(), nullptr); // drain avcodec_send_packet(_parentData->codec.get(), nullptr); // drain
return bytes::const_span(); return ReadError::Retry;
} }
auto res = avcodec_send_packet( AvErrorWrap error = avcodec_send_packet(
_parentData->codec.get(), _parentData->codec.get(),
&packet.fields()); &packet.fields());
if (res < 0) { if (error) {
char err[AV_ERROR_MAX_STRING_SIZE] = { 0 }; LogError(u"avcodec_send_packet"_q, error);
LOG(("Audio Error: Unable to avcodec_send_packet() file '%1', "
"data size '%2', error %3, %4"
).arg(_file.name()
).arg(_data.size()
).arg(res
).arg(av_make_error_string(err, sizeof(err), res)));
// There is a sample voice message where skipping such packet // There is a sample voice message where skipping such packet
// results in a crash (read_access to nullptr) in swr_convert(). // results in a crash (read_access to nullptr) in swr_convert().
if (res == AVERROR_INVALIDDATA) { if (error.code() == AVERROR_INVALIDDATA) {
return ReadError::NotYet; // try to skip bad packet return ReadError::Retry; // try to skip bad packet
} }
return ReadError::Other; return ReadError::Other;
} }
return bytes::const_span(); return ReadError::Retry;
} }
void ChildFFMpegLoader::enqueuePackets( void ChildFFMpegLoader::enqueuePackets(

View file

@ -15,8 +15,7 @@ namespace Media {
struct ExternalSoundData { struct ExternalSoundData {
FFmpeg::CodecPointer codec; FFmpeg::CodecPointer codec;
FFmpeg::FramePointer frame; FFmpeg::FramePointer frame;
int32 frequency = Media::Player::kDefaultFrequency; crl::time duration = 0;
int64 length = 0;
float64 speed = 1.; // 0.5 <= speed <= 2. float64 speed = 1.; // 0.5 <= speed <= 2.
}; };
@ -29,7 +28,7 @@ class ChildFFMpegLoader : public AbstractAudioFFMpegLoader {
public: public:
ChildFFMpegLoader(std::unique_ptr<ExternalSoundData> &&data); ChildFFMpegLoader(std::unique_ptr<ExternalSoundData> &&data);
bool open(crl::time positionMs) override; bool open(crl::time positionMs, float64 speed = 1.) override;
bool check(const Core::FileLocation &file, const QByteArray &data) override { bool check(const Core::FileLocation &file, const QByteArray &data) override {
return true; return true;

View file

@ -53,7 +53,10 @@ constexpr auto kRememberShuffledOrderItems = 16;
constexpr auto kMinLengthForSavePosition = 20 * TimeId(60); // 20 minutes. constexpr auto kMinLengthForSavePosition = 20 * TimeId(60); // 20 minutes.
auto VoicePlaybackSpeed() { auto VoicePlaybackSpeed() {
return std::clamp(Core::App().settings().voicePlaybackSpeed(), 0.6, 1.7); return std::clamp(
Core::App().settings().voicePlaybackSpeed(),
Media::Audio::kSpeedMin,
Media::Audio::kSpeedMax);
} }
base::options::toggle OptionDisableAutoplayNext({ base::options::toggle OptionDisableAutoplayNext({

View file

@ -133,8 +133,7 @@ void AudioTrack::mixerInit() {
auto data = std::make_unique<ExternalSoundData>(); auto data = std::make_unique<ExternalSoundData>();
data->frame = std::move(_stream.decodedFrame); data->frame = std::move(_stream.decodedFrame);
data->codec = std::move(_stream.codec); data->codec = std::move(_stream.codec);
data->frequency = _stream.frequency; data->duration = _stream.duration;
data->length = (_stream.duration * data->frequency) / 1000LL;
data->speed = _options.speed; data->speed = _options.speed;
Media::Player::mixer()->play( Media::Player::mixer()->play(

View file

@ -589,7 +589,7 @@ void System::showNext() {
alertThread->owner().notifySettings().sound(alertThread).id); alertThread->owner().notifySettings().sound(alertThread).id);
track->playOnce(); track->playOnce();
Media::Player::mixer()->suppressAll(track->getLengthMs()); Media::Player::mixer()->suppressAll(track->getLengthMs());
Media::Player::mixer()->faderOnTimer(); Media::Player::mixer()->scheduleFaderCallback();
} }
} }

2
cmake

@ -1 +1 @@
Subproject commit 31eb395967d5da6f07bd1e3f5f5f9bdaac94cb85 Subproject commit d3977a83a6c8d7a1551dc457a79e05a3f383aa6f