From 15620b5c2d97fadc2197a634df2391d2d0263a0b Mon Sep 17 00:00:00 2001 From: John Preston Date: Tue, 15 Dec 2020 14:16:44 +0400 Subject: [PATCH] Divide speaking status and background noise. --- Telegram/SourceFiles/api/api_updates.cpp | 10 +- .../SourceFiles/calls/calls_group_call.cpp | 64 ++++++----- Telegram/SourceFiles/calls/calls_group_call.h | 14 ++- .../SourceFiles/calls/calls_group_members.cpp | 104 +++++++++++------- Telegram/SourceFiles/data/data_group_call.cpp | 22 ++-- Telegram/SourceFiles/data/data_group_call.h | 16 ++- Telegram/ThirdParty/tgcalls | 2 +- 7 files changed, 139 insertions(+), 93 deletions(-) diff --git a/Telegram/SourceFiles/api/api_updates.cpp b/Telegram/SourceFiles/api/api_updates.cpp index 48013f6fc..99fdf520c 100644 --- a/Telegram/SourceFiles/api/api_updates.cpp +++ b/Telegram/SourceFiles/api/api_updates.cpp @@ -245,7 +245,10 @@ Updates::Updates(not_null session) for (const auto [userId, when] : *users) { call->applyActiveUpdate( userId, - when, + Data::LastSpokeTimes{ + .anything = when, + .voice = when + }, peer->owner().userLoaded(userId)); } } @@ -928,7 +931,10 @@ void Updates::handleSendActionUpdate( const auto call = peer->groupCall(); const auto now = crl::now(); if (call) { - call->applyActiveUpdate(userId, now, user); + call->applyActiveUpdate( + userId, + Data::LastSpokeTimes{ .anything = now, .voice = now }, + user); } else { const auto chat = peer->asChat(); const auto channel = peer->asChannel(); diff --git a/Telegram/SourceFiles/calls/calls_group_call.cpp b/Telegram/SourceFiles/calls/calls_group_call.cpp index 18e999db3..f4a71eafd 100644 --- a/Telegram/SourceFiles/calls/calls_group_call.cpp +++ b/Telegram/SourceFiles/calls/calls_group_call.cpp @@ -498,28 +498,30 @@ void GroupCall::handleUpdate(const MTPDupdateGroupCallParticipants &data) { } void GroupCall::createAndStartController() { - using AudioLevels = std::vector>; - const auto &settings = Core::App().settings(); const auto weak = base::make_weak(this); - const auto myLevel = std::make_shared(); + const auto myLevel = std::make_shared(); tgcalls::GroupInstanceDescriptor descriptor = { .config = tgcalls::GroupConfig{ }, .networkStateUpdated = [=](bool connected) { crl::on_main(weak, [=] { setInstanceConnected(connected); }); }, - .audioLevelsUpdated = [=](const AudioLevels &data) { - if (!data.empty()) { - crl::on_main(weak, [=] { audioLevelsUpdated(data); }); - } - }, - .myAudioLevelUpdated = [=](float level) { - if (*myLevel != level) { // Don't send many 0 while we're muted. - *myLevel = level; - crl::on_main(weak, [=] { myLevelUpdated(level); }); + .audioLevelsUpdated = [=](const tgcalls::GroupLevelsUpdate &data) { + const auto &updates = data.updates; + if (updates.empty()) { + return; + } else if (updates.size() == 1 && !updates.front().ssrc) { + const auto &value = updates.front().value; + // Don't send many 0 while we're muted. + if (myLevel->level == value.level + && myLevel->voice == value.voice) { + return; + } + *myLevel = updates.front().value; } + crl::on_main(weak, [=] { audioLevelsUpdated(data); }); }, .initialInputDeviceId = settings.callInputDeviceId().toStdString(), .initialOutputDeviceId = settings.callOutputDeviceId().toStdString(), @@ -555,24 +557,28 @@ void GroupCall::updateInstanceMuteState() { && state != MuteState::PushToTalk); } -void GroupCall::handleLevelsUpdated( - gsl::span> data) { - Expects(!data.empty()); +void GroupCall::audioLevelsUpdated(const tgcalls::GroupLevelsUpdate &data) { + Expects(!data.updates.empty()); auto check = false; auto checkNow = false; const auto now = crl::now(); - for (const auto &[ssrc, level] : data) { + for (const auto &[ssrcOrZero, value] : data.updates) { + const auto ssrc = ssrcOrZero ? ssrcOrZero : _mySsrc; + const auto level = value.level; + const auto voice = value.voice; const auto self = (ssrc == _mySsrc); _levelUpdates.fire(LevelUpdate{ .ssrc = ssrc, .value = level, + .voice = voice, .self = self }); if (level <= kSpeakLevelThreshold) { continue; } if (self + && voice && (!_lastSendProgressUpdate || _lastSendProgressUpdate + kUpdateSendActionEach < now)) { _lastSendProgressUpdate = now; @@ -584,13 +590,21 @@ void GroupCall::handleLevelsUpdated( check = true; const auto i = _lastSpoke.find(ssrc); if (i == _lastSpoke.end()) { - _lastSpoke.emplace(ssrc, now); + _lastSpoke.emplace(ssrc, Data::LastSpokeTimes{ + .anything = now, + .voice = voice ? now : 0, + }); checkNow = true; } else { - if (i->second + kCheckLastSpokeInterval / 3 <= now) { + if ((i->second.anything + kCheckLastSpokeInterval / 3 <= now) + || (voice + && i->second.voice + kCheckLastSpokeInterval / 3 <= now)) { checkNow = true; } - i->second = now; + i->second.anything = now; + if (voice) { + i->second.voice = now; + } } } if (checkNow) { @@ -600,16 +614,6 @@ void GroupCall::handleLevelsUpdated( } } -void GroupCall::myLevelUpdated(float level) { - const auto pair = std::pair{ _mySsrc, level }; - handleLevelsUpdated({ &pair, &pair + 1 }); -} - -void GroupCall::audioLevelsUpdated( - const std::vector> &data) { - handleLevelsUpdated(gsl::make_span(data)); -} - void GroupCall::checkLastSpoke() { const auto real = _peer->groupCall(); if (!real || real->id() != _id) { @@ -621,7 +625,7 @@ void GroupCall::checkLastSpoke() { auto list = base::take(_lastSpoke); for (auto i = list.begin(); i != list.end();) { const auto [ssrc, when] = *i; - if (when + kCheckLastSpokeInterval >= now) { + if (when.anything + kCheckLastSpokeInterval >= now) { hasRecent = true; ++i; } else { diff --git a/Telegram/SourceFiles/calls/calls_group_call.h b/Telegram/SourceFiles/calls/calls_group_call.h index 9b2b802e9..9a507b5e8 100644 --- a/Telegram/SourceFiles/calls/calls_group_call.h +++ b/Telegram/SourceFiles/calls/calls_group_call.h @@ -17,6 +17,7 @@ class History; namespace tgcalls { class GroupInstanceImpl; +struct GroupLevelsUpdate; } // namespace tgcalls namespace base { @@ -24,6 +25,10 @@ class GlobalShortcutManager; class GlobalShortcutValue; } // namespace base +namespace Data { +struct LastSpokeTimes; +} // namespace Data + namespace Calls { enum class MuteState { @@ -42,6 +47,7 @@ enum class MuteState { struct LevelUpdate { uint32 ssrc = 0; float value = 0.; + bool voice = false; bool self = false; }; @@ -146,11 +152,7 @@ private: void applySelfInCallLocally(); void rejoin(); - void myLevelUpdated(float level); - void audioLevelsUpdated( - const std::vector> &data); - void handleLevelsUpdated( - gsl::span> data); + void audioLevelsUpdated(const tgcalls::GroupLevelsUpdate &data); void setInstanceConnected(bool connected); void checkLastSpoke(); void pushToTalkCancel(); @@ -178,7 +180,7 @@ private: std::unique_ptr _instance; rpl::event_stream _levelUpdates; - base::flat_map _lastSpoke; + base::flat_map _lastSpoke; base::Timer _lastSpokeCheckTimer; base::Timer _checkJoinedTimer; diff --git a/Telegram/SourceFiles/calls/calls_group_members.cpp b/Telegram/SourceFiles/calls/calls_group_members.cpp index cfffc4696..64922c5e6 100644 --- a/Telegram/SourceFiles/calls/calls_group_members.cpp +++ b/Telegram/SourceFiles/calls/calls_group_members.cpp @@ -97,6 +97,9 @@ public: [[nodiscard]] uint32 ssrc() const { return _ssrc; } + [[nodiscard]] bool sounding() const { + return _sounding; + } [[nodiscard]] bool speaking() const { return _speaking; } @@ -147,7 +150,7 @@ private: Ui::Paint::Blobs blobs; crl::time lastTime = 0; - crl::time lastSpeakingUpdateTime = 0; + crl::time lastSoundingUpdateTime = 0; float64 enter = 0.; QImage userpicCache; @@ -156,6 +159,7 @@ private: rpl::lifetime lifetime; }; void refreshStatus() override; + void setSounding(bool sounding); void setSpeaking(bool speaking); void setState(State state); void setSsrc(uint32 ssrc); @@ -172,6 +176,7 @@ private: Ui::Animations::Simple _mutedAnimation; // For gray/red icon. Ui::Animations::Simple _activeAnimation; // For icon cross animation. uint32 _ssrc = 0; + bool _sounding = false; bool _speaking = false; bool _skipLevelUpdate = false; @@ -252,10 +257,10 @@ private: base::unique_qptr _menu; base::flat_set> _menuCheckRowsAfterHidden; - base::flat_map> _speakingRowBySsrc; - Ui::Animations::Basic _speakingAnimation; + base::flat_map> _soundingRowBySsrc; + Ui::Animations::Basic _soundingAnimation; - crl::time _speakingAnimationHideLastTime = 0; + crl::time _soundingAnimationHideLastTime = 0; bool _skipRowLevelUpdate = false; Ui::CrossLineAnimation _inactiveCrossLine; @@ -284,16 +289,20 @@ void Row::updateState(const Data::GroupCall::Participant *participant) { setCustomStatus(QString()); } setState(State::Inactive); + setSounding(false); setSpeaking(false); } else if (!participant->muted - || (participant->speaking && participant->ssrc != 0)) { + || (participant->sounding && participant->ssrc != 0)) { setState(State::Active); + setSounding(participant->sounding && participant->ssrc != 0); setSpeaking(participant->speaking && participant->ssrc != 0); } else if (participant->canSelfUnmute) { setState(State::Inactive); + setSounding(false); setSpeaking(false); } else { setState(State::Muted); + setSounding(false); setSpeaking(false); } } @@ -308,7 +317,14 @@ void Row::setSpeaking(bool speaking) { _speaking ? 0. : 1., _speaking ? 1. : 0., st::widgetFadeDuration); - if (!_speaking) { +} + +void Row::setSounding(bool sounding) { + if (_sounding == sounding) { + return; + } + _sounding = sounding; + if (!_sounding) { _blobsAnimation = nullptr; } else if (!_blobsAnimation) { _blobsAnimation = std::make_unique( @@ -358,7 +374,7 @@ void Row::updateLevel(float level) { } if (level >= GroupCall::kSpeakLevelThreshold) { - _blobsAnimation->lastSpeakingUpdateTime = crl::now(); + _blobsAnimation->lastSoundingUpdateTime = crl::now(); } _blobsAnimation->blobs.setLevel(level); } @@ -366,14 +382,14 @@ void Row::updateLevel(float level) { void Row::updateBlobAnimation(crl::time now) { Expects(_blobsAnimation != nullptr); - const auto speakingFinishesAt = _blobsAnimation->lastSpeakingUpdateTime - + Data::GroupCall::kSpeakStatusKeptFor; - const auto speakingStartsFinishing = speakingFinishesAt + const auto soundingFinishesAt = _blobsAnimation->lastSoundingUpdateTime + + Data::GroupCall::kSoundStatusKeptFor; + const auto soundingStartsFinishing = soundingFinishesAt - kBlobsEnterDuration; - const auto speakingFinishes = (speakingStartsFinishing < now); - if (speakingFinishes) { + const auto soundingFinishes = (soundingStartsFinishing < now); + if (soundingFinishes) { _blobsAnimation->enter = std::clamp( - (speakingFinishesAt - now) / float64(kBlobsEnterDuration), + (soundingFinishesAt - now) / float64(kBlobsEnterDuration), 0., 1.); } else if (_blobsAnimation->enter < 1.) { @@ -418,9 +434,15 @@ auto Row::generatePaintUserpicCallback() -> PaintRoundImageCallback { return [=](Painter &p, int x, int y, int outerWidth, int size) mutable { if (_blobsAnimation) { const auto shift = QPointF(x + size / 2., y + size / 2.); + const auto speaking = _speakingAnimation.value( + _speaking ? 1. : 0.); auto hq = PainterHighQualityEnabler(p); p.translate(shift); - _blobsAnimation->blobs.paint(p, st::groupCallMemberActiveStatus); + const auto brush = anim::brush( + st::groupCallMemberInactiveStatus, + st::groupCallMemberActiveStatus, + speaking); + _blobsAnimation->blobs.paint(p, brush); p.translate(-shift); p.setOpacity(1.); @@ -537,28 +559,28 @@ MembersController::MembersController( ) | rpl::start_with_next([=](bool animDisabled, bool deactivated) { const auto hide = !(!animDisabled && !deactivated); - if (!(hide && _speakingAnimationHideLastTime)) { - _speakingAnimationHideLastTime = hide ? crl::now() : 0; + if (!(hide && _soundingAnimationHideLastTime)) { + _soundingAnimationHideLastTime = hide ? crl::now() : 0; } - for (const auto [_, row] : _speakingRowBySsrc) { + for (const auto [_, row] : _soundingRowBySsrc) { if (hide) { updateRowLevel(row, 0.); } row->setSkipLevelUpdate(hide); } - if (!hide && !_speakingAnimation.animating()) { - _speakingAnimation.start(); + if (!hide && !_soundingAnimation.animating()) { + _soundingAnimation.start(); } _skipRowLevelUpdate = hide; }, _lifetime); - _speakingAnimation.init([=](crl::time now) { - if (const auto &last = _speakingAnimationHideLastTime; (last > 0) + _soundingAnimation.init([=](crl::time now) { + if (const auto &last = _soundingAnimationHideLastTime; (last > 0) && (now - last >= kBlobsEnterDuration)) { - _speakingAnimation.stop(); + _soundingAnimation.stop(); return false; } - for (const auto [ssrc, row] : _speakingRowBySsrc) { + for (const auto [ssrc, row] : _soundingRowBySsrc) { row->updateBlobAnimation(now); delegate()->peerListUpdateRow(row); } @@ -600,8 +622,8 @@ void MembersController::setupListChangeViewers(not_null call) { call->levelUpdates( ) | rpl::start_with_next([=](const LevelUpdate &update) { - const auto i = _speakingRowBySsrc.find(update.ssrc); - if (i != end(_speakingRowBySsrc)) { + const auto i = _soundingRowBySsrc.find(update.ssrc); + if (i != end(_soundingRowBySsrc)) { updateRowLevel(i->second, update.value); } }, _lifetime); @@ -699,41 +721,41 @@ void MembersController::checkSpeakingRowPosition(not_null row) { void MembersController::updateRow( not_null row, const Data::GroupCall::Participant *participant) { - const auto wasSpeaking = row->speaking(); + const auto wasSounding = row->sounding(); const auto wasSsrc = row->ssrc(); row->setSkipLevelUpdate(_skipRowLevelUpdate); row->updateState(participant); - const auto nowSpeaking = row->speaking(); + const auto nowSounding = row->sounding(); const auto nowSsrc = row->ssrc(); - const auto wasNoSpeaking = _speakingRowBySsrc.empty(); + const auto wasNoSounding = _soundingRowBySsrc.empty(); if (wasSsrc == nowSsrc) { - if (nowSpeaking != wasSpeaking) { - if (nowSpeaking) { - _speakingRowBySsrc.emplace(nowSsrc, row); + if (nowSounding != wasSounding) { + if (nowSounding) { + _soundingRowBySsrc.emplace(nowSsrc, row); } else { - _speakingRowBySsrc.remove(nowSsrc); + _soundingRowBySsrc.remove(nowSsrc); } } } else { - _speakingRowBySsrc.remove(wasSsrc); - if (nowSpeaking) { + _soundingRowBySsrc.remove(wasSsrc); + if (nowSounding) { Assert(nowSsrc != 0); - _speakingRowBySsrc.emplace(nowSsrc, row); + _soundingRowBySsrc.emplace(nowSsrc, row); } } - const auto nowNoSpeaking = _speakingRowBySsrc.empty(); - if (wasNoSpeaking && !nowNoSpeaking) { - _speakingAnimation.start(); - } else if (nowNoSpeaking && !wasNoSpeaking) { - _speakingAnimation.stop(); + const auto nowNoSounding = _soundingRowBySsrc.empty(); + if (wasNoSounding && !nowNoSounding) { + _soundingAnimation.start(); + } else if (nowNoSounding && !wasNoSounding) { + _soundingAnimation.stop(); } delegate()->peerListUpdateRow(row); } void MembersController::removeRow(not_null row) { - _speakingRowBySsrc.remove(row->ssrc()); + _soundingRowBySsrc.remove(row->ssrc()); delegate()->peerListRemoveRow(row); } diff --git a/Telegram/SourceFiles/data/data_group_call.cpp b/Telegram/SourceFiles/data/data_group_call.cpp index b754637b5..ed4fa9618 100644 --- a/Telegram/SourceFiles/data/data_group_call.cpp +++ b/Telegram/SourceFiles/data/data_group_call.cpp @@ -342,7 +342,10 @@ void GroupCall::applyParticipantsMutes( } } -void GroupCall::applyLastSpoke(uint32 ssrc, crl::time when, crl::time now) { +void GroupCall::applyLastSpoke( + uint32 ssrc, + LastSpokeTimes when, + crl::time now) { const auto i = _userBySsrc.find(ssrc); if (i == end(_userBySsrc)) { _unknownSpokenSsrcs[ssrc] = when; @@ -353,10 +356,13 @@ void GroupCall::applyLastSpoke(uint32 ssrc, crl::time when, crl::time now) { Assert(j != end(_participants)); _speakingByActiveFinishes.remove(j->user); - const auto speaking = (when + kSpeakStatusKeptFor >= now) + const auto sounding = (when.anything + kSoundStatusKeptFor >= now) && j->canSelfUnmute; - if (j->speaking != speaking) { + const auto speaking = sounding + && (when.voice + kSoundStatusKeptFor >= now); + if (j->sounding != sounding || j->speaking != speaking) { const auto was = *j; + j->sounding = sounding; j->speaking = speaking; _participantUpdates.fire({ .was = was, @@ -367,7 +373,7 @@ void GroupCall::applyLastSpoke(uint32 ssrc, crl::time when, crl::time now) { void GroupCall::applyActiveUpdate( UserId userId, - crl::time when, + LastSpokeTimes when, UserData *userLoaded) { if (inCall()) { return; @@ -387,9 +393,9 @@ void GroupCall::applyActiveUpdate( } const auto was = std::make_optional(*i); const auto now = crl::now(); - const auto elapsed = TimeId((now - when) / crl::time(1000)); + const auto elapsed = TimeId((now - when.anything) / crl::time(1000)); const auto lastActive = base::unixtime::now() - elapsed; - const auto finishes = when + kSpeakingAfterActive; + const auto finishes = when.anything + kSpeakingAfterActive; if (lastActive <= i->lastActive || finishes <= now) { return; } @@ -450,7 +456,7 @@ void GroupCall::requestUnknownParticipants() { if (_unknownSpokenSsrcs.size() < kRequestPerPage) { return base::take(_unknownSpokenSsrcs); } - auto result = base::flat_map(); + auto result = base::flat_map(); result.reserve(kRequestPerPage); while (result.size() < kRequestPerPage) { const auto [ssrc, when] = _unknownSpokenSsrcs.back(); @@ -463,7 +469,7 @@ void GroupCall::requestUnknownParticipants() { if (_unknownSpokenUids.size() + ssrcs.size() < kRequestPerPage) { return base::take(_unknownSpokenUids); } - auto result = base::flat_map(); + auto result = base::flat_map(); const auto available = (kRequestPerPage - int(ssrcs.size())); if (available > 0) { result.reserve(available); diff --git a/Telegram/SourceFiles/data/data_group_call.h b/Telegram/SourceFiles/data/data_group_call.h index 0010c6851..3b778f8ec 100644 --- a/Telegram/SourceFiles/data/data_group_call.h +++ b/Telegram/SourceFiles/data/data_group_call.h @@ -16,6 +16,11 @@ class ApiWrap; namespace Data { +struct LastSpokeTimes { + crl::time anything = 0; + crl::time voice = 0; +}; + class GroupCall final { public: GroupCall(not_null peer, uint64 id, uint64 accessHash); @@ -32,6 +37,7 @@ public: TimeId date = 0; TimeId lastActive = 0; uint32 ssrc = 0; + bool sounding = false; bool speaking = false; bool muted = false; bool canSelfUnmute = false; @@ -41,7 +47,7 @@ public: std::optional now; }; - static constexpr auto kSpeakStatusKeptFor = crl::time(350); + static constexpr auto kSoundStatusKeptFor = crl::time(350); [[nodiscard]] auto participants() const -> const std::vector &; @@ -56,10 +62,10 @@ public: void applyUpdate(const MTPDupdateGroupCallParticipants &update); void applyUpdateChecked( const MTPDupdateGroupCallParticipants &update); - void applyLastSpoke(uint32 ssrc, crl::time when, crl::time now); + void applyLastSpoke(uint32 ssrc, LastSpokeTimes when, crl::time now); void applyActiveUpdate( UserId userId, - crl::time when, + LastSpokeTimes when, UserData *userLoaded); [[nodiscard]] int fullCount() const; @@ -106,8 +112,8 @@ private: QString _nextOffset; rpl::variable _fullCount = 0; - base::flat_map _unknownSpokenSsrcs; - base::flat_map _unknownSpokenUids; + base::flat_map _unknownSpokenSsrcs; + base::flat_map _unknownSpokenUids; mtpRequestId _unknownUsersRequestId = 0; rpl::event_stream _participantUpdates; diff --git a/Telegram/ThirdParty/tgcalls b/Telegram/ThirdParty/tgcalls index e884535f1..b892eb58b 160000 --- a/Telegram/ThirdParty/tgcalls +++ b/Telegram/ThirdParty/tgcalls @@ -1 +1 @@ -Subproject commit e884535f1854aa57616b162a4901bd60c8287575 +Subproject commit b892eb58bc941a4a1a67303439df8ffd379d6051