From a6f379a17a6a2078388832c81670fc66903617b5 Mon Sep 17 00:00:00 2001 From: John Preston Date: Thu, 15 Apr 2021 17:55:32 +0400 Subject: [PATCH] Parse and serialize video parameters. --- Telegram/SourceFiles/calls/calls_call.cpp | 2 +- Telegram/SourceFiles/calls/calls_call.h | 2 +- .../SourceFiles/calls/calls_group_call.cpp | 208 +++++++++++++++++- Telegram/SourceFiles/calls/calls_group_call.h | 12 + Telegram/SourceFiles/calls/calls_instance.cpp | 8 + Telegram/SourceFiles/calls/calls_instance.h | 7 +- Telegram/SourceFiles/data/data_group_call.cpp | 15 +- Telegram/SourceFiles/data/data_group_call.h | 6 + 8 files changed, 251 insertions(+), 9 deletions(-) diff --git a/Telegram/SourceFiles/calls/calls_call.cpp b/Telegram/SourceFiles/calls/calls_call.cpp index cbf41b004..8f80c38b1 100644 --- a/Telegram/SourceFiles/calls/calls_call.cpp +++ b/Telegram/SourceFiles/calls/calls_call.cpp @@ -380,7 +380,7 @@ void Call::setupOutgoingVideo() { // Paused not supported right now. Assert(state == Webrtc::VideoState::Active); if (!_videoCapture) { - _videoCapture = _delegate->getVideoCapture(); + _videoCapture = _delegate->callGetVideoCapture(); _videoCapture->setOutput(_videoOutgoing->sink()); } if (_instance) { diff --git a/Telegram/SourceFiles/calls/calls_call.h b/Telegram/SourceFiles/calls/calls_call.h index b84f5c51f..c61015a8d 100644 --- a/Telegram/SourceFiles/calls/calls_call.h +++ b/Telegram/SourceFiles/calls/calls_call.h @@ -72,7 +72,7 @@ public: Fn onSuccess, bool video) = 0; - virtual auto getVideoCapture() + virtual auto callGetVideoCapture() -> std::shared_ptr = 0; virtual ~Delegate() = default; diff --git a/Telegram/SourceFiles/calls/calls_group_call.cpp b/Telegram/SourceFiles/calls/calls_group_call.cpp index 8edb10426..e730af122 100644 --- a/Telegram/SourceFiles/calls/calls_group_call.cpp +++ b/Telegram/SourceFiles/calls/calls_group_call.cpp @@ -27,12 +27,14 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "data/data_session.h" #include "base/global_shortcuts.h" #include "base/openssl_help.h" +#include "webrtc/webrtc_video_track.h" #include "webrtc/webrtc_media_devices.h" #include "webrtc/webrtc_create_adm.h" #include +#include #include - +#include #include #include #include @@ -76,6 +78,11 @@ constexpr auto kPlayConnectingEach = crl::time(1056) + 2 * crl::time(1000); } // namespace +struct VideoParams { + tgcalls::GroupParticipantDescription description; + uint32 hash = 0; +}; + class GroupCall::LoadPartTask final : public tgcalls::BroadcastPartTask { public: LoadPartTask( @@ -129,6 +136,114 @@ private: return false; } +std::shared_ptr ParseVideoParams( + const QByteArray &json, + const std::shared_ptr &existing) { + using namespace tgcalls; + + if (json.isEmpty()) { + return nullptr; + } + const auto hash = XXH32(json.data(), json.size(), uint32(0)); + if (existing && existing->hash == hash) { + return existing; + } + const auto data = existing ? existing : std::make_shared(); + data->hash = hash; + + auto error = QJsonParseError{ 0, QJsonParseError::NoError }; + const auto document = QJsonDocument::fromJson(json, &error); + if (error.error != QJsonParseError::NoError) { + LOG(("API Error: " + "Failed to parse group call video params, error: %1." + ).arg(error.errorString())); + return data; + } else if (!document.isObject()) { + LOG(("API Error: " + "Not an object received in group call video params.")); + return data; + } + + const auto readString = []( + const QJsonObject &object, + const char *key) { + return object.value(key).toString().toStdString(); + }; + const auto object = document.object(); + data->description.endpointId = readString(object, "endpoint"); + + const auto ssrcGroups = object.value("ssrc-groups").toArray(); + data->description.videoSourceGroups.reserve(ssrcGroups.size()); + for (const auto &value : ssrcGroups) { + const auto inner = value.toObject(); + auto sources = std::vector(); + { + const auto list = inner.value("sources").toArray(); + sources.reserve(list.size()); + for (const auto &source : list) { + sources.push_back(uint32_t(source.toDouble())); + } + } + data->description.videoSourceGroups.push_back({ + .ssrcs = std::move(sources), + .semantics = readString(inner, "semantics"), + }); + } + + const auto payloadTypes = object.value("payload-types").toArray(); + data->description.videoPayloadTypes.reserve(payloadTypes.size()); + for (const auto &value : payloadTypes) { + const auto inner = value.toObject(); + + auto types = std::vector(); + { + const auto list = inner.value("rtcp-fbs").toArray(); + types.reserve(list.size()); + for (const auto &type : list) { + const auto inside = type.toObject(); + types.push_back({ + .type = readString(inside, "type"), + .subtype = readString(inside, "subtype"), + }); + } + } + auto parameters = std::vector>(); + { + const auto list = inner.value("parameters").toArray(); + parameters.reserve(list.size()); + for (const auto ¶meter : list) { + const auto inside = parameter.toObject(); + for (auto i = inside.begin(); i != inside.end(); ++i) { + parameters.push_back({ + i.key().toStdString(), + i.value().toString().toStdString(), + }); + } + } + } + data->description.videoPayloadTypes.push_back({ + .id = uint32_t(inner.value("id").toDouble()), + .name = readString(inner, "name"), + .clockrate = uint32_t(inner.value("clockrate").toDouble()), + .channels = uint32_t(inner.value("channels").toDouble()), + .feedbackTypes = std::move(types), + .parameters = std::move(parameters), + }); + } + + const auto extensionMap = object.value("rtp-hdrexts").toArray(); + data->description.videoExtensionMap.reserve(extensionMap.size()); + for (const auto &extension : extensionMap) { + const auto inner = extension.toObject(); + data->description.videoExtensionMap.push_back({ + uint32_t(inner.value("id").toDouble()), + readString(inner, "uri"), + }); + } + + return data; +} + GroupCall::LoadPartTask::LoadPartTask( base::weak_ptr call, int64 time, @@ -186,6 +301,8 @@ GroupCall::GroupCall( , _joinHash(info.joinHash) , _id(inputCall.c_inputGroupCall().vid().v) , _scheduleDate(info.scheduleDate) +, _videoOutgoing(std::make_unique( + Webrtc::VideoState::Inactive)) , _lastSpokeCheckTimer([=] { checkLastSpoke(); }) , _checkJoinedTimer([=] { checkJoined(); }) , _pushToTalkCancelTimer([=] { pushToTalkCancel(); }) @@ -429,6 +546,13 @@ void GroupCall::join(const MTPInputGroupCall &inputCall) { const auto volumeChanged = was ? (was->volume != now.volume || was->mutedByMe != now.mutedByMe) : (now.volume != Group::kDefaultVolume || now.mutedByMe); + if (now.videoParams) { + auto participants = std::vector(); + participants.push_back(now.videoParams->description); + participants.back().audioSsrc = now.ssrc; + _instance->addParticipants(std::move(participants)); + } + if (volumeChanged) { _instance->setVolume( now.ssrc, @@ -511,12 +635,80 @@ void GroupCall::rejoin(not_null as) { fingerprints.push_back(object); } + auto extensionMap = QJsonArray(); + for (const auto &extension : payload.videoExtensionMap) { + auto object = QJsonObject(); + object.insert("id", int64(extension.first)); + object.insert( + "uri", + QString::fromStdString(extension.second)); + extensionMap.push_back(object); + } + + auto payloadTypes = QJsonArray(); + for (const auto &type : payload.videoPayloadTypes) { + auto object = QJsonObject(); + object.insert("id", int64(type.id)); + object.insert("name", QString::fromStdString(type.name)); + object.insert("clockrate", int64(type.clockrate)); + if (!type.parameters.empty()) { + auto parameters = QJsonObject(); + for (const auto ¶meter : type.parameters) { + parameters.insert( + QString::fromStdString(parameter.first), + QString::fromStdString(parameter.second)); + } + object.insert("parameters", parameters); + } + if (type.name != "rtx") { + object.insert("channels", int64(type.channels)); + auto fbs = QJsonArray(); + for (const auto &element : type.feedbackTypes) { + auto inner = QJsonObject(); + inner.insert( + "type", + QString::fromStdString(element.type)); + if (!element.subtype.empty()) { + inner.insert( + "subtype", + QString::fromStdString(element.subtype)); + } + fbs.push_back(inner); + } + object.insert("rtcp-fbs", fbs); + } + payloadTypes.push_back(object); + } + + auto sourceGroups = QJsonArray(); + for (const auto &group : payload.videoSourceGroups) { + auto object = QJsonObject(); + object.insert( + "semantics", + QString::fromStdString(group.semantics)); + auto list = QJsonArray(); + for (const auto source : group.ssrcs) { + list.push_back(int64(source)); + } + object.insert("sources", list); + sourceGroups.push_back(object); + } + auto root = QJsonObject(); const auto ssrc = payload.ssrc; root.insert("ufrag", QString::fromStdString(payload.ufrag)); root.insert("pwd", QString::fromStdString(payload.pwd)); root.insert("fingerprints", fingerprints); root.insert("ssrc", double(payload.ssrc)); + if (!extensionMap.isEmpty()) { + root.insert("rtp-hdrexts", extensionMap); + } + if (!payloadTypes.isEmpty()) { + root.insert("payload-types", payloadTypes); + } + if (!sourceGroups.isEmpty()) { + root.insert("ssrc-groups", sourceGroups); + } LOG(("Call Info: Join payload received, joining with ssrc: %1." ).arg(ssrc)); @@ -895,7 +1087,7 @@ void GroupCall::handlePossibleCreateOrJoinResponse( const auto candidates = root.value("candidates").toArray(); for (const auto &print : prints) { const auto object = print.toObject(); - payload.fingerprints.push_back(tgcalls::GroupJoinPayloadFingerprint{ + payload.fingerprints.push_back({ .hash = readString(object, "hash"), .setup = readString(object, "setup"), .fingerprint = readString(object, "fingerprint"), @@ -903,7 +1095,7 @@ void GroupCall::handlePossibleCreateOrJoinResponse( } for (const auto &candidate : candidates) { const auto object = candidate.toObject(); - payload.candidates.push_back(tgcalls::GroupJoinResponseCandidate{ + payload.candidates.push_back({ .port = readString(object, "port"), .protocol = readString(object, "protocol"), .network = readString(object, "network"), @@ -945,7 +1137,9 @@ void GroupCall::addParticipantsToInstance() { void GroupCall::prepareParticipantForAdding( const Data::GroupCallParticipant &participant) { - _preparedParticipants.push_back(tgcalls::GroupParticipantDescription()); + _preparedParticipants.push_back(participant.videoParams + ? participant.videoParams->description + : tgcalls::GroupParticipantDescription()); auto &added = _preparedParticipants.back(); added.audioSsrc = participant.ssrc; _unresolvedSsrcs.remove(added.audioSsrc); @@ -1142,6 +1336,11 @@ void GroupCall::ensureControllerCreated() { } const auto &settings = Core::App().settings(); + if (!_videoCapture) { + _videoCapture = _delegate->groupCallGetVideoCapture(); + _videoCapture->setOutput(_videoOutgoing->sink()); + } + const auto weak = base::make_weak(this); const auto myLevel = std::make_shared(); tgcalls::GroupInstanceDescriptor descriptor = { @@ -1170,6 +1369,7 @@ void GroupCall::ensureControllerCreated() { .initialOutputDeviceId = _audioOutputId.toStdString(), .createAudioDeviceModule = Webrtc::AudioDeviceModuleCreator( settings.callAudioBackend()), + .videoCapture = _videoCapture, .participantDescriptionsRequired = [=]( const std::vector &ssrcs) { crl::on_main(weak, [=] { diff --git a/Telegram/SourceFiles/calls/calls_group_call.h b/Telegram/SourceFiles/calls/calls_group_call.h index 90f0f8f9f..f2b7972aa 100644 --- a/Telegram/SourceFiles/calls/calls_group_call.h +++ b/Telegram/SourceFiles/calls/calls_group_call.h @@ -20,6 +20,7 @@ class GroupInstanceCustomImpl; struct GroupLevelsUpdate; struct GroupNetworkState; struct GroupParticipantDescription; +class VideoCaptureInterface; } // namespace tgcalls namespace base { @@ -29,6 +30,7 @@ class GlobalShortcutValue; namespace Webrtc { class MediaDevices; +class VideoTrack; } // namespace Webrtc namespace Data { @@ -72,6 +74,12 @@ struct LevelUpdate { bool me = false; }; +struct VideoParams; + +[[nodiscard]] std::shared_ptr ParseVideoParams( + const QByteArray &json, + const std::shared_ptr &existing); + class GroupCall final : public base::has_weak_ptr { public: class Delegate { @@ -90,6 +98,8 @@ public: Ended, }; virtual void groupCallPlaySound(GroupCallSound sound) = 0; + virtual auto groupCallGetVideoCapture() + -> std::shared_ptr = 0; }; using GlobalShortcutManager = base::GlobalShortcutManager; @@ -336,6 +346,8 @@ private: mtpRequestId _updateMuteRequestId = 0; std::unique_ptr _instance; + std::shared_ptr _videoCapture; + const std::unique_ptr _videoOutgoing; rpl::event_stream _levelUpdates; base::flat_map _lastSpoke; rpl::event_stream _rejoinEvents; diff --git a/Telegram/SourceFiles/calls/calls_instance.cpp b/Telegram/SourceFiles/calls/calls_instance.cpp index af869bd63..b59a9f42b 100644 --- a/Telegram/SourceFiles/calls/calls_instance.cpp +++ b/Telegram/SourceFiles/calls/calls_instance.cpp @@ -592,6 +592,14 @@ void Instance::requestPermissionOrFail(Platform::PermissionType type, Fn } } +std::shared_ptr Instance::callGetVideoCapture() { + return getVideoCapture(); +} + +std::shared_ptr Instance::groupCallGetVideoCapture() { + return getVideoCapture(); +} + std::shared_ptr Instance::getVideoCapture() { if (auto result = _videoCapture.lock()) { return result; diff --git a/Telegram/SourceFiles/calls/calls_instance.h b/Telegram/SourceFiles/calls/calls_instance.h index 96034fcaf..c7e66a47b 100644 --- a/Telegram/SourceFiles/calls/calls_instance.h +++ b/Telegram/SourceFiles/calls/calls_instance.h @@ -69,8 +69,7 @@ public: bool activateCurrentCall(const QString &joinHash = QString()); bool minimizeCurrentActiveCall(); bool closeCurrentActiveCall(); - auto getVideoCapture() - -> std::shared_ptr override; + std::shared_ptr getVideoCapture(); void requestPermissionsOrFail(Fn onSuccess, bool video = true); void setCurrentAudioDevice(bool input, const QString &deviceId); @@ -103,6 +102,8 @@ private: requestPermissionsOrFail(std::move(onSuccess), video); } void callPlaySound(CallSound sound) override; + auto callGetVideoCapture() + ->std::shared_ptr override; void groupCallFinished(not_null call) override; void groupCallFailed(not_null call) override; @@ -110,6 +111,8 @@ private: requestPermissionsOrFail(std::move(onSuccess), false); } void groupCallPlaySound(GroupCallSound sound) override; + auto groupCallGetVideoCapture() + ->std::shared_ptr override; void createCall(not_null user, Call::Type type, bool video); void destroyCall(not_null call); diff --git a/Telegram/SourceFiles/data/data_group_call.cpp b/Telegram/SourceFiles/data/data_group_call.cpp index 1ec22213c..c590b54f7 100644 --- a/Telegram/SourceFiles/data/data_group_call.cpp +++ b/Telegram/SourceFiles/data/data_group_call.cpp @@ -343,7 +343,7 @@ void GroupCall::applyLocalUpdate( const MTPDupdateGroupCallParticipants &update) { applyParticipantsSlice( update.vparticipants().v, - ApplySliceSource::UpdateReceived); + ApplySliceSource::UpdateConstructed); } void GroupCall::applyEnqueuedUpdate(const MTPUpdate &update) { @@ -529,8 +529,21 @@ void GroupCall::applyParticipantsSlice( && (!was || was->onlyMinLoaded); const auto raisedHandRating = data.vraise_hand_rating().value_or_empty(); + const auto hasVideoParamsInformation = (sliceSource + != ApplySliceSource::UpdateConstructed); const auto value = Participant{ .peer = participantPeer, + .videoParams = (hasVideoParamsInformation + ? Calls::ParseVideoParams( + (data.vparams() + ? data.vparams()->c_dataJSON().vdata().v + : QByteArray()), + (i != end(_participants) + ? i->videoParams + : nullptr)) + : (i != end(_participants)) + ? i->videoParams + : nullptr), .date = data.vdate().v, .lastActive = lastActive, .raisedHandRating = raisedHandRating, diff --git a/Telegram/SourceFiles/data/data_group_call.h b/Telegram/SourceFiles/data/data_group_call.h index 078f5634e..c2dd507d2 100644 --- a/Telegram/SourceFiles/data/data_group_call.h +++ b/Telegram/SourceFiles/data/data_group_call.h @@ -13,6 +13,10 @@ class PeerData; class ApiWrap; +namespace Calls { +struct VideoParams; +} // namespace Calls + namespace Data { struct LastSpokeTimes { @@ -22,6 +26,7 @@ struct LastSpokeTimes { struct GroupCallParticipant { not_null peer; + std::shared_ptr videoParams; TimeId date = 0; TimeId lastActive = 0; uint64 raisedHandRating = 0; @@ -131,6 +136,7 @@ private: SliceLoaded, UnknownLoaded, UpdateReceived, + UpdateConstructed, }; enum class QueuedType : uint8 { VersionedParticipant,