From 9514b6eecd9df649b30b8426043492026e526f4d Mon Sep 17 00:00:00 2001 From: John Preston Date: Tue, 22 Oct 2024 09:49:33 +0400 Subject: [PATCH] Show mini-thumbnails when pausing recording. --- .../history_view_voice_record_bar.cpp | 103 ++++++++++++----- .../controls/history_view_voice_record_bar.h | 3 +- .../ui/controls/round_video_recorder.cpp | 109 ++++++++++++++++-- .../ui/controls/round_video_recorder.h | 5 +- 4 files changed, 178 insertions(+), 42 deletions(-) diff --git a/Telegram/SourceFiles/history/view/controls/history_view_voice_record_bar.cpp b/Telegram/SourceFiles/history/view/controls/history_view_voice_record_bar.cpp index 8d7c9c0ff..b24d7ddd8 100644 --- a/Telegram/SourceFiles/history/view/controls/history_view_voice_record_bar.cpp +++ b/Telegram/SourceFiles/history/view/controls/history_view_voice_record_bar.cpp @@ -203,6 +203,44 @@ void PaintWaveform( } } +void FillWithMinithumbs( + QPainter &p, + not_null data, + QRect rect, + float64 progress) { + if (!data->minithumbsCount || !data->minithumbSize || rect.isEmpty()) { + return; + } + const auto size = rect.height(); + const auto single = data->minithumbSize; + const auto perrow = data->minithumbs.width() / single; + const auto thumbs = (rect.width() + size - 1) / size; + if (!thumbs || !perrow) { + return; + } + for (auto i = 0; i != thumbs - 1; ++i) { + const auto index = (i * data->minithumbsCount) / thumbs; + p.drawImage( + QRect(rect.x() + i * size, rect.y(), size, size), + data->minithumbs, + QRect( + (index % perrow) * single, + (index / perrow) * single, + single, + single)); + } + const auto last = rect.width() - (thumbs - 1) * size; + const auto index = ((thumbs - 1) * data->minithumbsCount) / thumbs; + p.drawImage( + QRect(rect.x() + (thumbs - 1) * size, rect.y(), last, size), + data->minithumbs, + QRect( + (index % perrow) * single, + (index / perrow) * single, + (last * single) / size, + single)); +} + [[nodiscard]] QRect DrawLockCircle( QPainter &p, const QRect &widgetRect, @@ -428,7 +466,7 @@ public: not_null parent, const style::RecordBar &st, not_null session, - ::Media::Capture::Result *data, + not_null data, const style::font &font); void requestPaintProgress(float64 progress); @@ -456,7 +494,7 @@ private: const not_null _document; const std::unique_ptr _voiceData; const std::shared_ptr _mediaView; - const not_null<::Media::Capture::Result*> _data; + const not_null _data; const base::unique_qptr _delete; const style::font &_durationFont; const QString _duration; @@ -486,7 +524,7 @@ ListenWrap::ListenWrap( not_null parent, const style::RecordBar &st, not_null session, - ::Media::Capture::Result *data, + not_null data, const style::font &font) : _parent(parent) , _st(st) @@ -604,20 +642,27 @@ void ListenWrap::init() { } // Waveform paint. - { - const auto rect = (progress == 1.) - ? _waveformFgRect - : computeWaveformRect(bgCenterRect); - if (rect.width() > 0) { - p.translate(rect.topLeft()); + const auto waveformRect = (progress == 1.) + ? _waveformFgRect + : computeWaveformRect(bgCenterRect); + if (!waveformRect.isEmpty()) { + const auto playProgress = _playProgress.current(); + if (_data->minithumbs.isNull()) { + p.translate(waveformRect.topLeft()); PaintWaveform( p, _voiceData.get(), - rect.width(), + waveformRect.width(), _activeWaveformBar, _inactiveWaveformBar, - _playProgress.current()); + playProgress); p.resetTransform(); + } else { + FillWithMinithumbs( + p, + _data, + waveformRect, + playProgress); } } } @@ -631,9 +676,11 @@ void ListenWrap::initPlayButton() { using namespace ::Media::Player; using State = TrackState; - _mediaView->setBytes(_data->bytes); - _document->size = _data->bytes.size(); - _document->type = _data->video ? RoundVideoDocument : VoiceDocument; + _mediaView->setBytes(_data->content); + _document->size = _data->content.size(); + _document->type = _data->minithumbs.isNull() + ? VoiceDocument + : RoundVideoDocument; const auto &play = _playPauseSt.playOuter; const auto &width = _waveformBgFinalCenterRect.height(); @@ -1688,10 +1735,7 @@ void VoiceRecordBar::startRecording() { instance()->pause(false, nullptr); if (_videoRecorder) { _videoRecorder->resume({ - .video = { - .content = _data.bytes, - .duration = _data.duration, - }, + .video = std::move(_data), }); } } else { @@ -1836,12 +1880,7 @@ void VoiceRecordBar::stopRecording(StopType type, bool ttlBeforeHide) { window()->activateWindow(); _paused = true; - _data = ::Media::Capture::Result{ - .bytes = std::move(data.content), - //.waveform = std::move(data.waveform), - .duration = data.duration, - .video = true, - }; + _data = std::move(data); _listen = std::make_unique( this, _st, @@ -1861,7 +1900,11 @@ void VoiceRecordBar::stopRecording(StopType type, bool ttlBeforeHide) { return; } _paused = true; - _data = std::move(data); + _data = Ui::RoundVideoResult{ + .content = std::move(data.bytes), + .waveform = std::move(data.waveform), + .duration = data.duration, + }; window()->raise(); window()->activateWindow(); @@ -1906,7 +1949,11 @@ void VoiceRecordBar::stopRecording(StopType type, bool ttlBeforeHide) { stop(false); return; } - _data = std::move(data); + _data = Ui::RoundVideoResult{ + .content = std::move(data.bytes), + .waveform = std::move(data.waveform), + .duration = data.duration, + }; window()->raise(); window()->activateWindow(); @@ -1916,7 +1963,7 @@ void VoiceRecordBar::stopRecording(StopType type, bool ttlBeforeHide) { : 0), }; _sendVoiceRequests.fire({ - _data.bytes, + _data.content, _data.waveform, _data.duration, options, @@ -1983,7 +2030,7 @@ void VoiceRecordBar::requestToSendWithOptions(Api::SendOptions options) { options.ttlSeconds = std::numeric_limits::max(); } _sendVoiceRequests.fire({ - _data.bytes, + _data.content, _data.waveform, _data.duration, options, diff --git a/Telegram/SourceFiles/history/view/controls/history_view_voice_record_bar.h b/Telegram/SourceFiles/history/view/controls/history_view_voice_record_bar.h index c2db218bf..9c9d66c55 100644 --- a/Telegram/SourceFiles/history/view/controls/history_view_voice_record_bar.h +++ b/Telegram/SourceFiles/history/view/controls/history_view_voice_record_bar.h @@ -11,6 +11,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "base/timer.h" #include "history/view/controls/compose_controls_common.h" #include "media/audio/media_audio_capture_common.h" +#include "ui/controls/round_video_recorder.h" #include "ui/effects/animations.h" #include "ui/round_rect.h" #include "ui/rp_widget.h" @@ -170,7 +171,7 @@ private: std::unique_ptr _ttlButton; std::unique_ptr _listen; - ::Media::Capture::Result _data; + Ui::RoundVideoResult _data; rpl::variable _paused; base::Timer _startTimer; diff --git a/Telegram/SourceFiles/ui/controls/round_video_recorder.cpp b/Telegram/SourceFiles/ui/controls/round_video_recorder.cpp index 3851f0bf8..e26e3af6c 100644 --- a/Telegram/SourceFiles/ui/controls/round_video_recorder.cpp +++ b/Telegram/SourceFiles/ui/controls/round_video_recorder.cpp @@ -16,6 +16,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "ui/painter.h" #include "ui/rp_widget.h" #include "webrtc/webrtc_video_track.h" +#include "styles/style_chat.h" #include "styles/style_chat_helpers.h" namespace Ui { @@ -30,6 +31,8 @@ constexpr auto kMinDuration = crl::time(200); constexpr auto kMaxDuration = 60 * crl::time(1000); constexpr auto kInitTimeout = 5 * crl::time(1000); constexpr auto kBlurredSize = 64; +constexpr auto kMinithumbsPerSecond = 5; +constexpr auto kMinithumbsInRow = 16; using namespace FFmpeg; @@ -66,11 +69,19 @@ struct ReadBytesWrap { }; }; +[[nodiscard]] int MinithumbSize() { + const auto full = st::historySendSize.height(); + const auto margin = st::historyRecordWaveformBgMargins; + const auto outer = full - margin.top() - margin.bottom(); + const auto inner = outer - 2 * st::msgWaveformMin; + return inner * style::DevicePixelRatio(); +} + } // namespace class RoundVideoRecorder::Private final { public: - Private(crl::weak_on_queue weak); + Private(crl::weak_on_queue weak, int minithumbSize); ~Private(); void push(int64 mcstimestamp, const QImage &frame); @@ -103,6 +114,13 @@ private: int64_t seek(int64_t offset, int whence); void initEncoding(); + void initCircleMask(); + void initMinithumbsCanvas(); + void maybeSaveMinithumb( + not_null frame, + const QImage &original, + QRect crop); + bool initVideo(); bool initAudio(); void notifyFinished(); @@ -122,7 +140,6 @@ private: void updateResultDuration(int64 pts, AVRational timeBase); void cutCircleFromYUV420P(not_null frame); - void initCircleMask(); [[nodiscard]] RoundVideoResult appendToPrevious(RoundVideoResult video); [[nodiscard]] static FormatPointer OpenInputContext( @@ -169,6 +186,11 @@ private: crl::time _lastUpdateDuration = 0; rpl::event_stream _updates; + crl::time _minithumbNextTimestamp = 0; + const int _minithumbSize = 0; + int _minithumbsCount = 0; + QImage _minithumbs; + crl::time _maxDuration = 0; RoundVideoResult _previous; @@ -185,12 +207,16 @@ RoundVideoRecorder::Private::CopyContext::CopyContext() { ranges::fill(lastDts, std::numeric_limits::min()); } -RoundVideoRecorder::Private::Private(crl::weak_on_queue weak) +RoundVideoRecorder::Private::Private( + crl::weak_on_queue weak, + int minithumbSize) : _weak(std::move(weak)) +, _minithumbSize(minithumbSize) , _maxDuration(kMaxDuration) , _timeoutTimer(_weak, [=] { timeout(); }) { initEncoding(); initCircleMask(); + initMinithumbsCanvas(); _timeoutTimer.callOnce(kInitTimeout); } @@ -452,8 +478,11 @@ RoundVideoResult RoundVideoRecorder::Private::finish() { finishEncoding(); auto result = appendToPrevious({ .content = base::take(_result), - .waveform = QByteArray(), .duration = base::take(_resultDuration), + //.waveform = {}, + .minithumbs = base::take(_minithumbs), + .minithumbsCount = base::take(_minithumbsCount), + .minithumbSize = _minithumbSize, }); if (result.duration < kMinDuration) { return {}; @@ -523,11 +552,9 @@ RoundVideoResult RoundVideoRecorder::Private::appendToPrevious( fail(Error::Encoding); return {}; } - return RoundVideoResult{ - .content = base::take(_result), - .waveform = QByteArray(), - .duration = _previous.duration + video.duration, - }; + video.content = base::take(_result); + video.duration += _previous.duration; + return video; } FormatPointer RoundVideoRecorder::Private::OpenInputContext( @@ -605,7 +632,11 @@ void RoundVideoRecorder::Private::restart(RoundVideoPartial partial) { return; } _previous = std::move(partial.video); + _minithumbs = std::move(_previous.minithumbs); + _minithumbsCount = _previous.minithumbsCount; + Assert(_minithumbSize == _previous.minithumbSize); _maxDuration = kMaxDuration - _previous.duration; + _minithumbNextTimestamp = 0; _finished = false; initEncoding(); _timeoutTimer.callOnce(kInitTimeout); @@ -720,6 +751,7 @@ void RoundVideoRecorder::Private::encodeVideoFrame( cutCircleFromYUV420P(_videoFrame.get()); _videoFrame->pts = mcstimestamp - _videoFirstTimestamp; + maybeSaveMinithumb(_videoFrame.get(), frame, crop); if (_videoFrame->pts >= _maxDuration * int64(1000)) { notifyFinished(); return; @@ -728,6 +760,49 @@ void RoundVideoRecorder::Private::encodeVideoFrame( } } +void RoundVideoRecorder::Private::maybeSaveMinithumb( + not_null frame, + const QImage &original, + QRect crop) { + if (frame->pts < _minithumbNextTimestamp * 1000) { + return; + } + _minithumbNextTimestamp += crl::time(1000) / kMinithumbsPerSecond; + const auto perline = original.bytesPerLine(); + const auto perpixel = original.depth() / 8; + const auto cropped = QImage( + original.constBits() + (crop.y() * perline) + (crop.x() * perpixel), + crop.width(), + crop.height(), + perline, + original.format() + ).scaled( + _minithumbSize, + _minithumbSize, + Qt::IgnoreAspectRatio, + Qt::SmoothTransformation); + + const auto row = _minithumbsCount / kMinithumbsInRow; + const auto column = _minithumbsCount % kMinithumbsInRow; + const auto fromPerLine = cropped.bytesPerLine(); + auto from = cropped.constBits(); + const auto toPerLine = _minithumbs.bytesPerLine(); + const auto toPerPixel = _minithumbs.depth() / 8; + auto to = _minithumbs.bits() + + (row * _minithumbSize * toPerLine) + + (column * _minithumbSize * toPerPixel); + + Assert(toPerPixel == perpixel); + for (auto y = 0; y != _minithumbSize; ++y) { + Assert(to + toPerLine - _minithumbs.constBits() + <= _minithumbs.bytesPerLine() * _minithumbs.height()); + memcpy(to, from, _minithumbSize * toPerPixel); + from += fromPerLine; + to += toPerLine; + } + ++_minithumbsCount; +} + void RoundVideoRecorder::Private::initCircleMask() { const auto width = kSide; const auto height = kSide; @@ -747,6 +822,16 @@ void RoundVideoRecorder::Private::initCircleMask() { } } +void RoundVideoRecorder::Private::initMinithumbsCanvas() { + const auto width = kMinithumbsInRow * _minithumbSize; + const auto seconds = (kMaxDuration + 999) / 1000; + const auto persecond = kMinithumbsPerSecond; + const auto frames = (seconds + persecond - 1) * persecond; + const auto rows = (frames + kMinithumbsInRow - 1) / kMinithumbsInRow; + const auto height = rows * _minithumbSize; + _minithumbs = QImage(width, height, QImage::Format_ARGB32_Premultiplied); +} + void RoundVideoRecorder::Private::cutCircleFromYUV420P( not_null frame) { const auto width = frame->width; @@ -888,9 +973,9 @@ bool RoundVideoRecorder::Private::writeFrame( while (true) { error = AvErrorWrap(avcodec_receive_packet(codec.get(), pkt)); if (error.code() == AVERROR(EAGAIN)) { - return true; // Need more input + return true; // Need more input } else if (error.code() == AVERROR_EOF) { - return true; // Encoding finished + return true; // Encoding finished } else if (error) { LogError("avcodec_receive_packet", error); fail(Error::Encoding); @@ -945,7 +1030,7 @@ RoundVideoRecorder::RoundVideoRecorder( RoundVideoRecorderDescriptor &&descriptor) : _descriptor(std::move(descriptor)) , _preview(std::make_unique(_descriptor.container)) -, _private() { +, _private(MinithumbSize()) { setup(); } diff --git a/Telegram/SourceFiles/ui/controls/round_video_recorder.h b/Telegram/SourceFiles/ui/controls/round_video_recorder.h index 914054435..51f0e537d 100644 --- a/Telegram/SourceFiles/ui/controls/round_video_recorder.h +++ b/Telegram/SourceFiles/ui/controls/round_video_recorder.h @@ -40,8 +40,11 @@ struct RoundVideoRecorderDescriptor { struct RoundVideoResult { QByteArray content; - QByteArray waveform; + QVector waveform; crl::time duration = 0; + QImage minithumbs; + int minithumbsCount = 0; + int minithumbSize = 0; }; struct RoundVideoPartial {