void/AyuGramDesktop
2
0
Fork 0
mirror of https://github.com/AyuGram/AyuGramDesktop.git synced 2025-06-05 06:33:57 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Render NV12 in OpenGL from hwaccel without swscale.

Browse source
This commit is contained in:
John Preston 2022-03-22 19:50:23 +04:00
parent 0dca556843
commit 41eb223bbf
10 changed files with 121 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
Telegram/SourceFiles/media/streaming/media_streaming_common.h
View file

@ -162,6 +162,7 @@ enum class FrameFormat {
None, None,
ARGB32, ARGB32,
YUV420, YUV420,
NV12,
}; };
struct FrameChannel { struct FrameChannel {
@ -169,7 +170,7 @@ struct FrameChannel {
int stride = 0; int stride = 0;
}; };
struct FrameYUV420 { struct FrameYUV {
QSize size; QSize size;
QSize chromaSize; QSize chromaSize;
FrameChannel y; FrameChannel y;
@ -179,7 +180,7 @@ struct FrameYUV420 {
struct FrameWithInfo { struct FrameWithInfo {
QImage image; QImage image;
FrameYUV420 *yuv420 = nullptr; FrameYUV *yuv = nullptr;
FrameFormat format = FrameFormat::None; FrameFormat format = FrameFormat::None;
int index = -1; int index = -1;
bool alpha = false; bool alpha = false;

2
Telegram/SourceFiles/media/streaming/media_streaming_utility.cpp
View file

@ -198,7 +198,7 @@ QImage ConvertFrame(
return storage; return storage;
} }
FrameYUV420 ExtractYUV420(Stream &stream, AVFrame *frame) { FrameYUV ExtractYUV(Stream &stream, AVFrame *frame) {
return { return {
.size = { frame->width, frame->height }, .size = { frame->width, frame->height },
.chromaSize = { .chromaSize = {

2
Telegram/SourceFiles/media/streaming/media_streaming_utility.h
View file

@ -64,7 +64,7 @@ struct Stream {
not_null<AVFrame*> frame, not_null<AVFrame*> frame,
QSize resize, QSize resize,
QImage storage); QImage storage);
[[nodiscard]] FrameYUV420 ExtractYUV420(Stream &stream, AVFrame *frame); [[nodiscard]] FrameYUV ExtractYUV(Stream &stream, AVFrame *frame);
[[nodiscard]] QImage PrepareByRequest( [[nodiscard]] QImage PrepareByRequest(
const QImage &original, const QImage &original,
bool alpha, bool alpha,

46
Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp
View file

@ -22,10 +22,12 @@ constexpr auto kDisplaySkipped = crl::time(-1);
constexpr auto kFinishedPosition = std::numeric_limits<crl::time>::max(); constexpr auto kFinishedPosition = std::numeric_limits<crl::time>::max();
static_assert(kDisplaySkipped != kTimeUnknown); static_assert(kDisplaySkipped != kTimeUnknown);
[[nodiscard]] QImage ConvertToARGB32(const FrameYUV420 &data) { [[nodiscard]] QImage ConvertToARGB32(
FrameFormat format,
const FrameYUV &data) {
Expects(data.y.data != nullptr); Expects(data.y.data != nullptr);
Expects(data.u.data != nullptr); Expects(data.u.data != nullptr);
Expects(data.v.data != nullptr); Expects((format == FrameFormat::NV12) || (data.v.data != nullptr));
Expects(!data.size.isEmpty()); Expects(!data.size.isEmpty());
//if (FFmpeg::RotationSwapWidthHeight(stream.rotation)) { //if (FFmpeg::RotationSwapWidthHeight(stream.rotation)) {
@ -35,7 +37,9 @@ static_assert(kDisplaySkipped != kTimeUnknown);
auto result = FFmpeg::CreateFrameStorage(data.size); auto result = FFmpeg::CreateFrameStorage(data.size);
const auto swscale = FFmpeg::MakeSwscalePointer( const auto swscale = FFmpeg::MakeSwscalePointer(
data.size, data.size,
AV_PIX_FMT_YUV420P, (format == FrameFormat::YUV420
? AV_PIX_FMT_YUV420P
: AV_PIX_FMT_NV12),
data.size, data.size,
AV_PIX_FMT_BGRA); AV_PIX_FMT_BGRA);
if (!swscale) { if (!swscale) {
@ -448,14 +452,16 @@ void VideoTrackObject::rasterizeFrame(not_null<Frame*> frame) {
const auto frameWithData = frame->transferred const auto frameWithData = frame->transferred
? frame->transferred.get() ? frame->transferred.get()
: frame->decoded.get(); : frame->decoded.get();
if (frameWithData->format == AV_PIX_FMT_YUV420P && !requireARGB32()) { if ((frameWithData->format == AV_PIX_FMT_YUV420P
|| frameWithData->format == AV_PIX_FMT_NV12) && !requireARGB32()) {
const auto nv12 = (frameWithData->format == AV_PIX_FMT_NV12);
frame->alpha = false; frame->alpha = false;
frame->yuv420 = ExtractYUV420(_stream, frameWithData); frame->yuv = ExtractYUV(_stream, frameWithData);
if (frame->yuv420.size.isEmpty() if (frame->yuv.size.isEmpty()
|| frame->yuv420.chromaSize.isEmpty() || frame->yuv.chromaSize.isEmpty()
|| !frame->yuv420.y.data || !frame->yuv.y.data
|| !frame->yuv420.u.data || !frame->yuv.u.data
|| !frame->yuv420.v.data) { || (!nv12 && !frame->yuv.v.data)) {
frame->prepared.clear(); frame->prepared.clear();
fail(Error::InvalidData); fail(Error::InvalidData);
return; return;
@ -466,11 +472,11 @@ void VideoTrackObject::rasterizeFrame(not_null<Frame*> frame) {
prepared.image = QImage(); prepared.image = QImage();
} }
} }
frame->format = FrameFormat::YUV420; frame->format = nv12 ? FrameFormat::NV12 : FrameFormat::YUV420;
} else { } else {
frame->alpha = (frameWithData->format == AV_PIX_FMT_BGRA) frame->alpha = (frameWithData->format == AV_PIX_FMT_BGRA)
|| (frameWithData->format == AV_PIX_FMT_YUVA420P); || (frameWithData->format == AV_PIX_FMT_YUVA420P);
frame->yuv420.size = { frame->yuv.size = {
frameWithData->width, frameWithData->width,
frameWithData->height frameWithData->height
}; };
@ -1173,7 +1179,7 @@ FrameWithInfo VideoTrack::frameWithInfo(const Instance *instance) {
} }
return { return {
.image = data.frame->original, .image = data.frame->original,
.yuv420 = &data.frame->yuv420, .yuv = &data.frame->yuv,
.format = data.frame->format, .format = data.frame->format,
.index = data.index, .index = data.index,
.alpha = data.frame->alpha, .alpha = data.frame->alpha,
@ -1197,8 +1203,9 @@ QImage VideoTrack::frameImage(
}); });
} }
if (frame->original.isNull() if (frame->original.isNull()
&& frame->format == FrameFormat::YUV420) { && (frame->format == FrameFormat::YUV420
frame->original = ConvertToARGB32(frame->yuv420); || frame->format == FrameFormat::NV12)) {
frame->original = ConvertToARGB32(frame->format, frame->yuv);
} }
if (GoodForRequest( if (GoodForRequest(
frame->original, frame->original,
@ -1235,8 +1242,10 @@ QImage VideoTrack::frameImage(
QImage VideoTrack::currentFrameImage() { QImage VideoTrack::currentFrameImage() {
const auto frame = _shared->frameForPaint(); const auto frame = _shared->frameForPaint();
if (frame->original.isNull() && frame->format == FrameFormat::YUV420) { if (frame->original.isNull()
frame->original = ConvertToARGB32(frame->yuv420); && (frame->format == FrameFormat::YUV420
|| frame->format == FrameFormat::NV12)) {
frame->original = ConvertToARGB32(frame->format, frame->yuv);
} }
return frame->original; return frame->original;
} }
@ -1293,7 +1302,8 @@ bool VideoTrack::IsDecoded(not_null<const Frame*> frame) {
bool VideoTrack::IsRasterized(not_null<const Frame*> frame) { bool VideoTrack::IsRasterized(not_null<const Frame*> frame) {
return IsDecoded(frame) return IsDecoded(frame)
&& (!frame->original.isNull() && (!frame->original.isNull()
|| frame->format == FrameFormat::YUV420); || frame->format == FrameFormat::YUV420
|| frame->format == FrameFormat::NV12);
} }
bool VideoTrack::IsStale(not_null<const Frame*> frame, crl::time trackTime) { bool VideoTrack::IsStale(not_null<const Frame*> frame, crl::time trackTime) {

2
Telegram/SourceFiles/media/streaming/media_streaming_video_track.h
View file

@ -84,7 +84,7 @@ private:
FFmpeg::FramePointer decoded = FFmpeg::MakeFramePointer(); FFmpeg::FramePointer decoded = FFmpeg::MakeFramePointer();
FFmpeg::FramePointer transferred; FFmpeg::FramePointer transferred;
QImage original; QImage original;
FrameYUV420 yuv420; FrameYUV yuv;
crl::time position = kTimeUnknown; crl::time position = kTimeUnknown;
crl::time displayed = kTimeUnknown; crl::time displayed = kTimeUnknown;
crl::time display = kTimeUnknown; crl::time display = kTimeUnknown;

64
Telegram/SourceFiles/media/view/media_view_overlay_opengl.cpp
View file

@ -106,6 +106,14 @@ void OverlayWidget::RendererGL::init(
FragmentSampleYUV420Texture(), FragmentSampleYUV420Texture(),
})); }));
_nv12Program.emplace();
LinkProgram(
&*_nv12Program,
_texturedVertexShader,
FragmentShader({
FragmentSampleNV12Texture(),
}));
_fillProgram.emplace(); _fillProgram.emplace();
LinkProgram( LinkProgram(
&*_fillProgram, &*_fillProgram,
@ -136,6 +144,7 @@ void OverlayWidget::RendererGL::deinit(
_texturedVertexShader = nullptr; _texturedVertexShader = nullptr;
_withTransparencyProgram = std::nullopt; _withTransparencyProgram = std::nullopt;
_yuv420Program = std::nullopt; _yuv420Program = std::nullopt;
_nv12Program = std::nullopt;
_fillProgram = std::nullopt; _fillProgram = std::nullopt;
_controlsProgram = std::nullopt; _controlsProgram = std::nullopt;
_contentBuffer = std::nullopt; _contentBuffer = std::nullopt;
@ -196,10 +205,13 @@ void OverlayWidget::RendererGL::paintTransformedVideoFrame(
data.alpha); data.alpha);
return; return;
} }
Assert(data.format == Streaming::FrameFormat::YUV420); Assert(!data.yuv->size.isEmpty());
Assert(!data.yuv420->size.isEmpty()); const auto program = (data.format == Streaming::FrameFormat::NV12)
const auto yuv = data.yuv420; ? &*_nv12Program
_yuv420Program->bind(); : &*_yuv420Program;
program->bind();
const auto nv12 = (data.format == Streaming::FrameFormat::NV12);
const auto yuv = data.yuv;
const auto upload = (_trackFrameIndex != data.index) const auto upload = (_trackFrameIndex != data.index)
|| (_streamedIndex != _owner->streamedIndex()); || (_streamedIndex != _owner->streamedIndex());
@ -223,32 +235,38 @@ void OverlayWidget::RendererGL::paintTransformedVideoFrame(
_textures.bind(*_f, 2); _textures.bind(*_f, 2);
if (upload) { if (upload) {
uploadTexture( uploadTexture(
GL_ALPHA, nv12 ? GL_RG : GL_ALPHA,
GL_ALPHA, nv12 ? GL_RG : GL_ALPHA,
yuv->chromaSize, yuv->chromaSize,
_chromaSize, _chromaSize,
yuv->u.stride, yuv->u.stride / (nv12 ? 2 : 1),
yuv->u.data); yuv->u.data);
} }
_f->glActiveTexture(GL_TEXTURE2); if (!nv12) {
_textures.bind(*_f, 3); _f->glActiveTexture(GL_TEXTURE2);
if (upload) { _textures.bind(*_f, 3);
uploadTexture( if (upload) {
GL_ALPHA, uploadTexture(
GL_ALPHA, GL_ALPHA,
yuv->chromaSize, GL_ALPHA,
_chromaSize, yuv->chromaSize,
yuv->v.stride, _chromaSize,
yuv->v.data); yuv->v.stride,
_chromaSize = yuv->chromaSize; yuv->v.data);
_f->glPixelStorei(GL_UNPACK_ALIGNMENT, 4); _chromaSize = yuv->chromaSize;
_f->glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
}
}
program->setUniformValue("y_texture", GLint(0));
if (nv12) {
program->setUniformValue("uv_texture", GLint(1));
} else {
program->setUniformValue("u_texture", GLint(1));
program->setUniformValue("v_texture", GLint(2));
} }
_yuv420Program->setUniformValue("y_texture", GLint(0));
_yuv420Program->setUniformValue("u_texture", GLint(1));
_yuv420Program->setUniformValue("v_texture", GLint(2));
toggleBlending(false); toggleBlending(false);
paintTransformedContent(&*_yuv420Program, geometry); paintTransformedContent(program, geometry);
} }
void OverlayWidget::RendererGL::paintTransformedStaticContent( void OverlayWidget::RendererGL::paintTransformedStaticContent(

1
Telegram/SourceFiles/media/view/media_view_overlay_opengl.h
View file

@ -107,6 +107,7 @@ private:
QOpenGLShader *_texturedVertexShader = nullptr; QOpenGLShader *_texturedVertexShader = nullptr;
std::optional<QOpenGLShaderProgram> _withTransparencyProgram; std::optional<QOpenGLShaderProgram> _withTransparencyProgram;
std::optional<QOpenGLShaderProgram> _yuv420Program; std::optional<QOpenGLShaderProgram> _yuv420Program;
std::optional<QOpenGLShaderProgram> _nv12Program;
std::optional<QOpenGLShaderProgram> _fillProgram; std::optional<QOpenGLShaderProgram> _fillProgram;
std::optional<QOpenGLShaderProgram> _controlsProgram; std::optional<QOpenGLShaderProgram> _controlsProgram;
Ui::GL::Textures<4> _textures; Ui::GL::Textures<4> _textures;

66
Telegram/SourceFiles/media/view/media_view_pip_opengl.cpp
View file

@ -195,6 +195,16 @@ void Pip::RendererGL::init(
FragmentRoundToShadow(), FragmentRoundToShadow(),
})); }));
_nv12Program.emplace();
LinkProgram(
&*_nv12Program,
_texturedVertexShader,
FragmentShader({
FragmentSampleNV12Texture(),
FragmentApplyFade(),
FragmentRoundToShadow(),
}));
_imageProgram.emplace(); _imageProgram.emplace();
LinkProgram( LinkProgram(
&*_imageProgram, &*_imageProgram,
@ -231,6 +241,7 @@ void Pip::RendererGL::deinit(
_texturedVertexShader = nullptr; _texturedVertexShader = nullptr;
_argb32Program = std::nullopt; _argb32Program = std::nullopt;
_yuv420Program = std::nullopt; _yuv420Program = std::nullopt;
_nv12Program = std::nullopt;
_controlsProgram = std::nullopt; _controlsProgram = std::nullopt;
_contentBuffer = std::nullopt; _contentBuffer = std::nullopt;
} }
@ -289,10 +300,13 @@ void Pip::RendererGL::paintTransformedVideoFrame(
paintTransformedStaticContent(data.image, geometry); paintTransformedStaticContent(data.image, geometry);
return; return;
} }
Assert(data.format == Streaming::FrameFormat::YUV420); Assert(!data.yuv->size.isEmpty());
Assert(!data.yuv420->size.isEmpty()); const auto program = (data.format == Streaming::FrameFormat::NV12)
const auto yuv = data.yuv420; ? &*_nv12Program
_yuv420Program->bind(); : &*_yuv420Program;
program->bind();
const auto nv12 = (data.format == Streaming::FrameFormat::NV12);
const auto yuv = data.yuv;
const auto upload = (_trackFrameIndex != data.index); const auto upload = (_trackFrameIndex != data.index);
_trackFrameIndex = data.index; _trackFrameIndex = data.index;
@ -314,31 +328,37 @@ void Pip::RendererGL::paintTransformedVideoFrame(
_textures.bind(*_f, 2); _textures.bind(*_f, 2);
if (upload) { if (upload) {
uploadTexture( uploadTexture(
GL_ALPHA, nv12 ? GL_RG : GL_ALPHA,
GL_ALPHA, nv12 ? GL_RG : GL_ALPHA,
yuv->chromaSize, yuv->chromaSize,
_chromaSize, _chromaSize,
yuv->u.stride, yuv->u.stride / (nv12 ? 2 : 1),
yuv->u.data); yuv->u.data);
} }
_f->glActiveTexture(GL_TEXTURE2); if (!nv12) {
_textures.bind(*_f, 3); _f->glActiveTexture(GL_TEXTURE2);
if (upload) { _textures.bind(*_f, 3);
uploadTexture( if (upload) {
GL_ALPHA, uploadTexture(
GL_ALPHA, GL_ALPHA,
yuv->chromaSize, GL_ALPHA,
_chromaSize, yuv->chromaSize,
yuv->v.stride, _chromaSize,
yuv->v.data); yuv->v.stride,
_chromaSize = yuv->chromaSize; yuv->v.data);
_f->glPixelStorei(GL_UNPACK_ALIGNMENT, 4); _chromaSize = yuv->chromaSize;
_f->glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
}
}
program->setUniformValue("y_texture", GLint(0));
if (nv12) {
program->setUniformValue("uv_texture", GLint(1));
} else {
program->setUniformValue("u_texture", GLint(1));
program->setUniformValue("v_texture", GLint(2));
} }
_yuv420Program->setUniformValue("y_texture", GLint(0));
_yuv420Program->setUniformValue("u_texture", GLint(1));
_yuv420Program->setUniformValue("v_texture", GLint(2));
paintTransformedContent(&*_yuv420Program, geometry); paintTransformedContent(program, geometry);
} }
void Pip::RendererGL::paintTransformedStaticContent( void Pip::RendererGL::paintTransformedStaticContent(

1
Telegram/SourceFiles/media/view/media_view_pip_opengl.h
View file

@ -101,6 +101,7 @@ private:
QOpenGLShader *_texturedVertexShader = nullptr; QOpenGLShader *_texturedVertexShader = nullptr;
std::optional<QOpenGLShaderProgram> _argb32Program; std::optional<QOpenGLShaderProgram> _argb32Program;
std::optional<QOpenGLShaderProgram> _yuv420Program; std::optional<QOpenGLShaderProgram> _yuv420Program;
std::optional<QOpenGLShaderProgram> _nv12Program;
Ui::GL::Textures<4> _textures; Ui::GL::Textures<4> _textures;
QSize _rgbaSize; QSize _rgbaSize;
QSize _lumaSize; QSize _lumaSize;

2
Telegram/lib_ui

@ -1 +1 @@
Subproject commit 5fa3d7a9daa62fb82713bc822a5138116a6015f2 Subproject commit 8700c2223ab60db994376b39e4e74a6309d5154a