void/AyuGramDesktop
2
0
Fork 0
mirror of https://github.com/AyuGram/AyuGramDesktop.git synced 2025-04-17 06:37:24 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Render NV12 in OpenGL from hwaccel without swscale.

Browse source
This commit is contained in:
John Preston 2022-03-22 19:50:23 +04:00
parent 0dca556843
commit 41eb223bbf
10 changed files with 121 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
Telegram/SourceFiles/media/streaming/media_streaming_common.h
View file

@ -162,6 +162,7 @@ enum class FrameFormat {
None,
ARGB32,
YUV420,
NV12,
};
struct FrameChannel {
@ -169,7 +170,7 @@ struct FrameChannel {
int stride = 0;
};
struct FrameYUV420 {
struct FrameYUV {
QSize size;
QSize chromaSize;
FrameChannel y;
@ -179,7 +180,7 @@ struct FrameYUV420 {
struct FrameWithInfo {
QImage image;
FrameYUV420 *yuv420 = nullptr;
FrameYUV *yuv = nullptr;
FrameFormat format = FrameFormat::None;
int index = -1;
bool alpha = false;

2
Telegram/SourceFiles/media/streaming/media_streaming_utility.cpp
View file

@ -198,7 +198,7 @@ QImage ConvertFrame(
return storage;
}
FrameYUV420 ExtractYUV420(Stream &stream, AVFrame *frame) {
FrameYUV ExtractYUV(Stream &stream, AVFrame *frame) {
return {
.size = { frame->width, frame->height },
.chromaSize = {

2
Telegram/SourceFiles/media/streaming/media_streaming_utility.h
View file

@ -64,7 +64,7 @@ struct Stream {
not_null<AVFrame*> frame,
QSize resize,
QImage storage);
[[nodiscard]] FrameYUV420 ExtractYUV420(Stream &stream, AVFrame *frame);
[[nodiscard]] FrameYUV ExtractYUV(Stream &stream, AVFrame *frame);
[[nodiscard]] QImage PrepareByRequest(
const QImage &original,
bool alpha,

46
Telegram/SourceFiles/media/streaming/media_streaming_video_track.cpp
View file

@ -22,10 +22,12 @@ constexpr auto kDisplaySkipped = crl::time(-1);
constexpr auto kFinishedPosition = std::numeric_limits<crl::time>::max();
static_assert(kDisplaySkipped != kTimeUnknown);
[[nodiscard]] QImage ConvertToARGB32(const FrameYUV420 &data) {
[[nodiscard]] QImage ConvertToARGB32(
FrameFormat format,
const FrameYUV &data) {
Expects(data.y.data != nullptr);
Expects(data.u.data != nullptr);
Expects(data.v.data != nullptr);
Expects((format == FrameFormat::NV12) || (data.v.data != nullptr));
Expects(!data.size.isEmpty());
//if (FFmpeg::RotationSwapWidthHeight(stream.rotation)) {
@ -35,7 +37,9 @@ static_assert(kDisplaySkipped != kTimeUnknown);
auto result = FFmpeg::CreateFrameStorage(data.size);
const auto swscale = FFmpeg::MakeSwscalePointer(
data.size,
AV_PIX_FMT_YUV420P,
(format == FrameFormat::YUV420
? AV_PIX_FMT_YUV420P
: AV_PIX_FMT_NV12),
data.size,
AV_PIX_FMT_BGRA);
if (!swscale) {
@ -448,14 +452,16 @@ void VideoTrackObject::rasterizeFrame(not_null<Frame*> frame) {
const auto frameWithData = frame->transferred
? frame->transferred.get()
: frame->decoded.get();
if (frameWithData->format == AV_PIX_FMT_YUV420P && !requireARGB32()) {
if ((frameWithData->format == AV_PIX_FMT_YUV420P
|| frameWithData->format == AV_PIX_FMT_NV12) && !requireARGB32()) {
const auto nv12 = (frameWithData->format == AV_PIX_FMT_NV12);
frame->alpha = false;
frame->yuv420 = ExtractYUV420(_stream, frameWithData);
if (frame->yuv420.size.isEmpty()
|| frame->yuv420.chromaSize.isEmpty()
|| !frame->yuv420.y.data
|| !frame->yuv420.u.data
|| !frame->yuv420.v.data) {
frame->yuv = ExtractYUV(_stream, frameWithData);
if (frame->yuv.size.isEmpty()
|| frame->yuv.chromaSize.isEmpty()
|| !frame->yuv.y.data
|| !frame->yuv.u.data
|| (!nv12 && !frame->yuv.v.data)) {
frame->prepared.clear();
fail(Error::InvalidData);
return;
@ -466,11 +472,11 @@ void VideoTrackObject::rasterizeFrame(not_null<Frame*> frame) {
prepared.image = QImage();
}
}
frame->format = FrameFormat::YUV420;
frame->format = nv12 ? FrameFormat::NV12 : FrameFormat::YUV420;
} else {
frame->alpha = (frameWithData->format == AV_PIX_FMT_BGRA)
|| (frameWithData->format == AV_PIX_FMT_YUVA420P);
frame->yuv420.size = {
frame->yuv.size = {
frameWithData->width,
frameWithData->height
};
@ -1173,7 +1179,7 @@ FrameWithInfo VideoTrack::frameWithInfo(const Instance *instance) {
}
return {
.image = data.frame->original,
.yuv420 = &data.frame->yuv420,
.yuv = &data.frame->yuv,
.format = data.frame->format,
.index = data.index,
.alpha = data.frame->alpha,
@ -1197,8 +1203,9 @@ QImage VideoTrack::frameImage(
});
}
if (frame->original.isNull()
&& frame->format == FrameFormat::YUV420) {
frame->original = ConvertToARGB32(frame->yuv420);
&& (frame->format == FrameFormat::YUV420
|| frame->format == FrameFormat::NV12)) {
frame->original = ConvertToARGB32(frame->format, frame->yuv);
}
if (GoodForRequest(
frame->original,
@ -1235,8 +1242,10 @@ QImage VideoTrack::frameImage(
QImage VideoTrack::currentFrameImage() {
const auto frame = _shared->frameForPaint();
if (frame->original.isNull() && frame->format == FrameFormat::YUV420) {
frame->original = ConvertToARGB32(frame->yuv420);
if (frame->original.isNull()
&& (frame->format == FrameFormat::YUV420
|| frame->format == FrameFormat::NV12)) {
frame->original = ConvertToARGB32(frame->format, frame->yuv);
}
return frame->original;
}
@ -1293,7 +1302,8 @@ bool VideoTrack::IsDecoded(not_null<const Frame*> frame) {
bool VideoTrack::IsRasterized(not_null<const Frame*> frame) {
return IsDecoded(frame)
&& (!frame->original.isNull()
|| frame->format == FrameFormat::YUV420);
|| frame->format == FrameFormat::YUV420
|| frame->format == FrameFormat::NV12);
}
bool VideoTrack::IsStale(not_null<const Frame*> frame, crl::time trackTime) {

2
Telegram/SourceFiles/media/streaming/media_streaming_video_track.h
View file

@ -84,7 +84,7 @@ private:
FFmpeg::FramePointer decoded = FFmpeg::MakeFramePointer();
FFmpeg::FramePointer transferred;
QImage original;
FrameYUV420 yuv420;
FrameYUV yuv;
crl::time position = kTimeUnknown;
crl::time displayed = kTimeUnknown;
crl::time display = kTimeUnknown;

64
Telegram/SourceFiles/media/view/media_view_overlay_opengl.cpp
View file

@ -106,6 +106,14 @@ void OverlayWidget::RendererGL::init(
FragmentSampleYUV420Texture(),
}));
_nv12Program.emplace();
LinkProgram(
&*_nv12Program,
_texturedVertexShader,
FragmentShader({
FragmentSampleNV12Texture(),
}));
_fillProgram.emplace();
LinkProgram(
&*_fillProgram,
@ -136,6 +144,7 @@ void OverlayWidget::RendererGL::deinit(
_texturedVertexShader = nullptr;
_withTransparencyProgram = std::nullopt;
_yuv420Program = std::nullopt;
_nv12Program = std::nullopt;
_fillProgram = std::nullopt;
_controlsProgram = std::nullopt;
_contentBuffer = std::nullopt;
@ -196,10 +205,13 @@ void OverlayWidget::RendererGL::paintTransformedVideoFrame(
data.alpha);
return;
}
Assert(data.format == Streaming::FrameFormat::YUV420);
Assert(!data.yuv420->size.isEmpty());
const auto yuv = data.yuv420;
_yuv420Program->bind();
Assert(!data.yuv->size.isEmpty());
const auto program = (data.format == Streaming::FrameFormat::NV12)
? &*_nv12Program
: &*_yuv420Program;
program->bind();
const auto nv12 = (data.format == Streaming::FrameFormat::NV12);
const auto yuv = data.yuv;
const auto upload = (_trackFrameIndex != data.index)
|| (_streamedIndex != _owner->streamedIndex());
@ -223,32 +235,38 @@ void OverlayWidget::RendererGL::paintTransformedVideoFrame(
_textures.bind(*_f, 2);
if (upload) {
uploadTexture(
GL_ALPHA,
GL_ALPHA,
nv12 ? GL_RG : GL_ALPHA,
nv12 ? GL_RG : GL_ALPHA,
yuv->chromaSize,
_chromaSize,
yuv->u.stride,
yuv->u.stride / (nv12 ? 2 : 1),
yuv->u.data);
}
_f->glActiveTexture(GL_TEXTURE2);
_textures.bind(*_f, 3);
if (upload) {
uploadTexture(
GL_ALPHA,
GL_ALPHA,
yuv->chromaSize,
_chromaSize,
yuv->v.stride,
yuv->v.data);
_chromaSize = yuv->chromaSize;
_f->glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
if (!nv12) {
_f->glActiveTexture(GL_TEXTURE2);
_textures.bind(*_f, 3);
if (upload) {
uploadTexture(
GL_ALPHA,
GL_ALPHA,
yuv->chromaSize,
_chromaSize,
yuv->v.stride,
yuv->v.data);
_chromaSize = yuv->chromaSize;
_f->glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
}
}
program->setUniformValue("y_texture", GLint(0));
if (nv12) {
program->setUniformValue("uv_texture", GLint(1));
} else {
program->setUniformValue("u_texture", GLint(1));
program->setUniformValue("v_texture", GLint(2));
}
_yuv420Program->setUniformValue("y_texture", GLint(0));
_yuv420Program->setUniformValue("u_texture", GLint(1));
_yuv420Program->setUniformValue("v_texture", GLint(2));
toggleBlending(false);
paintTransformedContent(&*_yuv420Program, geometry);
paintTransformedContent(program, geometry);
}
void OverlayWidget::RendererGL::paintTransformedStaticContent(

1
Telegram/SourceFiles/media/view/media_view_overlay_opengl.h
View file

@ -107,6 +107,7 @@ private:
QOpenGLShader *_texturedVertexShader = nullptr;
std::optional<QOpenGLShaderProgram> _withTransparencyProgram;
std::optional<QOpenGLShaderProgram> _yuv420Program;
std::optional<QOpenGLShaderProgram> _nv12Program;
std::optional<QOpenGLShaderProgram> _fillProgram;
std::optional<QOpenGLShaderProgram> _controlsProgram;
Ui::GL::Textures<4> _textures;

66
Telegram/SourceFiles/media/view/media_view_pip_opengl.cpp
View file

@ -195,6 +195,16 @@ void Pip::RendererGL::init(
FragmentRoundToShadow(),
}));
_nv12Program.emplace();
LinkProgram(
&*_nv12Program,
_texturedVertexShader,
FragmentShader({
FragmentSampleNV12Texture(),
FragmentApplyFade(),
FragmentRoundToShadow(),
}));
_imageProgram.emplace();
LinkProgram(
&*_imageProgram,
@ -231,6 +241,7 @@ void Pip::RendererGL::deinit(
_texturedVertexShader = nullptr;
_argb32Program = std::nullopt;
_yuv420Program = std::nullopt;
_nv12Program = std::nullopt;
_controlsProgram = std::nullopt;
_contentBuffer = std::nullopt;
}
@ -289,10 +300,13 @@ void Pip::RendererGL::paintTransformedVideoFrame(
paintTransformedStaticContent(data.image, geometry);
return;
}
Assert(data.format == Streaming::FrameFormat::YUV420);
Assert(!data.yuv420->size.isEmpty());
const auto yuv = data.yuv420;
_yuv420Program->bind();
Assert(!data.yuv->size.isEmpty());
const auto program = (data.format == Streaming::FrameFormat::NV12)
? &*_nv12Program
: &*_yuv420Program;
program->bind();
const auto nv12 = (data.format == Streaming::FrameFormat::NV12);
const auto yuv = data.yuv;
const auto upload = (_trackFrameIndex != data.index);
_trackFrameIndex = data.index;
@ -314,31 +328,37 @@ void Pip::RendererGL::paintTransformedVideoFrame(
_textures.bind(*_f, 2);
if (upload) {
uploadTexture(
GL_ALPHA,
GL_ALPHA,
nv12 ? GL_RG : GL_ALPHA,
nv12 ? GL_RG : GL_ALPHA,
yuv->chromaSize,
_chromaSize,
yuv->u.stride,
yuv->u.stride / (nv12 ? 2 : 1),
yuv->u.data);
}
_f->glActiveTexture(GL_TEXTURE2);
_textures.bind(*_f, 3);
if (upload) {
uploadTexture(
GL_ALPHA,
GL_ALPHA,
yuv->chromaSize,
_chromaSize,
yuv->v.stride,
yuv->v.data);
_chromaSize = yuv->chromaSize;
_f->glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
if (!nv12) {
_f->glActiveTexture(GL_TEXTURE2);
_textures.bind(*_f, 3);
if (upload) {
uploadTexture(
GL_ALPHA,
GL_ALPHA,
yuv->chromaSize,
_chromaSize,
yuv->v.stride,
yuv->v.data);
_chromaSize = yuv->chromaSize;
_f->glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
}
}
program->setUniformValue("y_texture", GLint(0));
if (nv12) {
program->setUniformValue("uv_texture", GLint(1));
} else {
program->setUniformValue("u_texture", GLint(1));
program->setUniformValue("v_texture", GLint(2));
}
_yuv420Program->setUniformValue("y_texture", GLint(0));
_yuv420Program->setUniformValue("u_texture", GLint(1));
_yuv420Program->setUniformValue("v_texture", GLint(2));
paintTransformedContent(&*_yuv420Program, geometry);
paintTransformedContent(program, geometry);
}
void Pip::RendererGL::paintTransformedStaticContent(

1
Telegram/SourceFiles/media/view/media_view_pip_opengl.h
View file

@ -101,6 +101,7 @@ private:
QOpenGLShader *_texturedVertexShader = nullptr;
std::optional<QOpenGLShaderProgram> _argb32Program;
std::optional<QOpenGLShaderProgram> _yuv420Program;
std::optional<QOpenGLShaderProgram> _nv12Program;
Ui::GL::Textures<4> _textures;
QSize _rgbaSize;
QSize _lumaSize;

2
Telegram/lib_ui

@ -1 +1 @@
Subproject commit 5fa3d7a9daa62fb82713bc822a5138116a6015f2
Subproject commit 8700c2223ab60db994376b39e4e74a6309d5154a