void/AyuGramDesktop
2
0
Fork 0
mirror of https://github.com/AyuGram/AyuGramDesktop.git synced 2025-04-16 06:07:06 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

OpenGL render of frames in single widget.

Browse source
This commit is contained in:
John Preston 2021-05-24 00:11:24 +04:00
parent ec468431b4
commit d44f923277
10 changed files with 349 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
Telegram/SourceFiles/calls/calls_panel.cpp
View file

@ -134,7 +134,9 @@ Ui::GL::ChosenRenderer Panel::Incoming::chooseRenderer(
}
void Panel::Incoming::paint(QPainter &p, const QRegion &clip, bool opengl) {
const auto [image, rotation] = _track->frameOriginalWithRotation();
const auto data = _track->frameWithInfo();
const auto &image = data.original;
const auto rotation = data.rotation;
if (image.isNull()) {
p.fillRect(clip.boundingRect(), Qt::black);
} else {
@ -517,10 +519,7 @@ void Panel::reinitWithCall(Call *call) {
_call->videoIncoming()->renderNextFrame(
) | rpl::start_with_next([=] {
const auto track = _call->videoIncoming();
const auto [frame, rotation] = track->frameOriginalWithRotation();
setIncomingSize((rotation == 90 || rotation == 270)
? QSize(frame.height(), frame.width())
: frame.size());
setIncomingSize(track->frameSize());
if (_incoming->widget()->isHidden()) {
return;
}

16
Telegram/SourceFiles/calls/group/calls_group_large_video.cpp
View file

@ -391,9 +391,11 @@ void LargeVideo::RendererGL::paint(
bg.blueF(),
bg.alphaF());
const auto [image, rotation] = _owner->_track
? _owner->_track.track->frameOriginalWithRotation()
: std::pair<QImage, int>();
const auto data = _owner->_track
? _owner->_track.track->frameWithInfo()
: Webrtc::FrameWithInfo();
const auto &image = data.original;
const auto rotation = data.rotation;
f->glEnable(GL_BLEND);
f->glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
@ -766,9 +768,11 @@ void LargeVideo::paint(Painter &p, QRect clip, bool opengl) {
p.fillRect(rect.intersected(clip), st::groupCallMembersBg);
}
};
const auto [image, rotation] = _track
? _track.track->frameOriginalWithRotation()
: std::pair<QImage, int>();
const auto data = _track
? _track.track->frameWithInfo()
: Webrtc::FrameWithInfo();
const auto &image = data.original;
const auto rotation = data.rotation;
if (image.isNull()) {
fill(clip);
return;

15
Telegram/SourceFiles/calls/group/calls_group_viewport.cpp
View file

@ -41,7 +41,11 @@ Viewport::Viewport(QWidget *parent, PanelMode mode)
}
Viewport::~Viewport() {
base::take(_tiles);
for (const auto &tile : base::take(_tiles)) {
if (const auto textures = tile->takeTextures()) {
_freeTextures(textures);
}
}
}
not_null<QWidget*> Viewport::widget() const {
@ -200,6 +204,9 @@ void Viewport::remove(const VideoEndpoint &endpoint) {
}
if (_pressed.tile == removing) {
setPressed({});
}
if (const auto textures = removing->takeTextures()) {
}
_tiles.erase(i);
updateTilesGeometry();
@ -378,8 +385,12 @@ Ui::GL::ChosenRenderer Viewport::chooseRenderer(
: capabilities.transparency;
LOG(("OpenGL: %1 (Calls::Group::Viewport)").arg(Logs::b(use)));
if (use) {
auto renderer = std::make_unique<RendererGL>(this);
_freeTextures = [raw = renderer.get()](const Textures &textures) {
raw->free(textures);
};
return {
.renderer = std::make_unique<Renderer>(this),
.renderer = std::move(renderer),
.backend = Ui::GL::Backend::OpenGL,
};
}

3
Telegram/SourceFiles/calls/group/calls_group_viewport.h
View file

@ -65,7 +65,7 @@ public:
[[nodiscard]] rpl::lifetime &lifetime();
private:
struct PinButton;
struct Textures;
class VideoTile;
class Renderer;
class RendererGL;
@ -100,6 +100,7 @@ private:
[[nodiscard]] Ui::GL::ChosenRenderer chooseRenderer(
Ui::GL::Capabilities capabilities);
Fn<void(const Textures &)> _freeTextures;
rpl::variable<PanelMode> _mode;
const std::unique_ptr<Ui::RpWidgetWrap> _content;
std::vector<std::unique_ptr<VideoTile>> _tiles;

333
Telegram/SourceFiles/calls/group/calls_group_viewport_opengl.cpp
View file

@ -7,6 +7,10 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
*/
#include "calls/group/calls_group_viewport_opengl.h"
#include "calls/group/calls_group_viewport_tile.h"
#include "webrtc/webrtc_video_track.h"
#include "media/view/media_view_pip.h"
#include <QtGui/QOpenGLShader>
namespace Calls::Group {
@ -49,31 +53,6 @@ void main() {
)";
}
[[nodiscard]] ShaderPart VertexPassTextureCoord() {
return {
.header = R"(
in vec2 texcoord;
out vec2 v_texcoord;
)",
.body = R"(
v_texcoord = texcoord;
)",
};
}
[[nodiscard]] ShaderPart FragmentSampleTexture() {
return {
.header = R"(
in vec2 v_texcoord;
uniform sampler2D s_texture;
)",
.body = R"(
result = texture(s_texture, v_texcoord);
result = vec4(result.b, result.g, result.r, result.a);
)",
};
}
[[nodiscard]] ShaderPart VertexViewportTransform() {
return {
.header = R"(
@ -94,21 +73,47 @@ vec4 transform(vec4 position) {
[[nodiscard]] ShaderPart FragmentRoundCorners() {
return {
.header = R"(
uniform vec2 viewport;
uniform vec4 roundRect;
uniform vec4 roundBg;
uniform float roundRadius;
float roundedCorner() {
vec2 viewportHalf = viewport / 2;
vec2 fromViewportCenter = abs(gl_FragCoord.xy - viewportHalf);
vec2 rectHalf = roundRect.zw / 2;
vec2 rectCenter = roundRect.xy + rectHalf;
vec2 fromRectCenter = abs(gl_FragCoord.xy - rectCenter);
vec2 vectorRadius = vec2(roundRadius + 0.5, roundRadius + 0.5);
vec2 fromCenterWithRadius = fromViewportCenter + vectorRadius;
vec2 fromRoundingCenter = max(fromCenterWithRadius, viewportHalf)
- viewportHalf;
vec2 fromCenterWithRadius = fromRectCenter + vectorRadius;
vec2 fromRoundingCenter = max(fromCenterWithRadius, rectHalf)
- rectHalf;
float d = length(fromRoundingCenter) - roundRadius;
return 1. - smoothstep(0., 1., d);
}
)",
.body = R"(
result = vec4(result.r, result.g, result.b, result.a * roundedCorner());
float rounded = roundedCorner();
result = result * rounded + roundBg * (1. - rounded);
)",
};
}
[[nodiscard]] ShaderPart FragmentFrameColor() {
return {
.header = R"(
uniform sampler2D s_texture;
uniform vec4 textureRect;
uniform vec4 frameBg;
)",
.body = R"(
vec2 texturePos = gl_FragCoord.xy - textureRect.xy;
vec2 textureCoord = vec2(texturePos.x, textureRect.w - texturePos.y)
/ textureRect.zw;
vec2 textureHalf = textureRect.zw / 2;
vec2 fromTextureCenter = abs(texturePos - textureHalf);
vec2 fromTextureEdge = max(fromTextureCenter, textureHalf) - textureHalf;
float outsideCheck = dot(fromTextureEdge, fromTextureEdge);
float inside = step(outsideCheck, 0);
result = texture(s_texture, textureCoord);
result = vec4(result.b, result.g, result.r, result.a);
result = result * inside + frameBg * (1. - inside);
)",
};
}
@ -149,69 +154,64 @@ void LinkProgram(
}
}
class Quads final {
public:
void fill(QRect rect);
void paint(
not_null<QOpenGLFunctions*> f,
not_null<QOpenGLBuffer*> buffer,
not_null<QOpenGLShaderProgram*> program,
QSize viewport,
const QColor &color,
Fn<void()> additional = nullptr);
private:
static constexpr auto kMaxTriangles = 8;
std::array<GLfloat, 6 * kMaxTriangles> coordinates{ 0 };
int triangles = 0;
};
void Quads::fill(QRect rect) {
Expects(triangles + 2 <= kMaxTriangles);
auto i = triangles * 6;
coordinates[i + 0] = coordinates[i + 10] = rect.x();
coordinates[i + 1] = coordinates[i + 11] = rect.y();
coordinates[i + 2] = rect.x() + rect.width();
coordinates[i + 3] = rect.y();
coordinates[i + 4] = coordinates[i + 6] = rect.x() + rect.width();
coordinates[i + 5] = coordinates[i + 7] = rect.y() + rect.height();
coordinates[i + 8] = rect.x();
coordinates[i + 9] = rect.y() + rect.height();
triangles += 2;
[[nodiscard]] QVector4D Uniform(const QRect &rect) {
return QVector4D(rect.x(), rect.y(), rect.width(), rect.height());
}
void Quads::paint(
not_null<QOpenGLFunctions*> f,
not_null<QOpenGLBuffer*> buffer,
not_null<QOpenGLShaderProgram*> program,
QSize viewport,
const QColor &color,
Fn<void()> additional) {
if (!triangles) {
return;
}
buffer->bind();
buffer->allocate(coordinates.data(), triangles * 6 * sizeof(GLfloat));
f->glUseProgram(program->programId());
program->setUniformValue("viewport", QSizeF(viewport));
program->setUniformValue("s_color", QVector4D(
[[nodiscard]] QVector4D Uniform(const QColor &color) {
return QVector4D(
color.redF(),
color.greenF(),
color.blueF(),
color.alphaF()));
color.alphaF());
}
void FillRectVertices(GLfloat *coords, QRect rect) {
coords[0] = coords[10] = rect.x();
coords[1] = coords[11] = rect.y();
coords[2] = rect.x() + rect.width();
coords[3] = rect.y();
coords[4] = coords[6] = rect.x() + rect.width();
coords[5] = coords[7] = rect.y() + rect.height();
coords[8] = rect.x();
coords[9] = rect.y() + rect.height();
}
void FillTriangles(
not_null<QOpenGLFunctions*> f,
gsl::span<const GLfloat> coords,
not_null<QOpenGLBuffer*> buffer,
not_null<QOpenGLShaderProgram*> program,
QSize viewport,
const QColor &color,
Fn<void()> additional = nullptr) {
Expects(coords.size() % 6 == 0);
if (coords.empty()) {
return;
}
buffer->bind();
buffer->allocate(coords.data(), coords.size() * sizeof(GLfloat));
f->glUseProgram(program->programId());
program->setUniformValue("viewport", QSizeF(viewport));
program->setUniformValue("s_color", Uniform(color));
GLint position = program->attributeLocation("position");
f->glVertexAttribPointer(position, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(GLfloat), (void*)0);
f->glVertexAttribPointer(
position,
2,
GL_FLOAT,
GL_FALSE,
2 * sizeof(GLfloat),
nullptr);
f->glEnableVertexAttribArray(position);
if (additional) {
additional();
}
f->glDrawArrays(GL_TRIANGLES, 0, triangles * 3);
f->glDrawArrays(GL_TRIANGLES, 0, coords.size() / 2);
f->glDisableVertexAttribArray(position);
}
@ -222,14 +222,50 @@ Viewport::RendererGL::RendererGL(not_null<Viewport*> owner)
: _owner(owner) {
}
void Viewport::RendererGL::free(const Textures &textures) {
_texturesToFree.push_back(textures);
}
void Viewport::RendererGL::init(
not_null<QOpenGLWidget*> widget,
not_null<QOpenGLFunctions*> f) {
_frameBuffer.emplace();
_frameBuffer->setUsagePattern(QOpenGLBuffer::DynamicDraw);
_frameBuffer->create();
_frameProgram.emplace();
LinkProgram(
&*_frameProgram,
VertexShader({
VertexViewportTransform(),
}),
FragmentShader({
FragmentFrameColor(),
FragmentRoundCorners(),
}));
_bgBuffer.emplace();
_bgBuffer->setUsagePattern(QOpenGLBuffer::DynamicDraw);
_bgBuffer->create();
_bgProgram.emplace();
LinkProgram(
&*_bgProgram,
VertexShader({ VertexViewportTransform() }),
FragmentShader({ FragmentStaticColor() }));
}
void Viewport::RendererGL::deinit(
not_null<QOpenGLWidget*> widget,
not_null<QOpenGLFunctions*> f) {
_frameBuffer = std::nullopt;
_bgBuffer = std::nullopt;
_frameProgram = std::nullopt;
_bgProgram = std::nullopt;
for (const auto &tile : _owner->_tiles) {
if (const auto textures = tile->takeTextures()) {
free(textures);
}
}
freeTextures(f);
}
void Viewport::RendererGL::resize(
@ -237,11 +273,144 @@ void Viewport::RendererGL::resize(
not_null<QOpenGLFunctions*> f,
int w,
int h) {
_viewport = QSize(w, h);
f->glViewport(0, 0, w, h);
}
void Viewport::RendererGL::paint(
not_null<QOpenGLWidget*> widget,
not_null<QOpenGLFunctions*> f) {
fillBackground(f);
for (const auto &tile : _owner->_tiles) {
paintTile(f, tile.get());
}
freeTextures(f);
}
void Viewport::RendererGL::fillBackground(not_null<QOpenGLFunctions*> f) {
const auto radius = st::roundRadiusLarge;
const auto radiuses = QMargins{ radius, radius, radius, radius };
auto bg = QRegion(QRect(QPoint(), _viewport));
for (const auto &tile : _owner->_tiles) {
bg -= tile->geometry().marginsRemoved(radiuses);
}
if (bg.isEmpty()) {
return;
}
_bgTriangles.resize((bg.end() - bg.begin()) * 12);
auto coords = _bgTriangles.data();
for (const auto rect : bg) {
FillRectVertices(coords, rect);
coords += 12;
}
FillTriangles(
f,
_bgTriangles,
&*_bgBuffer,
&*_bgProgram,
_viewport,
st::groupCallBg->c);
}
void Viewport::RendererGL::paintTile(
not_null<QOpenGLFunctions*> f,
not_null<VideoTile*> tile) {
const auto track = tile->track();
const auto data = track->frameWithInfo();
const auto &image = data.original;
if (image.isNull()) {
return;
}
const auto geometry = tile->geometry();
const auto x = geometry.x();
const auto y = geometry.y();
const auto width = geometry.width();
const auto height = geometry.height();
const auto scaled = Media::View::FlipSizeByRotation(
image.size(),
data.rotation
).scaled(QSize(width, height), Qt::KeepAspectRatio);
const auto left = (width - scaled.width()) / 2;
const auto top = (height - scaled.height()) / 2;
const auto right = left + scaled.width();
const auto bottom = top + scaled.height();
const auto radius = GLfloat(st::roundRadiusLarge * cIntRetinaFactor());
// #TODO rotation
//if (data.rotation > 0) {
// std::rotate(
// texcoords.begin(),
// texcoords.begin() + (data.rotation / 90),
// texcoords.end());
//}
const GLfloat coords[] = {
float(x), float(y),
float(x + width), float(y),
float(x + width), float(y + height),
float(x), float(y + height),
};
tile->ensureTexturesCreated(f);
const auto &textures = tile->textures();
const auto upload = (textures.trackIndex != data.index);
if (upload) {
textures.textureIndex = 1 - textures.textureIndex;
}
const auto texture = textures.values[textures.textureIndex];
f->glUseProgram(_frameProgram->programId());
f->glActiveTexture(GL_TEXTURE0);
f->glBindTexture(GL_TEXTURE_2D, texture);
if (upload) {
f->glPixelStorei(GL_UNPACK_ROW_LENGTH, image.bytesPerLine() / 4);
f->glTexImage2D(
GL_TEXTURE_2D,
0,
GL_RGB,
image.width(),
image.height(),
0,
GL_RGBA,
GL_UNSIGNED_BYTE,
image.constBits());
f->glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
}
tile->track()->markFrameShown();
_frameBuffer->bind();
_frameBuffer->allocate(coords, sizeof(coords));
_frameProgram->setUniformValue("viewport", QSizeF(_viewport));
_frameProgram->setUniformValue("s_texture", GLint(0));
_frameProgram->setUniformValue(
"textureRect",
Uniform(QRect(x + left, y + top, scaled.width(), scaled.height())));
_frameProgram->setUniformValue(
"frameBg",
Uniform(st::groupCallMembersBg->c));
_frameProgram->setUniformValue("roundRadius", radius);
_frameProgram->setUniformValue("roundRect", Uniform(geometry));
_frameProgram->setUniformValue("roundBg", Uniform(st::groupCallBg->c));
GLint position = _frameProgram->attributeLocation("position");
f->glVertexAttribPointer(
position,
2,
GL_FLOAT,
GL_FALSE,
2 * sizeof(GLfloat),
nullptr);
f->glEnableVertexAttribArray(position);
f->glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
f->glDisableVertexAttribArray(position);
}
void Viewport::RendererGL::freeTextures(not_null<QOpenGLFunctions*> f) {
for (const auto &textures : base::take(_texturesToFree)) {
f->glDeleteTextures(textures.values.size(), textures.values.data());
}
}
} // namespace Calls::Group

14
Telegram/SourceFiles/calls/group/calls_group_viewport_opengl.h
View file

@ -19,6 +19,8 @@ class Viewport::RendererGL final : public Ui::GL::Renderer {
public:
explicit RendererGL(not_null<Viewport*> owner);
void free(const Textures &textures);
void init(
not_null<QOpenGLWidget*> widget,
not_null<QOpenGLFunctions*> f) override;
@ -38,15 +40,23 @@ public:
not_null<QOpenGLFunctions*> f) override;
private:
void fillBackground(not_null<QOpenGLFunctions*> f);
void paintTile(
not_null<QOpenGLFunctions*> f,
not_null<VideoTile*> tile);
void freeTextures(not_null<QOpenGLFunctions*> f);
const not_null<Viewport*> _owner;
QSize _viewport;
std::optional<QOpenGLBuffer> _frameBuffer;
std::optional<QOpenGLBuffer> _fillBuffer;
std::optional<QOpenGLBuffer> _bgBuffer;
std::optional<QOpenGLShaderProgram> _frameProgram;
std::optional<QOpenGLShaderProgram> _fillProgram;
std::optional<QOpenGLShaderProgram> _bgProgram;
std::vector<GLfloat> _bgTriangles;
std::vector<Textures> _texturesToFree;
};
} // namespace Calls::Group

13
Telegram/SourceFiles/calls/group/calls_group_viewport_raster.cpp
View file

@ -40,18 +40,15 @@ void Viewport::Renderer::paintFallback(
const QRegion &clip,
Ui::GL::Backend backend) {
auto bg = clip;
const auto guard = gsl::finally([&] {
for (const auto rect : bg) {
p.fillRect(rect, st::groupCallBg);
}
});
auto hq = PainterHighQualityEnabler(p);
const auto bounding = clip.boundingRect();
const auto opengl = (backend == Ui::GL::Backend::OpenGL);
for (const auto &tile : _owner->_tiles) {
paintTile(p, tile.get(), bounding, opengl, bg);
}
for (const auto rect : bg) {
p.fillRect(rect, st::groupCallBg);
}
}
void Viewport::Renderer::paintTile(
@ -61,7 +58,9 @@ void Viewport::Renderer::paintTile(
bool opengl,
QRegion &bg) {
const auto track = tile->track();
const auto [image, rotation] = track->frameOriginalWithRotation();
const auto data = track->frameWithInfo();
const auto &image = data.original;
const auto rotation = data.rotation;
if (image.isNull()) {
return;
}

32
Telegram/SourceFiles/calls/group/calls_group_viewport_tile.cpp
View file

@ -11,6 +11,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include "lang/lang_keys.h"
#include "styles/style_calls.h"
#include <QtGui/QOpenGLFunctions>
namespace Calls::Group {
Viewport::VideoTile::VideoTile(
@ -27,6 +29,10 @@ Viewport::VideoTile::VideoTile(
setup(std::move(pinned));
}
Viewport::VideoTile::~VideoTile() {
Expects(!_textures);
}
QRect Viewport::VideoTile::pinInner() const {
return _pinInner.translated(0, -pinSlide());
}
@ -60,7 +66,7 @@ void Viewport::VideoTile::togglePinShown(bool shown) {
}
bool Viewport::VideoTile::updateRequestedQuality(VideoQuality quality) {
if (!_quality || *_quality == quality) {
if (_quality && *_quality == quality) {
return false;
}
_quality = quality;
@ -123,4 +129,28 @@ void Viewport::VideoTile::setup(rpl::producer<bool> pinned) {
}
}
void Viewport::VideoTile::ensureTexturesCreated(
not_null<QOpenGLFunctions*> f) {
if (_textures) {
return;
}
f->glGenTextures(_textures.values.size(), _textures.values.data());
for (const auto texture : _textures.values) {
f->glBindTexture(GL_TEXTURE_2D, texture);
const auto clamp = GL_CLAMP_TO_EDGE;
f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, clamp);
f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, clamp);
f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
f->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
}
}
const Viewport::Textures &Viewport::VideoTile::textures() const {
return _textures;
}
Viewport::Textures Viewport::VideoTile::takeTextures() {
return base::take(_textures);
}
} // namespace Calls::Group

19
Telegram/SourceFiles/calls/group/calls_group_viewport_tile.h
View file

@ -15,8 +15,20 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include "ui/effects/cross_line.h"
#include "ui/round_rect.h"
class QOpenGLFunctions;
namespace Calls::Group {
struct Viewport::Textures {
std::array<GLuint, 2> values = { { 0 } };
mutable int textureIndex = 0;
mutable int trackIndex = -1;
explicit operator bool() const {
return values[0] || values[1];
}
};
class Viewport::VideoTile final {
public:
VideoTile(
@ -24,6 +36,7 @@ public:
LargeVideoTrack track,
rpl::producer<bool> pinned,
Fn<void()> update);
~VideoTile();
[[nodiscard]] not_null<Webrtc::VideoTrack*> track() const {
return _track.track;
@ -53,6 +66,10 @@ public:
void togglePinShown(bool shown);
bool updateRequestedQuality(VideoQuality quality);
void ensureTexturesCreated(not_null<QOpenGLFunctions*> f);
[[nodiscard]] const Textures &textures() const;
[[nodiscard]] Textures takeTextures();
[[nodiscard]] rpl::lifetime &lifetime() {
return _lifetime;
}
@ -75,6 +92,8 @@ private:
bool _pinned = false;
std::optional<VideoQuality> _quality;
Textures _textures;
rpl::lifetime _lifetime;
};

2
Telegram/lib_webrtc

@ -1 +1 @@
Subproject commit 0c867b0b6ae29e6ae5d5c2fda3824cdb595900eb
Subproject commit 802145e0de64013b91a0c05e760ea10c0978a973