scenes/opengl: Calculate item transforms differently

Currently, a vertex coordinate is transformed first, then mapped to
the global screen coordinates. This causes a problem if a transform
is applied to the top-most item and child items are not at (0, 0).
For example, scaled windows may have popping out sub-surfaces, etc.

With this change, the item transforms will be computed differently. For
example, if the parent item is transformed, a child's transform will
look as follows

  [Parent's translation][Parent's transform][Child's translation]

instead of

  [Parent's translation][Child's translation][Parent's transform]

In the future, I'd like to get rid of the Item::setTransform() call in
OpenGLWindow::performPaint() and have either AnimationEffect or
libkwineffects call Item::setTransform().

BUG: 440201

src/item.cpp

@@ -179,6 +179,16 @@ QPoint Item::rootPosition() const
     return ret;
 }
 
+QMatrix4x4 Item::transform() const
+{
+    return m_transform;
+}
+
+void Item::setTransform(const QMatrix4x4 &transform)
+{
+    m_transform = transform;
+}
+
 QRegion Item::mapToGlobal(const QRegion &region) const
 {
     return region.translated(rootPosition());

src/item.h

@@ -54,6 +54,9 @@ public:
     Scene::Window *window() const;
     QPoint rootPosition() const;
 
+    QMatrix4x4 transform() const;
+    void setTransform(const QMatrix4x4 &transform);
+
     /**
      * Maps the given @a region from the item's coordinate system to the scene's coordinate
      * system.
@@ -119,6 +122,7 @@ private:
     Scene::Window *m_window;
     QPointer<Item> m_parentItem;
     QList<Item *> m_childItems;
+    QMatrix4x4 m_transform;
     QRect m_boundingRect;
     QPoint m_position;
     QSize m_size = QSize(0, 0);

src/plugins/scenes/opengl/scene_opengl.cpp

@@ -833,33 +833,6 @@ OpenGLWindow::~OpenGLWindow()
 {
 }
 
-static QMatrix4x4 transformation(const Item *item, const OpenGLWindow::RenderContext *context)
-{
-    const QPoint position = item->rootPosition();
-    QMatrix4x4 matrix;
-    matrix.translate(position.x(), position.y());
-
-    if (!(context->paintFlags & Scene::PAINT_WINDOW_TRANSFORMED))
-        return matrix;
-
-    const WindowPaintData *data = &context->paintData;
-    matrix.translate(data->translation());
-    const QVector3D scale = data->scale();
-    matrix.scale(scale.x(), scale.y(), scale.z());
-
-    if (data->rotationAngle() == 0.0)
-        return matrix;
-
-    // Apply the rotation
-    // cannot use data.rotation.applyTo(&matrix) as QGraphicsRotation uses projectedRotate to map back to 2D
-    matrix.translate(data->rotationOrigin());
-    const QVector3D axis = data->rotationAxis();
-    matrix.rotate(data->rotationAngle(), axis.x(), axis.y(), axis.z());
-    matrix.translate(-data->rotationOrigin());
-
-    return matrix;
-}
-
 QVector4D OpenGLWindow::modulate(float opacity, float brightness) const
 {
     const float a = opacity;
@@ -911,7 +884,7 @@ static WindowQuadList clipQuads(const Item *item, const OpenGLWindow::RenderCont
 {
     const WindowQuadList quads = item->quads();
     if (context->clip != infiniteRegion() && !context->hardwareClipping) {
-        const QPoint offset = item->rootPosition();
+        const QPoint offset = context->transforms.top().map(QPoint(0, 0));
 
         WindowQuadList ret;
         ret.reserve(quads.count());
@@ -942,6 +915,11 @@ void OpenGLWindow::createRenderNode(Item *item, RenderContext *context)
 {
     const QList<Item *> sortedChildItems = item->sortedChildItems();
 
+    QMatrix4x4 matrix;
+    matrix.translate(item->position().x(), item->position().y());
+    matrix *= item->transform();
+    context->transforms.push(context->transforms.top() * matrix);
+
     for (Item *childItem : sortedChildItems) {
         if (childItem->z() >= 0) {
             break;
@@ -959,7 +937,7 @@ void OpenGLWindow::createRenderNode(Item *item, RenderContext *context)
             context->renderNodes.append(RenderNode{
                 .texture = shadow->shadowTexture(),
                 .quads = quads,
-                .transformMatrix = transformation(item, context),
+                .transformMatrix = context->transforms.top(),
                 .opacity = context->paintData.opacity(),
                 .hasAlpha = true,
                 .coordinateType = NormalizedCoordinates,
@@ -972,7 +950,7 @@ void OpenGLWindow::createRenderNode(Item *item, RenderContext *context)
             context->renderNodes.append(RenderNode{
                 .texture = renderer->texture(),
                 .quads = quads,
-                .transformMatrix = transformation(item, context),
+                .transformMatrix = context->transforms.top(),
                 .opacity = context->paintData.opacity(),
                 .hasAlpha = true,
                 .coordinateType = UnnormalizedCoordinates,
@@ -988,7 +966,7 @@ void OpenGLWindow::createRenderNode(Item *item, RenderContext *context)
                 context->renderNodes.append(RenderNode{
                     .texture = bindSurfaceTexture(surfaceItem),
                     .quads = quads,
-                    .transformMatrix = transformation(item, context),
+                    .transformMatrix = context->transforms.top(),
                     .opacity = context->paintData.opacity(),
                     .hasAlpha = hasAlpha,
                     .coordinateType = UnnormalizedCoordinates,
@@ -1005,6 +983,8 @@ void OpenGLWindow::createRenderNode(Item *item, RenderContext *context)
             createRenderNode(childItem, context);
         }
     }
+
+    context->transforms.pop();
 }
 
 QMatrix4x4 OpenGLWindow::modelViewProjectionMatrix(int mask, const WindowPaintData &data) const
@@ -1031,6 +1011,33 @@ QMatrix4x4 OpenGLWindow::modelViewProjectionMatrix(int mask, const WindowPaintDa
     return scene->projectionMatrix() * mvMatrix;
 }
 
+static QMatrix4x4 transformForPaintData(int mask, const WindowPaintData &data)
+{
+    // TODO: Switch to QTransform.
+    QMatrix4x4 matrix;
+
+    if (!(mask & Scene::PAINT_WINDOW_TRANSFORMED)) {
+        return matrix;
+    }
+
+    matrix.translate(data.translation());
+    const QVector3D scale = data.scale();
+    matrix.scale(scale.x(), scale.y(), scale.z());
+
+    if (data.rotationAngle() == 0.0) {
+        return matrix;
+    }
+
+    // Apply the rotation
+    // cannot use data.rotation.applyTo(&matrix) as QGraphicsRotation uses projectedRotate to map back to 2D
+    matrix.translate(data.rotationOrigin());
+    const QVector3D axis = data.rotationAxis();
+    matrix.rotate(data.rotationAngle(), axis.x(), axis.y(), axis.z());
+    matrix.translate(-data.rotationOrigin());
+
+    return matrix;
+}
+
 void OpenGLWindow::performPaint(int mask, const QRegion &region, const WindowPaintData &data)
 {
     if (region.isEmpty()) {
@@ -1038,11 +1045,15 @@ void OpenGLWindow::performPaint(int mask, const QRegion &region, const WindowPai
     }
 
     RenderContext renderContext {
-        .paintFlags = mask,
         .clip = region,
         .paintData = data,
         .hardwareClipping = region != infiniteRegion() && (mask & Scene::PAINT_WINDOW_TRANSFORMED) && !(mask & Scene::PAINT_SCREEN_TRANSFORMED),
     };
+
+    renderContext.transforms.push(QMatrix4x4());
+
+    windowItem()->setTransform(transformForPaintData(mask, data));
+
     createRenderNode(windowItem(), &renderContext);
 
     int quadCount = 0;

src/plugins/scenes/opengl/scene_opengl.h

@@ -140,7 +140,7 @@ public:
     struct RenderContext
     {
         QVector<RenderNode> renderNodes;
-        const int paintFlags;
+        QStack<QMatrix4x4> transforms;
         const QRegion clip;
         const WindowPaintData &paintData;
         const bool hardwareClipping;