diff --git a/src/backends/drm/drm_egl_layer.cpp b/src/backends/drm/drm_egl_layer.cpp
index 7b7c19c215..d673b1c6db 100644
--- a/src/backends/drm/drm_egl_layer.cpp
+++ b/src/backends/drm/drm_egl_layer.cpp
@@ -57,7 +57,7 @@ std::optional<OutputLayerBeginFrameInfo> EglGbmLayer::doBeginFrame()
     m_scanoutBuffer.reset();
     m_colorPipeline = ColorPipeline{};
     return m_surface.startRendering(targetRect().size(), m_pipeline->output()->transform().combine(OutputTransform::FlipY), m_pipeline->formats(m_type), m_pipeline->output()->scanoutColorDescription(),
-                                    m_pipeline->output()->needsChannelFactorFallback() ? m_pipeline->output()->effectiveChannelFactors() : QVector3D(1, 1, 1), m_pipeline->iccProfile(), m_pipeline->output()->scale());
+                                    m_pipeline->output()->needsChannelFactorFallback() ? m_pipeline->output()->adaptedChannelFactors() : QVector3D(1, 1, 1), m_pipeline->iccProfile(), m_pipeline->output()->scale());
 }
 
 bool EglGbmLayer::doEndFrame(const QRegion &renderedRegion, const QRegion &damagedRegion, OutputFrame *frame)
@@ -105,7 +105,7 @@ bool EglGbmLayer::doImportScanoutBuffer(GraphicsBuffer *buffer, const ColorDescr
     ColorPipeline pipeline = ColorPipeline::create(color, m_pipeline->output()->scanoutColorDescription(), intent);
     if (m_pipeline->output()->needsChannelFactorFallback()) {
         pipeline.addTransferFunction(m_pipeline->output()->scanoutColorDescription().transferFunction());
-        pipeline.addMultiplier(m_pipeline->output()->effectiveChannelFactors());
+        pipeline.addMultiplier(m_pipeline->output()->adaptedChannelFactors());
         pipeline.addInverseTransferFunction(m_pipeline->output()->scanoutColorDescription().transferFunction());
     }
     m_colorPipeline = pipeline;
diff --git a/src/backends/drm/drm_output.cpp b/src/backends/drm/drm_output.cpp
index b102ad0d7c..470c425314 100644
--- a/src/backends/drm/drm_output.cpp
+++ b/src/backends/drm/drm_output.cpp
@@ -405,7 +405,10 @@ ColorDescription DrmOutput::createColorDescription(const std::shared_ptr<OutputC
     // HDR screens are weird, sending them the min. luminance from the EDID does *not* make all of them present the darkest luminance the display can show
     // to work around that, (unless overridden by the user), assume the min. luminance of the transfer function instead
     const double minBrightness = effectiveHdr ? props->minBrightnessOverride.value_or(m_state.minBrightnessOverride).value_or(TransferFunction::defaultMinLuminanceFor(TransferFunction::PerceptualQuantizer)) : transferFunction.minLuminance;
-    return ColorDescription(containerColorimetry, transferFunction, referenceLuminance, minBrightness, maxAverageBrightness, maxPeakBrightness, masteringColorimetry, sdrColorimetry);
+
+    const double brightnessFactor = !m_brightnessDevice || effectiveHdr ? props->brightness.value_or(m_state.brightness) : 1.0;
+    const double effectiveReferenceLuminance = 25 + (referenceLuminance - 25) * brightnessFactor;
+    return ColorDescription(containerColorimetry, transferFunction, effectiveReferenceLuminance, minBrightness, maxAverageBrightness, maxPeakBrightness, masteringColorimetry, sdrColorimetry);
 }
 
 void DrmOutput::applyQueuedChanges(const std::shared_ptr<OutputChangeSet> &props)
@@ -472,9 +475,12 @@ void DrmOutput::setBrightnessDevice(BrightnessDevice *device)
         } else {
             device->setBrightness(m_state.brightness);
         }
-        // reset the brightness factors
-        tryKmsColorOffloading();
     }
+    // reset the brightness factors
+    State next = m_state;
+    next.colorDescription = createColorDescription(std::make_shared<OutputChangeSet>());
+    setState(next);
+    tryKmsColorOffloading();
 }
 
 void DrmOutput::revertQueuedChanges()
@@ -516,7 +522,7 @@ void DrmOutput::tryKmsColorOffloading()
     }
     // TODO this doesn't allow using only a CTM for night light offloading
     // maybe relax correctness in that case and apply night light in non-linear space?
-    const QVector3D channelFactors = effectiveChannelFactors();
+    const QVector3D channelFactors = adaptedChannelFactors();
     const double maxLuminance = colorDescription().maxHdrLuminance().value_or(colorDescription().referenceLuminance());
     const ColorDescription optimal = colorDescription().transferFunction().type == TransferFunction::gamma22 ? colorDescription() : colorDescription().withTransferFunction(TransferFunction(TransferFunction::gamma22, 0, maxLuminance));
     ColorPipeline colorPipeline = ColorPipeline::create(optimal, colorDescription(), RenderingIntent::RelativeColorimetric);
@@ -543,19 +549,11 @@ bool DrmOutput::needsChannelFactorFallback() const
     return m_channelFactorsNeedShaderFallback;
 }
 
-QVector3D DrmOutput::effectiveChannelFactors() const
+QVector3D DrmOutput::adaptedChannelFactors() const
 {
-    QVector3D adaptedChannelFactors = ColorDescription::sRGB.toOther(colorDescription(), RenderingIntent::RelativeColorimetric) * m_channelFactors;
+    const QVector3D ret = ColorDescription::sRGB.toOther(colorDescription(), RenderingIntent::RelativeColorimetric) * m_channelFactors;
     // normalize red to be the original brightness value again
-    adaptedChannelFactors *= m_channelFactors.x() / adaptedChannelFactors.x();
-    if (m_state.highDynamicRange || !m_brightnessDevice) {
-        // enforce a minimum of 25 nits for the reference luminance
-        constexpr double minLuminance = 25;
-        const double brightnessFactor = (m_state.brightness * (1 - (minLuminance / m_state.referenceLuminance))) + (minLuminance / m_state.referenceLuminance);
-        return adaptedChannelFactors * brightnessFactor;
-    } else {
-        return adaptedChannelFactors;
-    }
+    return ret * m_channelFactors.x() / ret.x();
 }
 
 const ColorDescription &DrmOutput::scanoutColorDescription() const
diff --git a/src/backends/drm/drm_output.h b/src/backends/drm/drm_output.h
index 8c010d4606..af7e6bbb71 100644
--- a/src/backends/drm/drm_output.h
+++ b/src/backends/drm/drm_output.h
@@ -61,7 +61,7 @@ public:
     /**
      * channel factors adapted to the target color space + brightness setting multiplied in
      */
-    QVector3D effectiveChannelFactors() const;
+    QVector3D adaptedChannelFactors() const;
     void updateConnectorProperties();
 
     /**