From e3953e76020d37a0d450fd6ccc6aa3300ded110c Mon Sep 17 00:00:00 2001 From: Xaver Hugl Date: Tue, 13 Aug 2024 17:42:33 +0200 Subject: [PATCH] core/colorpipeline: allow rounding away more floating point errors Black point compensation introduced a tiny bit increased error in the floating point calculations, but the errors are still way too small to actually matter. To avoid those unnecessary calculations causing performance issues and more rounding errors, this commit changes the allowed error to 10^-5. --- src/core/colorpipeline.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/colorpipeline.cpp b/src/core/colorpipeline.cpp index b175e0ec53..af78412fbd 100644 --- a/src/core/colorpipeline.cpp +++ b/src/core/colorpipeline.cpp @@ -167,7 +167,7 @@ static bool isFuzzyIdentity(const QMatrix4x4 &mat) { // matrix calculations with floating point numbers can result in very small errors // -> ignore them, as that just causes inefficiencies and more rounding errors - constexpr float maxResolution = 0.0000001; + constexpr float maxResolution = 0.00001; for (int i = 0; i < 4; i++) { for (int j = 0; j < 4; j++) { const float targetValue = i == j ? 1 : 0; @@ -183,7 +183,7 @@ static bool isFuzzyScalingOnly(const QMatrix4x4 &mat) { // matrix calculations with floating point numbers can result in very small errors // -> ignore them, as that just causes inefficiencies and more rounding errors - constexpr float maxResolution = 0.0000001; + constexpr float maxResolution = 0.00001; for (int i = 0; i < 4; i++) { for (int j = 0; j < 4; j++) { if (i < 3 && i == j) {