diff --git a/kcms/colors/package/contents/ui/main.qml b/kcms/colors/package/contents/ui/main.qml
index 71f72d223..1f305a689 100644
--- a/kcms/colors/package/contents/ui/main.qml
+++ b/kcms/colors/package/contents/ui/main.qml
@@ -286,7 +286,8 @@ KCM.GridViewKCM {
                     ColorRadioButton {
                         id: customColorIndicator
 
-                        readonly property bool isCustomColor: root.accentColor
+                        readonly property bool isCustomColor: !wallpaperAccentBox.checked
+                            && root.accentColor
                             && !colorRepeater.model.some(color => Qt.colorEqual(color, root.accentColor))
 
                        /* The qt binding will keep the binding alive as well as uncheck the button