Add support for KHR_debug

Register a callback function for debug output when KHR_debug or
ARB_debug_output is supported.

Only error messages and warnings about undefined behavior are
enabled in release builds, while all debug messages are enabled
in debug builds.

scene_opengl.cpp

@@ -386,6 +386,7 @@ SceneOpenGL::SceneOpenGL(Workspace* ws, OpenGLBackend *backend)
 #endif
 
     m_debug = qstrcmp(qgetenv("KWIN_GL_DEBUG"), "1") == 0;
+    initDebugOutput();
 
     // set strict binding
     if (options->isGlStrictBindingFollowsDriver()) {
@@ -420,6 +421,58 @@ SceneOpenGL::~SceneOpenGL()
     }
 }
 
+void SceneOpenGL::initDebugOutput()
+{
+    const bool have_KHR_debug = hasGLExtension(QByteArrayLiteral("GL_KHR_debug"));
+    if (!have_KHR_debug && !hasGLExtension(QByteArrayLiteral("GL_ARB_debug_output")))
+        return;
+
+    // Set the callback function
+    auto callback = [](GLenum source, GLenum type, GLuint id,
+                       GLenum severity, GLsizei length,
+                       const GLchar *message,
+                       const GLvoid *userParam) {
+        Q_UNUSED(source)
+        Q_UNUSED(severity)
+        Q_UNUSED(userParam)
+
+        switch (type) {
+        case GL_DEBUG_TYPE_ERROR:
+        case GL_DEBUG_TYPE_UNDEFINED_BEHAVIOR:
+            qWarning("%#x: %.*s", id, length, message);
+            break;
+
+        case GL_DEBUG_TYPE_DEPRECATED_BEHAVIOR:
+        case GL_DEBUG_TYPE_PORTABILITY:
+        case GL_DEBUG_TYPE_PERFORMANCE:
+        case GL_DEBUG_TYPE_OTHER:
+        default:
+            qDebug("%#x: %.*s", id, length, message);
+            break;
+        }
+    };
+
+    glDebugMessageCallback(callback, nullptr);
+
+    // This state exists only in GL_KHR_debug
+    if (have_KHR_debug)
+        glEnable(GL_DEBUG_OUTPUT);
+
+#ifndef NDEBUG
+    // Enable all debug messages
+    glDebugMessageControl(GL_DONT_CARE, GL_DONT_CARE, GL_DONT_CARE, 0, nullptr, GL_TRUE);
+#else
+    // Enable error messages
+    glDebugMessageControl(GL_DONT_CARE, GL_DEBUG_TYPE_ERROR, GL_DONT_CARE, 0, nullptr, GL_TRUE);
+    glDebugMessageControl(GL_DONT_CARE, GL_DEBUG_TYPE_UNDEFINED_BEHAVIOR, GL_DONT_CARE, 0, nullptr, GL_TRUE);
+#endif
+
+    // Insert a test message
+    const QByteArray message = QByteArrayLiteral("OpenGL debug output initialized");
+    glDebugMessageInsert(GL_DEBUG_SOURCE_APPLICATION, GL_DEBUG_TYPE_OTHER, 0,
+                         GL_DEBUG_SEVERITY_LOW, message.length(), message.constData());
+}
+
 SceneOpenGL *SceneOpenGL::createScene()
 {
     OpenGLBackend *backend = NULL;

scene_opengl.h

@@ -66,6 +66,7 @@ public:
     void idle();
 
     bool debug() const { return m_debug; }
+    void initDebugOutput();
 
     /**
      * @brief Factory method to create a backend specific texture.