diff --git a/src/backends/CMakeLists.txt b/src/backends/CMakeLists.txt
index cccd1f5736..93e19918db 100644
--- a/src/backends/CMakeLists.txt
+++ b/src/backends/CMakeLists.txt
@@ -3,7 +3,4 @@ add_subdirectory(fakeinput)
 add_subdirectory(libinput)
 add_subdirectory(virtual)
 add_subdirectory(wayland)
-
-if (KWIN_BUILD_X11_BACKEND)
-    add_subdirectory(x11)
-endif()
+add_subdirectory(x11)
diff --git a/src/backends/x11/CMakeLists.txt b/src/backends/x11/CMakeLists.txt
index 145cee89b7..85263becd6 100644
--- a/src/backends/x11/CMakeLists.txt
+++ b/src/backends/x11/CMakeLists.txt
@@ -1,4 +1,6 @@
-add_subdirectory(standalone)
+if (KWIN_BUILD_X11_BACKEND)
+    add_subdirectory(standalone)
+endif()
 if (X11_XCB_FOUND)
     add_subdirectory(windowed)
 endif()