I faced the same problem and solved it by using Windows API to subclass a window. If you are allowed to use JNA, the following code would help you.
MyUser32.java
import com.sun.jna.Callback; import com.sun.jna.Native; import com.sun.jna.platform.win32.*; import com.sun.jna.win32.W32APIOptions; public interface MyUser32 extends User32 { int WM_MENUCHAR = 0x0120; int MNC_CLOSE = 1; int VK_RETURN = 0x0d; MyUser32 INSTANCE = Native.loadLibrary("user32", MyUser32.class, W32APIOptions.UNICODE_OPTIONS); LONG_PTR SetWindowLongPtr(WinDef.HWND hWnd, int nIndex, Callback callback); LRESULT CallWindowProc(LONG_PTR lpPrevWndProc, HWND hWnd, int Msg, WPARAM wParam, LPARAM lParam); }
App.java
import com.sun.jna.Pointer; import com.sun.jna.platform.win32.*; import com.sun.jna.platform.win32.WinDef.*; import com.sun.jna.win32.StdCallLibrary; import javafx.application.Application; import javafx.stage.Stage; import java.lang.reflect.Method; public class App extends Application implements StdCallLibrary.StdCallCallback { private BaseTSD.LONG_PTR baseWndProc; public static void main(String[] args) { launch(args); } @Override public void start(Stage primaryStage) { // set up scene ... primaryStage.show(); final HWND hWnd = new HWND(getWindowPointer(primaryStage)); baseWndProc = MyUser32.INSTANCE.SetWindowLongPtr(hWnd, User32.GWL_WNDPROC, this); } public LRESULT callback(HWND hWnd, int Msg, WPARAM wParam, LPARAM lParam) { if (Msg == MyUser32.WM_MENUCHAR && (wParam.longValue() & 0xffff) == MyUser32.VK_RETURN) { return new LRESULT(MyUser32.MNC_CLOSE << 16); } return MyUser32.INSTANCE.CallWindowProc(baseWndProc, hWnd, Msg, wParam, lParam); } private Pointer getWindowPointer(Stage stage) { try { Method getPeer = stage.getClass().getMethod("impl_getPeer"); final Object tkStage = getPeer.invoke(stage); Method getPlatformWindow = tkStage.getClass().getDeclaredMethod("getPlatformWindow"); getPlatformWindow.setAccessible(true); final Object platformWindow = getPlatformWindow.invoke(tkStage); Method getNativeHandle = platformWindow.getClass().getMethod("getNativeHandle"); return new Pointer((Long) getNativeHandle.invoke(platformWindow)); } catch (Throwable t) { return null; } } }
References
Disable MessageBeep on Invalid Syskeypress
Why my JNA using application doesn't react in a right way?
How can I get the window handle (hWnd) for a Stage in JavaFX?