works fine after i changed a couple things - you used '$WS_SYSMENU' twice and other styles you chose are breaking it - see the help file for gui control styles
#include <GUIConstantsEx.au3>
#include "F:\AutoIt\Resources\Funcs\_other\HotKey_Yashid\HotKey_2.1b.au3"
#include "F:\AutoIt\Resources\Funcs\_other\HotKey_Yashid\HotKey_vkConstants.au3"
Global $hWnd
$hWnd = GUICreate("HotKeys", 800, 600) ;, -1, -1, BitOR($WS_CAPTION, $WS_MINIMIZEBOX, $WS_MAXIMIZEBOX, $WS_SYSMENU, $DS_CONTEXTHELP, $WS_SYSMENU, $WS_VISIBLE, $WS_CLIPCHILDREN, $WS_THICKFRAME))
GUISetState(@SW_SHOW, $hWnd)
_HotKey_Assign($VK_RETURN, "PressedEnter", $HK_FLAG_NOBLOCKHOTKEY, $hWnd)
ConsoleWrite("error=" & @error & @LF)
While GUIGetMsg() <> $GUI_EVENT_CLOSE
Sleep(10)
WEnd
GUIDelete($hWnd)
Exit
Func PressedEnter()
ConsoleWrite("Pressed enter")
EndFunc
...