OCR_ClickText

[ioa747]

OCR_ClickText

Clicks on a text in a window using Optical Character Recognition (OCR) technology.

This function uses the _UWPOCR_GetWordsRectTo2DArray() function from the UWPOCR UDF ,  to get the words rectangles in the window. (Thanks to Danyfirex)

 

expandcollapse popup

; https://www.autoitscript.com/forum/topic/212167-ocr_clicktext/
;----------------------------------------------------------------------------------------
; Title...........: OCR_ClickText.au3
; Description.....: Clicks on a text in a window using Optical Character Recognition (OCR) technology.
; AutoIt Version..: 3.3.16.1   Author: ioa747  Script Version: 1.0
; Note............: Testet in Win10 22H2
;----------------------------------------------------------------------------------------
#AutoIt3Wrapper_Au3Check_Parameters=-d -w 1 -w 2 -w 3 -w 4 -w 5 -w 6 -w 7

#include <GUIConstantsEx.au3>
#include <ScreenCapture.au3>
#include <WindowsConstants.au3>
#include <Array.au3>
#include "UWPOCR.au3" ; * <-"https://www.autoitscript.com/forum/topic/207324-uwpocr-windows-platform-optical-character-recognition-api-implementation"

Global $g_idMemo, $MyGui, $a_idBtn[7]

;Run Example3() in separate process as example Window
;~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
If $CmdLine[0] > 0 Then
    If $CmdLine[1] = "Example3" Then Example3()
    Exit
EndIf
Run(@AutoItExe & ' /AutoIt3ExecuteScript "' & @ScriptFullPath & '" Example3')
;~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Global $hWin = WinWait("Example3")

; with "*" _ArrayDisplay all found words (initialize mode)
OCR_ClickText($hWin, "*")

; If it ends with a "*", it will be treated as a partial match (First match)
OCR_ClickText($hWin, "Button*")

OCR_ClickText($hWin, "Button2")
OCR_ClickText($hWin, "Button5")

;to me Button1 sees it as ButtonL. I overcome this by using the offset parameter
OCR_ClickText($hWin, "Button2", "primary", 1, 1, 0, -60)


;--------------------------------------------------------------------------------------------------------------------------------
Func Example3()
    $MyGui = GUICreate("Example3", 510, 400)
    GUISetFont($g_idMemo, 9)
    ConsoleWrite("$MyGui=" & $MyGui & @CRLF)
    $g_idMemo = GUICtrlCreateEdit("", 119, 10, 376, 374, $WS_VSCROLL)
    $a_idBtn[0] = 6
    Local $y = 20, $iWidth = 90, $iHeight = 50
    For $x = 1 To $a_idBtn[0]
        $a_idBtn[$x] = GUICtrlCreateButton("Button" & $x, 10, $y, $iWidth, $iHeight)
        $y += 60
    Next
    GUISetState(@SW_SHOW)

    Local $iMsg
    While 1
        $iMsg = GUIGetMsg()
        Switch $iMsg
            Case $GUI_EVENT_CLOSE
                ExitLoop
            Case $a_idBtn[1] To $a_idBtn[$a_idBtn[0]]
                MemoWrite("cklick-" & GUICtrlRead($a_idBtn[$iMsg - $a_idBtn[1] + 1]))
        EndSwitch
    WEnd
EndFunc   ;==>Example3
;--------------------------------------------------------------------------------------------------------------------------------
Func MemoWrite($sMessage) ; Write a line to the memo control
    GUICtrlSetData($g_idMemo, $sMessage & @CRLF, 1)
EndFunc   ;==>MemoWrite


; #FUNCTION# --------------------------------------------------------------------------------------------------------------------
; Name...........: OCR_ClickText
; Description....: Clicks on a text in a window using Optical Character Recognition (OCR) technology.
; Syntax.........: OCR_ClickText($hWnd, $sFindText = "*",  $sButton = "primary", $iClicks = 1, $iSpeed = 1, $sLanguageTag = Default)
; Parameters.....: $hWnd              Handle of the window to search in.
;                  $sFindText    [optional] Text to find in the window, with "*" _ArrayDisplay all found words (initialize mode) (default = "*").
;                                         If it ends with a "*", it will be treated as a partial match (First match).
;                  $sButton      [optional] Button to click on the found text (default = "primary").
;                  $iClicks      [optional] Number of clicks to perform on the button (default = 1).
;                  $iSpeed       [optional] Speed at which to click the button, in milliseconds (default = 1).
;                  $iXOffset     [optional] if it is positive it adds , if it is negative it subtracts to X coords  (default = 0).
;                  $iYOffset     [optional] if it is positive it adds , if it is negative it subtracts to Y coords  (default = 0).
;                  $sLanguageTag [optional] Language tag to use for OCR. (default = first OS language).
; Return values..: Success - 1 if text was found and clicked on, 0 otherwise.
;                  With "*" as $sFindText parameter return _ArrayDisplay with all found words with coords (initialize mode)
;                  Failure - Returns an error code
;                            @error = 1 if the function fails to capture a screenshot of the window.
;                            @error = 2 if the function fails to get the words rectangles using OCR.
; Author ........: ioa747
; Notes .........: This function uses the _UWPOCR_GetWordsRectTo2DArray() function from the UWPOCR UDF to get the words rectangles in the window.
; Link ..........: https://www.autoitscript.com/forum/topic/212167-ocr_clicktext/
; Dependencies...: #include "UWPOCR.au3" ;https://www.autoitscript.com/forum/topic/207324-uwpocr-windows-platform-optical-character-recognition-api-implementation
;--------------------------------------------------------------------------------------------------------------------------------
Func OCR_ClickText($hWnd, $sFindText = "*", $sButton = "primary", $iClicks = 1, $iSpeed = 1, $iXOffset = 0, $iYOffset = 0, $sLanguageTag = Default)

    Local $hHBitmap, $hBitmap
    Local $aWPos = WinGetPos($hWnd)
    Local $RetVal = 0

    _GDIPlus_Startup()
    $hHBitmap = _ScreenCapture_Capture("", $aWPos[0], $aWPos[1], $aWPos[0] + $aWPos[2], $aWPos[1] + $aWPos[3], False)
    If @error Then Return SetError(1, 0, 0)
    $hBitmap = _GDIPlus_BitmapCreateFromHBITMAP($hHBitmap)

    Local $aWords = _UWPOCR_GetWordsRectTo2DArray($hBitmap, $sLanguageTag)
    If @error Then Return SetError(2, 0, 0)

    Local $bPartial = StringRight($sFindText, 1) = "*" ? True : False

    If $sFindText = '*' Then
        $RetVal = _ArrayDisplay($aWords)
    Else
        For $i = 1 To UBound($aWords) - 1
            If $bPartial = True Then
                If StringLeft($aWords[$i][0], StringLen($sFindText) - 1) = StringTrimRight($sFindText, 1) Then ExitLoop
            Else
                If $aWords[$i][0] = $sFindText Then ExitLoop
            EndIf
        Next

        If $i < UBound($aWords) Then
            ConsoleWrite($aWords[$i][0] & " -> " & $aWords[$i][1] & ";" & $aWords[$i][2] & ";" & $aWords[$i][3] & ";" & $aWords[$i][4] & @CRLF)
            $RetVal = MouseClick($sButton, $aWPos[0] + $aWords[$i][1] + $iXOffset + $aWords[$i][3] / 2, _
                    $aWPos[1] + $aWords[$i][2] + $iYOffset + $aWords[$i][4] / 2, $iClicks, $iSpeed)
        EndIf
    EndIf

    _WinAPI_DeleteObject($hHBitmap)
    _GDIPlus_BitmapDispose($hBitmap)
    _GDIPlus_Shutdown()

    Return $RetVal

EndFunc   ;==>OCR_ClickText
;--------------------------------------------------------------------------------------------------------------------------------

Please, every comment is appreciated!
leave your comments and experiences here!
Thank you very much 

Edited August 15 by ioa747