autoit で指定したボタンなどの画像を監視してクリックする (OpenCV v4 画像認識)

2023年3月20日2023年10月14日

OpenCV はスピードが速いです

導入方法

https://memo.eightban.com/autoit/autoit-opencv-v4-udf

簡単なスクリプト

#include <Clipboard.au3>
#include <EditConstants.au3>
#include <MsgBoxConstants.au3>
#include <SendMessage.au3>
#include <WinAPISys.au3>

#Include <ScreenCapture.au3>

#include <GUIConstantsEx.au3>
#include <WindowsConstants.au3>
#include "..\autoit-opencv-com\udf\opencv_udf_utils.au3"

Global $aRect, $iLeft, $iTop, $iWidth, $iHeight
Global $cv

Global Const $Mouse_Move_On_Found = 1
Global Const $Mouse_Click_On_Found = 0
Global Const $Wait_mode = 1
Global Const $iSleep_Time=500
Global $sCount = 0
Global $mCount = 8
Global $iCount = 0
Global $aCount = 0

Dim $_Image[$mCount]
Dim $_Image1[$mCount]
For $iz=1 To $mCount Step 1
 $_Image[$iz-1] = @ScriptDir & "\butonimagej" & $iz & ".bmp"
Next
Global $_msg_0 =  "  image"
Global $_msg_1 =  "* image"
Global $_Image_0 =  ""
Global $_Pic_0 = 0

Global $ii =  0
Global $kflg =  0
Global $ix=0

dim $_msg[$mCount*2]
For $iz=1 To $mCount*2 Step 1
 $_msg[$iz-1] = $_msg_0
Next
 $_msg[8] = $_msg_1

Global $_title_1 =  "autoimageclosej"

_OpenCV_Open(_OpenCV_FindDLL("opencv_world470*"), _OpenCV_FindDLL("autoit_opencv_com470*"))
;_GDIPlus_Startup()
OnAutoItExitRegister("_OnAutoItExit")
HotKeySet("{Esc}", "_Exit") ; Press ESC for exit

Opt("GUIOnEventMode", 1)  ; Change to OnEvent mode
$mainwindow = GUICreate($_title_1, 140,300,100,50)
;$mainwindow = GUICreate($_title_1, 140,300)
GUISetOnEvent($GUI_EVENT_CLOSE, "CLOSEClicked")
$msg1="stopping"
$labelx=GUICtrlCreateLabel($msg1, 70, 25)


$buttonx = GUICtrlCreateButton("push", 10, 20,50, 25)
GUICtrlSetOnEvent($buttonx, "OKButtonx")

	#comments-start
	#comments-end

Dim $_Pic[$mCount]
Dim $_label[$mCount]
Dim $_button[$mCount]
For $iz=0 To $mCount-1 Step 1
 $_Pic[$iz] = GUICtrlCreatePic($_Image[$iz], 70, 50+30*$iz, 50, 25)
 $_label[$iz]=GUICtrlCreateLabel($_msg_0, 65,60+30*$iz)
 GUICtrlSetColor($_label[$iz], 0xff0000)
 GUICtrlSetBkColor($_label[$iz], $GUI_BKCOLOR_TRANSPARENT)
 $_button[$iz] = GUICtrlCreateButton("set"&($iz+1), 10, 50+30*$iz,50, 25)
 GUICtrlSetOnEvent($_button[$iz], "OKButton"&($iz+1))
Next
Global $_Pic_0 =  0
Dim $_tmp[$mCount]

GUIRegisterMsg($WM_CLIPBOARDUPDATE, 'WM_CLIPBOARDUPDATE')

GUISetState(@SW_SHOW)

_WinAPI_AddClipboardFormatListener($mainwindow)
_SendMessage($mainwindow, $WM_CLIPBOARDUPDATE)

WinWait($_title_1)
WinActivate($_title_1)
WinSetOnTop($_title_1, "", 1)


$cv = _OpenCV_get()
If Not IsObj($cv) Then Exit
_Initialize()

While 1
	if $kflg =  2 Then
		ConsoleWrite($_Pic_0 & $_Image_0 & @CRLF )
		Sleep(3000)
		_Initialize()
		if $_Pic_0 = 0 Then
			$_Pic[$ii] = GUICtrlCreatePic($_Image[$ii], 70, 50+30*$ii, 50, 25)
			GUISetState(@SW_HIDE )
			GUISetState(@SW_SHOW)
		Else
			GUICtrlSetImage ($_Pic_0,$_Image_0)
		EndIf
		$kflg =  0
	EndIf
	if $msg1="stopping"  Then
	Else
		For $iz=0 To $mCount-1 Step 1
			GUICtrlSetData ($_label[$iz], $_msg[$ix+8-$iz])
		Next
		_osu()
	EndIf
	if $ix >  $mCount-1 Then $ix = 0
	If $_Image1[$ix]<>"" Then Sleep($iSleep_Time)
WEnd

Func OKButtonx()
if $msg1="stopping"  Then
  $msg1="executing"
Else
  $msg1="stopping"
  $ix+=1
EndIf
GUICtrlSetData ($labelx, $msg1)
EndFunc

Func OKButton1()
 $ii=0
 _imageget($ii)
EndFunc
Func OKButton2()
 $ii=1
 _imageget($ii)
EndFunc
Func OKButton3()
 $ii=2
 _imageget($ii)
EndFunc
Func OKButton4()
 $ii=3
 _imageget($ii)
EndFunc
Func OKButton5()
 $ii=4
 _imageget($ii)
EndFunc
Func OKButton6()
 $ii=5
 _imageget($ii)
EndFunc
Func OKButton7()
 $ii=6
 _imageget($ii)
EndFunc
Func OKButton8()
 $ii=7
 _imageget($ii)
EndFunc

Func _imageget($ic=0)

$_Image_0 = $_Image[$ic]
$_Pic_0 =  $_Pic[$ic]
$kflg =  1

Send("#+S")
Sleep(2000)

EndFunc

Func _opt($ia=0)
Switch Int($ia+1)
	Case 0 To 3
	Case 4 To 5
		Sleep(3000)
		Send("{END}")
		Sleep(1000)
	Case Else
EndSwitch
EndFunc
Func _opt2($ib=0)
Switch Int($ib+1)
	Case 0 To 3
	Case 4 To 5
		Sleep(3000)
Send("!{LEFT}")
		Sleep(1000)
	Case Else
EndSwitch
EndFunc
;===========================
Func _osu()
If $_Image1[$ix]<>"" Then
;	_opt($ix)
	$sCount += 1
	ConsoleWrite("Rect: [" & $iWidth & " x " & $iHeight & " from (" & $iLeft & ", " & $iTop & ")]" & @CRLF)

	; The higher the value, the higher the match is exact
	Local $threshold = 0.95

	$img = _OpenCV_GetDesktopScreenMat($aRect)
	if $iCount=0 Then
	; Ignore the alpha channel when matching the template => $CV_COLOR_BGRA2BGR
	; Only consider gray scale colors when matching the template => $CV_COLOR_BGRA2GRAY
	; using cveCanny and/or gray scale can significantly reduce the matching time but also reduce the matching accuracy
		$aMatches = 0
		Global $aMatches = _OpenCV_FindTemplate($img, $_tmp[$ix], $threshold, Default, Default, Default, $CV_COLOR_BGRA2GRAY)
		Local $iMatches = UBound($aMatches)
		ConsoleWrite("Number of matches: " & $iMatches & $_Image[$ix]&@CRLF)
		$iCount = $iMatches
		$aCount = $iMatches
		if  $iMatches <>0 Then 		$ix+=1
	EndIf

	if $iCount<>0 Then
		Local $ixx = $aCount - $iCount
		$iCount -=1
		Local $aMatchRect = _OpenCV_Rect(0, 0, $_tmp[$ii].width, $_tmp[$ii].height)

		$aMatchRect[0] = $aMatches[$ixx][0]
		$aMatchRect[1] = $aMatches[$ixx][1]
        ConsoleWrite('- [' & $sCount & '] Image found:' & " X=" & $aMatchRect[0] & " Y=" & $aMatchRect[1] & @CRLF)
        If $Mouse_Move_On_Found Then
			MouseMove($aMatchRect[0], $aMatchRect[1])
			Sleep($iSleep_Time)
		EndIf
        If $Mouse_Click_On_Found Then MouseClick("left", $aMatchRect[0], $aMatchRect[1])
;        ToolTip('(' & $sCount & "] Image found:" &  " X=" & $aMatchRect[0] & " Y=" & $aMatchRect[1], 1, 1)
;		_opt2($ix)
	Else
        If Not $Wait_mode Then	$ix+=1
    EndIf
    Sleep(2000)
Else
	$ix+=1
EndIf
EndFunc

Func WM_CLIPBOARDUPDATE($hWnd, $iMsg, $wParam, $lParam)
        #forceref $hWnd, $iMsg, $wParam, $lParam
	if $kflg =  1 Then
	    Sleep(200)
        _ClipBoard_Open(0)
          $hBmp = _ClipBoard_GetDataEx  ($CF_BITMAP )
		_ScreenCapture_SaveImage ($_Image_0, $hBmp)
        _ClipBoard_Close()
		ConsoleWrite(" Save image: " & $_Image_0 & @CRLF )
		$kflg =  2
	EndIf
    Return 0
EndFunc   ;==>

Func _Initialize()
	For $iz=0 To $mCount-1 Step 1
		If FileExists($_Image[$iz]) Then
			$_tmp[$iz] = $cv.imread($_Image[$iz])
			$_Image1[$iz] = $_Image[$iz]
					ConsoleWrite(" mage: " & $_Image1[$iz] & @CRLF )

		Else
			$_tmp[$iz] = ""
			$_Image1[$iz] = ""
		EndIf
	;	$_tmp[$iz] = $cv.imread(_OpenCV_FindFile("samples\data\mario_coin.png"))
	Next
	; Search on the first screen. Leave empty to search on all screens
	; Reducing the search area improves the search time
	$aRect = _OpenCV_GetDesktopScreenRect(1)
	$iLeft = $aRect[0]
	$iTop = $aRect[1]
	$iWidth = $aRect[2]
	$iHeight = $aRect[3]
EndFunc   ;==>_Initialize

Func _Release()
	#comments-start
	#comments-end
	$cv.destroyAllWindows()
EndFunc   ;==>_Release

Func _OnAutoItExit()
;	_GDIPlus_Shutdown()
	_OpenCV_Close()
EndFunc   ;==>_OnAutoItExit

Func CLOSEClicked()
	_Release()
Exit
EndFunc

Func _Exit()
	_Release()
    Exit 0
EndFunc   ;==>_Exit

_OpenCV_FindTemplateこれを使って画像を見つけます。複数の画像を素早く見つけます。マウスを動かすサンプルなのでパラメータを変えてクリックする方法に変えてください。