AutoItとUI Automation UDFを使ってポップアップウィンドウのボタンにマウスカーソルを移動させる

2024年6月29日

チューチューマウスみたいなものです。標準の Windows ではボタンにマウスカーソルを移動することができますができないボタンもあるので対応します

準備

AutoItとUI Automation UDFを使って自動化する

UI Automationはツリー構造になっています

Conditionに条件を指定してデスクトップから要素まで辿って行きます

  ; Create UI Automation object
  ; Get Desktop element
  ; Get window element
  ; Get Button element
  ; Get Rect element

コード

#include <GUIConstantsEx.au3>
#include <WindowsConstants.au3>
#include <Array.au3>
#include "..\UIAIncludes\Includes\CUIAutomation2.au3"

Global $_title_1 =  "btnuia"

;[CLASS:Windows.UI.Core.CoreWindow; INSTANCE:0]
Opt("GUIOnEventMode", 1)  ; Change to OnEvent mode
$mainwindow = GUICreate($_title_1, 70, 70)
GUISetOnEvent($GUI_EVENT_CLOSE, "CLOSEClicked")
$msg1="stopping"
$label1=GUICtrlCreateLabel($msg1, 10, 10)
$button1 = GUICtrlCreateButton("push", 10, 20,50, 50)
GUICtrlSetOnEvent($button1, "OKButton")
GUISetState(@SW_SHOW)

Global $Paused
;

HotKeySet("{Esc}", "_Exit") ; Press ESC for exit
Func _Exit()
    Exit 0
EndFunc   ;==>_Exit

if    WinWait($_title_1, "",3) Then
		sleep(50)
    ControlClick ($_title_1,"","Button1")
	sleep(50)
	GUISetState(@SW_MINIMIZE)

EndIf


While 1
  Sleep(1000)  ; Idle around
  if $msg1="stopping"  Then
  Else
    _chk()
	Sleep(5000)
  EndIf
WEnd

Func OKButton()
;  MsgBox(0, "GUI Event", "You pressed OK!")
if $msg1="stopping"  Then
  $msg1="executing"

Else
  $msg1="stopping"
EndIf
GUICtrlSetData ($label1, $msg1)


EndFunc

Func CLOSEClicked()
 ; MsgBox(0, "GUI Event", "You clicked CLOSE! Exiting...")
Exit
EndFunc



;===========================
Func _chk()

  ; Create UI Automation object
  Local $oUIAutomation = ObjCreateInterface( $sCLSID_CUIAutomation, $sIID_IUIAutomation, $dtagIUIAutomation )
  If Not IsObj( $oUIAutomation ) Then Return ConsoleWrite( "$oUIAutomation ERR" & @CRLF )
  ConsoleWrite( "$oUIAutomation OK" & @CRLF )

  ; Get Desktop element
  Local $pDesktop, $oDesktop
  $oUIAutomation.GetRootElement( $pDesktop )
  $oDesktop = ObjCreateInterface( $pDesktop, $sIID_IUIAutomationElement, $dtagIUIAutomationElement )
  If Not IsObj( $oDesktop ) Then Return ConsoleWrite( "$oDesktop ERR" & @CRLF )
  ConsoleWrite( "$oDesktop OK" & @CRLF )

  ; --- Notepad window ---

  ConsoleWrite( "---  window ---" & @CRLF )

  Local $pCondition
  Local $pCondition1
  Local $pCondition2
  Local $pElements

  $oUIAutomation.CreatePropertyCondition( $UIA_ControlTypePropertyId, $UIA_WindowControlTypeId, $pCondition )

  If Not $pCondition Then Return ConsoleWrite( "$pCondition ERR" & @CRLF )
  ConsoleWrite( "$pCondition OK" & @CRLF )
  ; --- $oUIAutomation methods ---

  ConsoleWrite( "--- $oUIAutomation methods ---" & @CRLF )

  Local $pCondition
  $oUIAutomation.CreateTrueCondition( $pCondition )
  ConsoleWrite( "$oUIAutomation.CreateTrueCondition()" & @CRLF )

  ; --- $oUIElement methods ---

  ConsoleWrite( "--- $oUIElement methods ---" & @CRLF )

  Local $pElements
  $oDesktop.FindAll( $TreeScope_Children, $pCondition, $pElements )
  ConsoleWrite( "$oDesktop.FindAll()" & @CRLF )

  ; --- Code Snippets ---

  ConsoleWrite( "--- Code Snippets ---" & @CRLF )

  Local $oUIElementArray1, $iLength1 ; $pElements is a pointer to an UI Automation element array
  $oUIElementArray1 = ObjCreateInterFace( $pElements, $sIID_IUIAutomationElementArray, $dtagIUIAutomationElementArray )
  $oUIElementArray1.Length( $iLength1 )
  If Not $iLength1 Then Return ConsoleWrite( "$iLength1 = 0 ERR" & @CRLF )
  ConsoleWrite( "$iLength1 = " & $iLength1 & @CRLF )

  ; --- Code Snippets ---

  ConsoleWrite( "--- Code Snippets ---" & @CRLF )

  Local $pElement1, $oElement1, $sValue1, $sValue2, $sValue3
  For $i = 0 To $iLength1 - 1
    $oUIElementArray1.GetElement( $i, $pElement1 )
    $oElement1 = ObjCreateInterface( $pElement1, $sIID_IUIAutomationElement, $dtagIUIAutomationElement )

	If Not IsObj( $oElement1 ) Then
	  ConsoleWrite( "$oElement1 ERR" & @CRLF )
      ContinueLoop
 	EndIf

    $oElement1.GetCurrentPropertyValue( $UIA_ControlTypePropertyId, $sValue3 ) ; $UIA_ClassNamePropertyId is used as example
	If $sValue3 = $UIA_PaneControlTypeId Then ContinueLoop

;  ConsoleWrite( "$oElement1 OK" & @CRLF )
    $oElement1.GetCurrentPropertyValue( $UIA_ClassNamePropertyId, $sValue1 ) ; $UIA_ClassNamePropertyId is used as example
;    ConsoleWrite( "$sValue1 = " & $sValue1 & @CRLF )
#comments-start

	Select
        Case $sValue1 = "Notepad"
        Case $sValue1 = "QQuickView"
        Case $sValue1 = "ApplicationFrameWindow"
;        Case $sValue1 = ""
            ConsoleWrite( "$sValue1 = " & $sValue1 & @CRLF )
		Case Else
			ContinueLoop
	EndSelect

	If $sValue1 = "Shell_TrayWnd" Then ContinueLoop
	If $sValue1 = "TrayNotifyWnd" Then ContinueLoop
	If $sValue1 = "TextBlock" Then ContinueLoop
	If $sValue1 = "Image" Then ContinueLoop
	If $sValue1 = "Button" Then ContinueLoop
	If $sValue1 = "ToggleButton" Then ContinueLoop
	If $sValue1 = "" Then ContinueLoop
	If $sValue1 = Null Then ContinueLoop

	If StringInStr($sValue1, "SystemTray.") Then ContinueLoop
	If StringInStr($sValue1, "Taskbar.") Then ContinueLoop
#comments-end

    ConsoleWrite( "$sValue1 = " & $sValue1 & @CRLF )

    $oElement1.GetCurrentPropertyValue( $UIA_NamePropertyId, $sValue2 )
    ConsoleWrite( "$sValue2 = " & StringLeft($sValue2, 30) & @CRLF )
	If $sValue2 = "" Then ContinueLoop
	If $sValue2 = Null Then ContinueLoop

	If $sValue2 = "Microsoft OneDrive" Then

  		$oUIAutomation.CreatePropertyCondition( $UIA_NamePropertyId, "了解しました", $pCondition )
  		If  $pCondition Then
 		 Local $pSave, $oSave
  			$oElement1.FindFirst( $TreeScope_Descendants, $pCondition, $pSave )
			$oSave = ObjCreateInterface( $pSave, $sIID_IUIAutomationElement, $dtagIUIAutomationElement )
  			If Not IsObj( $oSave ) Then
	  			ConsoleWrite( "$oSave ERR" & @CRLF )
      			ContinueLoop
  			EndIf
  			ConsoleWrite( "$oSave OK" & @CRLF )
  		EndIf

	Else

		ConsoleWrite( "$pCondition ERR" & @CRLF )

  ; --- Save As window ---

		ConsoleWrite( "--- Save As window ---" & @CRLF )

  ; Note that $UIA_ClassNamePropertyId maybe ia a CASE SENSITIVE condition
  		$oUIAutomation.CreatePropertyCondition( $UIA_ClassNamePropertyId, "Popup", $pCondition1 )
  		If Not $pCondition1 Then
			  ConsoleWrite( "$pCondition1 ERR" & @CRLF )
		      ContinueLoop
		EndIf
		ConsoleWrite( "$pCondition1 OK" & @CRLF )

		  $oUIAutomation.CreatePropertyCondition( $UIA_WindowIsTopmostPropertyId, true, $pCondition2 )
		  If Not $pCondition2 Then
			  ConsoleWrite( "$pCondition2 ERR" & @CRLF )
		      ContinueLoop
		  EndIf
		  ConsoleWrite( "$pCondition2 OK" & @CRLF )

		  ; And condition
		  $oUIAutomation.CreateAndCondition( $pCondition1, $pCondition2, $pCondition )
		  If Not $pCondition Then
			  ConsoleWrite( "$pCondition ERR" & @CRLF )
		      ContinueLoop
		  EndIf
		  ConsoleWrite( "$pCondition OK" & @CRLF )

		  Local $pSaveAsWin, $oSaveAsWin
		  $oElement1.FindFirst( $TreeScope_Descendants, $pCondition, $pSaveAsWin )
		  $oSaveAsWin = ObjCreateInterface( $pSaveAsWin, $sIID_IUIAutomationElement, $dtagIUIAutomationElement )
		  If Not IsObj( $oSaveAsWin ) Then
			  ConsoleWrite( "$oSaveAsWin ERR" & @CRLF )
		      ContinueLoop
		  EndIf
		  ConsoleWrite( "$oSaveAsWin OK" & @CRLF )


		  ConsoleWrite( "--- Save Button ---" & @CRLF )

		  $oUIAutomation.CreatePropertyCondition( $UIA_ControlTypePropertyId, $UIA_ButtonControlTypeId, $pCondition1 )
		  If Not $pCondition1 Then
			  ConsoleWrite( "$pCondition1 ERR" & @CRLF )
		      ContinueLoop
		  EndIf
		  ConsoleWrite( "$pCondition1 OK" & @CRLF )

		  $oUIAutomation.CreatePropertyCondition( $UIA_ClassNamePropertyId, "Button", $pCondition2 )
		  If Not $pCondition2 Then
			  ConsoleWrite( "$pCondition2 ERR" & @CRLF )
		      ContinueLoop
		  EndIf
		  ConsoleWrite( "$pCondition2 OK" & @CRLF )

		  ; And condition
		  $oUIAutomation.CreateAndCondition( $pCondition1, $pCondition2, $pCondition )
		  If Not $pCondition Then
			  ConsoleWrite( "$pCondition ERR" & @CRLF )
		      ContinueLoop
		  EndIf
		  ConsoleWrite( "$pCondition OK" & @CRLF )

		  Local $pSave, $oSave
		  $oSaveAsWin.FindFirst( $TreeScope_Descendants, $pCondition, $pSave )
		  $oSave = ObjCreateInterface( $pSave, $sIID_IUIAutomationElement, $dtagIUIAutomationElement )
		  If Not IsObj( $oSave ) Then
			  ConsoleWrite( "$oSave ERR" & @CRLF )
		      ContinueLoop
		  EndIf
		  ConsoleWrite( "$oSave OK" & @CRLF )
	EndIf

		$oSave.SetFocus()
		Local $vValue, $sValue
		Local $fScale = 0
		; Rectangle
		Local $aRect ; l, t, w, h
		$oSave.GetCurrentPropertyValue( $UIA_BoundingRectanglePropertyId, $aRect )
		If Not IsArray( $aRect ) Then
		  ConsoleWrite( "$aRect ERR" & @CRLF )
	      ContinueLoop
	    EndIf
	    ConsoleWrite( "$aRect OK" & @CRLF )

		If $fScale > 1.00 Then
			$aRect[0] = Round( $aRect[0] * $fScale )
			$aRect[1] = Round( $aRect[1] * $fScale )
			$aRect[2] = Round( $aRect[2] * $fScale )
			$aRect[3] = Round( $aRect[3] * $fScale )
		EndIf
		$sValue = ""
		If IsArray( $aRect ) And UBound( $aRect ) Then
				$sValue = $aRect[0]
				For $j = 1 To UBound( $aRect ) - 1
					$sValue &= "," & $aRect[$j]
				Next
		EndIf
		ConsoleWrite( $sValue & @CRLF )

		MouseMove( $aRect[0]+$aRect[2]/2, $aRect[1]+$aRect[3]/2 )

		Sleep(10000)
  Next

EndFunc

PropertyId 例

よく使うもの$UIA_ClassNamePropertyId $UIA_NamePropertyId $UIA_BoundingRectanglePropertyId 

Treeview Element                                    Window: Microsoft OneDrive
                                                    
Element Properties (identification)                 
$UIA_AccessKeyPropertyId                            Microsoft OneDrive
$UIA_ClassNamePropertyId                            QQuickView
$UIA_ControlTypePropertyId                          $UIA_WindowControlTypeId
$UIA_NamePropertyId                                 Microsoft OneDrive
                                                    
Element Properties (session unique)                 
$UIA_NativeWindowHandlePropertyId                   0x000000000020045C
$UIA_ProcessIdPropertyId                            19348
$UIA_RuntimeIdPropertyId                            42,2098268
                                                    
Element Properties (information)                    
$UIA_BoundingRectanglePropertyId                    l=512,t=288,w=576,h=277
$UIA_FrameworkIdPropertyId                          Qt
$UIA_HelpTextPropertyId                             Microsoft OneDrive
$UIA_LocalizedControlTypePropertyId                 ウィンドウ
$UIA_ProviderDescriptionPropertyId                  [pid:37040,providerId:0x20045C Main:Nested [pid:19348,providerId:0x20045C Main(parent link):Unidentified Provider (unmanaged:qwindows.dll)]; Nonclient:Microsoft: Non-Client Proxy (unmanaged:uiautomationcore.dll); Hwnd(parent link):Microsoft: HWND Proxy (unmanaged:uiautomationcore.dll)]
                                                    
Element Properties (has/is info)                    
$UIA_HasKeyboardFocusPropertyId                     False
$UIA_IsContentElementPropertyId                     True
$UIA_IsControlElementPropertyId                     True
$UIA_IsDataValidForFormPropertyId                   False
$UIA_IsEnabledPropertyId                            True
$UIA_IsKeyboardFocusablePropertyId                  True
$UIA_IsOffscreenPropertyId                          False
$UIA_IsPasswordPropertyId                           False
$UIA_IsRequiredForFormPropertyId                    False
                                                    
Control Patterns (element actions)                  
$UIA_IsLegacyIAccessiblePatternAvailablePropertyId  True (LegacyIAccessiblePattern)
$UIA_IsTransformPatternAvailablePropertyId          True (TransformPattern)
$UIA_IsValuePatternAvailablePropertyId              True (ValuePattern)
$UIA_IsWindowPatternAvailablePropertyId             True (WindowPattern)
                                                    
Control Pattern Properties                          
$UIA_LegacyIAccessibleChildIdPropertyId             0
$UIA_LegacyIAccessibleDefaultActionPropertyId       
$UIA_LegacyIAccessibleDescriptionPropertyId         
$UIA_LegacyIAccessibleHelpPropertyId                Microsoft OneDrive
$UIA_LegacyIAccessibleKeyboardShortcutPropertyId    Microsoft OneDrive
$UIA_LegacyIAccessibleNamePropertyId                Microsoft OneDrive
$UIA_LegacyIAccessibleRolePropertyId                9 = $ROLE_SYSTEM_WINDOW
$UIA_LegacyIAccessibleStatePropertyId               1048576 = $STATE_SYSTEM_FOCUSABLE
$UIA_LegacyIAccessibleValuePropertyId               Microsoft OneDrive
$UIA_TransformCanMovePropertyId                     True
$UIA_TransformCanResizePropertyId                   True
$UIA_TransformCanRotatePropertyId                   False
$UIA_ValueIsReadOnlyPropertyId                      False
$UIA_ValueValuePropertyId                           Microsoft OneDrive
$UIA_WindowCanMaximizePropertyId                    False
$UIA_WindowCanMinimizePropertyId                    False
$UIA_WindowIsModalPropertyId                        False
$UIA_WindowIsTopmostPropertyId                      False
$UIA_WindowWindowInteractionStatePropertyId         $WindowInteractionState_Running
$UIA_WindowWindowVisualStatePropertyId              $WindowVisualState_Normal
                                                    
Control Pattern Methods                             
LegacyIAccessible Pattern Methods                   
                                                    DoDefaultAction()
                                                    Select(long)
                                                    SetValue(wstr)
                                                    GetIAccessible(idispatch*)
                                                    CurrentChildId(int*)
                                                    CurrentDefaultAction(bstr*)
                                                    CurrentDescription(bstr*)
                                                    CurrentHelp(bstr*)
                                                    CurrentKeyboardShortcut(bstr*)
                                                    CurrentName(bstr*)
                                                    CurrentRole(uint*)
                                                    CurrentState(uint*)
                                                    CurrentValue(bstr*)
                                                    GetCurrentSelection(ptr*)
Transform Pattern Methods                           
                                                    Move($fXPos,$fYPos)
                                                    Resize($fWidth,$fHeight)
                                                    Rotate($fDegrees)
                                                    CurrentCanMove($bCanMove*)
                                                    CurrentCanResize($bCanResize*)
                                                    CurrentCanRotate($bCanRotate*)
Value Pattern Methods                               
                                                    SetValue($sValue)
                                                    CurrentValue($sValue*)
                                                    CurrentIsReadOnly($bIsReadOnly*)
Window Pattern Methods                              
                                                    Close()
                                                    SetWindowVisualState($iWindowVisualState)
                                                    WaitForInputIdle($iMilliSeconds,$bSuccess*)
                                                    CurrentCanMaximize($bCanMaximize*)
                                                    CurrentCanMinimize($bCanMinimize*)
                                                    CurrentIsModal($bIsModal*)
                                                    CurrentIsTopmost($bIsTopmost*)
                                                    CurrentWindowVisualState($iWindowVisualState*)
                                                    CurrentWindowInteractionState($iWindowInteractionState*)
                                                    
Parents from Desktop                                Pane: デスクトップ 1
                                                    
Parent to child index

参考

$oDesktop.FindAll $oElement1.FindFirstなどで指定するものです

TreeScope (uiautomationclient.h) – Win32 apps | Microsoft Learn

 
TreeScope_None
値: 0
スコープは、検索からサブツリーを除外します。
TreeScope_Element
値: 0x1
スコープには、要素自体が含まれます。
TreeScope_Children
値: 0x2
スコープには、 要素の子が含まれます。
TreeScope_Descendants
値: 0x4
スコープには、 要素の子とより遠い子孫が含まれます。
TreeScope_Parent
値: 0x8
スコープには、 要素の親が含まれます。
TreeScope_Ancestors
値: 0x10
スコープには、 要素の親およびより遠い先祖が含まれます。
TreeScope_Subtree
スコープには、 要素とそのすべての子孫が含まれます。 このフラグは、TreeScope_Element値とTreeScope_Descendants値の組み合わせです。

AutoItとUI Automation UDFを使ってポップアップウィンドウのボタンにマウスカーソルを移動させる 複数ボタン対応版

AutoIt

Posted by eightban