feat(ui-query): add query-state.py for element state inspection
Detailed state querying for UI elements: - Find focused element with --focused - Show all state flags with descriptions (--all-states) - Reports actions, value ranges, text content, caret position - Human-readable state descriptions Completes ui-query skill (list-windows, get-text, find-element, query-state) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
dan
parent
6ed762ad05
commit
205e8b0dfa
314
skills/ui-query/scripts/query-state.py
Executable file
314
skills/ui-query/scripts/query-state.py
Executable file
|
|
@ -0,0 +1,314 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Query detailed state of UI elements via AT-SPI.
|
||||
|
||||
Usage:
|
||||
query-state.py --role button --name "OK"
|
||||
query-state.py --window "Firefox" --focused
|
||||
query-state.py --all-states
|
||||
|
||||
Options:
|
||||
--role ROLE Filter by role
|
||||
--name PATTERN Filter by name
|
||||
--window PATTERN Limit to matching window
|
||||
--focused Only show focused element
|
||||
--all-states Show all state flags (verbose)
|
||||
--json Output as JSON
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
|
||||
import pyatspi
|
||||
|
||||
# Human-readable state descriptions
|
||||
STATE_DESCRIPTIONS = {
|
||||
"active": "Currently active/foreground",
|
||||
"armed": "Ready to be activated",
|
||||
"busy": "Processing/loading",
|
||||
"checked": "Checkbox/toggle is checked",
|
||||
"collapsed": "Collapsed (can be expanded)",
|
||||
"defunct": "Object no longer valid",
|
||||
"editable": "Text can be edited",
|
||||
"enabled": "Interactive (not grayed out)",
|
||||
"expandable": "Can be expanded",
|
||||
"expanded": "Currently expanded",
|
||||
"focusable": "Can receive keyboard focus",
|
||||
"focused": "Has keyboard focus",
|
||||
"has-tooltip": "Has tooltip available",
|
||||
"horizontal": "Horizontal orientation",
|
||||
"iconified": "Minimized to icon",
|
||||
"indeterminate": "Neither checked nor unchecked",
|
||||
"invalid-entry": "Contains invalid data",
|
||||
"is-default": "Default action element",
|
||||
"modal": "Modal dialog",
|
||||
"multi-line": "Multi-line text",
|
||||
"multiselectable": "Multiple selection allowed",
|
||||
"opaque": "Fully opaque (not transparent)",
|
||||
"pressed": "Button is pressed",
|
||||
"resizable": "Can be resized",
|
||||
"selectable": "Can be selected",
|
||||
"selected": "Currently selected",
|
||||
"sensitive": "Responds to user input",
|
||||
"showing": "Visible on screen",
|
||||
"single-line": "Single-line text",
|
||||
"stale": "Data may be outdated",
|
||||
"transient": "Temporary element",
|
||||
"vertical": "Vertical orientation",
|
||||
"visible": "Rendered (may be scrolled off)",
|
||||
"visited": "Link has been visited",
|
||||
}
|
||||
|
||||
|
||||
def get_all_states(accessible):
|
||||
"""Get all states for an accessible."""
|
||||
state = accessible.getState()
|
||||
states = {}
|
||||
for s in pyatspi.STATE_VALUE_TO_NAME:
|
||||
name = pyatspi.STATE_VALUE_TO_NAME[s]
|
||||
if state.contains(s):
|
||||
states[name] = True
|
||||
return states
|
||||
|
||||
|
||||
def get_element_details(element, all_states=False):
|
||||
"""Get detailed info about an element."""
|
||||
info = {
|
||||
"role": element.getRoleName(),
|
||||
"name": element.name or "",
|
||||
}
|
||||
|
||||
if element.description:
|
||||
info["description"] = element.description
|
||||
|
||||
# States
|
||||
states = get_all_states(element)
|
||||
if all_states:
|
||||
info["states"] = states
|
||||
else:
|
||||
# Just key interactive states
|
||||
key_states = {k: v for k, v in states.items()
|
||||
if k in ("focused", "selected", "checked", "expanded",
|
||||
"enabled", "editable", "pressed", "active",
|
||||
"busy", "invalid-entry", "indeterminate")}
|
||||
info["states"] = key_states
|
||||
|
||||
# State summary for quick reading
|
||||
active_states = [k for k, v in states.items() if v]
|
||||
info["state_summary"] = ", ".join(sorted(active_states))
|
||||
|
||||
# Geometry
|
||||
try:
|
||||
component = element.queryComponent()
|
||||
if component:
|
||||
rect = component.getExtents(pyatspi.DESKTOP_COORDS)
|
||||
info["geometry"] = {
|
||||
"x": rect.x, "y": rect.y,
|
||||
"width": rect.width, "height": rect.height,
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Text content
|
||||
try:
|
||||
text_iface = element.queryText()
|
||||
if text_iface and text_iface.characterCount > 0:
|
||||
info["text"] = text_iface.getText(0, min(50, text_iface.characterCount))
|
||||
info["text_length"] = text_iface.characterCount
|
||||
# Caret position if editable
|
||||
if states.get("editable"):
|
||||
info["caret_offset"] = text_iface.caretOffset
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Value (for sliders, progress bars, etc.)
|
||||
try:
|
||||
value_iface = element.queryValue()
|
||||
if value_iface:
|
||||
info["value"] = {
|
||||
"current": value_iface.currentValue,
|
||||
"min": value_iface.minimumValue,
|
||||
"max": value_iface.maximumValue,
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Actions available
|
||||
try:
|
||||
action_iface = element.queryAction()
|
||||
if action_iface:
|
||||
actions = []
|
||||
for i in range(action_iface.nActions):
|
||||
actions.append(action_iface.getName(i))
|
||||
if actions:
|
||||
info["actions"] = actions
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def find_focused_element(accessible, depth=0, max_depth=20):
|
||||
"""Find the focused element in the tree."""
|
||||
if depth > max_depth:
|
||||
return None
|
||||
|
||||
try:
|
||||
states = get_all_states(accessible)
|
||||
if states.get("focused"):
|
||||
return accessible
|
||||
|
||||
for i in range(accessible.childCount):
|
||||
child = accessible.getChildAtIndex(i)
|
||||
if child:
|
||||
result = find_focused_element(child, depth + 1, max_depth)
|
||||
if result:
|
||||
return result
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def find_elements(accessible, role=None, name=None, results=None, max_depth=15, depth=0, limit=10):
|
||||
"""Find matching elements."""
|
||||
if results is None:
|
||||
results = []
|
||||
|
||||
if len(results) >= limit or depth > max_depth:
|
||||
return results
|
||||
|
||||
try:
|
||||
matches = True
|
||||
|
||||
if role:
|
||||
elem_role = accessible.getRoleName().lower().replace(" ", "-")
|
||||
if role.lower() not in elem_role:
|
||||
matches = False
|
||||
|
||||
if name and matches:
|
||||
elem_name = (accessible.name or "").lower()
|
||||
if name.lower() not in elem_name:
|
||||
matches = False
|
||||
|
||||
if matches and (role or name):
|
||||
results.append(accessible)
|
||||
|
||||
for i in range(accessible.childCount):
|
||||
if len(results) >= limit:
|
||||
break
|
||||
child = accessible.getChildAtIndex(i)
|
||||
if child:
|
||||
find_elements(child, role, name, results, max_depth, depth + 1, limit)
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def print_element(info, all_states=False):
|
||||
"""Print element info in readable format."""
|
||||
print(f"\n[{info['role']}] {info['name'] or '(unnamed)'}")
|
||||
print("-" * 40)
|
||||
|
||||
if info.get("description"):
|
||||
print(f"Description: {info['description']}")
|
||||
|
||||
if info.get("text"):
|
||||
text_preview = info['text'][:50] + "..." if len(info.get('text', '')) > 50 else info['text']
|
||||
print(f"Text: {text_preview}")
|
||||
if info.get("text_length", 0) > 50:
|
||||
print(f" ({info['text_length']} chars total)")
|
||||
if "caret_offset" in info:
|
||||
print(f"Caret: position {info['caret_offset']}")
|
||||
|
||||
if info.get("value"):
|
||||
v = info["value"]
|
||||
print(f"Value: {v['current']} (range: {v['min']} - {v['max']})")
|
||||
|
||||
geo = info.get("geometry", {})
|
||||
if geo:
|
||||
print(f"Position: ({geo['x']}, {geo['y']}) Size: {geo['width']}x{geo['height']}")
|
||||
|
||||
if info.get("actions"):
|
||||
print(f"Actions: {', '.join(info['actions'])}")
|
||||
|
||||
print(f"\nStates: {info.get('state_summary', 'none')}")
|
||||
|
||||
if all_states and info.get("states"):
|
||||
print("\nState Details:")
|
||||
for state, active in sorted(info["states"].items()):
|
||||
if active:
|
||||
desc = STATE_DESCRIPTIONS.get(state, "")
|
||||
print(f" ✓ {state}: {desc}" if desc else f" ✓ {state}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Query UI element states via AT-SPI")
|
||||
parser.add_argument("--role", "-r", help="Filter by role")
|
||||
parser.add_argument("--name", "-n", help="Filter by name")
|
||||
parser.add_argument("--window", "-w", help="Limit to window")
|
||||
parser.add_argument("--focused", "-f", action="store_true", help="Show focused element only")
|
||||
parser.add_argument("--all-states", "-a", action="store_true", help="Show all state flags")
|
||||
parser.add_argument("--json", action="store_true", help="Output as JSON")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.role and not args.name and not args.focused:
|
||||
parser.print_help()
|
||||
print("\nExamples:")
|
||||
print(" query-state.py --focused # Current focused element")
|
||||
print(" query-state.py --role checkbox # All checkboxes with states")
|
||||
print(" query-state.py --name 'Save' -a # 'Save' elements, all states")
|
||||
sys.exit(1)
|
||||
|
||||
desktop = pyatspi.Registry.getDesktop(0)
|
||||
|
||||
# Build list of windows to search
|
||||
windows = []
|
||||
for i in range(desktop.childCount):
|
||||
app = desktop.getChildAtIndex(i)
|
||||
if not app:
|
||||
continue
|
||||
for j in range(app.childCount):
|
||||
win = app.getChildAtIndex(j)
|
||||
if not win:
|
||||
continue
|
||||
if args.window and args.window.lower() not in (win.name or "").lower():
|
||||
continue
|
||||
windows.append(win)
|
||||
|
||||
if not windows:
|
||||
print("No matching windows found.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
results = []
|
||||
|
||||
if args.focused:
|
||||
# Find focused element
|
||||
for win in windows:
|
||||
focused = find_focused_element(win)
|
||||
if focused:
|
||||
results.append(get_element_details(focused, args.all_states))
|
||||
break
|
||||
if not results:
|
||||
print("No focused element found.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
else:
|
||||
# Find by role/name
|
||||
for win in windows:
|
||||
elements = find_elements(win, args.role, args.name, limit=10 - len(results))
|
||||
for elem in elements:
|
||||
results.append(get_element_details(elem, args.all_states))
|
||||
if len(results) >= 10:
|
||||
break
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(results, indent=2))
|
||||
else:
|
||||
for info in results:
|
||||
print_element(info, args.all_states)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -18,6 +18,9 @@ case "$CMD" in
|
|||
find-element|find|search)
|
||||
SCRIPT="find-element.py"
|
||||
;;
|
||||
query-state|state)
|
||||
SCRIPT="query-state.py"
|
||||
;;
|
||||
*)
|
||||
echo "Usage: ui-query <command> [options]"
|
||||
echo ""
|
||||
|
|
@ -25,6 +28,7 @@ case "$CMD" in
|
|||
echo " list-windows List all AT-SPI accessible windows"
|
||||
echo " get-text Extract text content from a window"
|
||||
echo " find-element Find elements by role or name"
|
||||
echo " query-state Query element states in detail"
|
||||
echo ""
|
||||
echo "Options are passed through to the underlying script."
|
||||
exit 1
|
||||
|
|
|
|||
Loading…
Reference in a new issue