From d0ba4b6702b825d68b159e1f6407881cab36e8e6 Mon Sep 17 00:00:00 2001 From: Matt Portune <59324545+mportune-bw@users.noreply.github.com> Date: Mon, 13 Jan 2020 17:14:57 -0500 Subject: [PATCH] Accessibility overlay support for username field and scroll tracking (#700) * Trigger overlay prompt when focusing on username field * Adjust accessibility overlay position in response to scroll events * Get username EditText with a single pass of the node tree, plus additional cleanup --- .../Accessibility/AccessibilityHelpers.cs | 92 ++++++++++-- .../Accessibility/AccessibilityService.cs | 132 +++++++++++------- .../Resources/xml/accessibilityservice.xml | 2 +- 3 files changed, 165 insertions(+), 61 deletions(-) diff --git a/src/Android/Accessibility/AccessibilityHelpers.cs b/src/Android/Accessibility/AccessibilityHelpers.cs index 35687bf45..92041674f 100644 --- a/src/Android/Accessibility/AccessibilityHelpers.cs +++ b/src/Android/Accessibility/AccessibilityHelpers.cs @@ -257,15 +257,48 @@ namespace Bit.Droid.Accessibility IEnumerable passwordNodes) { var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false); - var usernameEditText = GetUsernameEditText(allEditTexts); + var usernameEditText = GetUsernameEditTextIfPasswordExists(allEditTexts); FillCredentials(usernameEditText, passwordNodes); allEditTexts.Dispose(); usernameEditText = null; } - public static AccessibilityNodeInfo GetUsernameEditText(IEnumerable allEditTexts) + public static AccessibilityNodeInfo GetUsernameEditTextIfPasswordExists( + IEnumerable allEditTexts) { - return allEditTexts.TakeWhile(n => !n.Password).LastOrDefault(); + AccessibilityNodeInfo previousEditText = null; + foreach(var editText in allEditTexts) + { + if(editText.Password) + { + return previousEditText; + } + previousEditText = editText; + } + return null; + } + + public static bool IsUsernameEditText(AccessibilityNodeInfo root, AccessibilityEvent e) + { + var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false); + var usernameEditText = GetUsernameEditTextIfPasswordExists(allEditTexts); + if(usernameEditText != null) + { + var isUsernameEditText = IsSameNode(usernameEditText, e.Source); + allEditTexts.Dispose(); + usernameEditText = null; + return isUsernameEditText; + } + return false; + } + + public static bool IsSameNode(AccessibilityNodeInfo info1, AccessibilityNodeInfo info2) + { + if(info1 != null && info2 != null) + { + return info1.Equals(info2) || info1.GetHashCode() == info2.GetHashCode(); + } + return false; } public static bool OverlayPermitted() @@ -294,20 +327,59 @@ namespace Bit.Droid.Accessibility return view; } - public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo root, AccessibilityEvent e) + public static WindowManagerLayoutParams GetOverlayLayoutParams() + { + WindowManagerTypes windowManagerType; + if(Build.VERSION.SdkInt >= BuildVersionCodes.O) + { + windowManagerType = WindowManagerTypes.ApplicationOverlay; + } + else + { + windowManagerType = WindowManagerTypes.Phone; + } + + var layoutParams = new WindowManagerLayoutParams( + ViewGroup.LayoutParams.WrapContent, + ViewGroup.LayoutParams.WrapContent, + windowManagerType, + WindowManagerFlags.NotFocusable | WindowManagerFlags.NotTouchModal, + Format.Transparent); + layoutParams.Gravity = GravityFlags.Bottom | GravityFlags.Left; + + return layoutParams; + } + + public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo root, AccessibilityNodeInfo anchorView) { var rootRect = new Rect(); root.GetBoundsInScreen(rootRect); var rootRectHeight = rootRect.Height(); - var eSrcRect = new Rect(); - e.Source.GetBoundsInScreen(eSrcRect); - var eSrcRectLeft = eSrcRect.Left; - var eSrcRectTop = eSrcRect.Top; + var anchorViewRect = new Rect(); + anchorView.GetBoundsInScreen(anchorViewRect); + var anchorViewRectLeft = anchorViewRect.Left; + var anchorViewRectTop = anchorViewRect.Top; var navBarHeight = GetNavigationBarHeight(); - var calculatedTop = rootRectHeight - eSrcRectTop - navBarHeight; - return new Point(eSrcRectLeft, calculatedTop); + var calculatedTop = rootRectHeight - anchorViewRectTop - navBarHeight; + return new Point(anchorViewRectLeft, calculatedTop); + } + + public static Point GetOverlayAnchorPosition(int nodeHash, AccessibilityNodeInfo root, AccessibilityEvent e) + { + Point point = null; + var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false); + foreach(var node in allEditTexts) + { + if(node.GetHashCode() == nodeHash) + { + point = GetOverlayAnchorPosition(root, node); + break; + } + } + allEditTexts.Dispose(); + return point; } private static int GetStatusBarHeight() diff --git a/src/Android/Accessibility/AccessibilityService.cs b/src/Android/Accessibility/AccessibilityService.cs index d991c497e..c9f591ad6 100644 --- a/src/Android/Accessibility/AccessibilityService.cs +++ b/src/Android/Accessibility/AccessibilityService.cs @@ -35,6 +35,8 @@ namespace Bit.Droid.Accessibility private IWindowManager _windowManager = null; private LinearLayout _overlayView = null; + private int _anchorViewHash = 0; + private int _lastAnchorX, _lastAnchorY = 0; public override void OnAccessibilityEvent(AccessibilityEvent e) { @@ -68,33 +70,46 @@ namespace Bit.Droid.Accessibility { case EventTypes.ViewFocused: case EventTypes.ViewClicked: + case EventTypes.ViewScrolled: var isKnownBroswer = AccessibilityHelpers.SupportedBrowsers.ContainsKey(root.PackageName); if(e.EventType == EventTypes.ViewClicked && isKnownBroswer) { break; } - if(e.Source == null || !e.Source.Password) + if(e.Source == null || e.PackageName == BitwardenPackage) { CancelOverlayPrompt(); break; } - if(e.PackageName == BitwardenPackage) + if(e.EventType == EventTypes.ViewScrolled) { - CancelOverlayPrompt(); + AdjustOverlayForScroll(root, e); break; } - if(ScanAndAutofill(root, e)) - { - CancelOverlayPrompt(); - } else { - OverlayPromptToAutofill(root, e); + var isUsernameEditText1 = AccessibilityHelpers.IsUsernameEditText(root, e); + var isPasswordEditText1 = e.Source?.Password ?? false; + if(!isUsernameEditText1 && !isPasswordEditText1) + { + CancelOverlayPrompt(); + break; + } + if(ScanAndAutofill(root, e)) + { + CancelOverlayPrompt(); + } + else + { + OverlayPromptToAutofill(root, e); + } } break; case EventTypes.WindowContentChanged: case EventTypes.WindowStateChanged: - if(e.Source == null || e.Source.Password) + var isUsernameEditText2 = AccessibilityHelpers.IsUsernameEditText(root, e); + var isPasswordEditText2 = e.Source?.Password ?? false; + if(e.Source == null || isUsernameEditText2 || isPasswordEditText2) { break; } @@ -188,7 +203,10 @@ namespace Bit.Droid.Accessibility System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Removed"); _overlayView = null; + _anchorViewHash = 0; _lastNotificationUri = null; + _lastAnchorX = 0; + _lastAnchorY = 0; } private void OverlayPromptToAutofill(AccessibilityNodeInfo root, AccessibilityEvent e) @@ -206,64 +224,78 @@ namespace Bit.Droid.Accessibility return; } - WindowManagerTypes windowManagerType; - if(Build.VERSION.SdkInt >= BuildVersionCodes.O) - { - windowManagerType = WindowManagerTypes.ApplicationOverlay; - } - else - { - windowManagerType = WindowManagerTypes.Phone; - } - - var layoutParams = new WindowManagerLayoutParams( - ViewGroup.LayoutParams.WrapContent, - ViewGroup.LayoutParams.WrapContent, - windowManagerType, - WindowManagerFlags.NotFocusable | WindowManagerFlags.NotTouchModal, - Format.Transparent); - - var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(root, e); - - layoutParams.Gravity = GravityFlags.Bottom | GravityFlags.Left; + var layoutParams = AccessibilityHelpers.GetOverlayLayoutParams(); + var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(root, e.Source); layoutParams.X = anchorPosition.X; layoutParams.Y = anchorPosition.Y; - var intent = new Intent(this, typeof(AccessibilityActivity)); - intent.PutExtra("uri", uri); - intent.SetFlags(ActivityFlags.NewTask | ActivityFlags.SingleTop | ActivityFlags.ClearTop); - if(_windowManager == null) { _windowManager = GetSystemService(WindowService).JavaCast(); } - var updateView = false; - if(_overlayView != null) + if(_overlayView == null) { - updateView = true; - } + var intent = new Intent(this, typeof(AccessibilityActivity)); + intent.PutExtra("uri", uri); + intent.SetFlags(ActivityFlags.NewTask | ActivityFlags.SingleTop | ActivityFlags.ClearTop); - _overlayView = AccessibilityHelpers.GetOverlayView(this); - _overlayView.Click += (sender, eventArgs) => - { - CancelOverlayPrompt(); - StartActivity(intent); - }; + _overlayView = AccessibilityHelpers.GetOverlayView(this); + _overlayView.Click += (sender, eventArgs) => + { + CancelOverlayPrompt(); + StartActivity(intent); + }; - _lastNotificationUri = uri; + _lastNotificationUri = uri; - if(updateView) - { - _windowManager.UpdateViewLayout(_overlayView, layoutParams); + _windowManager.AddView(_overlayView, layoutParams); + + System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Added at X:{0} Y:{1}", + layoutParams.X, layoutParams.Y); } else { - _windowManager.AddView(_overlayView, layoutParams); + _windowManager.UpdateViewLayout(_overlayView, layoutParams); + + System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Updated to X:{0} Y:{1}", + layoutParams.X, layoutParams.Y); } - System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View {0} X:{1} Y:{2}", - updateView ? "Updated to" : "Added at", layoutParams.X, layoutParams.Y); + _anchorViewHash = e.Source.GetHashCode(); + _lastAnchorX = anchorPosition.X; + _lastAnchorY = anchorPosition.Y; + } + + private void AdjustOverlayForScroll(AccessibilityNodeInfo root, AccessibilityEvent e) + { + if(_overlayView == null || _anchorViewHash <= 0) + { + return; + } + + var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(_anchorViewHash, root, e); + if(anchorPosition == null) + { + return; + } + + if(anchorPosition.X == _lastAnchorX && anchorPosition.Y == _lastAnchorY) + { + return; + } + + var layoutParams = AccessibilityHelpers.GetOverlayLayoutParams(); + layoutParams.X = anchorPosition.X; + layoutParams.Y = anchorPosition.Y; + + _windowManager.UpdateViewLayout(_overlayView, layoutParams); + + _lastAnchorX = anchorPosition.X; + _lastAnchorY = anchorPosition.Y; + + System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Updated to X:{0} Y:{1}", + layoutParams.X, layoutParams.Y); } private bool SkipPackage(string eventPackageName) diff --git a/src/Android/Resources/xml/accessibilityservice.xml b/src/Android/Resources/xml/accessibilityservice.xml index 5b1e70019..674759ad6 100644 --- a/src/Android/Resources/xml/accessibilityservice.xml +++ b/src/Android/Resources/xml/accessibilityservice.xml @@ -2,7 +2,7 @@