vaultwarden/Bitwarden-app-android-iphone-ios
1
0
Fork 0
mirror of https://github.com/bitwarden/mobile synced 2025-01-29 17:59:23 +01:00
Code Issues 2 Releases Wiki Activity

Accessibility fixes (#709)

Browse Source 
* Show/hide accessibility overlay on scroll based on several visibility factors

* Improvements to accessibility overlay anchor view tracking

* Increase recursion limit and check for null children when walking the node tree

* Cleanup

* Hide overlay when expanding status (notification) bar

* use .Any() instead of .Count()
This commit is contained in:
Matt Portune 2020-01-27 17:36:20 -05:00 committed by Kyle Spearrin
parent c2e34a8b0e
commit 34e32403b0
3 changed files with 261 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
src/Android/Accessibility/AccessibilityHelpers.cs
View File

@ -120,7 +120,7 @@ namespace Bit.Droid.Accessibility
if(addressNode != null)
{
uri = ExtractUri(uri, addressNode, browser);
addressNode.Dispose();
addressNode.Recycle();
}
else
{
@ -217,7 +217,7 @@ namespace Bit.Droid.Accessibility
nodes = new NodeList();
}
var dispose = disposeIfUnused;
if(n != null && recursionDepth < 50)
if(n != null && recursionDepth < 100)
{
var add = n.WindowId == e.WindowId &&
!(n.ViewIdResourceName?.StartsWith(SystemUiPackage) ?? false) &&
@ -231,7 +231,11 @@ namespace Bit.Droid.Accessibility
for(var i = 0; i < n.ChildCount; i++)
{
var childNode = n.GetChild(i);
if(i > 100)
if(childNode == null)
{
continue;
}
else if(i > 100)
{
Android.Util.Log.Info(BitwardenTag, "Too many child iterations.");
break;
@ -248,7 +252,7 @@ namespace Bit.Droid.Accessibility
}
if(dispose)
{
n?.Dispose();
n?.Recycle();
}
return nodes;
}
@ -282,21 +286,23 @@ namespace Bit.Droid.Accessibility
{
var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false);
var usernameEditText = GetUsernameEditTextIfPasswordExists(allEditTexts);
var isUsernameEditText = false;
if(usernameEditText != null)
{
var isUsernameEditText = IsSameNode(usernameEditText, e.Source);
allEditTexts.Dispose();
usernameEditText = null;
return isUsernameEditText;
{
isUsernameEditText = IsSameNode(usernameEditText, e.Source);
usernameEditText.Recycle();
}
return false;
allEditTexts.Dispose();
return isUsernameEditText;
}
public static bool IsSameNode(AccessibilityNodeInfo info1, AccessibilityNodeInfo info2)
public static bool IsSameNode(AccessibilityNodeInfo node1, AccessibilityNodeInfo node2)
{
if(info1 != null && info2 != null)
if(node1 != null && node2 != null)
{
return info1.Equals(info2) || info1.GetHashCode() == info2.GetHashCode();
return node1.Equals(node2) || node1.GetHashCode() == node2.GetHashCode();
}
return false;
}
@ -350,38 +356,117 @@ namespace Bit.Droid.Accessibility
return layoutParams;
}
public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo root, AccessibilityNodeInfo anchorView)
public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo root, AccessibilityNodeInfo anchorView,
int rootRectHeight = 0)
{
var rootRect = new Rect();
root.GetBoundsInScreen(rootRect);
var rootRectHeight = rootRect.Height();
if(rootRectHeight == 0)
{
rootRectHeight = GetNodeHeight(root);
}
var anchorViewRect = new Rect();
anchorView.GetBoundsInScreen(anchorViewRect);
var anchorViewRectLeft = anchorViewRect.Left;
var anchorViewRectTop = anchorViewRect.Top;
anchorViewRect.Dispose();
var navBarHeight = GetNavigationBarHeight();
var calculatedTop = rootRectHeight - anchorViewRectTop - navBarHeight;
var calculatedTop = rootRectHeight - anchorViewRectTop - GetNavigationBarHeight();
return new Point(anchorViewRectLeft, calculatedTop);
}
public static Point GetOverlayAnchorPosition(int nodeHash, AccessibilityNodeInfo root, AccessibilityEvent e)
public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo anchorNode, AccessibilityNodeInfo root,
IEnumerable<AccessibilityWindowInfo> windows)
{
Point point = null;
var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false);
foreach(var node in allEditTexts)
if(anchorNode != null)
{
if(node.GetHashCode() == nodeHash)
anchorNode.Refresh(); // update node's info since this is still a reference from an older event
if(!anchorNode.VisibleToUser)
{
point = GetOverlayAnchorPosition(root, node);
break;
return new Point(-1, -1);
}
// node.VisibleToUser doesn't always give us exactly what we want, so attempt to tighten up the range
// of visibility
var rootNodeHeight = GetNodeHeight(root);
var limitLowY = 0;
var limitHighY = rootNodeHeight - GetNodeHeight(anchorNode);
if(windows != null)
{
if(IsStatusBarExpanded(windows))
{
return new Point(-1, -1);
}
Rect inputWindowRect = GetInputMethodWindowRect(windows);
if(inputWindowRect != null)
{
limitLowY += inputWindowRect.Height();
if(Build.VERSION.SdkInt >= BuildVersionCodes.Q)
{
limitLowY += GetNavigationBarHeight() + GetStatusBarHeight();
}
inputWindowRect.Dispose();
}
}
point = GetOverlayAnchorPosition(root, anchorNode, rootNodeHeight);
if(point.Y < limitLowY || point.Y > limitHighY)
{
point.X = -1;
point.Y = -1;
}
}
allEditTexts.Dispose();
return point;
}
public static bool IsStatusBarExpanded(IEnumerable<AccessibilityWindowInfo> windows)
{
if(windows != null && windows.Any())
{
var isSystemWindowsOnly = true;
foreach(var window in windows)
{
if(window.Type != AccessibilityWindowType.System)
{
isSystemWindowsOnly = false;
break;
}
}
return isSystemWindowsOnly;
}
return false;
}
public static Rect GetInputMethodWindowRect(IEnumerable<AccessibilityWindowInfo> windows)
{
Rect windowRect = null;
if(windows != null)
{
foreach(var window in windows)
{
if(window.Type == AccessibilityWindowType.InputMethod)
{
windowRect = new Rect();
window.GetBoundsInScreen(windowRect);
window.Recycle();
break;
}
window.Recycle();
}
}
return windowRect;
}
public static int GetNodeHeight(AccessibilityNodeInfo node)
{
var nodeRect = new Rect();
node.GetBoundsInScreen(nodeRect);
var nodeRectHeight = nodeRect.Height();
nodeRect.Dispose();
return nodeRectHeight;
}
private static int GetStatusBarHeight()
{
return GetSystemResourceDimenPx("status_bar_height");

251
src/Android/Accessibility/AccessibilityService.cs
View File

@ -15,6 +15,7 @@ using Bit.App.Resources;
using Bit.Core;
using Bit.Core.Abstractions;
using Bit.Core.Utilities;
using Java.Util;
namespace Bit.Droid.Accessibility
{
@ -27,17 +28,19 @@ namespace Bit.Droid.Accessibility
private const string BitwardenPackage = "com.x8bit.bitwarden";
private const string BitwardenWebsite = "vault.bitwarden.com";
private string _lastNotificationUri = null;
private AccessibilityNodeInfo _anchorNode = null;
private int _lastAnchorX, _lastAnchorY = 0;
private static bool _overlayAnchorObserverRunning = false;
private IWindowManager _windowManager = null;
private LinearLayout _overlayView = null;
private long _lastAutoFillTime = 0;
private Java.Lang.Runnable _overlayAnchorObserverRunnable = null;
private Handler _handler = new Handler(Looper.MainLooper);
private HashSet<string> _launcherPackageNames = null;
private DateTime? _lastLauncherSetBuilt = null;
private TimeSpan _rebuildLauncherSpan = TimeSpan.FromHours(1);
private IWindowManager _windowManager = null;
private LinearLayout _overlayView = null;
private int _anchorViewHash = 0;
private int _lastAnchorX, _lastAnchorY = 0;
public override void OnAccessibilityEvent(AccessibilityEvent e)
{
try
@ -54,106 +57,95 @@ namespace Bit.Droid.Accessibility
if(SkipPackage(e?.PackageName))
{
CancelOverlayPrompt();
return;
}
var root = RootInActiveWindow;
if(root == null || root.PackageName != e.PackageName)
{
if(e?.PackageName != "com.android.systemui")
{
CancelOverlayPrompt();
}
return;
}
// AccessibilityHelpers.PrintTestData(root, e);
AccessibilityNodeInfo root = null;
switch(e.EventType)
{
case EventTypes.ViewFocused:
case EventTypes.ViewClicked:
case EventTypes.ViewScrolled:
var isKnownBroswer = AccessibilityHelpers.SupportedBrowsers.ContainsKey(root.PackageName);
if(e.EventType == EventTypes.ViewClicked && isKnownBroswer)
{
break;
}
if(e.Source == null || e.PackageName == BitwardenPackage)
{
CancelOverlayPrompt();
e.Recycle();
break;
}
if(e.EventType == EventTypes.ViewScrolled)
root = RootInActiveWindow;
if(root == null || root.PackageName != e.PackageName)
{
AdjustOverlayForScroll(root, e);
e.Recycle();
break;
}
var isKnownBroswer = AccessibilityHelpers.SupportedBrowsers.ContainsKey(root.PackageName);
if(e.EventType == EventTypes.ViewClicked && isKnownBroswer)
{
e.Recycle();
break;
}
if(!(e.Source?.Password ?? false) && !AccessibilityHelpers.IsUsernameEditText(root, e))
{
CancelOverlayPrompt();
e.Recycle();
break;
}
if(ScanAndAutofill(root, e))
{
CancelOverlayPrompt();
e.Recycle();
}
else
{
var isUsernameEditText1 = AccessibilityHelpers.IsUsernameEditText(root, e);
var isPasswordEditText1 = e.Source?.Password ?? false;
if(!isUsernameEditText1 && !isPasswordEditText1)
{
CancelOverlayPrompt();
break;
}
if(ScanAndAutofill(root, e))
{
CancelOverlayPrompt();
}
else
{
OverlayPromptToAutofill(root, e);
}
OverlayPromptToAutofill(root, e);
}
break;
case EventTypes.WindowContentChanged:
case EventTypes.WindowStateChanged:
var isUsernameEditText2 = AccessibilityHelpers.IsUsernameEditText(root, e);
var isPasswordEditText2 = e.Source?.Password ?? false;
if(e.Source == null || isUsernameEditText2 || isPasswordEditText2)
if(AccessibilityHelpers.LastCredentials == null)
{
e.Recycle();
break;
}
else if(AccessibilityHelpers.LastCredentials == null)
{
if(string.IsNullOrWhiteSpace(_lastNotificationUri))
{
CancelOverlayPrompt();
break;
}
var uri = AccessibilityHelpers.GetUri(root);
if(uri != null && uri != _lastNotificationUri)
{
CancelOverlayPrompt();
}
else if(uri != null && uri.StartsWith(Constants.AndroidAppProtocol))
{
CancelOverlayPrompt();
}
break;
}
if(e.PackageName == BitwardenPackage)
{
CancelOverlayPrompt();
e.Recycle();
break;
}
root = RootInActiveWindow;
if(root == null || root.PackageName != e.PackageName)
{
e.Recycle();
break;
}
if(ScanAndAutofill(root, e))
{
CancelOverlayPrompt();
}
e.Recycle();
break;
default:
break;
}
root.Dispose();
e.Dispose();
if(root != null)
{
root.Recycle();
}
}
// Suppress exceptions so that service doesn't crash.
catch(Exception ex)
{
System.Diagnostics.Debug.WriteLine(">>> Exception: " + ex.StackTrace);
System.Diagnostics.Debug.WriteLine(">>> {0}: {1}", ex.GetType(), ex.StackTrace);
}
}
@ -175,8 +167,8 @@ namespace Bit.Droid.Accessibility
{
AccessibilityHelpers.GetNodesAndFill(root, e, passwordNodes);
filled = true;
_lastAutoFillTime = Java.Lang.JavaSystem.CurrentTimeMillis();
}
}
AccessibilityHelpers.LastCredentials = null;
}
@ -194,19 +186,23 @@ namespace Bit.Droid.Accessibility
private void CancelOverlayPrompt()
{
if(_windowManager == null || _overlayView == null)
_overlayAnchorObserverRunning = false;
if(_windowManager != null && _overlayView != null)
{
return;
_windowManager.RemoveViewImmediate(_overlayView);
System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Removed");
}
_windowManager.RemoveViewImmediate(_overlayView);
System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Removed");
_overlayView = null;
_anchorViewHash = 0;
_lastNotificationUri = null;
_lastAnchorX = 0;
_lastAnchorY = 0;
if(_anchorNode != null)
{
_anchorNode.Recycle();
_anchorNode = null;
}
}
private void OverlayPromptToAutofill(AccessibilityNodeInfo root, AccessibilityEvent e)
@ -215,12 +211,25 @@ namespace Bit.Droid.Accessibility
{
System.Diagnostics.Debug.WriteLine(">>> Overlay Permission not granted");
Toast.MakeText(this, AppResources.AccessibilityOverlayPermissionAlert, ToastLength.Long).Show();
e.Recycle();
return;
}
if(_overlayView != null || _anchorNode != null || _overlayAnchorObserverRunning)
{
CancelOverlayPrompt();
}
if(Java.Lang.JavaSystem.CurrentTimeMillis() - _lastAutoFillTime < 1000)
{
e.Recycle();
return;
}
var uri = AccessibilityHelpers.GetUri(root);
if(string.IsNullOrWhiteSpace(uri))
{
e.Recycle();
return;
}
@ -234,54 +243,85 @@ namespace Bit.Droid.Accessibility
_windowManager = GetSystemService(WindowService).JavaCast<IWindowManager>();
}
if(_overlayView == null)
var intent = new Intent(this, typeof(AccessibilityActivity));
intent.PutExtra("uri", uri);
intent.SetFlags(ActivityFlags.NewTask | ActivityFlags.SingleTop | ActivityFlags.ClearTop);
_overlayView = AccessibilityHelpers.GetOverlayView(this);
_overlayView.Click += (sender, eventArgs) =>
{
var intent = new Intent(this, typeof(AccessibilityActivity));
intent.PutExtra("uri", uri);
intent.SetFlags(ActivityFlags.NewTask | ActivityFlags.SingleTop | ActivityFlags.ClearTop);
CancelOverlayPrompt();
StartActivity(intent);
};
_overlayView = AccessibilityHelpers.GetOverlayView(this);
_overlayView.Click += (sender, eventArgs) =>
{
CancelOverlayPrompt();
StartActivity(intent);
};
_lastNotificationUri = uri;
_windowManager.AddView(_overlayView, layoutParams);
System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Added at X:{0} Y:{1}",
layoutParams.X, layoutParams.Y);
}
else
{
_windowManager.UpdateViewLayout(_overlayView, layoutParams);
System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Updated to X:{0} Y:{1}",
layoutParams.X, layoutParams.Y);
}
_anchorViewHash = e.Source.GetHashCode();
_anchorNode = e.Source;
_lastAnchorX = anchorPosition.X;
_lastAnchorY = anchorPosition.Y;
_windowManager.AddView(_overlayView, layoutParams);
System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Added at X:{0} Y:{1}",
layoutParams.X, layoutParams.Y);
StartOverlayAnchorObserver();
}
private void AdjustOverlayForScroll(AccessibilityNodeInfo root, AccessibilityEvent e)
private void StartOverlayAnchorObserver()
{
if(_overlayView == null || _anchorViewHash <= 0)
if(_overlayAnchorObserverRunning)
{
return;
}
_overlayAnchorObserverRunning = true;
_overlayAnchorObserverRunnable = new Java.Lang.Runnable(() =>
{
if(_overlayAnchorObserverRunning)
{
AdjustOverlayForScroll();
_handler.PostDelayed(_overlayAnchorObserverRunnable, 250);
}
});
_handler.PostDelayed(_overlayAnchorObserverRunnable, 250);
}
private void AdjustOverlayForScroll()
{
if(_overlayView == null || _anchorNode == null)
{
CancelOverlayPrompt();
return;
}
var root = RootInActiveWindow;
IEnumerable<AccessibilityWindowInfo> windows = null;
if(Build.VERSION.SdkInt > BuildVersionCodes.Kitkat)
{
windows = Windows;
}
var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(_anchorNode, root, windows);
root.Recycle();
var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(_anchorViewHash, root, e);
if(anchorPosition == null)
{
CancelOverlayPrompt();
return;
}
if(anchorPosition.X == _lastAnchorX && anchorPosition.Y == _lastAnchorY)
else if(anchorPosition.X == -1 && anchorPosition.Y == -1)
{
if(_overlayView.Visibility != ViewStates.Gone)
{
_overlayView.Visibility = ViewStates.Gone;
}
return;
}
else if(anchorPosition.X == _lastAnchorX && anchorPosition.Y == _lastAnchorY)
{
if(_overlayView.Visibility != ViewStates.Visible)
{
_overlayView.Visibility = ViewStates.Visible;
}
return;
}
@ -289,11 +329,16 @@ namespace Bit.Droid.Accessibility
layoutParams.X = anchorPosition.X;
layoutParams.Y = anchorPosition.Y;
_windowManager.UpdateViewLayout(_overlayView, layoutParams);
_lastAnchorX = anchorPosition.X;
_lastAnchorY = anchorPosition.Y;
_windowManager.UpdateViewLayout(_overlayView, layoutParams);
if(_overlayView.Visibility != ViewStates.Visible)
{
_overlayView.Visibility = ViewStates.Visible;
}
System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Updated to X:{0} Y:{1}",
layoutParams.X, layoutParams.Y);
}

4
src/Android/Resources/xml/accessibilityservice.xml
View File

@ -2,8 +2,8 @@
<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
android:summary="@string/AutoFillServiceSummary"
android:description="@string/AutoFillServiceDescription"
android:accessibilityEventTypes="typeWindowStateChanged|typeWindowContentChanged|typeViewFocused|typeViewClicked|typeViewScrolled"
android:accessibilityEventTypes="typeWindowStateChanged|typeWindowContentChanged|typeViewFocused|typeViewClicked"
android:accessibilityFeedbackType="feedbackGeneric"
android:accessibilityFlags="flagReportViewIds"
android:accessibilityFlags="flagReportViewIds|flagRetrieveInteractiveWindows"
android:notificationTimeout="100"
android:canRetrieveWindowContent="true"/>