diff --git a/packages/scrollable_positioned_list/lib/src/scrollable_positioned_list.dart b/packages/scrollable_positioned_list/lib/src/scrollable_positioned_list.dart
index 3b41baac..4d12ee21 100644
--- a/packages/scrollable_positioned_list/lib/src/scrollable_positioned_list.dart
+++ b/packages/scrollable_positioned_list/lib/src/scrollable_positioned_list.dart
@@ -338,9 +338,8 @@ class _ScrollablePositionedListState extends State<ScrollablePositionedList>
     return LayoutBuilder(
       builder: (context, constraints) {
         final cacheExtent = _cacheExtent(constraints);
-        return GestureDetector(
-          onPanDown: (_) => _stopScroll(canceled: true),
-          excludeFromSemantics: true,
+        return Listener(
+          onPointerDown: (_) => _stopScroll(canceled: true),
           child: Stack(
             children: <Widget>[
               PostMountCallback(