How to combine a scrollview with a collapsable modal?

[Snickleblor]

I am trying to combine a pan gesture with a modal to make it closable when I swipe downwards (this should only be true if the offset in the y direction is zero or atleast close to zero, and if the path is postive, meaning it is a swipe down gesture) and also make it compatible with the ScrollView. First I wrapped the scrollView in an extra GestureDetector, and used Native() to add it into the gesture handler system. Then added a state that switches between the two gestures. The issue I am getting is that I need to swipe down a few times to even activate the scrollGesture, and after scrolling it stops working, and I need swipe down again for it to work. Is this the right approach in implementing this? Or is this a bug?

Ask if I need to explain anything. Sincerly, Blor.

react-native-gesture-handler 2.24.0
react-native-reanimated 3.17.5
react-native-worklets 0.7.1

import { Dispatch, ReactNode, SetStateAction, useState } from "react";
import {
  Dimensions,
  Modal,
  NativeScrollEvent,
  NativeSyntheticEvent,
  ScrollView,
} from "react-native";
import {
  Gesture,
  GestureDetector,
  GestureHandlerRootView,
} from "react-native-gesture-handler";
import Animated, {
  runOnJS,
  useAnimatedStyle,
  useSharedValue,
  withTiming,
} from "react-native-reanimated";

const SCREEN_HEIGHT = Dimensions.get("screen").height; // Tror ikke denne endrer seg
const SWIPE_THRESHOLD = 300;

export default function ScrollableModal({
  isModal,
  setIsModal,
  children,
}: {
  isModal: boolean;
  setIsModal: Dispatch<SetStateAction<boolean>>;
  children: ReactNode;
}) {
  const startY = useSharedValue(0);
  const pathTraveled = useSharedValue(0);
  const translateY = useSharedValue(0);
  const scrollStartY = useSharedValue(0);
  const scrollY = useSharedValue(0);
  const [isPanEnabled, setIsPanEnabled] = useState(true);

  const collapseStyles = useAnimatedStyle(() => {
    return {
      transform: [{ translateY: translateY.value }],
    };
  });

  const collapseGesture = Gesture.Pan()
    .enabled(isPanEnabled)
    .maxPointers(1) 
    .onStart((event) => {
      startY.value = event.absoluteY;
    })
    .onUpdate((event) => {
      // const pathTraveledScroll = Math.abs(scrollY.value - scrollStartY.value);

      pathTraveled.value = event.absoluteY - startY.value;

      if (scrollY.value == 0 && pathTraveled.value > 0) {
        translateY.value = pathTraveled.value;
      } else if (pathTraveled.value < 0) {
        runOnJS(setIsPanEnabled)(false);
      }

      if (pathTraveled.value > SWIPE_THRESHOLD) {
        translateY.value = withTiming(
          SCREEN_HEIGHT,
          { duration: 300 },
          (finished) => {
            if (finished) {
              runOnJS(setIsModal)(false);
            }
          }
        );
      }
    })
    .onEnd(() => {
      if (pathTraveled.value < SWIPE_THRESHOLD) {
        translateY.value = withTiming(0);
      }
    });

  const scrollGesture = Gesture.Native()
    .enabled(!isPanEnabled)
    .onTouchesDown(() => {
      if (scrollY.value == 0) {
        runOnJS(setIsPanEnabled)(true);
      }
    });

  const onScroll = (event: NativeSyntheticEvent<NativeScrollEvent>) => {
    scrollY.value = event.nativeEvent.contentOffset.y;
  };

  return (
    <Modal visible={isModal} transparent={false} animationType="slide">
      <GestureHandlerRootView>
        <GestureDetector gesture={collapseGesture}>
          <Animated.View style={[collapseStyles]}>
            <GestureDetector gesture={scrollGesture}>
              <ScrollView
                onScroll={onScroll}
              >
                {children}
              </ScrollView>
            </GestureDetector>
          </Animated.View>
        </GestureDetector>
      </GestureHandlerRootView>
    </Modal>
  );
}

    <ScrollableModal isModal={isModal} setIsModal={setIsModal}>
      <View
        style={{ height: 100 }}
      >
      </View>
    </ScrollableModal>