bump requirements and add exlusion to depdendabot

.github/dependabot.yml

@@ -11,6 +11,9 @@ updates:
     schedule:
       interval: "weekly"
     open-pull-requests-limit: 10
+    ignore:
+      - dependency-name: "twisted"
+      - dependency-name: "cryptography"
 
   # Update GitHub Actions
   - package-ecosystem: "github-actions"

requirements.txt

@@ -7,38 +7,38 @@ google-auth-httplib2==0.1.1
 
 # Scrapy and related packages
 Scrapy==2.11.0
-Twisted==22.10.0  # Compatible with Scrapy 2.11.0
+Twisted==22.10.0  # Keep this version to avoid breaking changes
 scrapy-playwright==0.0.33
 playwright==1.40.0
 
 # IPython for interactive development
-ipython==8.17.2
+ipython==8.27.0  # Updated from 8.17.2
 
 # Other dependencies
 asttokens==2.4.1
 attrs==23.1.0
-Automat==22.10.0
+Automat==24.8.1  # Updated from 22.10.0
 backcall==0.2.0
 certifi==2023.11.17
-cffi==1.16.0
+cffi==1.17.1  # Updated from 1.16.0
 charset-normalizer==3.3.2
 constantly==23.10.4
-cryptography==41.0.5
+cryptography==41.0.5  # Keep this version to avoid breaking changes
 cssselect==1.2.0
 decorator==5.1.1
 executing==2.0.1
 filelock==3.13.1
 greenlet==3.0.1
 hyperlink==21.0.0
 idna==3.4
-incremental==22.10.0
+incremental==24.7.2  # Updated from 22.10.0
 itemadapter==0.8.0
 itemloaders==1.1.0
 jedi==0.19.1
 jmespath==1.0.1
 lxml==4.9.3
 matplotlib-inline==0.1.6
-packaging==23.2
+packaging==24.1  # Updated from 23.2
 parsel==1.8.1
 parso==0.8.3
 pexpect==4.8.0
@@ -48,7 +48,7 @@ Protego==0.3.0
 ptyprocess==0.7.0
 pure-eval==0.2.2
 pyasn1==0.5.0
-pyasn1-modules==0.3.0
+pyasn1-modules==0.4.0  # Updated from 0.3.0
 pycparser==2.21
 PyDispatcher==2.0.7
 pyee==11.0.1
@@ -62,8 +62,8 @@ six==1.16.0
 stack-data==0.6.3
 tldextract==5.1.1
 traitlets==5.13.0
-typing_extensions==4.8.0
+typing_extensions==4.12.2  # Updated from 4.8.0
 urllib3==2.1.0
-w3lib==2.1.2
+w3lib==2.2.1  # Updated from 2.1.2
 wcwidth==0.2.10
 zope.interface==6.1