feat: update js rendering

Author: VinciGit00

scrapegraph-py/scrapegraph_py/async_client.py

@@ -439,16 +439,18 @@ def new_id(prefix: str) -> str:
         return {"status": "mock", "url": url, "method": method, "kwargs": kwargs}
 
     async def markdownify(
-        self, website_url: str, headers: Optional[dict[str, str]] = None, stealth: bool = False
+        self, website_url: str, headers: Optional[dict[str, str]] = None, mock: bool = False, render_heavy_js: bool = False, stealth: bool = False
     ):
         """Send a markdownify request"""
         logger.info(f"🔍 Starting markdownify request for {website_url}")
         if headers:
             logger.debug("🔧 Using custom headers")
         if stealth:
             logger.debug("🥷 Stealth mode enabled")
+        if render_heavy_js:
+            logger.debug("⚡ Heavy JavaScript rendering enabled")
 
-        request = MarkdownifyRequest(website_url=website_url, headers=headers, stealth=stealth)
+        request = MarkdownifyRequest(website_url=website_url, headers=headers, mock=mock, render_heavy_js=render_heavy_js, stealth=stealth)
         logger.debug("✅ Request validation passed")
 
         result = await self._make_request(
@@ -741,6 +743,8 @@ async def crawl(
         same_domain_only: bool = True,
         batch_size: Optional[int] = None,
         sitemap: bool = False,
+        headers: Optional[dict[str, str]] = None,
+        render_heavy_js: bool = False,
         stealth: bool = False,
     ):
         """Send a crawl request with support for both AI extraction and
@@ -764,6 +768,8 @@ async def crawl(
         logger.debug(f"🗺️ Use sitemap: {sitemap}")
         if stealth:
             logger.debug("🥷 Stealth mode enabled")
+        if render_heavy_js:
+            logger.debug("⚡ Heavy JavaScript rendering enabled")
         if batch_size is not None:
             logger.debug(f"📦 Batch size: {batch_size}")
 
@@ -776,6 +782,7 @@ async def crawl(
             "max_pages": max_pages,
             "same_domain_only": same_domain_only,
             "sitemap": sitemap,
+            "render_heavy_js": render_heavy_js,
             "stealth": stealth,
         }
 
@@ -786,6 +793,8 @@ async def crawl(
             request_data["data_schema"] = data_schema
         if batch_size is not None:
             request_data["batch_size"] = batch_size
+        if headers is not None:
+            request_data["headers"] = headers
 
         request = CrawlRequest(**request_data)
         logger.debug("✅ Request validation passed")

scrapegraph-py/scrapegraph_py/client.py

@@ -452,15 +452,17 @@ def new_id(prefix: str) -> str:
         # Generic fallback
         return {"status": "mock", "url": url, "method": method, "kwargs": kwargs}
 
-    def markdownify(self, website_url: str, headers: Optional[dict[str, str]] = None, mock:bool=False, stealth:bool=False):
+    def markdownify(self, website_url: str, headers: Optional[dict[str, str]] = None, mock: bool = False, render_heavy_js: bool = False, stealth: bool = False):
         """Send a markdownify request"""
         logger.info(f"🔍 Starting markdownify request for {website_url}")
         if headers:
             logger.debug("🔧 Using custom headers")
         if stealth:
             logger.debug("🥷 Stealth mode enabled")
+        if render_heavy_js:
+            logger.debug("⚡ Heavy JavaScript rendering enabled")
 
-        request = MarkdownifyRequest(website_url=website_url, headers=headers, mock=mock, stealth=stealth)
+        request = MarkdownifyRequest(website_url=website_url, headers=headers, mock=mock, render_heavy_js=render_heavy_js, stealth=stealth)
         logger.debug("✅ Request validation passed")
 
         result = self._make_request(
@@ -749,6 +751,8 @@ def crawl(
         same_domain_only: bool = True,
         batch_size: Optional[int] = None,
         sitemap: bool = False,
+        headers: Optional[dict[str, str]] = None,
+        render_heavy_js: bool = False,
         stealth: bool = False,
     ):
         """Send a crawl request with support for both AI extraction and
@@ -772,6 +776,8 @@ def crawl(
         logger.debug(f"🗺️ Use sitemap: {sitemap}")
         if stealth:
             logger.debug("🥷 Stealth mode enabled")
+        if render_heavy_js:
+            logger.debug("⚡ Heavy JavaScript rendering enabled")
         if batch_size is not None:
             logger.debug(f"📦 Batch size: {batch_size}")
 
@@ -784,6 +790,7 @@ def crawl(
             "max_pages": max_pages,
             "same_domain_only": same_domain_only,
             "sitemap": sitemap,
+            "render_heavy_js": render_heavy_js,
             "stealth": stealth,
         }
 
@@ -794,6 +801,8 @@ def crawl(
             request_data["data_schema"] = data_schema
         if batch_size is not None:
             request_data["batch_size"] = batch_size
+        if headers is not None:
+            request_data["headers"] = headers
 
         request = CrawlRequest(**request_data)
         logger.debug("✅ Request validation passed")

scrapegraph-py/scrapegraph_py/models/markdownify.py

@@ -27,6 +27,8 @@ class MarkdownifyRequest(BaseModel):
         website_url: URL of the website to convert to markdown
         headers: Optional HTTP headers including cookies
         mock: Whether to use mock mode for testing
+        render_heavy_js: Whether to render heavy JavaScript on the page
+        stealth: Enable stealth mode to avoid bot detection
 
     Example:
         >>> request = MarkdownifyRequest(website_url="https://example.com")
@@ -42,6 +44,7 @@ class MarkdownifyRequest(BaseModel):
         "and user agent",
     )
     mock: bool = Field(default=False, description="Whether to use mock mode for the request")
+    render_heavy_js: bool = Field(default=False, description="Whether to render heavy JavaScript on the page")
     stealth: bool = Field(default=False, description="Enable stealth mode to avoid bot detection")
 
     @model_validator(mode="after")

scrapegraph-py/test_async_render_heavy_js.py

@@ -0,0 +1 @@
+ 

scrapegraph-py/test_render_heavy_js.py

@@ -0,0 +1 @@
+