Exclude anchor and query strung from URLs in sitemap connector (#86)

nuclia · Feb 14, 2025 · 85067b4 · 85067b4
1 parent 680be49
commit 85067b4
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 2 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "nuclia-sync-agent-app",
-  "version": "1.6.0",
+  "version": "1.6.1",
   "description": "This is a Nuclia Sync Agent App",
   "main": "build/index.js",
   "scripts": {

diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md
@@ -1,3 +1,7 @@
+# 1.6.1 (2025-02-14)
+
+- Exclude anchor and query strung from URLs in sitemap connector
+
 # 1.6.0 (2025-01-28)
 
 - Support files download when using sitemap connector with local extraction

diff --git a/server/src/logic/connector/infrastructure/connectors/sitemap.connector.ts b/server/src/logic/connector/infrastructure/connectors/sitemap.connector.ts
@@ -26,7 +26,7 @@ export function parseSitemap(sitemapContent: string): Promise<SiteMapModel[]> {
     const $ = cheerio.load(sitemapContent, { xml: true });
 
     $('url').each((_, element) => {
-      const loc = $(element).find('loc').text();
+      const loc = $(element).find('loc').text().split('?')[0].split('#')[0];
       const lastmod = $(element).find('lastmod').text();
       urls.push({ loc, lastmod });
     });