Delete and Logging

xbapps · Feb 26, 2025 · 36abd8d · 36abd8d
1 parent e7615f4
commit 36abd8d
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 21 deletions.
diff --git a/pkg/api/options.go b/pkg/api/options.go
@@ -325,6 +325,7 @@ func (i ConfigResource) WebService() *restful.WebService {
 	// "Collector Config endpoints"
 	ws.Route(ws.GET("/collector-config-list").To(i.getCollectorConfigs))
 	ws.Route(ws.POST("/save-collector-config").To(i.saveCollectorConfigs))
+	ws.Route(ws.DELETE("/delete-collector-config").To(i.deleteCollectorConfig))
 
 	return ws
 }
@@ -1117,3 +1118,21 @@ func (i ConfigResource) saveCollectorConfigs(req *restful.Request, resp *restful
 	config.Other = r.Other
 	scrape.SaveScrapeHttpConfig(r.DomainKey, config)
 }
+func (i ConfigResource) deleteCollectorConfig(req *restful.Request, resp *restful.Response) {
+	var r RequestSaveCollectorConfig
+
+	if err := req.ReadEntity(&r); err != nil {
+		APIError(req, resp, http.StatusInternalServerError, err)
+		log.Error(err)
+		return
+	}
+
+	if r.DomainKey == "" {
+		APIError(req, resp, http.StatusInternalServerError, nil)
+		log.Error("Could not delete collector config - name for config not found")
+		return
+	}
+	var kv models.KV
+	kv.Key = r.DomainKey
+	kv.Delete()
+}
diff --git a/pkg/api/trailers.go b/pkg/api/trailers.go
@@ -33,6 +33,7 @@ func LoadHeresphereScene(scrapeParams string) HeresphereVideo {
 	if params.KVHttpConfig == "" {
 		params.KVHttpConfig = scrape.GetCoreDomain(params.SceneUrl) + "-trailers"
 	}
+	log.Debugf("Using Header/Cookies from %s", params.KVHttpConfig)
 	scrape.SetupHtmlRequest(params.KVHttpConfig, req)
 	response, err := client.Do(req)
 
@@ -70,6 +71,7 @@ func LoadDeovrScene(scrapeParams string) DeoScene {
 	if params.KVHttpConfig == "" {
 		params.KVHttpConfig = scrape.GetCoreDomain(params.SceneUrl) + "-trailers"
 	}
+	log.Debugf("Using Header/Cookies from %s", params.KVHttpConfig)
 	scrape.SetupHtmlRequest(params.KVHttpConfig, req)
 
 	response, err := client.Do(req)
@@ -102,6 +104,7 @@ func ScrapeHtml(scrapeParams string) models.VideoSourceResponse {
 	if params.KVHttpConfig == "" {
 		params.KVHttpConfig = scrape.GetCoreDomain(params.SceneUrl) + "-trailers"
 	}
+	log.Debugf("Using Header/Cookies from %s", params.KVHttpConfig)
 	scrape.SetupCollector(params.KVHttpConfig, c)
 
 	var srcs []models.VideoSource
@@ -144,6 +147,7 @@ func ScrapeJson(scrapeParams string) models.VideoSourceResponse {
 	if params.KVHttpConfig == "" {
 		params.KVHttpConfig = scrape.GetCoreDomain(params.SceneUrl) + "-trailers"
 	}
+	log.Debugf("Using Header/Cookies from %s", params.KVHttpConfig)
 	scrape.SetupCollector(params.KVHttpConfig, c)
 	var srcs []models.VideoSource
 	c.OnHTML(`html`, func(e *colly.HTMLElement) {
@@ -182,6 +186,7 @@ func LoadJson(scrapeParams string) models.VideoSourceResponse {
 	if params.KVHttpConfig == "" {
 		params.KVHttpConfig = scrape.GetCoreDomain(params.SceneUrl) + "-trailers"
 	}
+	log.Debugf("Using Header/Cookies from %s", params.KVHttpConfig)
 	scrape.SetupHtmlRequest(params.KVHttpConfig, req)
 	response, err := client.Do(req)
 

diff --git a/pkg/scrape/scrape.go b/pkg/scrape/scrape.go
@@ -38,6 +38,7 @@ func createCollector(domains ...string) *colly.Collector {
 	// see if the domain has a limit and set it
 	for _, domain := range domains {
 		SetupCollector(GetCoreDomain(domain)+"-scraper", c)
+		log.Debugf("Using Header/Cookies from %s", GetCoreDomain(domain)+"-scraper")
 		if Limiters == nil {
 			LoadScraperRateLimits()
 		}
@@ -202,5 +203,6 @@ func GetCoreDomain(domain string) string {
 	if len(parts) > 2 && parts[0] == "www" {
 		parts = parts[1:]
 	}
+
 	return strings.Join(parts[:len(parts)-1], ".")
 }
diff --git a/ui/src/views/options/sections/InterfaceAdvanced.vue b/ui/src/views/options/sections/InterfaceAdvanced.vue
@@ -191,6 +191,7 @@
                   </b-field>
                 </b-tooltip>
                 <b-button v-if="showConfigField" type="is-primary" style="margin-left: 1em;" @click="saveCollectorConfig">Save</b-button>
+                <b-button v-if="showConfigField" type="is-primary" style="margin-left: 1em;" @click="deleteCollectorConfig" icon-right="delete"></b-button>
               </b-field>
               <b-autocomplete v-model="kvName" ref="autocompleteconfig" :data="filteredCollectorConfigList" :open-on-focus="true" :clearable="true" 
                 placeholder="e.g. domainname-scraper or domainname-trailers " 
@@ -361,7 +362,6 @@ export default {
       }      
       this.body= matched.config.body
       this.showCollectorConfigFields = true
-      console.log("in showConfigField", this.showCollectorConfigFields )
     },
     addHeaderRow(){
       this.headers.push({ key: "", value: "" });
@@ -373,8 +373,6 @@ export default {
       this.cookies.push({ name: "", value: "", domain: "", path: "", host:"" });
     },
     delCookieRow(props){      
-      console.log("deleteing ", props.row)
-      console.log("deleteing ", props.index)
       this.cookies.splice(props.index,1)
     },
     saveCollectorConfig() {
@@ -387,15 +385,6 @@ export default {
           other: [],
         }
       })
-      console.log("saving",
-      {
-          domain_key: this.kvName,
-          body: this.body,
-          cookies: this.cookies,
-          headers: this.headers,
-          other: [],
-        }
-      )      
       let row = this.$store.state.optionsAdvanced.advanced.collectorConfigs.find((config) => {
         return config.domain_key
           .toString()
@@ -406,7 +395,6 @@ export default {
       row.config.cookies = this.cookies
       row.config.headers=this.headers
       row.config.body = this.body      
-      console.log("updated",this.$store.state.optionsAdvanced.advanced.collectorConfigs)
     },
     showAddCollectorConfig() {
       this.$buefy.dialog.prompt({
@@ -417,14 +405,11 @@ export default {
                     value: this.kvName
                 },
                 confirmText: 'Add',
-                onConfirm: (value) => {
-                  console.log(this.$store.state.optionsAdvanced.advanced.collectorConfigs)
+                onConfirm: (value) => {                  
                     this.kvName=value
-                    this.$store.state.optionsAdvanced.advanced.collectorConfigs.push({config: {body: "", cookies: [], headers: [],  other: ""},domain_key: value} )
-                    console.log(this.$store.state.optionsAdvanced.advanced.collectorConfigs)
+                    this.$store.state.optionsAdvanced.advanced.collectorConfigs.push({config: {body: "", cookies: [], headers: [],  other: ""},domain_key: value} )                    
                     this.$refs.autocompleteconfig.setSelected(value)
-                    this.showCollectorConfigFields = true
-                    console.log("in showAddCollectorConfig confirm", this.showCollectorConfigFields )
+                    this.showCollectorConfigFields = true                    
                 }
       })
     },
@@ -442,6 +427,15 @@ export default {
         this.$store.dispatch('optionsAdvanced/load')
       }
     },
+    async deleteCollectorConfig() {
+      const response = await ky.delete('/api/options/delete-collector-config', {
+        json: {
+          domain_key: this.kvName,
+        }
+      })
+      this.kvName=""
+      this.$store.dispatch('optionsAdvanced/load')
+    },
   },
   computed: {
     showInternalSceneId: {
@@ -566,11 +560,9 @@ export default {
       } else {
         this.showCollectorConfigFields = false
       }
-      console.log("in filteredCollectorConfigList", this.showCollectorConfigFields, this.kvName )
       return matched.map(item => item.domain_key)
     },
     showConfigField () {
-      console.log("in showConfigField", this.showCollectorConfigFields)
       return this.showCollectorConfigFields
     },
     isLoading: function () {