Stashdb scraper

pkg/api/options.go

@@ -1017,6 +1017,7 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res
 	scrapers := make(map[string][]config.ScraperConfig)
 	scrapers["povr"] = scraperConfig.CustomScrapers.PovrScrapers
 	scrapers["slr"] = scraperConfig.CustomScrapers.SlrScrapers
+	scrapers["stashdb"] = scraperConfig.CustomScrapers.StashDbScrapers
 	scrapers["vrphub"] = scraperConfig.CustomScrapers.VrphubScrapers
 	scrapers["vrporn"] = scraperConfig.CustomScrapers.VrpornScrapers
 
@@ -1040,6 +1041,8 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res
 			scrapers["povr"] = append(scrapers["povr"], scraper)
 		case "sexlikereal":
 			scrapers["slr"] = append(scrapers["slr"], scraper)
+		case "stashdb":
+			scrapers["stashdb"] = append(scrapers["stashdb"], scraper)
 		case "vrphub":
 			scrapers["vrphub"] = append(scrapers["vrphub"], scraper)
 		case "vrporn":
@@ -1048,6 +1051,7 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res
 	}
 	scraperConfig.CustomScrapers.PovrScrapers = scrapers["povr"]
 	scraperConfig.CustomScrapers.SlrScrapers = scrapers["slr"]
+	scraperConfig.CustomScrapers.StashDbScrapers = scrapers["stashdb"]
 	scraperConfig.CustomScrapers.VrphubScrapers = scrapers["vrphub"]
 	scraperConfig.CustomScrapers.VrpornScrapers = scrapers["vrporn"]
 	fName := filepath.Join(common.AppDir, "scrapers.json")

pkg/config/scraper_list.go

@@ -21,16 +21,18 @@ type ScraperList struct {
 	XbvrScrapers   XbvrScrapers   `json:"xbvr"`
 }
 type XbvrScrapers struct {
-	PovrScrapers   []ScraperConfig `json:"povr"`
-	SlrScrapers    []ScraperConfig `json:"slr"`
-	VrpornScrapers []ScraperConfig `json:"vrporn"`
-	VrphubScrapers []ScraperConfig `json:"vrphub"`
+	PovrScrapers    []ScraperConfig `json:"povr"`
+	SlrScrapers     []ScraperConfig `json:"slr"`
+	StashDbScrapers []ScraperConfig `json:"stashdb"`
+	VrpornScrapers  []ScraperConfig `json:"vrporn"`
+	VrphubScrapers  []ScraperConfig `json:"vrphub"`
 }
 type CustomScrapers struct {
-	PovrScrapers   []ScraperConfig `json:"povr"`
-	SlrScrapers    []ScraperConfig `json:"slr"`
-	VrpornScrapers []ScraperConfig `json:"vrporn"`
-	VrphubScrapers []ScraperConfig `json:"vrphub"`
+	PovrScrapers    []ScraperConfig `json:"povr"`
+	SlrScrapers     []ScraperConfig `json:"slr"`
+	StashDbScrapers []ScraperConfig `json:"stashdb"`
+	VrpornScrapers  []ScraperConfig `json:"vrporn"`
+	VrphubScrapers  []ScraperConfig `json:"vrphub"`
 }
 type ScraperConfig struct {
 	ID           string `json:"-"`
@@ -71,16 +73,19 @@ func (o *ScraperList) Load() error {
 
 	SetSiteId(&o.XbvrScrapers.PovrScrapers, "")
 	SetSiteId(&o.XbvrScrapers.SlrScrapers, "")
+	SetSiteId(&o.XbvrScrapers.StashDbScrapers, "")
 	SetSiteId(&o.XbvrScrapers.VrphubScrapers, "")
 	SetSiteId(&o.XbvrScrapers.VrpornScrapers, "")
 	SetSiteId(&o.CustomScrapers.PovrScrapers, "povr")
 	SetSiteId(&o.CustomScrapers.SlrScrapers, "slr")
+	SetSiteId(&o.CustomScrapers.StashDbScrapers, "stashdb")
 	SetSiteId(&o.CustomScrapers.VrphubScrapers, "vrphub")
 	SetSiteId(&o.CustomScrapers.VrpornScrapers, "vrporn")
 
 	// remove custom sites that are now offical for the same aggregation site
 	o.CustomScrapers.PovrScrapers = RemoveCustomListNowOffical(o.CustomScrapers.PovrScrapers, o.XbvrScrapers.PovrScrapers)
 	o.CustomScrapers.SlrScrapers = RemoveCustomListNowOffical(o.CustomScrapers.SlrScrapers, o.XbvrScrapers.SlrScrapers)
+	o.CustomScrapers.StashDbScrapers = RemoveCustomListNowOffical(o.CustomScrapers.StashDbScrapers, o.XbvrScrapers.StashDbScrapers)
 	o.CustomScrapers.VrphubScrapers = RemoveCustomListNowOffical(o.CustomScrapers.VrphubScrapers, o.XbvrScrapers.VrphubScrapers)
 	o.CustomScrapers.VrpornScrapers = RemoveCustomListNowOffical(o.CustomScrapers.VrpornScrapers, o.XbvrScrapers.VrpornScrapers)
 

pkg/externalreference/stashdb.go

@@ -91,22 +91,39 @@ func matchOnSceneUrl() {
 		var xbvrId uint
 		var xbvrSceneId string
 
+		for _, unmatchedXbvrScene := range unmatchedXbvrScenes {
+			stashId := strings.TrimPrefix(unmatchedXbvrScene.SceneID, "stash-")
+			if stashId == stashScene.ExternalId {
+				// xbvrLink := models.ExternalReferenceLink{InternalTable: "scenes", InternalDbId: scene.ID, InternalNameId: scene.SceneID,
+				// 	ExternalReferenceID: stashScene.ID, ExternalSource: stashScene.ExternalSource, ExternalId: stashScene.ExternalId, MatchType: 20}
+				//stashScene.XbvrLinks = append(stashScene.XbvrLinks, xbvrLink)
+				//stashScene.Save()
+				//var externalData models.StashScene
+				//json.Unmarshal([]byte(stashScene.ExternalData), &externalData)
+				//matchPerformerName(externalData, scene, 20)
+				xbvrId = unmatchedXbvrScene.ID
+				xbvrSceneId = unmatchedXbvrScene.SceneID
+			}
+		}
+
 		// see if we can link to an xbvr scene based on the urls
-		for _, url := range scene.URLs {
-			if url.Type == "STUDIO" {
-				var xbvrScene models.Scene
-				for _, scene := range unmatchedXbvrScenes {
-					sceneurl := removeQueryFromURL(scene.SceneURL)
-					tmpurl := removeQueryFromURL(url.URL)
-					sceneurl = simplifyUrl(sceneurl)
-					tmpurl = simplifyUrl(tmpurl)
-					if strings.EqualFold(sceneurl, tmpurl) {
-						xbvrScene = scene
+		if xbvrId == 0 {
+			for _, url := range scene.URLs {
+				if url.Type == "STUDIO" {
+					var xbvrScene models.Scene
+					for _, scene := range unmatchedXbvrScenes {
+						sceneurl := removeQueryFromURL(scene.SceneURL)
+						tmpurl := removeQueryFromURL(url.URL)
+						sceneurl = simplifyUrl(sceneurl)
+						tmpurl = simplifyUrl(tmpurl)
+						if strings.EqualFold(sceneurl, tmpurl) {
+							xbvrScene = scene
+						}
+					}
+					if xbvrScene.ID != 0 {
+						xbvrId = xbvrScene.ID
+						xbvrSceneId = xbvrScene.SceneID
 					}
-				}
-				if xbvrScene.ID != 0 {
-					xbvrId = xbvrScene.ID
-					xbvrSceneId = xbvrScene.SceneID
 				}
 			}
 		}

pkg/models/model_scene.go

@@ -462,8 +462,12 @@ func SceneCreateUpdateFromExternal(db *gorm.DB, ext ScrapedScene) error {
 			}
 		}
 		if ext.ActorDetails[name].ProfileUrl != "" {
-			if tmpActor.AddToActorUrlArray(ActorLink{Url: ext.ActorDetails[name].ProfileUrl, Type: ext.ActorDetails[name].Source}) {
-				saveActor = true
+			if strings.HasPrefix(ext.ActorDetails[name].ProfileUrl, "https://stashdb.org/performers/") {
+
+			} else {
+				if tmpActor.AddToActorUrlArray(ActorLink{Url: ext.ActorDetails[name].ProfileUrl, Type: ext.ActorDetails[name].Source}) {
+					saveActor = true
+				}
 			}
 		}
 		if saveActor {

pkg/models/model_scraper.go

@@ -54,6 +54,7 @@ type ActorDetails struct {
 	ImageUrl   string
 	ProfileUrl string
 	Source     string
+	StashData  string
 }
 type TrailerScrape struct {
 	SceneUrl       string `json:"scene_url"`        // url of the page to be scrapped

pkg/models/model_stashdb.go

@@ -73,14 +73,25 @@ type StashScene struct {
 	Studio     StashStudio        `json:"studio"`
 	Duration   int                `json:"duration"`
 	Code       string             `json:"code"`
-	Images     []Image            `json:"images"`
+	Images     []StashImages      `json:"images"`
+	Tags       []StashTags        `json:"tags"`
 }
 
 type StashPerformerAs struct {
 	Performer StashPerformer ``
 	As        string         ``
 }
 
+type StashImages struct {
+	URL    string `json:"url"`
+	Width  string `json:"width"`
+	Height Site   `json:"height"`
+}
+type StashTags struct {
+	ID   string `json:"id"`
+	Name string `json:"name"`
+}
+
 type DELETEStashPerformerMin struct {
 	ID      string `json:"id"`
 	Updated string `json:"updated"`

pkg/scrape/stashdb.go

@@ -76,6 +76,12 @@ type QueryPerformerResultType struct {
 	Count      int                     `json:"count"`
 	Performers []models.StashPerformer `json:"performers"`
 }
+type FindSceneResult struct {
+	Data FindSceneData `json:"data"`
+}
+type FindSceneData struct {
+	Scene models.StashScene `json:"findScene"`
+}
 type FindPerformerScenesData struct {
 	Performer models.StashPerformer `json:"findPerformer"`
 }
@@ -135,6 +141,11 @@ fingerprints{
   submissions
 }
 duration
+  tags {
+    id
+	name
+}
+=======
 code  
 deleted
 `
@@ -379,7 +390,65 @@ func GetScenePage(variables string) QueryScenesResult {
 		queryScenes(input: $input) {
 		  count
 		 scenes{
-` + sceneFieldList + `
+		  id
+		  title
+		  details
+		  release_date
+		  date
+		  updated
+		  urls{
+			url
+			type
+			site {
+                id
+                name
+                description
+                url
+                regex
+                valid_types
+            }
+		  }
+		  studio{
+			  id
+			  name
+			  updated
+			  parent {
+				id
+				name  
+			  }
+		  }
+		  images{
+			url
+			width
+			height
+		  }
+		  performers{
+			performer{
+			  id
+			  updated
+			  gender
+			  name
+			  aliases
+			  images{
+				url
+				width
+				height
+			  }
+			}
+			as
+		  }
+		  fingerprints{
+			hash
+			duration
+			submissions
+		  }
+		  tags {
+			id
+			name
+		  }
+		  duration
+		  code  
+		  deleted
 		}
 	  }
 	  }
@@ -392,6 +461,81 @@ func GetScenePage(variables string) QueryScenesResult {
 	return data
 }
 
+func GetSceneFromStash(sceneId string) models.StashScene {
+	variables := `{"id": "` + sceneId + `"} `
+	query := `
+	 query  findScene($id: ID!) {
+		findScene(id: $id) {
+			id
+		   title
+		   details
+		   release_date
+		   date
+		   updated
+		   urls{
+			 url
+			 type
+			 site {
+				 id
+				 name
+				 description
+				 url
+				 regex
+				 valid_types
+			 }
+		   }
+		   studio{
+			   id
+			   name
+			   updated
+			   parent {
+				 id
+				 name  
+			   }
+		   }
+		   images{
+			 url
+			 width
+			 height
+		   }
+		   performers{
+			 performer{
+			   id
+			   updated
+			   gender
+			   name
+			   aliases
+			   images{
+				 url
+				 width
+				 height
+			   }
+			 }
+			 as
+		   }
+		   fingerprints{
+			 hash
+			 duration
+			 submissions
+		   }
+		   tags {
+			 id
+			 name
+		   }
+		   duration
+		   code  
+		   deleted
+		 }
+	   }
+	   `
+
+	// Define the variables needed for your query as a Go map
+	resp := CallStashDb(query, variables)
+	var data FindSceneResult
+	json.Unmarshal(resp, &data)
+	return data.Data.Scene
+}
+
 func saveScenesToExternalReferences(scenes QueryScenesResult, studioId string) {
 	tlog := log.WithField("task", "scrape")
 	startTime := time.Now()