diff --git a/pkg/api/heresphere.go b/pkg/api/heresphere.go index 290c63ee6..6f7ea98d8 100644 --- a/pkg/api/heresphere.go +++ b/pkg/api/heresphere.go @@ -89,12 +89,13 @@ type HeresphereMedia struct { Sources []HeresphereSource `json:"sources"` } +type StringOrInt string type HeresphereSource struct { - Resolution int `json:"resolution"` - Height int `json:"height"` - Width int `json:"width"` - Size int64 `json:"size"` - URL string `json:"url"` + Resolution StringOrInt `json:"resolution"` + Height int `json:"height"` + Width int `json:"width"` + Size int64 `json:"size"` + URL string `json:"url"` } type HereSphereAlphaPackedSettings struct { @@ -214,11 +215,11 @@ func (i HeresphereResource) getHeresphereFile(req *restful.Request, resp *restfu var file models.File db.Where(&models.File{ID: uint(fileId)}).First(&file) - resolution := file.VideoHeight + resolution := strconv.Itoa(file.VideoHeight) height := file.VideoHeight width := file.VideoWidth if file.VideoProjection == "360_tb" { - resolution = resolution / 2 + resolution = strconv.Itoa(file.VideoHeight / 2) } var media []HeresphereMedia @@ -226,7 +227,7 @@ func (i HeresphereResource) getHeresphereFile(req *restful.Request, resp *restfu Name: fmt.Sprintf("File 1/1 %vp - %v", resolution, humanize.Bytes(uint64(file.Size))), Sources: []HeresphereSource{ { - Resolution: resolution, + Resolution: StringOrInt(resolution), Height: height, Width: width, Size: file.Size, @@ -322,11 +323,11 @@ func (i HeresphereResource) getHeresphereScene(req *restful.Request, resp *restf for i, file := range videoFiles { height := file.VideoHeight width := file.VideoWidth - resolution := file.VideoHeight + resolution := strconv.Itoa(file.VideoHeight) vertresolution := file.VideoWidth if file.VideoProjection == "360_tb" { - resolution = resolution / 2 + resolution = strconv.Itoa(file.VideoHeight / 2) vertresolution = vertresolution * 2 } @@ -341,7 +342,7 @@ func (i HeresphereResource) getHeresphereScene(req *restful.Request, resp *restf Name: fmt.Sprintf("File %v/%v %vp - %v", i+1, len(videoFiles), resolution, humanize.Bytes(uint64(file.Size))), Sources: []HeresphereSource{ { - Resolution: resolution, + Resolution: StringOrInt(resolution), Height: height, Width: width, Size: file.Size, @@ -366,7 +367,7 @@ func (i HeresphereResource) getHeresphereScene(req *restful.Request, resp *restf if len(encoding.VideoSources) > 0 { hsp.Name = encoding.Name for _, source := range encoding.VideoSources { - hspSource := HeresphereSource(source) + hspSource := HeresphereSource(HeresphereSource{Resolution: StringOrInt(strconv.Itoa(source.Resolution)), Height: source.Height, Width: source.Width, Size: source.Size, URL: source.URL}) hsp.Sources = append(hsp.Sources, hspSource) } media = append(media, hsp) @@ -1099,3 +1100,20 @@ func getTrackAssignment(name string, trackAssignments *[]string) int { *trackAssignments = append(*trackAssignments, name) return len(*trackAssignments) } + +func (fs *StringOrInt) UnmarshalJSON(data []byte) error { + var str string + // Try to unmarshal as a string + if err := json.Unmarshal(data, &str); err == nil { + *fs = StringOrInt(str) + return nil + } + // Try to unmarshal as an int + var num int + if err := json.Unmarshal(data, &num); err == nil { + *fs = StringOrInt(fmt.Sprintf("%d", num)) + return nil + } + // Return an error if neither worked + return fmt.Errorf("invalid resolution format: %s", data) +} diff --git a/pkg/api/options.go b/pkg/api/options.go index 50f0ed464..b390752a4 100644 --- a/pkg/api/options.go +++ b/pkg/api/options.go @@ -27,6 +27,7 @@ import ( "github.com/xbapps/xbvr/pkg/common" "github.com/xbapps/xbvr/pkg/config" "github.com/xbapps/xbvr/pkg/models" + "github.com/xbapps/xbvr/pkg/scrape" "github.com/xbapps/xbvr/pkg/tasks" ) @@ -205,6 +206,14 @@ type RequestSaveOptionsStorage struct { MatchOhash bool `json:"match_ohash"` } +type RequestSaveCollectorConfig struct { + DomainKey string `json:"domain_key"` + Headers []scrape.ScrapeHttpKeyValue `json:"headers"` + Cookies []scrape.ScrapeHttpCookieDetail `json:"cookies"` + Body string `json:"body"` + Other []scrape.ScrapeHttpKeyValue `json:"other"` +} + type ConfigResource struct{} func (i ConfigResource) WebService() *restful.WebService { @@ -313,6 +322,11 @@ func (i ConfigResource) WebService() *restful.WebService { ws.Route(ws.PUT("/custom-sites/create").To(i.createCustomSite). Metadata(restfulspec.KeyOpenAPITags, tags)) + // "Collector Config endpoints" + ws.Route(ws.GET("/collector-config-list").To(i.getCollectorConfigs)) + ws.Route(ws.POST("/save-collector-config").To(i.saveCollectorConfigs)) + ws.Route(ws.DELETE("/delete-collector-config").To(i.deleteCollectorConfig)) + return ws } @@ -1077,3 +1091,48 @@ func (i ConfigResource) saveOptionsStorage(req *restful.Request, resp *restful.R resp.WriteHeaderAndEntity(http.StatusOK, r) } + +func (i ConfigResource) getCollectorConfigs(req *restful.Request, resp *restful.Response) { + list := scrape.GetAllScrapeHttpConfigs() + resp.WriteHeaderAndEntity(http.StatusOK, &list) +} + +func (i ConfigResource) saveCollectorConfigs(req *restful.Request, resp *restful.Response) { + var r RequestSaveCollectorConfig + + if err := req.ReadEntity(&r); err != nil { + APIError(req, resp, http.StatusInternalServerError, err) + log.Error(err) + return + } + + if r.DomainKey == "" { + APIError(req, resp, http.StatusInternalServerError, nil) + log.Error("Could not save collector config - name for config not found") + return + } + var config scrape.ScrapeHttpConfig + config.Cookies = r.Cookies + config.Headers = r.Headers + config.Body = r.Body + config.Other = r.Other + scrape.SaveScrapeHttpConfig(r.DomainKey, config) +} +func (i ConfigResource) deleteCollectorConfig(req *restful.Request, resp *restful.Response) { + var r RequestSaveCollectorConfig + + if err := req.ReadEntity(&r); err != nil { + APIError(req, resp, http.StatusInternalServerError, err) + log.Error(err) + return + } + + if r.DomainKey == "" { + APIError(req, resp, http.StatusInternalServerError, nil) + log.Error("Could not delete collector config - name for config not found") + return + } + var kv models.KV + kv.Key = r.DomainKey + kv.Delete() +} diff --git a/pkg/api/trailers.go b/pkg/api/trailers.go index 17f7235ab..a5f04aa92 100644 --- a/pkg/api/trailers.go +++ b/pkg/api/trailers.go @@ -17,41 +17,78 @@ import ( "github.com/xbapps/xbvr/pkg/scrape" ) -func LoadHeresphereScene(url string) HeresphereVideo { - response, err := http.Get(url) +func LoadHeresphereScene(scrapeParams string) HeresphereVideo { + var params models.TrailerScrape + if strings.HasPrefix(scrapeParams, "http") { + // keep backwards compatible with just url + params.SceneUrl = scrapeParams + } else { + json.Unmarshal([]byte(scrapeParams), ¶ms) + } + + method := "POST" + client := &http.Client{} + req, _ := http.NewRequest(method, params.SceneUrl, nil) + + if params.KVHttpConfig == "" { + params.KVHttpConfig = scrape.GetCoreDomain(params.SceneUrl) + "-trailers" + } + log.Debugf("Using Header/Cookies from %s", params.KVHttpConfig) + scrape.SetupHtmlRequest(params.KVHttpConfig, req) + response, err := client.Do(req) + if err != nil { return HeresphereVideo{} } responseData, err := io.ReadAll(response.Body) if err != nil { - log.Errorf("Error from %s %s", url, err) + log.Errorf("Error from %s %s", params.SceneUrl, err) } var video HeresphereVideo err = json.Unmarshal(responseData, &video) if err != nil { - log.Errorf("Error from %s %s", url, err) + log.Errorf("Error from %s %s", params.SceneUrl, err) } return video } -func LoadDeovrScene(url string) DeoScene { - response, err := http.Get(url) +func LoadDeovrScene(scrapeParams string) DeoScene { + var params models.TrailerScrape + if strings.HasPrefix(scrapeParams, "http") { + // keep backwards compatible with just url + params.SceneUrl = scrapeParams + } else { + json.Unmarshal([]byte(scrapeParams), ¶ms) + } + + method := "GET" + client := &http.Client{} + req, _ := http.NewRequest(method, params.SceneUrl, nil) + + if params.KVHttpConfig == "" { + params.KVHttpConfig = scrape.GetCoreDomain(params.SceneUrl) + "-trailers" + } + log.Debugf("Using Header/Cookies from %s", params.KVHttpConfig) + scrape.SetupHtmlRequest(params.KVHttpConfig, req) + + response, err := client.Do(req) + if err != nil { return DeoScene{} } responseData, err := io.ReadAll(response.Body) if err != nil { - log.Errorf("Error from %s %s", url, err) + log.Errorf("Error from %s %s", params.SceneUrl, err) } var video DeoScene err = json.Unmarshal(responseData, &video) if err != nil { - log.Errorf("Error from %s %s", url, err) + log.Errorf("Error from %s %s response: %s", params.SceneUrl, err, string(responseData)) } db, _ := models.GetDB() @@ -64,6 +101,11 @@ func ScrapeHtml(scrapeParams string) models.VideoSourceResponse { c := colly.NewCollector(colly.UserAgent(scrape.UserAgent)) var params models.TrailerScrape json.Unmarshal([]byte(scrapeParams), ¶ms) + if params.KVHttpConfig == "" { + params.KVHttpConfig = scrape.GetCoreDomain(params.SceneUrl) + "-trailers" + } + log.Debugf("Using Header/Cookies from %s", params.KVHttpConfig) + scrape.SetupCollector(params.KVHttpConfig, c) var srcs []models.VideoSource c.OnHTML(`html`, func(e *colly.HTMLElement) { @@ -102,7 +144,11 @@ func ScrapeJson(scrapeParams string) models.VideoSourceResponse { c := colly.NewCollector(colly.UserAgent(scrape.UserAgent)) var params models.TrailerScrape json.Unmarshal([]byte(scrapeParams), ¶ms) - + if params.KVHttpConfig == "" { + params.KVHttpConfig = scrape.GetCoreDomain(params.SceneUrl) + "-trailers" + } + log.Debugf("Using Header/Cookies from %s", params.KVHttpConfig) + scrape.SetupCollector(params.KVHttpConfig, c) var srcs []models.VideoSource c.OnHTML(`html`, func(e *colly.HTMLElement) { e.ForEach(params.HtmlElement, func(id int, e *colly.HTMLElement) { @@ -133,7 +179,17 @@ func LoadJson(scrapeParams string) models.VideoSourceResponse { var params models.TrailerScrape json.Unmarshal([]byte(scrapeParams), ¶ms) - response, err := http.Get(params.SceneUrl) + method := "GET" + client := &http.Client{} + req, err := http.NewRequest(method, params.SceneUrl, nil) + + if params.KVHttpConfig == "" { + params.KVHttpConfig = scrape.GetCoreDomain(params.SceneUrl) + "-trailers" + } + log.Debugf("Using Header/Cookies from %s", params.KVHttpConfig) + scrape.SetupHtmlRequest(params.KVHttpConfig, req) + response, err := client.Do(req) + if err != nil { return models.VideoSourceResponse{} } diff --git a/pkg/migrations/migrations.go b/pkg/migrations/migrations.go index 402c4f5e7..11441f52f 100644 --- a/pkg/migrations/migrations.go +++ b/pkg/migrations/migrations.go @@ -2159,7 +2159,96 @@ func Migrate() { return e }, }, - }) + { + ID: "0084-Migrate-Trailer-Sources", + Migrate: func(tx *gorm.DB) error { + // declare common update function + updateScenesWithTrailer := func(scenes []models.Scene, trailerType string) error { + for _, scene := range scenes { + scene.TrailerType = trailerType + params := models.TrailerScrape{SceneUrl: scene.TrailerSource} + strParams, _ := json.Marshal(params) + scene.TrailerSource = string(strParams) + + if err := scene.Save(); err != nil { + return err + } + } + return nil + } + var scenes []models.Scene + // naughty america + err := db.Where("scene_id like ?", "naughtyamerica-%").Find(&scenes).Error + if err != nil { + return err + } + for _, scene := range scenes { + scene.TrailerType = "heresphere" + params := models.TrailerScrape{SceneUrl: "https://api.naughtyapi.com/heresphere/" + strings.TrimLeft(scene.SceneID, "naughtyamerica-vr-")} + strParams, _ := json.Marshal(params) + scene.TrailerSource = string(strParams) + err = scene.Save() + if err != nil { + return err + } + } + // czech network + err = db.Where("trailer_type='heresphere' and trailer_source like 'https://www.czechvrnetwork.com/heresphere%'").Find(&scenes).Error + if err != nil { + return err + } + err = updateScenesWithTrailer(scenes, "heresphere") + if err != nil { + return err + } + // povr + err = db.Where("trailer_type='heresphere' and trailer_source like 'https://www.povr.com/heresphere%'").Find(&scenes).Error + if err != nil { + return err + } + err = updateScenesWithTrailer(scenes, "heresphere") + if err != nil { + return err + } + // stasyqvr + err = db.Where("trailer_type='deovr' and trailer_source like 'http://stasyqvr.com/deovr%'").Find(&scenes).Error + if err != nil { + return err + } + err = updateScenesWithTrailer(scenes, "deovr") + if err != nil { + return err + } + // zexyvr + err = db.Where("trailer_type='deovr' and trailer_source like 'https://zexyvr.com/deovr%'").Find(&scenes).Error + if err != nil { + return err + } + err = updateScenesWithTrailer(scenes, "deovr") + if err != nil { + return err + } + // wankitnowvr + err = db.Where("trailer_type='deovr' and trailer_source like 'https://wankitnowvr.com/deovr%'").Find(&scenes).Error + if err != nil { + return err + } + err = updateScenesWithTrailer(scenes, "deovr") + if err != nil { + return err + } + // slr + err = db.Where("trailer_type='slr'").Find(&scenes).Error + if err != nil { + return err + } + err = updateScenesWithTrailer(scenes, "slr") + if err != nil { + return err + } + return nil + }, + }}) if err := m.Migrate(); err != nil { common.Log.Fatalf("Could not migrate: %v", err) diff --git a/pkg/models/model_scraper.go b/pkg/models/model_scraper.go index 2af2b0292..1c5f8e445 100644 --- a/pkg/models/model_scraper.go +++ b/pkg/models/model_scraper.go @@ -65,6 +65,7 @@ type TrailerScrape struct { ContentPath string `json:"content_path"` // points to the content url uses jsonpath syntax EncodingPath string `json:"encoding_path"` // optional, points to the encoding for the source using jsonpath syntax, eg h264, h265 QualityPath string `json:"quality_path"` // points to the quality using jsonpath syntax, eg 1440p, 5k + KVHttpConfig string `json:"kv_http_config"` } func (s *ScrapedScene) ToJSON() ([]byte, error) { diff --git a/pkg/scrape/czechvr.go b/pkg/scrape/czechvr.go index bdb17b074..4087d02bf 100644 --- a/pkg/scrape/czechvr.go +++ b/pkg/scrape/czechvr.go @@ -1,6 +1,7 @@ package scrape import ( + "encoding/json" "regexp" "strconv" "strings" @@ -110,7 +111,9 @@ func CzechVR(wg *models.ScrapeWG, updateSite bool, knownScenes []string, out cha r := re.FindStringSubmatch(sc.HomepageURL) if len(r) > 0 { sc.TrailerType = "heresphere" - sc.TrailerSrc = "https://www.czechvrnetwork.com/heresphere/videoID" + r[1] + params := models.TrailerScrape{SceneUrl: "https://www.czechvrnetwork.com/heresphere/videoID" + r[1]} + strParams, _ := json.Marshal(params) + sc.TrailerSrc = string(strParams) } // Filenames diff --git a/pkg/scrape/navr.go b/pkg/scrape/navr.go index 4ee8f4aba..bdf516545 100644 --- a/pkg/scrape/navr.go +++ b/pkg/scrape/navr.go @@ -1,6 +1,7 @@ package scrape import ( + "encoding/json" "html" "strconv" "strings" @@ -83,8 +84,13 @@ func NaughtyAmericaVR(wg *models.ScrapeWG, updateSite bool, knownScenes []string base[8] = "vertical" base[9] = "1182x1788c.jpg" sc.Covers = append(sc.Covers, "https://"+strings.Join(base, "/")) - sc.TrailerSrc = `https://videos.naughtycdn.com/` + base[5] + `/trailers/vr/` + base[5] + base[6] + `/` + base[5] + base[6] + `teaser_vrdesktophd.mp4` }) + + sc.TrailerType = "heresphere" + params := models.TrailerScrape{SceneUrl: "https://api.naughtyapi.com/heresphere/" + sc.SiteID} + strParams, _ := json.Marshal(params) + sc.TrailerSrc = string(strParams) + // Old video element e.ForEach(`a.play-trailer img.start-card.desktop-only`, func(id int, e *colly.HTMLElement) { // images5.naughtycdn.com/cms/nacmscontent/v1/scenes/2cst/nikkijaclynmarco/scene/horizontal/1252x708c.jpg @@ -112,7 +118,6 @@ func NaughtyAmericaVR(wg *models.ScrapeWG, updateSite bool, knownScenes []string base[8] = "vertical" base[9] = "1182x1788c.jpg" sc.Covers = append(sc.Covers, "https://"+strings.Join(base, "/")) - sc.TrailerSrc = `https://videos.naughtycdn.com/` + base[5] + `/trailers/vr/` + base[5] + base[6] + `/` + base[5] + base[6] + `teaser_vrdesktophd.mp4` }) // Gallery diff --git a/pkg/scrape/povr.go b/pkg/scrape/povr.go index 092ef5c7f..1ece3b517 100644 --- a/pkg/scrape/povr.go +++ b/pkg/scrape/povr.go @@ -1,6 +1,7 @@ package scrape import ( + "encoding/json" "fmt" "path" "regexp" @@ -108,7 +109,9 @@ func POVR(wg *models.ScrapeWG, updateSite bool, knownScenes []string, out chan<- // trailer details sc.TrailerType = "heresphere" - sc.TrailerSrc = "https://www.povr.com/heresphere/" + sc.SiteID + params := models.TrailerScrape{SceneUrl: "https://www.povr.com/heresphere/" + sc.SiteID} + strParams, _ := json.Marshal(params) + sc.TrailerSrc = string(strParams) // Cast sc.ActorDetails = make(map[string]models.ActorDetails) diff --git a/pkg/scrape/scrape.go b/pkg/scrape/scrape.go index 355958cce..42aa7b544 100644 --- a/pkg/scrape/scrape.go +++ b/pkg/scrape/scrape.go @@ -37,6 +37,8 @@ func createCollector(domains ...string) *colly.Collector { // see if the domain has a limit and set it for _, domain := range domains { + SetupCollector(GetCoreDomain(domain)+"-scraper", c) + log.Debugf("Using Header/Cookies from %s", GetCoreDomain(domain)+"-scraper") if Limiters == nil { LoadScraperRateLimits() } @@ -191,3 +193,16 @@ func getTextFromHTMLWithSelector(data string, sel string) string { func CreateCollector(domains ...string) *colly.Collector { return createCollector(domains...) } + +func GetCoreDomain(domain string) string { + if strings.HasPrefix(domain, "http") { + parsedURL, _ := url.Parse(domain) + domain = parsedURL.Hostname() + } + parts := strings.Split(domain, ".") + if len(parts) > 2 && parts[0] == "www" { + parts = parts[1:] + } + + return strings.Join(parts[:len(parts)-1], ".") +} diff --git a/pkg/scrape/scrape_http_config.go b/pkg/scrape/scrape_http_config.go new file mode 100644 index 000000000..9584420d9 --- /dev/null +++ b/pkg/scrape/scrape_http_config.go @@ -0,0 +1,105 @@ +package scrape + +import ( + "encoding/json" + "io" + "net/http" + "strings" + + "github.com/gocolly/colly/v2" + "github.com/xbapps/xbvr/pkg/models" +) + +type ScrapeHttpKeyValue struct { + Key string `json:"key"` + Value string `json:"value"` +} +type ScrapeHttpCookieDetail struct { + Name string `json:"name"` + Value string `json:"value"` + Domain string `json:"domain"` + Path string `json:"path"` + Host string `json:"host"` +} +type ScrapeHttpConfig struct { + Headers []ScrapeHttpKeyValue `json:"headers"` + Cookies []ScrapeHttpCookieDetail `json:"cookies"` + Body string `json:"body"` + Other []ScrapeHttpKeyValue `json:"other"` +} + +type ScrapeHttpKeyAndConfig struct { + Id string `json:"domain_key"` + Config ScrapeHttpConfig `json:"config"` +} + +func SetupHtmlRequest(kvKey string, req *http.Request) *http.Request { + conf := GetScrapeHttpConfig(kvKey) + for _, header := range conf.Headers { + req.Header.Add(header.Key, header.Value) + } + for _, cookie := range conf.Cookies { + req.AddCookie(&http.Cookie{Name: cookie.Name, Value: cookie.Value, Domain: cookie.Domain, Path: cookie.Path}) + } + if conf.Body != "" { + body := strings.NewReader(conf.Body) + req.Body = io.NopCloser(body) + } + return req +} +func SetupCollector(kvKey string, collector *colly.Collector) *colly.Collector { + conf := GetScrapeHttpConfig(kvKey) + // setup header for the OnRequest Function + if len(conf.Headers) > 0 || conf.Body != "" { + collector.OnRequest(func(r *colly.Request) { + for _, header := range conf.Headers { + r.Headers.Set(header.Key, header.Value) + } + if conf.Body != "" { + body := strings.NewReader(conf.Body) + r.Body = io.NopCloser(body) + } + }) + } + // setup cookies + for _, cookie := range conf.Cookies { + c := collector.Cookies(cookie.Host) + newcookie := http.Cookie{Name: cookie.Name, Value: cookie.Value, Domain: cookie.Domain, Path: cookie.Path} + c = append(c, &newcookie) + collector.SetCookies(cookie.Host, c) + } + return collector + +} +func GetScrapeHttpConfig(kvKey string) ScrapeHttpConfig { + db, _ := models.GetCommonDB() + + c := ScrapeHttpConfig{} + var kv models.KV + kv.Key = kvKey + db.Find(&kv) + json.Unmarshal([]byte(kv.Value), &c) + return c +} + +func SaveScrapeHttpConfig(kvKey string, config ScrapeHttpConfig) { + var kv models.KV + kv.Key = kvKey + jsonStr, _ := json.MarshalIndent(config, "", " ") + kv.Value = string(jsonStr) + kv.Save() +} + +func GetAllScrapeHttpConfigs() []ScrapeHttpKeyAndConfig { + db, _ := models.GetCommonDB() + + c := ScrapeHttpConfig{} + configList := []ScrapeHttpKeyAndConfig{} + var kvs []models.KV + db.Where("(`value` like '%headers%' and `value` like '%cookies%') or (`key` like '%-scraper' and `key` like '%-trailers')").Find(&kvs) + for _, kv := range kvs { + json.Unmarshal([]byte(kv.Value), &c) + configList = append(configList, ScrapeHttpKeyAndConfig{kv.Key, c}) + } + return configList +} diff --git a/pkg/scrape/slrstudios.go b/pkg/scrape/slrstudios.go index e4a9e8f9c..69c1d7caa 100644 --- a/pkg/scrape/slrstudios.go +++ b/pkg/scrape/slrstudios.go @@ -121,7 +121,10 @@ func SexLikeReal(wg *models.ScrapeWG, updateSite bool, knownScenes []string, out // sc.TrailerSrc = string(jsonStr) sc.TrailerType = "slr" - sc.TrailerSrc = "https://api.sexlikereal.com/virtualreality/video/id/" + sc.SiteID + params := models.TrailerScrape{SceneUrl: "https://api.sexlikereal.com/virtualreality/video/id/" + sc.SiteID} + strParams, _ := json.Marshal(params) + sc.TrailerSrc = string(strParams) + s, _ := resty.New().R(). SetHeader("User-Agent", UserAgent). Get(sc.TrailerSrc) diff --git a/pkg/scrape/stasyqvr.go b/pkg/scrape/stasyqvr.go index e55708dbd..5d7966713 100644 --- a/pkg/scrape/stasyqvr.go +++ b/pkg/scrape/stasyqvr.go @@ -1,6 +1,7 @@ package scrape import ( + "encoding/json" "net/url" "strconv" "strings" @@ -66,7 +67,9 @@ func StasyQVR(wg *models.ScrapeWG, updateSite bool, knownScenes []string, out ch // trailer details sc.TrailerType = "deovr" - sc.TrailerSrc = `http://stasyqvr.com/deovr_feed/json/id/` + sc.SiteID + params := models.TrailerScrape{SceneUrl: `http://stasyqvr.com/deovr_feed/json/id/` + sc.SiteID} + strParams, _ := json.Marshal(params) + sc.TrailerSrc = string(strParams) // Cast sc.ActorDetails = make(map[string]models.ActorDetails) diff --git a/pkg/scrape/zexywankitnow.go b/pkg/scrape/zexywankitnow.go index 7aa1e8790..de73d1ffb 100644 --- a/pkg/scrape/zexywankitnow.go +++ b/pkg/scrape/zexywankitnow.go @@ -1,6 +1,7 @@ package scrape import ( + "encoding/json" "regexp" "strconv" "strings" @@ -87,7 +88,9 @@ func TwoWebMediaSite(wg *models.ScrapeWG, updateSite bool, knownScenes []string, // trailer details sc.TrailerType = "deovr" - sc.TrailerSrc = strings.Replace(URL, `/videos/`, ``, 1) + `/deovr/video/` + sc.SiteID + params := models.TrailerScrape{SceneUrl: strings.Replace(URL, `/videos/`, ``, 1) + `/deovr/video/` + sc.SiteID} + strParams, _ := json.Marshal(params) + sc.TrailerSrc = string(strParams) // Cast & Tags // Note: Cast/Tags links are currently all inside the same div element... diff --git a/ui/src/locales/en-GB.json b/ui/src/locales/en-GB.json index 6efb0854d..8e453883a 100644 --- a/ui/src/locales/en-GB.json +++ b/ui/src/locales/en-GB.json @@ -240,5 +240,6 @@ "Linked to Alternate Sites":"Linked to Alternate Sites", "Go": "Go", "Limit scraping to newest scenes on the website. Turn off if you are missing scenes.": "Limit scraping to newest scenes on the website. Turn off if you are missing scenes.", - "Highlights this studio in the scene view and includes scenes in the "Has subscription" attribute filter": "Highlights this studio in the scene view and includes scenes in the "Has subscription" attribute filter" + "Highlights this studio in the scene view and includes scenes in the "Has subscription" attribute filter": "Highlights this studio in the scene view and includes scenes in the "Has subscription" attribute filter", + "Cookies/Headers":"Cookies/Headers" } diff --git a/ui/src/store/optionsAdvanced.js b/ui/src/store/optionsAdvanced.js index 971252bd2..be416f90d 100644 --- a/ui/src/store/optionsAdvanced.js +++ b/ui/src/store/optionsAdvanced.js @@ -13,6 +13,7 @@ const state = { useAltSrcInFileMatching: true, useAltSrcInScriptFilters: true, ignoreReleasedBefore: null, + collectorConfigs: null, } } @@ -21,6 +22,11 @@ const mutations = {} const actions = { async load ({ state }) { state.loading = true + ky.get('/api/options/collector-config-list') + .json() + .then(data => { + state.advanced.collectorConfigs = data + }) ky.get('/api/options/state') .json() .then(data => { diff --git a/ui/src/views/options/sections/InterfaceAdvanced.vue b/ui/src/views/options/sections/InterfaceAdvanced.vue index e841f2aaf..d807668e7 100644 --- a/ui/src/views/options/sections/InterfaceAdvanced.vue +++ b/ui/src/views/options/sections/InterfaceAdvanced.vue @@ -9,6 +9,7 @@ + @@ -165,6 +166,90 @@ + +
+
+
+ +

+ Domain Config + Wiki +

+ + + + + + + Import + + + {{ file.name }} + + + + + Save + + + + + + + +

Headers

+ + + + + + + + + + + + + + +

Cookies

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ @@ -185,7 +270,35 @@ export default { scraperAvatar: '', scraperFieldsValid: false, masterSiteId: '', + kvName: "", + headers: [], + cookies: [], + body: "", + showCollectorConfigFields: false, + file: null, + uploadData: '', + } + }, + watch: { + // when a file is selected, then this will fire the upload process + file: function (o, n) { + try { + if (this.file != null) { + const reader = new FileReader() + reader.onload = (event) => { + try { + this.uploadData = JSON.stringify(JSON.parse(event.target.result)) + this.restoreCollectorConfig() + } catch (error) { + this.$buefy.toast.open({message: `Error: ${error.message}`, type: 'is-danger', duration: 30000}) + } + } + reader.readAsText(this.file) + } + } catch (error) { + this.$buefy.toast.open({message: `Error: ${error.message}`, type: 'is-danger', duration: 30000}) } + }, }, methods: { save () { @@ -228,6 +341,101 @@ export default { relinkAltSrc () { ky.get('/api/task/relink_alt_aource_scenes') }, + showConfigDetails(option) { + let matched = this.$store.state.optionsAdvanced.advanced.collectorConfigs.find((config) => { + return config.domain_key + .toString() + .toLowerCase() + .trim() + .indexOf(option.toLowerCase()) >= 0 + }) + if (matched == null ) return + if (matched.config.headers==null) { + this.headers = [] + } else { + this.headers = matched.config.headers + } + if (matched.config.cookies==null) { + this.cookies = [] + } else { + this.cookies = matched.config.cookies + } + this.body= matched.config.body + this.showCollectorConfigFields = true + }, + addHeaderRow(){ + this.headers.push({ key: "", value: "" }); + }, + delHeaderRow(props){ + this.headers.splice(props.index,1) + }, + addCookieRow(){ + this.cookies.push({ name: "", value: "", domain: "", path: "", host:"" }); + }, + delCookieRow(props){ + this.cookies.splice(props.index,1) + }, + saveCollectorConfig() { + ky.post('/api/options/save-collector-config', { + json: { + domain_key: this.kvName, + body: this.body, + cookies: this.cookies, + headers: this.headers, + other: [], + } + }) + let row = this.$store.state.optionsAdvanced.advanced.collectorConfigs.find((config) => { + return config.domain_key + .toString() + .toLowerCase() + .trim() + .indexOf(this.kvName.toString().toLowerCase()) >= 0 + }) + row.config.cookies = this.cookies + row.config.headers=this.headers + row.config.body = this.body + }, + showAddCollectorConfig() { + this.$buefy.dialog.prompt({ + message: `Add new config`, + inputAttrs: { + placeholder: 'domainname-scraper or domainname-trailers e.g. naughtyamerica-trailers', + maxlength: 20, + value: this.kvName + }, + confirmText: 'Add', + onConfirm: (value) => { + this.kvName=value + this.$store.state.optionsAdvanced.advanced.collectorConfigs.push({config: {body: "", cookies: [], headers: [], other: ""},domain_key: value} ) + this.$refs.autocompleteconfig.setSelected(value) + this.showCollectorConfigFields = true + } + }) + }, + async restoreCollectorConfig () { + if (this.uploadData !== '') { + try { + const response = await ky.post('/api/options/save-collector-config', { + json: JSON.parse(this.uploadData) + }) + } catch (error) { + this.$buefy.toast.open({message: `Error: Failed to import file`, type: 'is-danger', duration: 30000}) + return + } + this.file = null + this.$store.dispatch('optionsAdvanced/load') + } + }, + async deleteCollectorConfig() { + const response = await ky.delete('/api/options/delete-collector-config', { + json: { + domain_key: this.kvName, + } + }) + this.kvName="" + this.$store.dispatch('optionsAdvanced/load') + }, }, computed: { showInternalSceneId: { @@ -311,7 +519,7 @@ export default { }, set (value) { this.$store.state.optionsAdvanced.advanced.ignoreReleasedBefore = value - } + }, }, listOfMainSites: { get () { @@ -336,6 +544,27 @@ export default { showMatchParamsOverlay () { return this.$store.state.overlay.sceneMatchParams.show }, + filteredCollectorConfigList () { + // filter the list based on what has been entered so far + if (this.$store.state.optionsAdvanced.advanced.collectorConfigs.length==0) return + + let matched = this.$store.state.optionsAdvanced.advanced.collectorConfigs.filter((config) => { + return config.domain_key + .toLowerCase() + .trim() + .indexOf(this.kvName.toLowerCase()) >= 0 + }) + if (matched.length == 1 && matched[0].domain_key.toLowerCase() == this.kvName.toLowerCase()){ + this.showCollectorConfigFields = true + this.showConfigDetails(this.kvName) + } else { + this.showCollectorConfigFields = false + } + return matched.map(item => item.domain_key) + }, + showConfigField () { + return this.showCollectorConfigFields + }, isLoading: function () { return this.$store.state.optionsAdvanced.loading }