Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Configure requests and collectors #1946

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Configure requests and collectors #1946

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f8a40d8
Ability to Config Header/Cookies for Colly Collectors or Http Requests
toshski Jan 19, 2025
9045670
Add option for other params, willl depend on specific site implmentation
toshski Jan 19, 2025
a89e2a4
set the html body in the collector
toshski Jan 19, 2025
e7615f4
Options UI, Migration and fixes
toshski Feb 26, 2025
36abd8d
Delete and Logging
toshski Feb 26, 2025
cf82f2b
Handle the some sites implementing the resolution as an int and some …
toshski Feb 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 30 additions & 12 deletions pkg/api/heresphere.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,12 +89,13 @@ type HeresphereMedia struct {
Sources []HeresphereSource `json:"sources"`
}

type StringOrInt string
type HeresphereSource struct {
Resolution int `json:"resolution"`
Height int `json:"height"`
Width int `json:"width"`
Size int64 `json:"size"`
URL string `json:"url"`
Resolution StringOrInt `json:"resolution"`
Height int `json:"height"`
Width int `json:"width"`
Size int64 `json:"size"`
URL string `json:"url"`
}

type HereSphereAlphaPackedSettings struct {
Expand Down Expand Up @@ -214,19 +215,19 @@ func (i HeresphereResource) getHeresphereFile(req *restful.Request, resp *restfu
var file models.File
db.Where(&models.File{ID: uint(fileId)}).First(&file)

resolution := file.VideoHeight
resolution := strconv.Itoa(file.VideoHeight)
height := file.VideoHeight
width := file.VideoWidth
if file.VideoProjection == "360_tb" {
resolution = resolution / 2
resolution = strconv.Itoa(file.VideoHeight / 2)
}

var media []HeresphereMedia
media = append(media, HeresphereMedia{
Name: fmt.Sprintf("File 1/1 %vp - %v", resolution, humanize.Bytes(uint64(file.Size))),
Sources: []HeresphereSource{
{
Resolution: resolution,
Resolution: StringOrInt(resolution),
Height: height,
Width: width,
Size: file.Size,
Expand Down Expand Up @@ -322,11 +323,11 @@ func (i HeresphereResource) getHeresphereScene(req *restful.Request, resp *restf
for i, file := range videoFiles {
height := file.VideoHeight
width := file.VideoWidth
resolution := file.VideoHeight
resolution := strconv.Itoa(file.VideoHeight)
vertresolution := file.VideoWidth

if file.VideoProjection == "360_tb" {
resolution = resolution / 2
resolution = strconv.Itoa(file.VideoHeight / 2)
vertresolution = vertresolution * 2
}

Expand All @@ -341,7 +342,7 @@ func (i HeresphereResource) getHeresphereScene(req *restful.Request, resp *restf
Name: fmt.Sprintf("File %v/%v %vp - %v", i+1, len(videoFiles), resolution, humanize.Bytes(uint64(file.Size))),
Sources: []HeresphereSource{
{
Resolution: resolution,
Resolution: StringOrInt(resolution),
Height: height,
Width: width,
Size: file.Size,
Expand All @@ -366,7 +367,7 @@ func (i HeresphereResource) getHeresphereScene(req *restful.Request, resp *restf
if len(encoding.VideoSources) > 0 {
hsp.Name = encoding.Name
for _, source := range encoding.VideoSources {
hspSource := HeresphereSource(source)
hspSource := HeresphereSource(HeresphereSource{Resolution: StringOrInt(strconv.Itoa(source.Resolution)), Height: source.Height, Width: source.Width, Size: source.Size, URL: source.URL})
hsp.Sources = append(hsp.Sources, hspSource)
}
media = append(media, hsp)
Expand Down Expand Up @@ -1099,3 +1100,20 @@ func getTrackAssignment(name string, trackAssignments *[]string) int {
*trackAssignments = append(*trackAssignments, name)
return len(*trackAssignments)
}

func (fs *StringOrInt) UnmarshalJSON(data []byte) error {
var str string
// Try to unmarshal as a string
if err := json.Unmarshal(data, &str); err == nil {
*fs = StringOrInt(str)
return nil
}
// Try to unmarshal as an int
var num int
if err := json.Unmarshal(data, &num); err == nil {
*fs = StringOrInt(fmt.Sprintf("%d", num))
return nil
}
// Return an error if neither worked
return fmt.Errorf("invalid resolution format: %s", data)
}
59 changes: 59 additions & 0 deletions pkg/api/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/xbapps/xbvr/pkg/common"
"github.com/xbapps/xbvr/pkg/config"
"github.com/xbapps/xbvr/pkg/models"
"github.com/xbapps/xbvr/pkg/scrape"
"github.com/xbapps/xbvr/pkg/tasks"
)

Expand Down Expand Up @@ -205,6 +206,14 @@ type RequestSaveOptionsStorage struct {
MatchOhash bool `json:"match_ohash"`
}

type RequestSaveCollectorConfig struct {
DomainKey string `json:"domain_key"`
Headers []scrape.ScrapeHttpKeyValue `json:"headers"`
Cookies []scrape.ScrapeHttpCookieDetail `json:"cookies"`
Body string `json:"body"`
Other []scrape.ScrapeHttpKeyValue `json:"other"`
}

type ConfigResource struct{}

func (i ConfigResource) WebService() *restful.WebService {
Expand Down Expand Up @@ -313,6 +322,11 @@ func (i ConfigResource) WebService() *restful.WebService {
ws.Route(ws.PUT("/custom-sites/create").To(i.createCustomSite).
Metadata(restfulspec.KeyOpenAPITags, tags))

// "Collector Config endpoints"
ws.Route(ws.GET("/collector-config-list").To(i.getCollectorConfigs))
ws.Route(ws.POST("/save-collector-config").To(i.saveCollectorConfigs))
ws.Route(ws.DELETE("/delete-collector-config").To(i.deleteCollectorConfig))

return ws
}

Expand Down Expand Up @@ -1077,3 +1091,48 @@ func (i ConfigResource) saveOptionsStorage(req *restful.Request, resp *restful.R

resp.WriteHeaderAndEntity(http.StatusOK, r)
}

func (i ConfigResource) getCollectorConfigs(req *restful.Request, resp *restful.Response) {
list := scrape.GetAllScrapeHttpConfigs()
resp.WriteHeaderAndEntity(http.StatusOK, &list)
}

func (i ConfigResource) saveCollectorConfigs(req *restful.Request, resp *restful.Response) {
var r RequestSaveCollectorConfig

if err := req.ReadEntity(&r); err != nil {
APIError(req, resp, http.StatusInternalServerError, err)
log.Error(err)
return
}

if r.DomainKey == "" {
APIError(req, resp, http.StatusInternalServerError, nil)
log.Error("Could not save collector config - name for config not found")
return
}
var config scrape.ScrapeHttpConfig
config.Cookies = r.Cookies
config.Headers = r.Headers
config.Body = r.Body
config.Other = r.Other
scrape.SaveScrapeHttpConfig(r.DomainKey, config)
}
func (i ConfigResource) deleteCollectorConfig(req *restful.Request, resp *restful.Response) {
var r RequestSaveCollectorConfig

if err := req.ReadEntity(&r); err != nil {
APIError(req, resp, http.StatusInternalServerError, err)
log.Error(err)
return
}

if r.DomainKey == "" {
APIError(req, resp, http.StatusInternalServerError, nil)
log.Error("Could not delete collector config - name for config not found")
return
}
var kv models.KV
kv.Key = r.DomainKey
kv.Delete()
}
76 changes: 66 additions & 10 deletions pkg/api/trailers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,41 +17,78 @@ import (
"github.com/xbapps/xbvr/pkg/scrape"
)

func LoadHeresphereScene(url string) HeresphereVideo {
response, err := http.Get(url)
func LoadHeresphereScene(scrapeParams string) HeresphereVideo {
var params models.TrailerScrape
if strings.HasPrefix(scrapeParams, "http") {
// keep backwards compatible with just url
params.SceneUrl = scrapeParams
} else {
json.Unmarshal([]byte(scrapeParams), &params)
}

method := "POST"
client := &http.Client{}
req, _ := http.NewRequest(method, params.SceneUrl, nil)

if params.KVHttpConfig == "" {
params.KVHttpConfig = scrape.GetCoreDomain(params.SceneUrl) + "-trailers"
}
log.Debugf("Using Header/Cookies from %s", params.KVHttpConfig)
scrape.SetupHtmlRequest(params.KVHttpConfig, req)
response, err := client.Do(req)

if err != nil {
return HeresphereVideo{}
}

responseData, err := io.ReadAll(response.Body)
if err != nil {
log.Errorf("Error from %s %s", url, err)
log.Errorf("Error from %s %s", params.SceneUrl, err)
}

var video HeresphereVideo
err = json.Unmarshal(responseData, &video)
if err != nil {
log.Errorf("Error from %s %s", url, err)
log.Errorf("Error from %s %s", params.SceneUrl, err)
}

return video
}

func LoadDeovrScene(url string) DeoScene {
response, err := http.Get(url)
func LoadDeovrScene(scrapeParams string) DeoScene {
var params models.TrailerScrape
if strings.HasPrefix(scrapeParams, "http") {
// keep backwards compatible with just url
params.SceneUrl = scrapeParams
} else {
json.Unmarshal([]byte(scrapeParams), &params)
}

method := "GET"
client := &http.Client{}
req, _ := http.NewRequest(method, params.SceneUrl, nil)

if params.KVHttpConfig == "" {
params.KVHttpConfig = scrape.GetCoreDomain(params.SceneUrl) + "-trailers"
}
log.Debugf("Using Header/Cookies from %s", params.KVHttpConfig)
scrape.SetupHtmlRequest(params.KVHttpConfig, req)

response, err := client.Do(req)

if err != nil {
return DeoScene{}
}

responseData, err := io.ReadAll(response.Body)
if err != nil {
log.Errorf("Error from %s %s", url, err)
log.Errorf("Error from %s %s", params.SceneUrl, err)
}

var video DeoScene
err = json.Unmarshal(responseData, &video)
if err != nil {
log.Errorf("Error from %s %s", url, err)
log.Errorf("Error from %s %s response: %s", params.SceneUrl, err, string(responseData))
}

db, _ := models.GetDB()
Expand All @@ -64,6 +101,11 @@ func ScrapeHtml(scrapeParams string) models.VideoSourceResponse {
c := colly.NewCollector(colly.UserAgent(scrape.UserAgent))
var params models.TrailerScrape
json.Unmarshal([]byte(scrapeParams), &params)
if params.KVHttpConfig == "" {
params.KVHttpConfig = scrape.GetCoreDomain(params.SceneUrl) + "-trailers"
}
log.Debugf("Using Header/Cookies from %s", params.KVHttpConfig)
scrape.SetupCollector(params.KVHttpConfig, c)

var srcs []models.VideoSource
c.OnHTML(`html`, func(e *colly.HTMLElement) {
Expand Down Expand Up @@ -102,7 +144,11 @@ func ScrapeJson(scrapeParams string) models.VideoSourceResponse {
c := colly.NewCollector(colly.UserAgent(scrape.UserAgent))
var params models.TrailerScrape
json.Unmarshal([]byte(scrapeParams), &params)

if params.KVHttpConfig == "" {
params.KVHttpConfig = scrape.GetCoreDomain(params.SceneUrl) + "-trailers"
}
log.Debugf("Using Header/Cookies from %s", params.KVHttpConfig)
scrape.SetupCollector(params.KVHttpConfig, c)
var srcs []models.VideoSource
c.OnHTML(`html`, func(e *colly.HTMLElement) {
e.ForEach(params.HtmlElement, func(id int, e *colly.HTMLElement) {
Expand Down Expand Up @@ -133,7 +179,17 @@ func LoadJson(scrapeParams string) models.VideoSourceResponse {
var params models.TrailerScrape
json.Unmarshal([]byte(scrapeParams), &params)

response, err := http.Get(params.SceneUrl)
method := "GET"
client := &http.Client{}
req, err := http.NewRequest(method, params.SceneUrl, nil)

if params.KVHttpConfig == "" {
params.KVHttpConfig = scrape.GetCoreDomain(params.SceneUrl) + "-trailers"
}
log.Debugf("Using Header/Cookies from %s", params.KVHttpConfig)
scrape.SetupHtmlRequest(params.KVHttpConfig, req)
response, err := client.Do(req)

if err != nil {
return models.VideoSourceResponse{}
}
Expand Down
Loading
Loading