ScratchScrape/main.py at main · spooketti/ScratchScrape · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
import requests
import sqlalchemy
from init import init, Session, db
from model.user import User
from model.project import Project
import time
import atexit
import signal
import os
# from bs4 import BeautifulSoup

# def grabDataFromURL(url):
#     response = requests.get(url)
#     if response.status_code != 200:
#         print("Did not return success code")
#         print(response.text)
#     soup = BeautifulSoup(response.content,)

currentProjectIndex = open("currentProjectIndex.txt","r")
currentProject = int(currentProjectIndex.read())
currentProjectIndex.close()


projectApiLink = "https://api.scratch.mit.edu/projects/"
userApiLink = "https://api.scratch.mit.edu/users/"
testProjectID = [1072618379,938868806,983800322]

def grabProjectData(projectID):
    response = requests.get(projectApiLink+str(projectID))

    if response.status_code != 200:
        print("Unable to reach project")
        print(response.text)
        return

    data = response.json()
    dbProjectID = projectID
    dbTitle = data["title"]
    dbDesc = data["description"]
    dbAuthor = data["author"]["username"]
    dbCreateOn = data["history"]["created"]
    dbLastMod = data["history"]["modified"]
    dbRemixRoot = data["remix"]["root"]
    dbViews = data["stats"]["views"]
    dbRemixes = data["stats"]["remixes"]
    projectToken = data["project_token"]

    session.add(Project(projectID=dbProjectID,
    title=dbTitle,description=dbDesc,author=dbAuthor,createdOn=dbCreateOn,
    lastModified=dbLastMod,remixRoot=dbRemixRoot,views=dbViews,remixes=dbRemixes))
    session.commit()

    grabProjectFile(projectID,projectToken)
    grabUserData(dbAuthor)

def grabProjectFile(projectID, projectToken):
    filename = os.path.join("./projects/",str(projectID)+".json")
    open(filename,"wb").write(requests.get("https://projects.scratch.mit.edu/"+str(projectID)+"?token="+str(projectToken)).content)

def grabUserData(scratchun):
    if bool(session.query(User).filter_by(username=scratchun).first()):
        return
    response = requests.get(userApiLink+str(scratchun))
    if response.status_code != 200:
        print("Unable to reach User")
        print(response.text)
        return
    data = response.json()
    dbProfileID = data["profile"]["id"]
    dbUsername = scratchun
    dbScratchTeam = data["scratchteam"]
    dbjoinDate = data["history"]["joined"]
    dbstatus = data["profile"]["status"]
    dbBio = data["profile"]["bio"]
    dbCountry = data["profile"]["country"]

    session.add(User(profileID=dbProfileID,username=dbUsername,
    scratchteam=dbScratchTeam,joinDate = dbjoinDate,status=dbstatus,
    bio=dbBio,country=dbCountry))
    session.commit()


init()
session = Session()

def saveCurrentIndex(*args):
    currentProjectIndex = open("currentProjectIndex.txt","w")
    currentProjectIndex.write(str(currentProject+1))
    currentProjectIndex.close()

atexit.register(saveCurrentIndex)
signal.signal(signal.SIGTERM, saveCurrentIndex)
signal.signal(signal.SIGINT, saveCurrentIndex)

def __del__(self):
    saveCurrentIndex()

while True:
    grabProjectData(currentProject)
    currentProject+=1
    time.sleep(0.5)