From 7eea85fedae536bbcb836acdff4c8704098c341d Mon Sep 17 00:00:00 2001 From: Jools Wills Date: Fri, 27 Jun 2014 17:28:15 +0100 Subject: [PATCH 1/7] cosmetic: tabs -> spaces / strip trailing spaces --- scraper.py | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/scraper.py b/scraper.py index e8cfef5..d0cfebf 100644 --- a/scraper.py +++ b/scraper.py @@ -21,7 +21,7 @@ def normalize(s): return ''.join((c for c in unicodedata.normalize('NFKD', unicode(s)) if unicodedata.category(c) != 'Mn')) -def fixExtension(file): +def fixExtension(file): newfile="%s.%s" % (os.path.splitext(file)[0],imghdr.what(file)) os.rename(file, newfile) return newfile @@ -112,20 +112,20 @@ def getGameInfo(file,platformID): else: URL = "http://thegamesdb.net/api/GetGame.php" platform = getPlatformName(platformID) - if SCUMMVM: + if SCUMMVM: title = getScummvmTitle(title) args.fix = True #Scummvm doesn't have a proper platformID so we search all - if platform == "Arcade" or platform == "NeoGeo": title = getRealArcadeTitle(title) - + if platform == "Arcade" or platform == "NeoGeo": title = getRealArcadeTitle(title) + if args.fix: - try: + try: fixreq = urllib2.Request("http://thegamesdb.net/api/GetGamesList.php", urllib.urlencode({'name' : title, 'platform' : platform}), headers={'User-Agent' : "RetroPie Scraper Browser"}) fixdata=ET.parse(urllib2.urlopen(fixreq)).getroot() - if fixdata.find("Game") is not None: + if fixdata.find("Game") is not None: #values={ 'id': fixdata.findall("Game/id")[chooseResult(fixdata)].text if args.m else fixdata.find("Game/id").text } values={ 'id': fixdata.findall("Game/id")[chooseResult(fixdata)].text if args.m else fixdata.findall("Game/id")[autoChooseBestResult(fixdata,title)].text } - + except: return None else: @@ -166,7 +166,7 @@ def getGamePlatform(nodes): return getText(nodes.find("system_title")) else: return getText(nodes.find("Platform")) - + def getScummvmTitle(title): print "Fetching real title for %s from scummvm.org" % title URL = "http://scummvm.org/compatibility/DEV/%s" % title.split("-")[0] @@ -177,7 +177,7 @@ def getScummvmTitle(title): return m.groups()[0] else: print "No title found for %s on scummvm.org" % title - return title + return title def getRealArcadeTitle(title): print "Fetching real title for %s from mamedb.com" % title @@ -267,8 +267,8 @@ def chooseResult(nodes): return int(raw_input("Select a result (or press Enter to skip): ")) else: return 0 - - + + def autoChooseBestResult(nodes,t): results=nodes.findall('Game') t = t.split('(', 1)[0] @@ -341,20 +341,20 @@ def scanFiles(SystemInfo): try: filepath=os.path.abspath(os.path.join(root, files)) filename = os.path.splitext(files)[0] - + if gamelistExists and not args.f: if skipGame(existinglist,filepath): continue - + print "Trying to identify %s.." % files - + data=getGameInfo(filepath, platformID) - + if data is None: continue else: result=data - + str_title=getTitle(result) str_des=getDescription(result) str_img=getImage(result) @@ -362,7 +362,7 @@ def scanFiles(SystemInfo): str_pub=getPublisher(result) str_dev=getDeveloper(result) lst_genres=getGenres(result) - + if str_title is not None: game = SubElement(gamelist, 'game') path = SubElement(game, 'path') @@ -373,41 +373,41 @@ def scanFiles(SystemInfo): publisher=SubElement(game, 'publisher') developer=SubElement(game, 'developer') genres=SubElement(game, 'genres') - + path.text=filepath name.text=str_title print "Game Found: %s" % str_title - + if str_des is not None: desc.text=str_des - + if str_img is not None and args.noimg is False: if args.newpath is True: imgpath="./" + filename+os.path.splitext(str_img)[1] else: imgpath=os.path.abspath(os.path.join(root, filename+os.path.splitext(str_img)[1])) - + print "Downloading boxart.." - + downloadBoxart(str_img,imgpath) imgpath=fixExtension(imgpath) image.text=imgpath - + if args.w: try: resizeImage(Image.open(imgpath),imgpath) except: print "Image resize error" - + if str_rd is not None: releasedate.text=str_rd - + if str_pub is not None: publisher.text=str_pub - + if str_dev is not None: developer.text=str_dev - + if lst_genres is not None: for genre in lst_genres: newgenre = SubElement(genres, 'genre') From 55a06ec24d4455981918ed0db0ed6f42c5b9de8c Mon Sep 17 00:00:00 2001 From: Jools Wills Date: Fri, 27 Jun 2014 22:28:08 +0100 Subject: [PATCH 2/7] get mame rom name from mame4all -listfull if possible --- scraper.py | 60 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 20 deletions(-) mode change 100644 => 100755 scraper.py diff --git a/scraper.py b/scraper.py old mode 100644 new mode 100755 index d0cfebf..aa73c34 --- a/scraper.py +++ b/scraper.py @@ -1,5 +1,6 @@ #!/usr/bin/env python import os, imghdr, urllib, urllib2, sys, Image, argparse, zlib, unicodedata, re +import subprocess import difflib from xml.etree import ElementTree as ET from xml.etree.ElementTree import Element, SubElement @@ -37,14 +38,19 @@ def readConfig(file): name=line.split('=')[1] if "PATH=" in line: path=line.split('=')[1] + if "COMMAND=" in line: + command=line.split('=')[1] + command=re.sub(r'.*?runcommand.sh\s*[0-9]*\s*', r'', command) + command=re.sub(r'\s*%.*?%\s*', r'', command) + command=command.replace('"','') elif "EXTENSION" in line: ext=line.split('=')[1] elif "PLATFORMID" in line: - pid=line.split('=')[1] + pid=int(line.split('=')[1]) if not pid: continue else: - system=(name,path,ext,pid) + system=(name,path,command,ext,pid) systems.append(system) config.close() return systems @@ -98,8 +104,9 @@ def getFiles(base): dict.add(filepath) return dict -def getGameInfo(file,platformID): +def getGameInfo(file,platformID,titlesDict): title=re.sub(r'\[.*?\]|\(.*?\)', '', os.path.splitext(os.path.basename(file))[0]).strip() + keeptitle = False if args.crc: crcvalue=crc(file) if args.v: @@ -115,7 +122,10 @@ def getGameInfo(file,platformID): if SCUMMVM: title = getScummvmTitle(title) args.fix = True #Scummvm doesn't have a proper platformID so we search all - if platform == "Arcade" or platform == "NeoGeo": title = getRealArcadeTitle(title) + if platformID == 23 or platformID == 24: + if title in titlesDict: + title = titlesDict[title] + keeptitle = True if args.fix: try: @@ -145,7 +155,13 @@ def getGameInfo(file,platformID): if result is not None and result.find("title").text is not None: return result elif data.find("Game") is not None: - return data.findall("Game")[chooseResult(data)] if args.m else data.findall("Game")[autoChooseBestResult(data,title)] + if args.m: + game = data.findall("Game")[chooseResult(data)] + else: + game = data.findall("Game")[autoChooseBestResult(data,title)] + if keeptitle: + game.find("GameTitle").text = title + return game else: return None except Exception, err: @@ -179,18 +195,6 @@ def getScummvmTitle(title): print "No title found for %s on scummvm.org" % title return title -def getRealArcadeTitle(title): - print "Fetching real title for %s from mamedb.com" % title - URL = "http://www.mamedb.com/game/%s" % title - data = "".join(urllib2.urlopen(URL).readlines()) - m = re.search('Name:.*(.+) .*
Year', data) - if m: - print "Found real title %s for %s on mamedb.com" % (m.group(1), title) - return m.group(1) - else: - print "No title found for %s on mamedb.com" % title - return title - def getDescription(nodes): if args.crc: return getText(nodes.find("description")) @@ -296,14 +300,30 @@ def autoChooseBestResult(nodes,t): else: return 0 +def getMameTitles(command): + titlesDict = {} + if "mame4all" in command: + output = subprocess.check_output([command, '-listfull']) + r = re.compile('\s+') + for line in output.splitlines(): + file = line[0:10].strip() + title = line[10:].strip('"') + titlesDict[file] = title + return titlesDict + def scanFiles(SystemInfo): name=SystemInfo[0] if name == "scummvm": global SCUMMVM SCUMMVM = True folderRoms=SystemInfo[1] - extension=SystemInfo[2] - platformID=SystemInfo[3] + command=SystemInfo[2] + extension=SystemInfo[3] + platformID=SystemInfo[4] + + titlesDict = {} + if platformID == 23: + titlesDict = getMameTitles(command) global gamelistExists global existinglist @@ -348,7 +368,7 @@ def scanFiles(SystemInfo): print "Trying to identify %s.." % files - data=getGameInfo(filepath, platformID) + data=getGameInfo(filepath, platformID, titlesDict) if data is None: continue From 89109676a3fd1d428d1a875a41c4fa335ec10447 Mon Sep 17 00:00:00 2001 From: Jools Wills Date: Fri, 27 Jun 2014 22:42:51 +0100 Subject: [PATCH 3/7] actually allow breaking out of processing with ctrl+c --- scraper.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) mode change 100644 => 100755 scraper.py diff --git a/scraper.py b/scraper.py old mode 100644 new mode 100755 index e8cfef5..0b7bc32 --- a/scraper.py +++ b/scraper.py @@ -297,6 +297,8 @@ def autoChooseBestResult(nodes,t): return 0 def scanFiles(SystemInfo): + status = "ok" + name=SystemInfo[0] if name == "scummvm": global SCUMMVM @@ -414,15 +416,22 @@ def scanFiles(SystemInfo): newgenre.text=genre.strip() except KeyboardInterrupt: print "Ctrl+C detected. Closing work now..." + status = "break" + break except Exception as e: print "Exception caught! %s" % e - + else: + continue + break + if gamelist.find("game") is None: print "No new games added." else: print "{} games added.".format(len(gamelist)) exportList(gamelist) + return status + try: if os.getuid()==0: os.environ['HOME']="/home/"+os.getenv("SUDO_USER") @@ -454,6 +463,8 @@ def scanFiles(SystemInfo): sys.exit() else: for i,v in enumerate(ES_systems): - scanFiles(ES_systems[i]) + result = scanFiles(ES_systems[i]) + if result == "break": + break print "All done!" From 3ca077c0adb84d93e2b69ce89951f1c3bc3e188d Mon Sep 17 00:00:00 2001 From: Jools Wills Date: Fri, 27 Jun 2014 22:43:55 +0100 Subject: [PATCH 4/7] cosmetic: trailing spaces --- scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scraper.py b/scraper.py index 0b7bc32..ff847d2 100755 --- a/scraper.py +++ b/scraper.py @@ -423,7 +423,7 @@ def scanFiles(SystemInfo): else: continue break - + if gamelist.find("game") is None: print "No new games added." else: From 31f23024ff5400688292ca18072b1375b8307e9b Mon Sep 17 00:00:00 2001 From: Jools Wills Date: Fri, 27 Jun 2014 23:10:19 +0100 Subject: [PATCH 5/7] support extracting titles from fba2x directly --- scraper.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scraper.py b/scraper.py index e97a368..0d0a7d1 100755 --- a/scraper.py +++ b/scraper.py @@ -309,6 +309,16 @@ def getMameTitles(command): file = line[0:10].strip() title = line[10:].strip('"') titlesDict[file] = title + if "fba2x" in command: + subprocess.call([command, '--gamelist']) + file = os.path.join(os.path.dirname(command), 'gamelist.txt') + with open(file) as f: + output = f.readlines() + for line in output: + entries = line.split('|') + if len(entries) == 10: + titlesDict[entries[1].strip()] = entries[3].strip() + return titlesDict def scanFiles(SystemInfo): @@ -324,7 +334,7 @@ def scanFiles(SystemInfo): platformID=SystemInfo[4] titlesDict = {} - if platformID == 23: + if platformID == 23 or platformID == 24: titlesDict = getMameTitles(command) global gamelistExists From 385cf0513e4e032e9ba3fda34593ab5c054f535e Mon Sep 17 00:00:00 2001 From: Jools Wills Date: Fri, 27 Jun 2014 23:15:52 +0100 Subject: [PATCH 6/7] add entry for rom even if not found in thegamesdb (we may have the title from mame4all/fba2x anyway) --- scraper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scraper.py b/scraper.py index 0d0a7d1..1444a29 100755 --- a/scraper.py +++ b/scraper.py @@ -163,7 +163,10 @@ def getGameInfo(file,platformID,titlesDict): game.find("GameTitle").text = title return game else: - return None + game = ET.Element('Game') + gameTitle = ET.SubElement(game, 'GameTitle') + gameTitle.text = title + return game except Exception, err: print "Skipping game..(%s)" % str(err) return None From 7e497db26a81117daec99c0090f5839274119164 Mon Sep 17 00:00:00 2001 From: Jools Wills Date: Fri, 27 Jun 2014 23:27:15 +0100 Subject: [PATCH 7/7] search thegamesdb for mame titles without additional data in brackets --- scraper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scraper.py b/scraper.py index 1444a29..8240383 100755 --- a/scraper.py +++ b/scraper.py @@ -139,7 +139,9 @@ def getGameInfo(file,platformID,titlesDict): except: return None else: - values={'name':title,'platform':platform} + searchTitle = title.split('(',1) + searchTitle = searchTitle[0].strip() + values={'name':searchTitle,'platform':platform} try: req = urllib2.Request(URL,urllib.urlencode(values), headers={'User-Agent' : "RetroPie Scraper Browser"})