diff --git a/db.py b/db.py index 5fff7b6fa4..2386e923f3 100644 --- a/db.py +++ b/db.py @@ -4,7 +4,7 @@ import time from sqlalchemy import create_engine -from sqlalchemy import Column, Integer, String, ForeignKey, UniqueConstraint +from sqlalchemy import Column, Integer, String, ForeignKey, UniqueConstraint, func, DateTime from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker, relationship @@ -106,9 +106,37 @@ def __contains__(self, sighting): params[2] == sighting['guard_pokemon_id'] ) return is_the_same + +class StopCache(object): + """Simple cache for storing fort sightings""" + def __init__(self): + self.store = {} + + @staticmethod + def _make_key(stop_sighting): + return stop_sighting['external_id'] + + def add(self, sighting): + self.store[self._make_key(sighting)] = ( + sighting['lure_expires_timestamp_ms'], + sighting['encounter_id'], + sighting['active_pokemon_id'], + ) + + def __contains__(self, sighting): + params = self.store.get(self._make_key(sighting)) + if not params: + return False + is_the_same = ( + params[0] == sighting['lure_expires_timestamp_ms'] and + params[1] == sighting['encounter_id'] and + params[2] == sighting['active_pokemon_id'] + ) + return is_the_same SIGHTING_CACHE = SightingCache() FORT_CACHE = FortCache() +STOP_CACHE = StopCache() class Sighting(Base): @@ -122,7 +150,37 @@ class Sighting(Base): lat = Column(String(16), index=True) lon = Column(String(16), index=True) +class Stop(Base): + __tablename__ = 'stops' + + id = Column(Integer, primary_key=True) + external_id = Column(String(64), unique=True) + lat = Column(String(16), index=True) + lon = Column(String(16), index=True) + sightings = relationship( + 'StopSighting', + backref='stop', + order_by='StopSighting.last_modified' + ) + +class StopSighting(Base): + __tablename__ = 'stop_sightings' + id = Column(Integer, primary_key=True) + stop_id = Column(Integer, ForeignKey('stops.id')) + last_modified = Column(Integer) + lure_expires_timestamp_ms = Column(Integer) + encounter_id = Column(String(32)) + active_pokemon_id = Column(Integer) + sighting_time = Column(DateTime, server_default=func.now()) + __table_args__ = ( + UniqueConstraint( + 'stop_id', + 'last_modified', + name='stop_id_last_modified_unique' + ), + ) + class Fort(Base): __tablename__ = 'forts' @@ -248,6 +306,47 @@ def add_fort_sighting(session, raw_fort): else: FORT_CACHE.add(raw_fort) +def add_stop_sighting(session, raw_stop): + if raw_stop in STOP_CACHE: + return + # Check if stop exists + #logger.info('Logging the stop sighting: ' + raw_stop['lure_expires_timestamp_ms'] + ' - ' + raw_stop['encounter_id'] + ' - ' + raw_stop['active_pokemon_id']) + stop = session.query(Stop) \ + .filter(Stop.external_id == raw_stop['external_id']) \ + .first() + if not stop: + stop = Stop( + external_id=raw_stop['external_id'], + lat=raw_stop['lat'], + lon=raw_stop['lon'], + ) + session.add(stop) + if stop.id: + existing = session.query(StopSighting) \ + .filter(StopSighting.stop_id == stop.id) \ + .filter(StopSighting.lure_expires_timestamp_ms == raw_stop['lure_expires_timestamp_ms']) \ + .filter(StopSighting.encounter_id == str(raw_stop['encounter_id'])) \ + .filter(StopSighting.active_pokemon_id == + raw_stop['active_pokemon_id']) \ + .first() + if existing: + # Why it's not in cache? It should be there! + STOP_CACHE.add(raw_stop) + return + obj = StopSighting( + stop=stop, + lure_expires_timestamp_ms=raw_stop['lure_expires_timestamp_ms'], + encounter_id=str(raw_stop['encounter_id']), + active_pokemon_id=raw_stop['active_pokemon_id'], + last_modified=raw_stop['last_modified'], + ) + session.add(obj) + try: + session.commit() + except IntegrityError: # skip adding stop this time + session.rollback() + else: + STOP_CACHE.add(raw_stop) def get_sightings(session): return session.query(Sighting) \ @@ -257,31 +356,51 @@ def get_sightings(session): def get_forts(session): query = session.execute(''' - SELECT * FROM ( - SELECT - fs.fort_id, - fs.id, - fs.team, - fs.prestige, - fs.guard_pokemon_id, - fs.last_modified, - f.lat, - f.lon - FROM fort_sightings fs - JOIN forts f ON f.id=fs.fort_id - ORDER BY fs.last_modified DESC - ) t GROUP BY fort_id + + SELECT + fs.fort_id, + fs.id, + fs.team, + fs.prestige, + fs.guard_pokemon_id, + fs.last_modified, + f.lat, + f.lon + FROM fort_sightings fs + inner join (select max(fs.last_modified) as last_modified, fs.fort_id FROM fort_sightings fs group by fort_id) lfs on lfs.fort_id = fs.fort_id and lfs.last_modified = fs.last_modified + inner JOIN forts f ON f.id=fs.fort_id ''') return query.fetchall() +def get_stops(session): + query = session.execute(''' + SELECT + ss.stop_id, + ss.id, + ss.lure_expires_timestamp_ms, + ss.encounter_id, + ss.active_pokemon_id, + ss.last_modified, + s.lat, + s.lon + FROM stop_sightings ss + inner join (SELECT max(id) as maxid, stop_id, max(lure_expires_timestamp_ms) as ltime, max(sighting_time) stime from stop_sightings + where dateadd(mi, datediff(mi, getutcdate(), getDate()), dateadd(S, lure_expires_timestamp_ms, '1970-01-01')) > getdate() or lure_expires_timestamp_ms = 0 +group by stop_id) mss on ss.id = mss.maxid + JOIN stops s ON s.id=ss.stop_id + ORDER BY ss.last_modified DESC + ''') + return query.fetchall() + + def get_session_stats(session): query = ''' SELECT MIN(expire_timestamp) ts_min, MAX(expire_timestamp) ts_max, COUNT(*) - FROM `sightings` + FROM sightings {report_since} ''' min_max_query = session.execute(query.format( @@ -308,11 +427,11 @@ def get_punch_card(session): bigint = 'UNSIGNED' query = session.execute(''' SELECT - CAST((expire_timestamp / 300) AS {bigint}) ts_date, + CAST((expire_timestamp / 300) AS bigint) as ts_date, COUNT(*) how_many - FROM `sightings` + FROM sightings {report_since} - GROUP BY ts_date + GROUP BY CAST((expire_timestamp / 300) AS bigint) ORDER BY ts_date '''.format(bigint=bigint, report_since=get_since_query_part())) results = query.fetchall() @@ -326,14 +445,13 @@ def get_punch_card(session): def get_top_pokemon(session, count=30, order='DESC'): query = session.execute(''' - SELECT + SELECT top {count} pokemon_id, COUNT(*) how_many FROM sightings {report_since} GROUP BY pokemon_id ORDER BY how_many {order} - LIMIT {count} '''.format(order=order, count=count, report_since=get_since_query_part())) return query.fetchall() @@ -379,7 +497,7 @@ def get_spawns_per_hour(session, pokemon_id): if get_engine_name(session) == 'sqlite': ts_hour = 'STRFTIME("%H", expire_timestamp)' else: - ts_hour = 'HOUR(FROM_UNIXTIME(expire_timestamp))' + ts_hour = "datepart(hour,dateadd(mi, -14,dateadd(mi, datediff(mi, getutcdate(), getDate()), dateadd(S, expire_timestamp, '1970-01-01'))))" query = session.execute(''' SELECT {ts_hour} AS ts_hour, @@ -387,7 +505,7 @@ def get_spawns_per_hour(session, pokemon_id): FROM sightings WHERE pokemon_id = {pokemon_id} {report_since} - GROUP BY ts_hour + GROUP BY {ts_hour} ORDER BY ts_hour '''.format( pokemon_id=pokemon_id, diff --git a/newmap.html b/newmap.html new file mode 100644 index 0000000000..36fe5a069e --- /dev/null +++ b/newmap.html @@ -0,0 +1,284 @@ + + + + + Pokeminer - {{ area_name }} + + + + + +

Pokeminer is initializing, please wait.

+ +
+ + + + + + + + diff --git a/static/stops/Great_Ball.png b/static/stops/Great_Ball.png new file mode 100644 index 0000000000..96f6919b4f Binary files /dev/null and b/static/stops/Great_Ball.png differ diff --git a/static/stops/pokeball.png b/static/stops/pokeball.png new file mode 100644 index 0000000000..6ef77b7ee0 Binary files /dev/null and b/static/stops/pokeball.png differ diff --git a/templates/newmap.html b/templates/newmap.html index 7b5e3dd1ae..8c97ebcad7 100644 --- a/templates/newmap.html +++ b/templates/newmap.html @@ -70,6 +70,14 @@

Pokeminer is initializing, please wait.

className: 'fort-icon' } }); + + var StopIcon = L.Icon.extend({ + options: { + iconSize: [20, 20], + popupAnchor: [0, -10], + className: 'fort-icon' + } + }); function getMarkers () { return new Promise(function (resolve, reject) { @@ -84,7 +92,8 @@

Pokeminer is initializing, please wait.

Pokemon: L.layerGroup([]), Trash: L.layerGroup([]), Gyms: L.layerGroup([]), - Workers: L.layerGroup([]) + Workers: L.layerGroup([]), + Stops: L.layerGroup([]) }; function getPopupContent (item) { @@ -154,6 +163,35 @@

Pokeminer is initializing, please wait.

return marker; } + function StopMarker (raw) { + if (raw.pokemon_id != 0) + { + //var icon = new StopIcon({iconUrl: '/static/stops/Great_Ball.png'}); //uncomment this and comment below for pokeball icon + var icon = new PokemonIcon({iconUrl: '/static/icons/' + raw.pokemon_id + '.png'}); + } + else + { + var icon = new StopIcon({iconUrl: '/static/stops/pokeball.png'}); + } + var marker = L.marker([raw.lat, raw.lon], {icon: icon, opacity: 1}); + marker.raw = raw; + markers[raw.id] = marker; + marker.on('popupopen',function popupopen (event) { + var pokemonName; + var diff = (raw.lure_expires_timestamp - new Date().getTime() / 1000); + var minutes = parseInt(diff / 60); + var seconds = parseInt(diff - (minutes * 60)); + var expires_at = minutes + 'm ' + seconds + 's'; + if (raw.pokemon_id === 0) { + event.popup.setContent('No Lure'); + } else { + event.popup.setContent('Lure Pokemon expires at: ' + expires_at + '
Current Pokemon:
' + '#' + raw.pokemon_id + ' ' + raw.pokemon_name + ''); + } + }); + marker.bindPopup(); + return marker; + } + function addMarkersToMap (data, map) { // Pokemons data.forEach(function (item) { @@ -169,7 +207,23 @@

Pokeminer is initializing, please wait.

} else { marker.addTo(overlays.Pokemon); } - } else { + + } + else if (item.type === 'stop') { + var existing = markers[item.id]; + if (typeof existing !== 'undefined') + { + if (existing.raw.sighting_id === item.sighting_id) + { + return; + } + existing.removeFrom(overlays.Stops); + markers[item.id] = undefined; + } + marker = StopMarker(item); + marker.addTo(overlays.Stops); + } + else { // No change since last time? Then don't do anything var existing = markers[item.id]; if (typeof existing !== 'undefined') { diff --git a/web.py b/web.py index b9fa3f201f..5ee3793816 100644 --- a/web.py +++ b/web.py @@ -84,6 +84,7 @@ def get_pokemarkers(): session = db.Session() pokemons = db.get_sightings(session) forts = db.get_forts(session) + stops = db.get_stops(session) session.close() for pokemon in pokemons: @@ -113,7 +114,21 @@ def get_pokemarkers(): 'lat': fort['lat'], 'lon': fort['lon'], }) - + for stop in stops: + if stop['active_pokemon_id'] != 0: + pokemon_name = pokemon_names[str(stop['active_pokemon_id'])] + else: + pokemon_name = 'No Lure' + markers.append({ + 'id': 'stop-{}'.format(stop['stop_id']), + 'sighting_id': stop['id'], + 'type': 'stop', + 'lure_expires_timestamp': stop['lure_expires_timestamp_ms'], + 'pokemon_id': stop['active_pokemon_id'], + 'pokemon_name': pokemon_name, + 'lat': stop['lat'], + 'lon': stop['lon'], + }) return markers diff --git a/worker.py b/worker.py index aa17a32f42..0f86f5fcae 100644 --- a/worker.py +++ b/worker.py @@ -7,6 +7,7 @@ import sys import threading import time +import pprint from pgoapi import ( exceptions as pgoapi_exceptions, @@ -170,6 +171,7 @@ def main(self): map_objects = response_dict['responses'].get('GET_MAP_OBJECTS', {}) pokemons = [] forts = [] + stops = [] if map_objects.get('status') == 1: for map_cell in map_objects['map_cells']: for pokemon in map_cell.get('wild_pokemons', []): @@ -187,6 +189,7 @@ def main(self): if not fort.get('enabled'): continue if fort.get('type') == 1: # probably pokestops + stops.append(self.normalize_stop(fort, fort.get('lure_info', fort))) continue forts.append(self.normalize_fort(fort)) for raw_pokemon in pokemons: @@ -196,6 +199,8 @@ def main(self): session.commit() for raw_fort in forts: db.add_fort_sighting(session, raw_fort) + for raw_stop in stops: + db.add_stop_sighting(session, raw_stop) # Commit is not necessary here, it's done by add_fort_sighting logger.info( 'Point processed, %d Pokemons and %d forts seen!', @@ -227,6 +232,7 @@ def normalize_pokemon(raw, now): @staticmethod def normalize_fort(raw): + return { 'external_id': raw['id'], 'lat': raw['latitude'], @@ -236,7 +242,19 @@ def normalize_fort(raw): 'guard_pokemon_id': raw.get('guard_pokemon_id', 0), 'last_modified': raw['last_modified_timestamp_ms'] / 1000.0, } - + + @staticmethod + def normalize_stop(raw, lure): + return { + 'external_id': raw['id'], + 'lat': raw['latitude'], + 'lon': raw['longitude'], + 'lure_expires_timestamp_ms': lure.get('lure_expires_timestamp_ms', 0) / 1000, + 'encounter_id': lure.get('encounter_id', 0), + 'active_pokemon_id': lure.get('active_pokemon_id', 0), + 'last_modified': raw['last_modified_timestamp_ms'] / 1000.0, + } + @property def status(self): """Returns status message to be displayed in status screen"""