RocketMap · SiteEffect · Mar 30, 2017 · Mar 30, 2017 · Mar 30, 2017 · Mar 30, 2017
diff --git a/pogom/models.py b/pogom/models.py
@@ -43,7 +43,7 @@
 flaskDb = FlaskDB()
 cache = TTLCache(maxsize=100, ttl=60 * 5)
 
-db_schema_version = 18
+db_schema_version = 19
 
 
 class MyRetryDB(RetryOperationalError, PooledMySQLDatabase):
@@ -420,8 +420,8 @@ def get_spawnpoints_in_hex(cls, center, steps):
         filtered = []
 
         for idx, sp in enumerate(s):
-            if geopy.distance.distance(
-                    center, (sp['lat'], sp['lng'])).meters <= step_distance:
+            if (geopy.distance.distance(
+                    center, (sp['lat'], sp['lng'])).meters <= step_distance):
                 filtered.append(s[idx])
 
         # At this point, 'time' is DISAPPEARANCE time, we're going to morph it
@@ -531,6 +531,42 @@ def get_stops(swLat, swLng, neLat, neLng, timestamp=0, oSwLat=None,
 
         return pokestops
 
+    @classmethod
+    def get_stops_in_hex(cls, center, steps):
+        log.info('Finding pokestops {} steps away.'.format(steps))
+
+        n, e, s, w = hex_bounds(center, steps)
+
+        query = (Pokestop
+                 .select(Pokestop.latitude.alias('lat'),
+                         Pokestop.longitude.alias('lng'),
+                         Pokestop.pokestop_id
+                         ))
+        query = (query.where((Pokestop.latitude <= n) &
+                             (Pokestop.latitude >= s) &
+                             (Pokestop.longitude >= w) &
+                             (Pokestop.longitude <= e)
+                             ))
+        # Sqlite doesn't support distinct on columns.
+        if args.db_type == 'mysql':
+            query = query.distinct(Pokestop.pokestop_id)
+        else:
+            query = query.group_by(Pokestop.pokestop_id)
+
+        s = list(query.dicts())
+
+        step_distance = ((steps - 1) * 779.4231) + 450
+        filtered = []
+
+        for pokestop in s:
+            if (geopy.distance.distance(
+                    center,
+                    (pokestop['lat'], pokestop['lng'])).meters <=
+                    step_distance):
+                filtered.append(pokestop)
+
+        return filtered
+
 
 class Gym(BaseModel):
     UNCONTESTED = 0
@@ -704,6 +740,40 @@ def get_gym(id):
 
         return result
 
+    @classmethod
+    def get_gyms_in_hex(cls, center, steps):
+        log.info('Finding gyms {} steps away.'.format(steps))
+
+        n, e, s, w = hex_bounds(center, steps)
+
+        query = (Gym
+                 .select(Gym.latitude.alias('lat'),
+                         Gym.longitude.alias('lng'),
+                         Gym.gym_id
+                         ))
+        query = (query.where((Gym.latitude <= n) &
+                             (Gym.latitude >= s) &
+                             (Gym.longitude >= w) &
+                             (Gym.longitude <= e)
+                             ))
+        # Sqlite doesn't support distinct on columns.
+        if args.db_type == 'mysql':
+            query = query.distinct(Gym.gym_id)
+        else:
+            query = query.group_by(Gym.gym_id)
+
+        s = list(query.dicts())
+
+        step_distance = ((steps - 1) * 779.4231) + 450
+        filtered = []
+
+        for gym in s:
+            if (geopy.distance.distance(
+                    center, (gym['lat'], gym['lng'])).meters <= step_distance):
+                filtered.append(gym)
+
+        return filtered
+
 
 class LocationAltitude(BaseModel):
     cellid = CharField(primary_key=True, max_length=50)
@@ -781,6 +851,9 @@ class ScannedLocation(BaseModel):
     # is 0.4 minutes in minsec.
     width = SmallIntegerField(default=0)
 
+    radius_default = 450 if args.no_pokemon else 70
+    radius = SmallIntegerField(default=radius_default)
+
     class Meta:
         indexes = ((('latitude', 'longitude'), False),)
         constraints = [Check('band1 >= -1'), Check('band1 < 3600'),
@@ -848,7 +921,8 @@ def new_loc(loc):
                 'band5': -1,
                 'width': 0,
                 'midpoint': 0,
-                'last_modified': None}
+                'last_modified': None,
+                'radius': 450 if args.no_pokemon else 70}
 
     # Used to update bands.
     @staticmethod
@@ -1799,7 +1873,12 @@ def getStoredPeak(key):
 def hex_bounds(center, steps=None, radius=None):
     # Make a box that is (70m * step_limit * 2) + 70m away from the
     # center point.  Rationale is that you need to travel.
-    sp_dist = 0.07 * (2 * steps + 1) if steps else radius
+    if args.no_pokemon:
+        step_diameter = 0.45
+    else:
+        step_diameter = 0.07
+
+    sp_dist = step_diameter * (2 * steps + 1) if steps else radius
     n = get_new_coords(center, sp_dist, 0)[0]
     e = get_new_coords(center, sp_dist, 90)[1]
     s = get_new_coords(center, sp_dist, 180)[0]
@@ -1880,8 +1959,26 @@ def parse_map(args, map_dict, step_location, db_update_queue, wh_update_queue,
             # a speed violation.
             log.warning('No nearby or wild Pokemon but there are visible gyms '
                         'or pokestops. Possible speed violation.')
+            if not (config['parse_pokestops'] or config['parse_gyms']):
+                # If we're not going to parse the forts, then we'll just
+                # exit here.
+                abandon_loc = True
+
+        if abandon_loc:
+            scan_loc = ScannedLocation.get_by_loc(step_location)
+            scan_loc['radius'] = 450 if args.no_pokemon else 70
+            ScannedLocation.update_band(scan_loc)
+            db_update_queue.put((ScannedLocation, {0: scan_loc}))
+
+            return {
+                'count': 0,
+                'gyms': gyms,
+                'spawn_points': spawn_points,
+                'bad_scan': True
+            }
 
     scan_loc = ScannedLocation.get_by_loc(step_location)
+    scan_loc['radius'] = 450 if args.no_pokemon else 70
     done_already = scan_loc['done']
     ScannedLocation.update_band(scan_loc, now_date)
     just_completed = not done_already and scan_loc['done']
@@ -2936,5 +3033,11 @@ def database_migrate(db, old_ver):
                                 SmallIntegerField(null=True))
         )
 
+    if old_ver < 19:
+        migrate(
+            migrator.add_column(
+                'scannedlocation', 'radius', SmallIntegerField(default=70))
+        )
+
     # Always log that we're done.
     log.info('Schema upgrade complete.')
diff --git a/pogom/schedulers.py b/pogom/schedulers.py
@@ -60,8 +60,8 @@
 from operator import itemgetter
 from datetime import datetime, timedelta
 from .transform import get_new_coords
-from .models import (hex_bounds, Pokemon, SpawnPoint, ScannedLocation,
-                     ScanSpawnPoint, HashKeys)
+from .models import (hex_bounds, Pokemon, Pokestop, Gym, SpawnPoint,
+                     ScannedLocation, ScanSpawnPoint, HashKeys)
 from .utils import now, cur_sec, cellid, equi_rect_distance
 from .altitude import get_altitude
 
@@ -334,6 +334,58 @@ def _generate_locations(self):
         return locations
 
 
+# Fort Only Hex Search works like Hex Search, but skips locations that
+# have no known forts.
+class HexSearchFort(HexSearch):
+
+    def _any_forts_in_range(self, coords, forts):
+        return any(
+            geopy.distance.distance(coords, x).meters <= 450
+            for x in forts)
+
+    # Extend the generate_locations function to remove locations with no forts.
+    def _generate_locations(self):
+
+        fort_locations = {}
+        # Attempt to load forts from file.
+        if self.args.fort_scanning != 'nofile':
+            log.debug('Loading forts from json file @ %s',
+                      self.args.fort_scanning)
+            try:
+                with open(self.args.fort_scanning) as file:
+                    fort_locations = json.load(file)
+            except ValueError as e:
+                log.error('JSON error: %s; will fallback to database', repr(e))
+            except IOError as e:
+                log.error(
+                    'Error opening json file: %s; will fallback to database',
+                    repr(e))
+        else:  # Load all forts in step limit from DB
+            if not self.args.no_pokestops:
+                fort_locations = Pokestop.get_stops_in_hex(
+                                    self.scan_location,
+                                    self.step_limit)
+            if not self.args.no_gyms:
+                fort_locations = fort_locations + (Gym.get_gyms_in_hex(
+                                    self.scan_location,
+                                    self.step_limit))
+
+        forts = set((d['lat'], d['lng']) for d in fort_locations)
+
+        if len(forts) == 0:
+            log.warning('No forts found in the specified area!  (Did ' +
+                        'you forget to run a normal scan in this area first?)')
+
+        # Call the original _generate_locations.
+        locations = super(HexSearchFort, self)._generate_locations()
+
+        # Remove items with no fort in range.
+        locations = [
+            coords for coords in locations
+            if self._any_forts_in_range(coords[1], forts)]
+        return locations
+
+
 # Spawn Scan searches known spawnpoints at the specific time they spawn.
 class SpawnScan(BaseScheduler):
 
@@ -1160,6 +1212,7 @@ class SchedulerFactory():
     __schedule_classes = {
         "hexsearch": HexSearch,
         "hexsearchspawnpoint": HexSearchSpawnpoint,
+        "hexsearchfort": HexSearchFort,
         "spawnscan": SpawnScan,
         "speedscan": SpeedScan,
     }

diff --git a/pogom/utils.py b/pogom/utils.py
@@ -287,8 +287,16 @@ def get_args():
                         action='store_true', default=False)
     parser.add_argument('-ss', '--spawnpoint-scanning',
                         help=('Use spawnpoint scanning (instead of hex ' +
-                              'grid). Scans in a circle based on step_limit ' +
-                              'when on DB.'),
+                              'grid or speed scan). Scans in a circle based ' +
+                              'on step_limit when on DB.'),
+                        nargs='?', const='nofile', default=False)
+    parser.add_argument('-fs', '--fort-scanning',
+                        help=('Use fort scanning (instead of hex ' +
+                              'grid or speed scan). Scans in a circle based ' +
+                              'on step_limit and skips cells without ' +
+                              'forts, which are retrieved from DB or file ' +
+                              'if added as argument. Best used with -np to ' +
+                              'increase cell size.'),
                         nargs='?', const='nofile', default=False)
     parser.add_argument('-speed', '--speed-scan',
                         help=('Use speed scanning to identify spawn points ' +
@@ -308,7 +316,15 @@ def get_args():
                               'duration.'), type=int, default=30)
     parser.add_argument('--dump-spawnpoints',
                         help=('Dump the spawnpoints from the db to json ' +
-                              '(only for use with -ss).'),
+                              '(only for use with -ss and file).'),
+                        action='store_true', default=False)
+    parser.add_argument('--dump-gyms',
+                        help=('Dump the gyms from the db to json ' +
+                              '(only for use with -fs and file).'),
+                        action='store_true', default=False)
+    parser.add_argument('--dump-pokestops',
+                        help=('Dump the pokestops from the db to json ' +
+                              '(only for use with -fs and file).'),
                         action='store_true', default=False)
     parser.add_argument('-pd', '--purge-data',
                         help=('Clear Pokemon from database this many hours ' +
@@ -718,6 +734,8 @@ def get_args():
             args.scheduler = 'SpawnScan'
         elif args.skip_empty:
             args.scheduler = 'HexSearchSpawnpoint'
+        elif args.fort_scanning:
+            args.scheduler = 'HexSearchFort'
         elif args.speed_scan:
             args.scheduler = 'SpeedScan'
         else:

diff --git a/runserver.py b/runserver.py
@@ -23,7 +23,7 @@
 
 from pogom.search import search_overseer_thread
 from pogom.models import (init_database, create_tables, drop_tables,
-                          Pokemon, db_updater, clean_db_loop)
+                          Pokemon, Gym, Pokestop, db_updater, clean_db_loop)
 from pogom.webhook import wh_updater
 
 from pogom.proxy import check_proxies, proxies_refresher
@@ -294,7 +294,7 @@ def main():
 
         # Gather the Pokemon!
 
-        # Attempt to dump the spawn points (do this before starting threads of
+        # Attempt to dump the DB data (do this before starting threads of
         # endure the woe).
         if (args.spawnpoint_scanning and
                 args.spawnpoint_scanning != 'nofile' and
@@ -303,9 +303,33 @@ def main():
                 log.info('Saving spawn points to %s', args.spawnpoint_scanning)
                 spawns = Pokemon.get_spawnpoints_in_hex(
                     position, args.step_limit)
-                file.write(json.dumps(spawns))
+                file.write(json.dumps(
+                    spawns, sort_keys=True, indent=4, separators=(',', ': ')))
                 log.info('Finished exporting spawn points')
 
+        if args.fort_scanning:
+            if args.fort_scanning != 'nofile':
+                if args.dump_gyms:
+                    with open(args.fort_scanning, 'w+') as file:
+                        log.info('Saving gyms to %s', args.fort_scanning)
+                        gyms = Gym.get_gyms_in_hex(
+                            position, args.step_limit)
+                        file.write(json.dumps(
+                            gyms, sort_keys=True, indent=4,
+                            separators=(',', ': ')))
+                        log.info('Finished exporting gyms')
+                if args.dump_pokestops:
+                    with open(args.fort_scanning, 'w+') as file:
+                        log.info('Saving pokestops to %s', args.fort_scanning)
+                        pokestops = Pokestop.get_stops_in_hex(
+                            position, args.step_limit)
+                        file.write(json.dumps(
+                            pokestops, sort_keys=True, indent=4,
+                            separators=(',', ': ')))
+                        log.info('Finished exporting Pokestops')
+            else:
+                log.warning('Output file must be provided for a dump.')
+
         argset = (args, new_location_queue, pause_bit,
                   heartbeat, db_updates_queue, wh_updates_queue)
 

diff --git a/static/js/map.js b/static/js/map.js
@@ -911,7 +911,7 @@ function setupScannedMarker(item) {
         map: map,
         clickable: false,
         center: circleCenter,
-        radius: 70, // metres
+        radius: item['radius'], // metres
         fillColor: getColorByDate(item['last_modified']),
         fillOpacity: 0.1,
         strokeWeight: 1,