From 092be2d5bf5854dd02420fcf4523ed42d55d5fbb Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 23 Jul 2026 08:26:00 -0700 Subject: [PATCH] feat(contacts): implement pagination, search, and sorting for contacts API - Enhanced the `/api/contacts` endpoint to support pagination, allowing users to specify `page` and `page_size` parameters. - Added search functionality to filter contacts based on a search term. - Implemented sorting options for the contacts list, enabling users to sort by various fields. - Updated the frontend to reflect these changes, including a new pagination UI and improved loading states. - Introduced caching for multibyte hop chunks to optimize performance when retrieving contact badge evidence. - Added tests to ensure the new features work as expected and maintain existing functionality. --- docs/web-viewer.md | 4 +- modules/web_viewer/app.py | 250 ++++++++++++-- .../web_viewer/templates/api_explorer.html | 2 +- modules/web_viewer/templates/contacts.html | 289 ++++++++-------- package.json | 3 +- tests/js/contacts_manager.test.mjs | 322 ++++++++++++++++++ tests/test_contacts_frontend.py | 26 ++ tests/test_web_viewer.py | 302 ++++++++++++++++ tests/test_web_viewer_app.py | 34 ++ 9 files changed, 1043 insertions(+), 189 deletions(-) create mode 100644 tests/js/contacts_manager.test.mjs create mode 100644 tests/test_contacts_frontend.py diff --git a/docs/web-viewer.md b/docs/web-viewer.md index 4a36f48..17078b5 100644 --- a/docs/web-viewer.md +++ b/docs/web-viewer.md @@ -193,7 +193,9 @@ proxy_set_header X-Forwarded-Proto $scheme; The viewer also provides JSON API endpoints: - `GET /api/stats` - Database statistics -- `GET /api/contacts` - Repeater contacts data +- `GET /api/contacts` - Repeater contacts data. The contacts page uses optional + `page`, `page_size` (maximum 200), `search`, `sort`, and `direction` parameters; + callers that omit pagination retain the legacy full-list response. - `GET /api/tracking` - Contact tracking data Example usage: diff --git a/modules/web_viewer/app.py b/modules/web_viewer/app.py index 072bcf9..70bd7dd 100644 --- a/modules/web_viewer/app.py +++ b/modules/web_viewer/app.py @@ -256,6 +256,13 @@ class BotDataViewer: self._db_last_used = 0 self._db_timeout = 300 # 5 minutes connection timeout + # The contacts list needs all-time multibyte hop-prefix evidence for its + # capability badge. Cache that derived set between requests and invalidate + # it when the underlying multibyte-path population changes. + self._contacts_badge_cache_lock = threading.Lock() + self._contacts_badge_cache_signature = None + self._contacts_badge_cache_chunks: set[str] = set() + # Load configuration self.config = self._load_config(config_path) self.config_path = config_path # kept for config.ini write-back endpoints @@ -2084,12 +2091,34 @@ class BotDataViewer: @self.app.route('/api/contacts') def api_contacts(): - """Get contact data. Optional query param: since=24h|7d|30d|90d|all (default 30d).""" + """Get filtered contact data, optionally paginated for the interactive list.""" try: since = request.args.get('since', '30d') if since not in ('24h', '7d', '30d', '90d', 'all'): since = '30d' - contacts = self._get_tracking_data(since=since) + paginate = 'page' in request.args or 'page_size' in request.args + page = None + page_size = None + if paginate: + try: + page = max(1, int(request.args.get('page', '1'))) + except (TypeError, ValueError): + page = 1 + try: + page_size = max(1, min(200, int(request.args.get('page_size', '100')))) + except (TypeError, ValueError): + page_size = 100 + search = request.args.get('search', '').strip()[:100] + sort = request.args.get('sort', 'last_seen') + direction = request.args.get('direction', 'desc').lower() + contacts = self._get_tracking_data( + since=since, + page=page, + page_size=page_size, + search=search, + sort=sort, + direction=direction, + ) return jsonify(contacts) except Exception as e: self.logger.error(f"Error getting contacts: {e}") @@ -5203,7 +5232,8 @@ class BotDataViewer: cursor.execute( f""" SELECT DISTINCT path_hex, bytes_per_hop FROM observed_paths - WHERE bytes_per_hop IN (2, 3) AND path_hex IS NOT NULL AND length(path_hex) > 0 + WHERE bytes_per_hop >= 2 AND bytes_per_hop <= 3 + AND path_hex IS NOT NULL AND length(path_hex) > 0 {extra} """ ) @@ -5221,6 +5251,35 @@ class BotDataViewer: self.logger.debug(f"Could not load multibyte hop chunks: {e}") return chunks + def _get_cached_contact_multibyte_hop_chunks(self, cursor) -> set[str]: + """Return all-time contact badge evidence without rebuilding it per page request.""" + try: + # last_seen changes when an existing path is observed again; count changes on + # inserts and retention deletes. Together they cheaply invalidate the cache while + # using the multibyte covering index rather than the wide observed_paths table. + cursor.execute( + """ + SELECT MAX(last_seen), COUNT(*) FROM observed_paths + WHERE bytes_per_hop >= 2 AND bytes_per_hop <= 3 + """ + ) + signature_row = cursor.fetchone() + signature = tuple(signature_row) if signature_row else (None, 0) + except Exception as e: + self.logger.debug(f"Could not fingerprint multibyte hop chunks: {e}") + return self._collect_multibyte_hop_chunks(cursor) + + lock = getattr(self, '_contacts_badge_cache_lock', None) + if lock is None: + lock = self._contacts_badge_cache_lock = threading.Lock() + with lock: + if getattr(self, '_contacts_badge_cache_signature', None) == signature: + return self._contacts_badge_cache_chunks + chunks = self._collect_multibyte_hop_chunks(cursor) + self._contacts_badge_cache_signature = signature + self._contacts_badge_cache_chunks = chunks + return chunks + @staticmethod def _bucket_hop_chunks(multibyte_hop_chunks: set[str]) -> dict[int, set[str]]: """Bucket hop-prefix chunks by length (4 or 6) for O(1) prefix matching. @@ -5881,12 +5940,23 @@ class BotDataViewer: if conn: conn.close() - def _get_tracking_data(self, since='30d', include_detail=False): + def _get_tracking_data( + self, + since='30d', + include_detail=False, + page: int | None = None, + page_size: int | None = None, + search: str = '', + sort: str = 'last_seen', + direction: str = 'desc', + ): """Get contact tracking data. since: 24h, 7d, 30d, 90d, or all (heard in that window). include_detail=False (the interactive /api/contacts list) omits per-contact ``all_paths`` and ``raw_advert_data`` to keep the payload small; the UI fetches those on demand via - /api/contact-detail. include_detail=True (the export endpoint) keeps the full fields. + /api/contact-detail. The interactive route supplies ``page`` and ``page_size`` so path + enrichment is limited to visible contacts. include_detail=True (the export endpoint) + keeps the legacy full-result behavior and full fields. """ conn = None try: @@ -5908,11 +5978,75 @@ class BotDataViewer: '30d': "'-30 days'", '90d': "'-90 days'", } + where_parts = [] + where_params: list[Any] = [] if since in datetime_offsets: - where_clause = f" WHERE c.last_heard >= datetime('now', 'localtime', {datetime_offsets[since]})" - else: # 'all' - where_clause = '' - params = () + where_parts.append( + f"c.last_heard >= datetime('now', 'localtime', {datetime_offsets[since]})" + ) + + search = (search or '').strip().lower()[:100] + if search and not include_detail: + # Match the former client-side behavior: public keys are prefix-only, + # while names, roles, device types, and locations match anywhere. + escaped = search.replace('\\', '\\\\').replace('%', '\\%').replace('_', '\\_') + where_parts.append( + "(" + "LOWER(COALESCE(c.public_key, '')) LIKE ? ESCAPE '\\' OR " + "LOWER(COALESCE(c.name, '')) LIKE ? ESCAPE '\\' OR " + "LOWER(COALESCE(c.role, '')) LIKE ? ESCAPE '\\' OR " + "LOWER(COALESCE(c.device_type, '')) LIKE ? ESCAPE '\\' OR " + "LOWER(COALESCE(c.city, '')) LIKE ? ESCAPE '\\' OR " + "LOWER(COALESCE(c.state, '')) LIKE ? ESCAPE '\\' OR " + "LOWER(COALESCE(c.country, '')) LIKE ? ESCAPE '\\'" + ")" + ) + where_params.extend([f'{escaped}%'] + [f'%{escaped}%'] * 6) + + where_clause = (' WHERE ' + ' AND '.join(where_parts)) if where_parts else '' + + pagination = None + filtered_stats = None + if page is not None and page_size is not None and not include_detail: + page_size = max(1, min(200, int(page_size))) + page = max(1, int(page)) + cursor.execute( + """ + SELECT + COUNT(*) AS total_items, + SUM(CASE WHEN c.last_heard >= datetime('now', 'localtime', '-24 hours') THEN 1 ELSE 0 END) AS contacts_24h, + SUM(CASE WHEN c.last_heard >= datetime('now', 'localtime', '-7 days') THEN 1 ELSE 0 END) AS contacts_7d, + SUM(CASE WHEN c.first_heard >= datetime('now', 'localtime', '-7 days') + AND LOWER(COALESCE(c.device_type, '')) LIKE '%companion%' THEN 1 ELSE 0 END) AS new_companions, + SUM(CASE WHEN c.first_heard >= datetime('now', 'localtime', '-7 days') + AND LOWER(COALESCE(c.device_type, '')) LIKE '%repeater%' THEN 1 ELSE 0 END) AS new_repeaters, + SUM(CASE WHEN c.first_heard >= datetime('now', 'localtime', '-7 days') + AND (LOWER(COALESCE(c.device_type, '')) LIKE '%room%' + OR LOWER(COALESCE(c.device_type, '')) LIKE '%server%') THEN 1 ELSE 0 END) AS new_room_servers + FROM complete_contact_tracking c + """ + where_clause, + tuple(where_params), + ) + aggregate = cursor.fetchone() + total_items = int(aggregate['total_items'] or 0) + total_pages = max(1, (total_items + page_size - 1) // page_size) + page = min(page, total_pages) + pagination = { + 'page': page, + 'page_size': page_size, + 'total_items': total_items, + 'total_pages': total_pages, + 'has_previous': page > 1, + 'has_next': page < total_pages, + } + filtered_stats = { + 'contacts_24h': int(aggregate['contacts_24h'] or 0), + 'contacts_7d': int(aggregate['contacts_7d'] or 0), + 'contacts_total': total_items, + 'new_companions': int(aggregate['new_companions'] or 0), + 'new_repeaters': int(aggregate['new_repeaters'] or 0), + 'new_room_servers': int(aggregate['new_room_servers'] or 0), + } # Fetch contacts directly (no join/group-by). The recent paths per contact are # loaded in a second query below and assembled in Python. This avoids materializing @@ -5920,6 +6054,44 @@ class BotDataViewer: # column (incl. the raw_advert_data blob) on every request. last_advert_timestamp is # the per-contact value, so it matches the old MAX(...) over a single contact's rows. detail_cols = "c.raw_advert_data," if include_detail else "" + sort_expressions = { + 'username': "LOWER(COALESCE(c.name, ''))", + 'device_type': "LOWER(COALESCE(c.device_type, ''))", + 'location': ( + "LOWER(CASE " + "WHEN c.city IS NOT NULL AND c.city != '' AND c.state IS NOT NULL AND c.state != '' " + "THEN c.city || ', ' || c.state " + "WHEN c.city IS NOT NULL AND c.city != '' THEN c.city " + "WHEN c.latitude IS NOT NULL AND c.longitude IS NOT NULL " + "AND c.latitude != 0 AND c.longitude != 0 THEN printf('%s, %s', c.latitude, c.longitude) " + "ELSE '' END)" + ), + 'snr': 'COALESCE(c.snr, 0)', + 'hop_count': 'COALESCE(c.hop_count, 0)', + 'first_heard': "COALESCE(c.first_heard, '')", + 'last_seen': "COALESCE(c.last_heard, '')", + 'advert_count': 'COALESCE(c.advert_count, 0)', + } + sort = sort if sort in (*sort_expressions.keys(), 'distance') else 'last_seen' + direction = 'asc' if direction == 'asc' else 'desc' + if sort == 'distance': + if bot_lat is None or bot_lon is None: + sort_expression = '0' + else: + conn.create_function('contacts_distance_km', 2, lambda lat, lon: ( + self._calculate_distance(bot_lat, bot_lon, lat, lon) + if lat is not None and lon is not None else 0 + )) + sort_expression = 'contacts_distance_km(c.latitude, c.longitude)' + else: + sort_expression = sort_expressions[sort] + + query_params = list(where_params) + limit_clause = '' + if pagination is not None: + limit_clause = ' LIMIT ? OFFSET ?' + query_params.extend([page_size, (page - 1) * page_size]) + cursor.execute(""" SELECT c.public_key, c.name, c.role, c.device_type, @@ -5932,31 +6104,41 @@ class BotDataViewer: c.last_advert_timestamp as last_message FROM complete_contact_tracking c """ + where_clause + """ - ORDER BY c.last_heard DESC - """, params) + ORDER BY """ + sort_expression + f" {direction.upper()}, c.public_key ASC" + limit_clause, + tuple(query_params), + ) main_rows = cursor.fetchall() - # Fetch the 50 most recent advert paths per contact and group them in Python keyed by - # public_key. With idx_observed_paths_advert_pk_seen this runs as an ordered covering - # index scan (no temp B-tree). We don't filter to the windowed keys here: the assembly - # loop only looks up paths for contacts in main_rows, and adding a JOIN to the key set - # pushes the planner off the covering index into a much slower nested-loop plan. - cursor.execute(""" - WITH recent_paths AS ( - SELECT public_key, path_hex, path_length, bytes_per_hop, - observation_count, last_seen, - ROW_NUMBER() OVER (PARTITION BY public_key ORDER BY last_seen DESC) as rn - FROM observed_paths - WHERE packet_type = 'advert' AND public_key IS NOT NULL - ) - SELECT public_key, path_hex, path_length, bytes_per_hop, observation_count, last_seen - FROM recent_paths WHERE rn <= 50 - ORDER BY public_key, last_seen DESC - """) - paths_by_key = {} - for prow in cursor.fetchall(): + path_rows = [] + if main_rows: + path_params: list[Any] = [] + page_key_clause = '' + if pagination is not None: + # The interactive list enriches only the visible page. At most 200 keys are + # supplied, staying comfortably below SQLite's parameter limit and turning + # the former all-history window scan into targeted index lookups. + page_keys = [row['public_key'] for row in main_rows] + placeholders = ','.join('?' for _ in page_keys) + page_key_clause = f' AND public_key IN ({placeholders})' + path_params.extend(page_keys) + cursor.execute(""" + WITH recent_paths AS ( + SELECT public_key, path_hex, path_length, bytes_per_hop, + observation_count, last_seen, + ROW_NUMBER() OVER (PARTITION BY public_key ORDER BY last_seen DESC) as rn + FROM observed_paths + WHERE packet_type = 'advert' AND public_key IS NOT NULL + """ + page_key_clause + """ + ) + SELECT public_key, path_hex, path_length, bytes_per_hop, observation_count, last_seen + FROM recent_paths WHERE rn <= 50 + ORDER BY public_key, last_seen DESC + """, tuple(path_params)) + path_rows = cursor.fetchall() + + for prow in path_rows: if not prow['path_hex']: # Skip empty paths continue bph = None @@ -5975,7 +6157,7 @@ class BotDataViewer: 'last_seen': prow['last_seen'] if prow['last_seen'] is not None else None }) - multibyte_hop_chunks = self._collect_multibyte_hop_chunks(cursor) + multibyte_hop_chunks = self._get_cached_contact_multibyte_hop_chunks(cursor) chunk_buckets = self._bucket_hop_chunks(multibyte_hop_chunks) tracking = [] @@ -6216,10 +6398,14 @@ class BotDataViewer: except Exception as e: self.logger.debug(f"Could not get server stats: {e}") - return { + result = { 'tracking_data': tracking, 'server_stats': server_stats } + if pagination is not None: + result['pagination'] = pagination + result['filtered_stats'] = filtered_stats + return result except Exception as e: self.logger.error(f"Error getting tracking data: {e}") return {'error': str(e)} diff --git a/modules/web_viewer/templates/api_explorer.html b/modules/web_viewer/templates/api_explorer.html index 810c187..9a793b4 100644 --- a/modules/web_viewer/templates/api_explorer.html +++ b/modules/web_viewer/templates/api_explorer.html @@ -66,7 +66,7 @@ ("POST", "/api/optimize-database","Run VACUUM + ANALYZE on the SQLite database"), ]), ("Contacts", "fa-users", [ - ("GET", "/api/contacts", "List all contacts with last-seen and position"), + ("GET", "/api/contacts", "List contacts; supports paging, search, and sorting"), ("GET", "/api/export/contacts", "Export contacts as CSV"), ("POST", "/api/geocode-contact", "Reverse-geocode a contact's GPS position"), ("POST", "/api/toggle-star-contact", "Star or unstar a contact"), diff --git a/modules/web_viewer/templates/contacts.html b/modules/web_viewer/templates/contacts.html index 86b56ba..235a7fc 100644 --- a/modules/web_viewer/templates/contacts.html +++ b/modules/web_viewer/templates/contacts.html @@ -496,6 +496,18 @@
Loading contacts data...
+
+ Loading contacts… +
+ + +
+
@@ -540,12 +552,22 @@