From ec49ab12a41f52a5a60328b87bc9580db1b2e969 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:12:11 -0700 Subject: [PATCH] Remove Android build fingerprint from crawler patterns Removes the `AP3A\.240617\.008` crawler entry and its ordinary Instagram example. Adds a regression test asserting that this Android user agent is not classified as a crawler. Fixes #466 --- crawler-user-agents.json | 12 ------------ test_harness.py | 6 ++++++ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/crawler-user-agents.json b/crawler-user-agents.json index 82760315..64853d9d 100644 --- a/crawler-user-agents.json +++ b/crawler-user-agents.json @@ -8318,18 +8318,6 @@ "ai-crawler" ] }, - { - "pattern": "AP3A\\.240617\\.008", - "url": "https://knownagents.com/agents/008", - "instances": [ - "Mozilla/5.0 (Linux; Android 15; CPH2557 Build/AP3A.240617.008; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/142.0.7444.142 Mobile Safari/537.36 Instagram 406.0.0.58.159 Android (35/15; 480dpi; 1080x2400; OPPO; CPH2557; OP573DL1; mt6833; en_MY; 822918295; IABMV/1) NV/1" - ], - "addition_date": "2026/04/26", - "description": "80legs web scraping service", - "tags": [ - "scanner" - ] - }, { "pattern": "ApifyBot", "url": "https://knownagents.com/agents/apifybot", diff --git a/test_harness.py b/test_harness.py index ecf00166..42f29aa0 100644 --- a/test_harness.py +++ b/test_harness.py @@ -14,6 +14,12 @@ def test_nomatch(): assert is_crawler("!!!!!!!!!!!!") is False +def test_android_instagram_build_fingerprint_is_not_crawler(): + # AP3A.240617.008 is an Android 15 platform build id on ordinary Instagram traffic. + user_agent = "Mozilla/5.0 (Linux; Android 15; CPH2557 Build/AP3A.240617.008; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/142.0.7444.142 Mobile Safari/537.36 Instagram 406.0.0.58.159 Android (35/15; 480dpi; 1080x2400; OPPO; CPH2557; OP573DL1; mt6833; en_MY; 822918295; IABMV/1) NV/1" + assert is_crawler(user_agent) is False + + def test_case_sensitive(): assert is_crawler("test googlebot/2.0 test") is False