diff --git a/etc/anubis/botPolicy.yaml b/etc/anubis/botPolicy.yaml index f117432..2274a85 100644 --- a/etc/anubis/botPolicy.yaml +++ b/etc/anubis/botPolicy.yaml @@ -31,6 +31,30 @@ bots: action: ALLOW user_agent_regex: frontend-instances-scraper remote_addresses: ["172.182.196.51/32"] + + - name: allow-git-clients + action: ALLOW + expression: + all: + - > + ( + userAgent.startsWith("git/") || + userAgent.contains("libgit") || + userAgent.startsWith("go-git") || + userAgent.startsWith("JGit/") || + userAgent.startsWith("JGit-") + ) + - '"Git-Protocol" in headers' + - headers["Git-Protocol"] == "version=2" + + - name: empty-ua + action: DENY + expression: userAgent == "" + + - name: cloudflare + headers_regex: + CF-Worker: .* + action: DENY # Proactively deny all apps, scripts, etc where the devs didnt even properly change the default ua to something valid, so they can just fuck off imo - name: lazy-user-agents @@ -40,6 +64,7 @@ bots: # Pathological bots to deny - import: (data)/bots/_deny-pathological.yaml + - import: (data)/crawlers/tencent-cloud.yaml - import: (data)/bots/aggressive-brazilian-scrapers.yaml # Aggressively block AI/LLM related bots/agents by default @@ -60,20 +85,36 @@ bots: # Challenge Firefox AI previews - import: (data)/clients/x-firefox-ai.yaml - # Allow common "keeping the internet working" routes (well-known, favicon, robots.txt) - - import: (data)/common/keep-internet-working.yaml - # Block abusive actors and bots - name: abusive-actors action: DENY user_agent_regex: >- 01h4x\.com|360Spider|404checker|404enemy|80legs|ADmantX|AIBOT|ALittleClient|Alexa|Alexibot|AllSubmitter|Alligator|AlphaBot|Amazonbot|Anthill|Anthropic-ai|Anarchie|Anarchy|Anarchy99|Ankit|Apexoo|Aspiegel|Asterias|Atomseobot|Attach|AwarioBot|AwarioRssBot|AwarioSmartBot|BBBike|BDCbot|BDFetch|BLEXBot|BackDoorBot|BackStreet|BackWeb|Backlink-Ceck|BacklinkCrawler|Backlink-check|BacklinksExtendedBot|Badass|Bandit|Barkrowler|BatchFTP|BattleztarBazinga|BetaBot|Bigfoot|Bitacle|BlackHole|BlackWidow|Blackboard|Blow|BlowFish|Blazer|Boardreader|Bolt|BotALot|Brandprotect|Brandwatch|Buck|Buddy|BuiltBotTough|BuiltWith|Bullseye|BunnySlippers|BuzzSumo|Bytespider|BUbiNG|CATExplorador|CCBot|Chrome\ Privacy\ Preserving\ Prefetch\ Proxy|CODE87|CSHttp|Calculon|Cah\.io\.community|CazoodleBot|Cegbfeieh|Censys|censys\.io|ChatGPT-User|CheTeam|CheeseBot|CherryPicker|ChinaClaw|Chlooe|Citoid|Claritybot|Clark-crawler|Cliqzbot|Cloudmapping|Cocolyzebot|Coccocbot|Cognitiveseo|Coher-ai|Collector|Com\.plumanalytics|Copier|CopyRightCheck|Copyscape|Cosmos|Craftbot|Crawl\.sogou\.com|Crawler\.feedback|Crawler4j|CrawlingatHomeProject|CrazyWebCrawler|Crescent|CrunchBot|Curious|Custo|CyberFind|CyotekWebCopy|DBLBot|DIIbot|DataCha0s|Dataforseo\.com|Dataforseobot|DatabaseDriverMysqli|Demon|Demandbase-bot|Deusu|Devil|Digincore|DigitalPebble|Dirbuster|Disco|Discobot|Discoverybot|Dispatch|DittoSpyder|DnBCrawler-Analytics|DnyzBot|DomCopBot|DomainAppender|DomainCrawler|DomainSigmaCrawler|DomainStatsBot|DomainsProject|Dotbot|DownloadWonder|Dragonfly|Drip|Ecatch|ECCP/1\.0|EMailSiphon|EMailWolf|EasyDL|Ebingbong|Ecxi|EirGrabber|EroCrawler|Evil|Ev-crawler|Evc-batch|Exabot|Expanse|ExpressWebPictures|ExtLinksBot|Extractor|ExtractorPro|ExtremePictureFinder|EyeNetIE|Ezooms|FDM|FHscan|FacebookBot|facebookexternalhit|Facebookplatform|Facebookscraper|FemtosearchBot|Fimap|Firefox/7\.0|FlashGet|Flunky|Foobot|Freeuploader|FrontPage|Fuzz|FyberSpider|Fyrebot|G-i-g-a-b-o-t|GPTBot|GRFZbot|GT::WWW|GalaxyBot|GeedoProductSearch|Genieo|GermCrawler|GetRight|GetWeb|Getintent|Gigabot|Go!Zilla|Go-Ahead-Got-It|GoZilla|Gotit|GrabNet|Grabber|Grafula|GrapeFX|GrapeshotCrawler|GridBot|Gopher|Haansoft|HaosouSpider|Harvest|Havij|Heritrix|Hloader|HonoluluBot|Humanlinks|HybridBot|IDBTE4M|IDBot|IRLbot|Iblog|Id-search|IlseBot|ImageFetch|ImagesiftBot|Imagesift\.com|IndeedBot|IndyLibrary|InfoNaviRobot|InfoTekies|InformationSecurityTeamInfraSecScanner|InfraSecScanner|Instabid|Intelliseek|InterGET|InternetMeasurement|InternetNinja|InternetSeer|InternetVista\ monitor|Iria|Iskanie|Isitwp\.com|IstellaBot|Iubenda-radar|JOCWebSpider|JamesBOT|Jbrofuzz|JennyBot|JetCar|Jetty|JikeSpider|Joomla|Jorgee|JustView|Jyxobot|KenjinSpider|KeybotTranslation-Search-Machine|KeywordDensity|Kinza|Kozmosbot|LNSpiderguy|LWP::Simple|Lanshanbot|Larbin|Leap|LeechFTP|LeechGet|LexiBot|Lftp|LibWeb|Libwhisker|LieBaoFast|Lightspeedsystems|Likse|LinkedInBot|LinkScan|Linkdexbot|Linkfluence|LinkWalker|Linkbot|LinkextractorPro|LinkpadBot|LinksManager|LinqiaMetadataDownloaderBot|LinqiaRSSBot|LinqiaScrapeBot|Lipperhey|LipperheySpider|Litemage_walker|Lmspider|Ltx71|Mag-Net|Magnet|Mail\.RU_Bot|Majestic-SEO|Majestic12|MajesticSEO|Magpie-crawler|MarkMonitor|MarkWatch|MassDownloader|Masscan|MataHari|MauiBot|Mb2345Browser|MeanPathBot|Meanpathbot|Mediatoolkitbot|MegaIndex\.ru|Metauri|Mediawords|MicroMessenger|MicrosoftDataAccess|MicrosoftURLControl|Minefield|MisterPiX|MoblieSafari|Mojolicious|MolokaiBot|MorfeusFuckingScanner|Mozlila|Mr\.4x3|Msrabot|Muhstik-scan|Musobot|MOptimizer|NICErsPRO|NPbot|NameIntelligence|Nameprotect|Navroad|NearSite|Needle|Nessus|NetAnts|NetEstate\ NE\ Crawler|NetLyzer|NetMechanic|NetSpider|NetZIP|NetVampire|Netcraft|Nettrack|Netvibes|NextGenSearchBot|Nibbler|Niki-bot|Nikto|NimbleCrawler|Nimbostratus|Ninja|Nmap|Nuclei|Nutch|OBot|OfflineExplorer|OfflineNavigator|getodin|OMGili|OnCrawl|OpenAI|OpenAI\.com|OpenLinkProfiler|OpenVAS|Openfind|OrangeBot|OrangeSpider|OutclicksBot|OutfoxBot|PalmSource|Page\ Analyzer|PageGrabber|PageScorer|PageThing\.com|Pandalytics|Panscient|PapaFoto|Pavuk|PcBrowser|PeoplePal|Petalbot|Picscout|Picsearch|PictureFinder|Piepmatz|Pimonster|Pinterestbot|Pixray|PleaseCrawl|Pockey|Polaris\ version|Probe-image-size|ProPowerBot|ProWebWalker|Probethenet|Proximic|Psbot|Pu_iN|Pump|PxBroker|PyCurl|QueryNMetasearch|Quick-Crawler|Quora-Bot|Rainbot|RankActive|RankActiveLinkBot|RankFlex|RankingBot|RankingBot2|Rankivabot|RankurBot|Rere|ReGet|RealDownload|Reaper|RebelMouse|Recorder|RedesScrapy|RepoMonkey|Researchscan|Ripz|Ripper|RocketCrawler|Rogerbot|RSSingBot|S1z\.ru|SBIder|Scan\.lol|Scalaj-http|Seocompany\.store|Seoscanners|Seostar|Serpstatbot|Sexsearcher|SentiBot|SenutoBot|SeoCherryBot|SeoSiteCheckup|Seobility|SeobilityBot|Seomoz|Shodan|Siphon|Slack-ImgProxy|Slackbot|Slackbot-LinkExpanding|SlySearch|SmartDownload|Snake|Snapbot|Snoopy|SocialRankIOBot|Sociscraper|Sogouwebspider|Sogouspider|Sosospider|Sottopop|SpaceBison|Sp_auditbot|Spammen|SpankBot|Spanner|Spbot|Spider_Bot|Spider_Bot/3\.0|Spinn3r|SplitSignalBot|SputnikBot|Sqlmap|Sqlworm|Sqworm|Steeler|Stripper|Sucker|Sucuri|SummalyBot|SuperBot|SuperHTTP|Surfbot|SurdotlyBot|SurveyBot|Suzuran|Swiftbot|Szukacz|T0PHackTeam|T8Abot|TAkeOut|TelegramBot|Teleport|TeleportPro|Telesoft|Telesphoreo|Telesphorep|TheIntraformant|TheNomad|Thumbor|TightTwatBot|TinEye|TinyTestBot|Titan|Toata|Toweyabot|Tracemyfile|Trendiction|Trendictionbot|Trendiction\.com|Trendiction\.de|True_Robot|Turingos|Turnitin|TurnitinBot|TwengaBot|Twice|Twitterbot|Typhoeus|UBermetrics-technologies\.com|URLy\.Warning|URLyWarning|UnisterBot|Upflow|V-BOT|VBProject|VCI|Vacuum|Vagabondo|VelenPublicWebCrawler|VeriCiteCrawler|VidibleScraper|Virusdie|VoidEYE|Voil|Voltron|Voyagerx\.com|WASALive-Bot|WBSearchBot|WEBDAV|WISENutbot|WPScan|WWW-Collector-E|WWW-Mechanize|WWW::Mechanize|WWWOFFLE|Wallpapers|Wallpapers/3\.0|WallpapersHD|WeSEE|WebAuto|WebBandit|WebCollage|WebCopier|WebEnhancer|WebFetch|WebFuck|WebGains-bot|WebGoIS|WebImageCollector|WebLeacher|WebMeup-crawler|WebPix|WebPros\.com|WebProsbot|WebReaper|WebSauger|WebShag|WebStripper|WebSucker|WebVigil|WebWhacker|WebZIP|Webalta|WebmasterWorldForumBot|WebsiteExtractor|WebsiteQuester|Webster|Whack|Whacker|Whatweb|WhatsApp|Who\.isBot|Widow|WinHTTrack|WiseGuysRobot|Wonderbot|Woobot|Wotbox|Wprecon|X09Mozilla|X22Mozilla|XaldonWebSpider|Xaldon_WebSpider|Xenu|XPymep1\.exe|YaK|YoudaoBot|Zade|Zauba|Zauba\.io|Zermelo|Zeus|Zitebot|ZmEu|ZoomBot|ZoominfoBot|ZumBot|ZyBorg|zgrab|YandexAccessibilityBot|YandexCalendar|YandexDialogs|YandexDirect|YandexDirectDyn|YandexFavicons|YaDirectFetcher|YandexMobileBot|YandexMarket|YandexMetrika|YandexMobileScreenShotBot|YandexOntoDBAPI|YandexPartner|YandexRCA|YandexRenderResourcesBot|YandexSearchShop|YandexTracker|YandexUserproxy|YandexVideoParser|YandexScreenshotBot|YandexComBot|TikTok|Custom-AsyncHttpClient|Applebot|Aliyun|AliyunSecBoti|Webmaster\.petalsearch|Seokicks|StepstoneCrawlBot|Keydrop\.io|fhms-its-research-scanner|meta-webindexer|Thinkbot|NetworkingExtension|com.apple.WebKit.Networking + # Allow common "keeping the internet working" routes (well-known, favicon, robots.txt) + - import: (data)/common/keep-internet-working.yaml # Catch all known webbrowsers - name: modern-browser - user_agent_regex: >- - Chrome|Firefox|Safari|Opera|OPR|Edg|Edge + expression: + all: + - '"User-Agent" in headers' + - > + ( + userAgent.contains("Firefox") || + userAgent.contains("Chrome") || + userAgent.contains("Safari") || + userAgent.contains("Opera") || + userAgent.contains("OPR") || + userAgent.contains("Edg") + ) + - '"Accept" in headers' + - '"Sec-Fetch-Dest" in headers' + - '"Sec-Fetch-Mode" in headers' + - '"Sec-Fetch-Site" in headers' + - '"Accept-Encoding" in headers' + - '( headers["Accept-Encoding"].contains("zstd") || headers["Accept-Encoding"].contains("br") )' + - '"Accept-Language" in headers' action: CHALLENGE challenge: algorithm: preact @@ -83,7 +124,7 @@ bots: # Catch all generic browsers - name: generic-browser user_agent_regex: >- - ^Mozilla + ^Mozilla|^Opera action: CHALLENGE challenge: algorithm: fast @@ -103,8 +144,55 @@ openGraph: store: backend: valkey parameters: - url: "unix://default:REDACTED/data/redis-server.sock" + url: "" status_codes: CHALLENGE: 200 DENY: 403 + +thresholds: + - name: zero + expression: weight <= 0 + action: ALLOW + + - name: mild + expression: + all: + - weight > 0 + - weight < 10 + action: CHALLENGE + challenge: + algorithm: preact + difficulty: 4 + report_as: 4 + + - name: moderate + expression: + all: + - weight >= 10 + - weight < 20 + action: CHALLENGE + challenge: + algorithm: fast + difficulty: 4 + report_as: 4 + + - name: high + expression: + all: + - weight >= 20 + - weight < 30 + action: CHALLENGE + challenge: + algorithm: fast + difficulty: 5 + report_as: 5 + + # Whoever ends up here needs to fucking choke + - name: impossible + expression: weight >= 30 + action: CHALLENGE + challenge: + algorithm: fast + difficulty: 6 + report_as: 6