feat: 加入 robots 看能不能優化 ai 爬蟲

sexyoung · Feb 20, 2024 · 43e5f21 · 43e5f21
1 parent 55cf2a4
commit 43e5f21
Showing 1 changed file with 22 additions and 0 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -0,0 +1,22 @@
+User-agent: *
+Allow: /
+User-agent: Googlebot
+Disallow: 
+Disallow: /cgi-bin/
+
+# Optimization for Google Ads Bot
+User-agent: AdsBot-Google-Mobile
+User-agent: AdsBot-Google
+Disallow: /_api/*
+Disallow: /_partials*
+Disallow: /pro-gallery-webapp/v1/galleries/*
+
+# Block PetalBot
+User-agent: PetalBot
+Disallow: /
+
+# Crawl delay for overly enthusiastic bots
+User-agent: AhrefsBot
+Crawl-delay: 10
+
+Sitemap: https://twsdm.com