Merge branch 'dev'

sexyoung · Feb 20, 2024 · d0df6de · d0df6de
2 parents 91e15f0 + 43e5f21
commit d0df6de
Showing 1 changed file with 22 additions and 0 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -0,0 +1,22 @@
+User-agent: *
+Allow: /
+User-agent: Googlebot
+Disallow: 
+Disallow: /cgi-bin/
+
+# Optimization for Google Ads Bot
+User-agent: AdsBot-Google-Mobile
+User-agent: AdsBot-Google
+Disallow: /_api/*
+Disallow: /_partials*
+Disallow: /pro-gallery-webapp/v1/galleries/*
+
+# Block PetalBot
+User-agent: PetalBot
+Disallow: /
+
+# Crawl delay for overly enthusiastic bots
+User-agent: AhrefsBot
+Crawl-delay: 10
+
+Sitemap: https://twsdm.com