Update robots.txt for robots that ignore rel=nofollow

fisharebest · Dec 19, 2023 · e96e110 · e96e110
1 parent cf522ed
commit e96e110
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 5 deletions.
diff --git a/app/Http/RequestHandlers/RobotsTxt.php b/app/Http/RequestHandlers/RobotsTxt.php
@@ -22,6 +22,8 @@
 use Fisharebest\Webtrees\Http\Middleware\BadBotBlocker;
 use Fisharebest\Webtrees\Module\SiteMapModule;
 use Fisharebest\Webtrees\Services\ModuleService;
+use Fisharebest\Webtrees\Services\TreeService;
+use Fisharebest\Webtrees\Tree;
 use Fisharebest\Webtrees\Validator;
 use Psr\Http\Message\ResponseInterface;
 use Psr\Http\Message\ServerRequestInterface;
@@ -48,12 +50,15 @@ class RobotsTxt implements RequestHandlerInterface
 
     private ModuleService $module_service;
 
+    private TreeService $tree_service;
+
     /**
      * @param ModuleService $module_service
      */
-    public function __construct(ModuleService $module_service)
+    public function __construct(ModuleService $module_service, TreeService $tree_service)
     {
         $this->module_service = $module_service;
+        $this->tree_service   = $tree_service;
     }
 
     /**
@@ -64,13 +69,15 @@ public function __construct(ModuleService $module_service)
     public function handle(ServerRequestInterface $request): ResponseInterface
     {
         $base_url = Validator::attributes($request)->string('base_url');
+        $trees    = $this->tree_service->all()->map(static fn (Tree $tree): string => $tree->name());
 
         $data = [
             'bad_user_agents'  => BadBotBlocker::BAD_ROBOTS,
             'base_url'         => $base_url,
             'base_path'        => parse_url($base_url, PHP_URL_PATH) ?? '',
             'disallowed_paths' => self::DISALLOWED_PATHS,
             'sitemap_url'      => '',
+            'trees'            => $trees,
         ];
 
         $sitemap_module = $this->module_service->findByInterface(SiteMapModule::class)->first();

diff --git a/resources/views/robots-txt.phtml b/resources/views/robots-txt.phtml
@@ -3,10 +3,11 @@
 declare(strict_types=1);
 
 /**
- * @var array<string> $bad_user_agents
- * @var string        $base_path
- * @var string        $base_url
- * @var string        $sitemap_url
+ * @var array<string>    $bad_user_agents
+ * @var string           $base_path
+ * @var string           $base_url
+ * @var string           $sitemap_url
+ * @var iterable<string> $trees
  */
 
 ?>
@@ -27,6 +28,21 @@ Disallow: <?= $base_path ?>/admin
 Disallow: <?= $base_path ?>/manager
 Disallow: <?= $base_path ?>/editor
 Disallow: <?= $base_path ?>/account
+<?php foreach ($trees as $tree) : ?>
+Disallow <?= $base_path ?>/tree/<?= $tree ?>/ancestors
+Disallow <?= $base_path ?>/tree/<?= $tree ?>/calendar
+Disallow <?= $base_path ?>/tree/<?= $tree ?>/compact
+Disallow <?= $base_path ?>/tree/<?= $tree ?>/descendants
+Disallow <?= $base_path ?>/tree/<?= $tree ?>/family-book
+Disallow <?= $base_path ?>/tree/<?= $tree ?>/family-list
+Disallow <?= $base_path ?>/tree/<?= $tree ?>/fan-chart
+Disallow <?= $base_path ?>/tree/<?= $tree ?>/hourglass
+Disallow <?= $base_path ?>/tree/<?= $tree ?>/individual-list
+Disallow <?= $base_path ?>/tree/<?= $tree ?>/lifespans
+Disallow <?= $base_path ?>/tree/<?= $tree ?>/pedigree
+Disallow <?= $base_path ?>/tree/<?= $tree ?>/report
+Disallow <?= $base_path ?>/tree/<?= $tree ?>/timeline
+<?php endforeach ?>
 Crawl-delay: 10
 
 <?php if ($sitemap_url !== '') : ?>