fixed duplicate content penalties from googlebot for the revisions pa…

…ges by adding a X-Robots-Tag header, closes #254
Erudika · Aug 19, 2021 · c2a0539 · c2a0539
1 parent 2e9f22f
commit c2a0539
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 3 deletions.
diff --git a/src/main/java/com/erudika/scoold/api/ApiController.java b/src/main/java/com/erudika/scoold/api/ApiController.java
@@ -346,7 +346,7 @@ public List<Comment> getPostComments(@PathVariable String id,
 	public List<Map<String, Object>> getPostRevisions(@PathVariable String id,
 			HttpServletRequest req, HttpServletResponse res) {
 		Model model = new ExtendedModelMap();
-		revisionsController.get(id, req, model);
+		revisionsController.get(id, req, res, model);
 		Post post = (Post) model.getAttribute("showPost");
 		if (post == null) {
 			res.setStatus(HttpStatus.NOT_FOUND.value());

diff --git a/src/main/java/com/erudika/scoold/controllers/RevisionsController.java b/src/main/java/com/erudika/scoold/controllers/RevisionsController.java
@@ -31,6 +31,7 @@
 import org.springframework.web.bind.annotation.RequestMapping;
 import static com.erudika.scoold.ScooldServer.QUESTIONSLINK;
 import com.erudika.scoold.core.Profile;
+import javax.servlet.http.HttpServletResponse;
 
 /**
  *
@@ -48,11 +49,12 @@ public RevisionsController(ScooldUtils utils) {
 	}
 
 	@GetMapping("/{postid}")
-	public String get(@PathVariable String postid, HttpServletRequest req, Model model) {
+	public String get(@PathVariable String postid, HttpServletRequest req, HttpServletResponse res, Model model) {
 		Post showPost = utils.getParaClient().read(postid);
 		if (showPost == null) {
 			return "redirect:" + QUESTIONSLINK;
 		}
+		res.setHeader("X-Robots-Tag", "noindex, nofollow"); // https://github.com/Erudika/scoold/issues/254
 		Profile authUser = utils.getAuthUser(req);
 		if (!utils.canAccessSpace(authUser, showPost.getSpace())) {
 			return "redirect:" + QUESTIONSLINK;

diff --git a/src/main/resources/static/robots.txt b/src/main/resources/static/robots.txt
@@ -2,9 +2,19 @@
 
 User-agent: *
 
-Disallow: /click/
+Disallow: /error
+Disallow: /feedback
+Disallow: /api/*
 Disallow: /p/
 Disallow: /signout/
 Disallow: /search/
 Disallow: /*?getreportform*
 Allow: /
+User-agent: Yahoo Pipes 1.0
+Disallow: /
+User-agent: 008
+Disallow: /
+User-agent: voltron
+Disallow: /
+User-agent: Bytespider
+Disallow: /