From 744e72efc9e405ee86ab962fdfa5a8a99a5cb162 Mon Sep 17 00:00:00 2001
From: Alex Lebens <alexanderlebens@gmail.com>
Date: Tue, 17 Feb 2026 22:32:22 -0600
Subject: [PATCH] feat: update robots.txt

---
 src/pages/robots.txt.ts | 29 ++++++-----------------------
 1 file changed, 6 insertions(+), 23 deletions(-)

diff --git a/src/pages/robots.txt.ts b/src/pages/robots.txt.ts
index 583a0b1..e49fce0 100644
--- a/src/pages/robots.txt.ts
+++ b/src/pages/robots.txt.ts
@@ -1,31 +1,14 @@
 // https://docs.astro.build/en/guides/integrations-guide/sitemap/#usage
 import type { APIRoute } from 'astro';
 
-const robotsTxt = `
-User-agent: Googlebot
-Disallow:
-Allow: /
-Crawl-delay: 10
-
-User-agent: Yandex
-Disallow:
-Allow: /
-Crawl-delay: 2
-
-User-agent: archive.org_bot
-Disallow:
-Allow: /
-Crawl-delay: 2
-
+const getRobotsTxt = (sitemapURL: URL) => `\
 User-agent: *
 Allow: /
 
-Sitemap: ${new URL('sitemap-index.xml', import.meta.env.SITE).href}`.trim();
+Sitemap: ${sitemapURL.href}
+`;
 
-export const GET: APIRoute = () => {
-  return new Response(robotsTxt, {
-    headers: {
-      'Content-Type': 'text/plain; charset=utf-8',
-    },
-  });
+export const GET: APIRoute = ({ site }) => {
+  const sitemapURL = new URL('sitemap-index.xml', site);
+  return new Response(getRobotsTxt(sitemapURL));
 };