merge in new changes

src/pages/robots.txt.ts

@@ -0,0 +1,31 @@
+// https://docs.astro.build/en/guides/integrations-guide/sitemap/#usage
+import type { APIRoute } from 'astro';
+
+const robotsTxt = `
+User-agent: Googlebot
+Disallow:
+Allow: /
+Crawl-delay: 10
+
+User-agent: Yandex
+Disallow:
+Allow: /
+Crawl-delay: 2
+
+User-agent: archive.org_bot
+Disallow:
+Allow: /
+Crawl-delay: 2
+
+User-agent: *
+Allow: /
+
+Sitemap: ${new URL('sitemap-index.xml', import.meta.env.SITE).href}`.trim();
+
+export const GET: APIRoute = () => {
+  return new Response(robotsTxt, {
+    headers: {
+      'Content-Type': 'text/plain; charset=utf-8',
+    },
+  });
+};