feat: add robots

src/pages/robots.txt.ts

@@ -0,0 +1,14 @@
+// https://docs.astro.build/en/guides/integrations-guide/sitemap/#usage
+import type { APIRoute } from 'astro';
+
+const getRobotsTxt = (sitemapURL: URL) => `\
+User-agent: *
+Allow: /
+
+Sitemap: ${sitemapURL.href}
+`;
+
+export const GET: APIRoute = ({ site }) => {
+  const sitemapURL = new URL('sitemap-index.xml', site);
+  return new Response(getRobotsTxt(sitemapURL));
+};