initial commit

app/robots.ts

@@ -0,0 +1,47 @@
+import { MetadataRoute } from 'next'
+
+export default function robots(): MetadataRoute.Robots {
+  const baseUrl = process.env.NEXT_PUBLIC_SITE_URL || 'http://localhost:4023'
+
+  return {
+    rules: [
+      {
+        userAgent: '*',
+        allow: [
+          '/',
+          '/about',
+          '/services',
+          '/tools',
+          '/topics',
+          '/contact',
+          '/feedback',
+          '/terms',
+          '/privacy',
+          '/legal-agreement',
+          '/refund-policy',
+        ],
+        disallow: [
+          '/api/',
+          '/admin/',
+          '/_next/',
+          '/private/',
+          '/profile',
+          '/auth',
+          '/*.tmp$',
+          '/*.bak$',
+          '/*.json$',
+        ],
+      },
+      {
+        userAgent: 'GPTBot',
+        disallow: '/',
+      },
+      {
+        userAgent: 'Google-Extended',
+        disallow: '/',
+      },
+    ],
+    sitemap: `${baseUrl}/sitemap.xml`,
+    host: baseUrl,
+  }
+}