Implement multi-domain SEO improvements:

- Add metadata to login and login-verify pages to prevent indexing.
- Update robots.txt to disallow additional sensitive paths.
- Enhance sitemap generation to filter by city and include accurate last modified dates.
- Refactor canonical URL generation to support multi-domain architecture, ensuring self-canonicalization for city domains.
- Remove deprecated CanonicalHandler component and streamline canonical URL handling.
- Introduce safe JSON-LD output to prevent XSS vulnerabilities.
- Add payment layout with appropriate metadata to prevent indexing.
- Conduct a comprehensive technical SEO audit and implement necessary fixes across the application.

app/robots.js

@@ -1,25 +1,17 @@
 import { headers } from 'next/headers';
+import { getBaseUrl } from '../utils/sitemap';
 
-// Get current domain from headers
-function getCurrentDomain() {
-    const headersList = headers();
-    const host = headersList.get('host');
-    return host || 'nobat724.com';
+async function getCurrentDomain() {
+    try {
+        const headersList = await headers();
+        return headersList.get('host') || 'nobat724.com';
+    } catch {
+        return 'nobat724.com';
+    }
 }
 
-// Get base URL based on current domain
-function getBaseUrl(domain) {
-    // Remove port number if present (for development)
-    const cleanDomain = domain.replace(/:\d+$/, '');
-
-    // For localhost domains, use http, otherwise https
-    const protocol = cleanDomain.includes('localhost') ? 'http' : 'https';
-
-    return `${protocol}://${domain}`;
-}
-
-export default function robots() {
-    const domain = getCurrentDomain();
+export default async function robots() {
+    const domain = await getCurrentDomain();
     const baseUrl = getBaseUrl(domain);
 
     // If DEV_MODE is TRUE, disallow all crawlers
@@ -32,13 +24,18 @@ export default function robots() {
         };
     }
 
-    // Normal behavior when DEV_MODE is FALSE or not set
     return {
         rules: {
             userAgent: '*',
             allow: '/',
-            disallow: ['/dashboard'],
+            disallow: [
+                '/dashboard',
+                '/login',
+                '/login-verify',
+                '/payment',
+                '/appointment',
+            ],
         },
         sitemap: `${baseUrl}/sitemap.xml`,
     };
-}
+}