add DEV_MODE support to metadata generation and robots handling; update .gitignore and docker-compose for environment variables

app/robots.js

@@ -22,6 +22,17 @@ export default function robots() {
     const domain = getCurrentDomain();
     const baseUrl = getBaseUrl(domain);
 
+    // If DEV_MODE is TRUE, disallow all crawlers
+    if (process.env.DEV_MODE === 'TRUE') {
+        return {
+            rules: {
+                userAgent: '*',
+                disallow: '/',
+            },
+        };
+    }
+
+    // Normal behavior when DEV_MODE is FALSE or not set
     return {
         rules: {
             userAgent: '*',