Merge pull request #3052 from SalesforceCommerceCloud/jbothra-sf/@W-17514374-add-robot-txt-signed

@W-17514374 Add robots.txt to mrt-reference-app.

Author: jbothra-sf

packages/template-mrt-reference-app/app/ssr.js

@@ -350,6 +350,7 @@ const runtime = getRuntime()
 
 const {handler, app, server} = runtime.createHandler(options, (app) => {
     app.get('/favicon.ico', runtime.serveStaticFile('static/favicon.ico'))
+    app.get('/robots.txt', runtime.serveStaticFile('static/robots.txt'))
 
     // Add middleware to explicitly suppress caching on all responses (done
     // before we invoke the handlers)

packages/template-mrt-reference-app/app/static/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

packages/template-mrt-reference-app/package.json

@@ -45,7 +45,9 @@
       "node_modules/**/*.*"
     ],
     "ssrShared": [
-      "static/example.txt"
+      "static/example.txt", 
+      "static/favicon.ico",
+      "static/robots.txt"
     ]
   },
   "dependencies": {