Add permissive robots.txt to website.

main.js

@@ -134,6 +134,11 @@ async function main() {
 			<strong>Web Pages (Alphabetical)</strong>
 			<ul>
 	`;
+	let robots_txt = `
+		User-agent: *
+		Allow: /
+		Sitemap: https://hyperling.com/sitemap.xml
+	`;
 
 	console.log("...Adding Routes...");
 	let router = express.Router();
@@ -199,6 +204,13 @@ async function main() {
 		res.send(sitemap_html);
 	});
 
+	// Provide a robots.txt for where search engines should find things.
+	console.log(" * Creating router for sitemap*");
+	router.get('/robots.txt', function (req, res) {
+		console.log("robots.txt being provided to", req.socket.remoteAddress)
+		res.send(robots_txt);
+	});
+
 	// Return a resource from the files folder.
 	// TBD/TODO would be great to log this as "DATE: ADDRESS hit ROUTER requesting RESOURCE"
 	console.log(" * Creating router for files");