diff --git a/main.js b/main.js
index f44b4de..10c54d4 100755
--- a/main.js
+++ b/main.js
@@ -134,6 +134,11 @@ async function main() {
Web Pages (Alphabetical)
`;
+ let robots_txt = `
+ User-agent: *
+ Allow: /
+ Sitemap: https://hyperling.com/sitemap.xml
+ `;
console.log("...Adding Routes...");
let router = express.Router();
@@ -199,6 +204,13 @@ async function main() {
res.send(sitemap_html);
});
+ // Provide a robots.txt for where search engines should find things.
+ console.log(" * Creating router for sitemap*");
+ router.get('/robots.txt', function (req, res) {
+ console.log("robots.txt being provided to", req.socket.remoteAddress)
+ res.send(robots_txt);
+ });
+
// Return a resource from the files folder.
// TBD/TODO would be great to log this as "DATE: ADDRESS hit ROUTER requesting RESOURCE"
console.log(" * Creating router for files");