From a4bf722bdc7ac7d1e60f95a7e9e9b79b59097ee4 Mon Sep 17 00:00:00 2001
From: Hyperling <me@hyperling.com>
Date: Fri, 15 May 2026 13:10:35 -0700
Subject: [PATCH] Add permissive robots.txt to website.

---
 main.js | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/main.js b/main.js
index f44b4de..10c54d4 100755
--- a/main.js
+++ b/main.js
@@ -134,6 +134,11 @@ async function main() {
 			<strong>Web Pages (Alphabetical)</strong>
 			<ul>
 	`;
+	let robots_txt = `
+		User-agent: *
+		Allow: /
+		Sitemap: https://hyperling.com/sitemap.xml
+	`;
 
 	console.log("...Adding Routes...");
 	let router = express.Router();
@@ -199,6 +204,13 @@ async function main() {
 		res.send(sitemap_html);
 	});
 
+	// Provide a robots.txt for where search engines should find things.
+	console.log(" * Creating router for sitemap*");
+	router.get('/robots.txt', function (req, res) {
+		console.log("robots.txt being provided to", req.socket.remoteAddress)
+		res.send(robots_txt);
+	});
+
 	// Return a resource from the files folder.
 	// TBD/TODO would be great to log this as "DATE: ADDRESS hit ROUTER requesting RESOURCE"
 	console.log(" * Creating router for files");