me/nodejs-website
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add permissive robots.txt to website.

Browse Source
This commit is contained in:
Hyperling
2026-05-15 13:10:35 -07:00
parent ce0dcb0f46
commit a4bf722bdc
1 changed files with 12 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
main.js
View File

@@ -134,6 +134,11 @@ async function main() {
<strong>Web Pages (Alphabetical)</strong> <strong>Web Pages (Alphabetical)</strong>
<ul> <ul>
`; `;
let robots_txt = `
User-agent: *
Allow: /
Sitemap: https://hyperling.com/sitemap.xml
`;
console.log("...Adding Routes..."); console.log("...Adding Routes...");
let router = express.Router(); let router = express.Router();
@@ -199,6 +204,13 @@ async function main() {
res.send(sitemap_html); res.send(sitemap_html);
}); });
// Provide a robots.txt for where search engines should find things.
console.log(" * Creating router for sitemap*");
router.get('/robots.txt', function (req, res) {
console.log("robots.txt being provided to", req.socket.remoteAddress)
res.send(robots_txt);
});
// Return a resource from the files folder. // Return a resource from the files folder.
// TBD/TODO would be great to log this as "DATE: ADDRESS hit ROUTER requesting RESOURCE" // TBD/TODO would be great to log this as "DATE: ADDRESS hit ROUTER requesting RESOURCE"
console.log(" * Creating router for files"); console.log(" * Creating router for files");