To generate a robots.txt file from our Express app, we can create a robots.txt file that returns the text content of the robots.txt file.
For instance, we can write:
const express = require('express')
const app = express()
const port = 3000
app.get('/', (req, res) => {
res.send('hello world')
});
app.use('/robots.txt', function(req, res, next) {
res.type('text/plain')
res.send("User-agent: *\nDisallow: /");
});
app.listen(port, () => {
console.log(`Example app listening at http://localhost:${port}`)
})
to create the /robots.txt route by writing:
app.use('/robots.txt', function(req, res, next) {
res.type('text/plain')
res.send("User-agent: *\nDisallow: /");
});
We call res.type to set the MIME type of the response to text/plain .
And we call res.send with the robots.txt content we want to return.
So when we make a GET request to the /robots.txt route, we get:
User-agent: *
Disallow: /
