From e9360033f62a681aea2542998f4cf3f9a8688d23 Mon Sep 17 00:00:00 2001 From: redrun Date: Sat, 17 May 2025 23:09:54 -0500 Subject: [PATCH 1/2] Some boundaries for nice bots --- public_html/robots.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public_html/robots.txt b/public_html/robots.txt index 2891e227..16cb83b5 100644 --- a/public_html/robots.txt +++ b/public_html/robots.txt @@ -2,3 +2,7 @@ # http://code.google.com/web/controlcrawlindex/ User-agent: * +Disallow: /uploads/ +Disallow: /librivox-validator-books/ +Disallow: /public_readers_iframe/ +Disallow: /auth/ From 3adf0ac1d8dd2853d779ec414281282f85968218 Mon Sep 17 00:00:00 2001 From: redrun Date: Sat, 17 May 2025 23:12:40 -0500 Subject: [PATCH 2/2] Give 404 status on missing catalog pages --- application/controllers/catalog/Page.php | 1 + 1 file changed, 1 insertion(+) diff --git a/application/controllers/catalog/Page.php b/application/controllers/catalog/Page.php index a5229795..56d0cee7 100644 --- a/application/controllers/catalog/Page.php +++ b/application/controllers/catalog/Page.php @@ -31,6 +31,7 @@ public function index($slug) { $this->data['message'] = 'We weren\'t able to find that project. You must include either the Project Id or the Project Slug (i.e., "tom_sawyer_by_mark_twain" , without any other link info)'; $this->_render('catalog/not_found'); + $this->output->set_status_header(404); return; }