dynamic /robots.txt

pandora/app/views.py

@@ -89,11 +89,17 @@ def opensearch_xml(request):
         'application/xml'
     )
 
-def robots_txt(request, url):
-    return HttpResponse(
-        'User-agent: *\nDisallow:\nSitemap: %s\n' % request.build_absolute_uri('/sitemap.xml'),
-        'text/plain'
-    )
+def robots_txt(request):
+    if settings.CONFIG['site'].get('public'):
+        robots_txt = '''User-agent: *
+Disallow:
+Sitemap: {}
+'''.format(request.build_absolute_uri('/sitemap.xml'))
+    else:
+        robots_txt = '''User-agent: *
+Disallow: /
+'''
+    return HttpResponse(robots_txt, 'text/plain')
 
 def getPage(request, data):
     '''

pandora/config.0xdb.jsonc

@@ -1110,6 +1110,7 @@
         "https": true,
         "id": "0xdb",
         "name": "0xDB",
+        "public": true,
         "sendReferrer": false,
         "url": "0xDB.org"
     },

pandora/config.indiancinema.jsonc

@@ -1583,6 +1583,7 @@
         "https": true,
         "id": "indiancinema",
         "name": "Indiancine.ma",
+        "public": true,
         "sendReferrer": true,
         "url": "indiancine.ma"
     },

pandora/config.padma.jsonc

@@ -1063,6 +1063,7 @@
         "https": true,
         "id": "padma",
         "name": "Pad.ma",
+        "public": false,
         "sendReferrer": true,
         "url": "pad.ma"
     },

pandora/config.pandora.jsonc

@@ -1002,6 +1002,8 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
         "https": false,
         "id": "pandora",
         "name": "pan.do/ra",
+        // Set to true to allow search engines to index the site
+        "public": false,
         "sendReferrer": true,
         "url": "pandora.local"
     },

pandora/urls.py

@@ -54,10 +54,6 @@ urlpatterns = [
     url(r'^texts/$', text.views.text),
     url(r'^texts/(?P<id>.*?)/\d+$', text.views.text),
     url(r'^texts/(?P<id>.*?)$', text.views.text),
-    url(r'^robots.txt$', serve_static_file, {
-        'location': os.path.join(settings.STATIC_ROOT, 'robots.txt'),
-        'content_type': 'text/plain'
-    }),
     url(r'^favicon.ico$', serve_static_file, {
         'location': os.path.join(settings.STATIC_ROOT, 'png/icon.16.png'),
         'content_type': 'image/x-icon'