datasette-block-robots/datasette_block_robots/__init__.py

from datasette import hookimpl
from datasette.utils.asgi import Response
from datasette.utils import await_me_maybe
from datasette.plugins import pm
from . import hookspecs
 
pm.add_hookspecs(hookspecs)
 
 
async def robots_txt(datasette, request):
    config = datasette.plugin_config("datasette-block-robots") or {}
    literal = config.get("literal")
    disallow = []
    if literal:
        lines = literal.split("\n")
    else:
        disallow = config.get("disallow") or []
        if isinstance(disallow, str):
            disallow = [disallow]
        allow_only_index = config.get("allow_only_index")
        if allow_only_index:
            for database_name in datasette.databases:
                if database_name != "_internal":
                    disallow.append(datasette.urls.database(database_name))
        if not disallow:
            disallow = ["/"]
        lines = ["User-agent: *"] + ["Disallow: {}".format(item) for item in disallow]
 
    # Add lines generated by plugins
    for more_lines in pm.hook.block_robots_extra_lines(
        datasette=datasette, request=request
    ):
        more_lines = await await_me_maybe(more_lines)
        if more_lines:
            lines.extend(more_lines)
 
    return Response.text("\n".join(lines))
 
 
@hookimpl
def register_routes():
    return [
        (r"^/robots\.txt$", robots_txt),
    ]
 
 
@hookimpl
def startup(datasette):
    config = datasette.plugin_config("datasette-block-robots") or {}
    assert not (
        config.get("disallow") and config.get("literal")
    ), "datasette-block-robots cannot be configured with both disallow: and literal:"