Robots txt tester | agent view

The same page content and links, with explicit tool capabilities. Tools marked with an API can run directly through HTTP. Other tools require the browser interface or are not connected yet.

Tool capabilities

{
  "id": "robots-txt-tester",
  "category": "web-tools",
  "built": true,
  "requirements": "Paste-first rules evaluation by user-agent and URL; show winning rule.",
  "implementation": {
    "fields": [
      {
        "key": "url",
        "label": "URL to test",
        "value": "https://happytails.ai/path",
        "type": "text"
      },
      {
        "key": "agent",
        "label": "Crawler user agent",
        "value": "Googlebot",
        "type": "text"
      }
    ],
    "note": "Paste robots.txt content; this tool does not fetch or change the remote file.",
    "experience": {
      "automatic": false,
      "single": false,
      "label": "Your text",
      "placeholder": "Type or paste your input…",
      "help": "Up to 1,000,000 characters. Review the settings before running.",
      "action": null
    },
    "mode": "explicit",
    "interactive": false
  },
  "execution": "POST /api/v1/tools/robots-txt-tester/run",
  "api": {
    "id": "robots-txt-tester",
    "name": "Robots txt tester",
    "category": "web-tools",
    "description": "Check whether robots.txt rules allow a crawler to visit a URL.",
    "notes": "Paste robots.txt content; this tool does not fetch or change the remote file.",
    "human_path": "/en/web-tools/robots-txt-tester/",
    "method": "POST",
    "endpoint": "/api/v1/tools/robots-txt-tester/run",
    "schema_url": "/api/v1/tools/robots-txt-tester",
    "input_schema": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "input": {
          "type": "string",
          "maxLength": 1000000,
          "default": "",
          "description": "Plain text input. File tools use files instead unless otherwise documented."
        },
        "options": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "url": {
              "type": "string",
              "description": "URL to test",
              "default": "https://happytails.ai/path"
            },
            "agent": {
              "type": "string",
              "description": "Crawler user agent",
              "default": "Googlebot"
            }
          }
        }
      }
    },
    "output_schema": {
      "type": "object",
      "required": [
        "tool",
        "result"
      ],
      "properties": {
        "tool": {
          "type": "string"
        },
        "result": {
          "type": "object",
          "required": [
            "text",
            "files"
          ],
          "properties": {
            "text": {
              "type": [
                "string",
                "null"
              ]
            },
            "data": {
              "description": "Parsed JSON when the textual result is JSON."
            },
            "name": {
              "type": "string"
            },
            "mime": {
              "type": "string"
            },
            "files": {
              "type": "array",
              "items": {
                "type": "object",
                "required": [
                  "name",
                  "mime",
                  "bytes",
                  "base64"
                ],
                "properties": {
                  "name": {
                    "type": "string"
                  },
                  "mime": {
                    "type": "string"
                  },
                  "bytes": {
                    "type": "integer"
                  },
                  "base64": {
                    "type": "string",
                    "contentEncoding": "base64"
                  }
                }
              }
            }
          }
        }
      }
    },
    "processing": "Server; same transformation code as browser tools",
    "limits": {
      "request_bytes": 8388608,
      "input_characters": 1000000,
      "files_bytes": 5242880,
      "max_files": 0,
      "timeout_seconds": 12
    }
  },
  "agent_processing": "Server; same transformation code as browser tools",
  "browser_agent": {
    "supported": true,
    "name": "run_current_tool",
    "discovery": "WebMCP on the human page in a supporting browser",
    "verification": "See audit/API-AUDIT.md; availability is not verification"
  }
}

Page guide

Check whether robots.txt rules allow a crawler to visit a URL.