Robots.txt Tester

Open Robots.txt Tester on Happytails.ai. This view has the same page content and links, with explicit tool capabilities. Read the processing notes before using a tool.

Tool capabilities

{
  "id": "robots-txt-tester",
  "category": "web-tools",
  "built": true,
  "requirements": "Paste-first rules evaluation by user-agent and URL; show winning rule.",
  "implementation": {
    "fields": [
      {
        "key": "url",
        "label": "URL to test",
        "value": "https://happytails.ai/path",
        "type": "text"
      },
      {
        "key": "agent",
        "label": "Crawler user agent",
        "value": "Googlebot",
        "type": "text"
      }
    ],
    "note": "Paste robots.txt content; this tool does not fetch or change the remote file.",
    "experience": {
      "automatic": false,
      "single": false,
      "inputMaxLength": 1000000,
      "inputMode": "text",
      "label": "Your text",
      "placeholder": "Type or paste your input…",
      "help": "Up to 1,000,000 characters. Review the settings before running.",
      "helpParts": [
        "Up to 1,000,000 characters.",
        "Review the settings before running."
      ],
      "action": null
    },
    "mode": "explicit",
    "interactive": false
  },
  "execution": "POST /api/v1/tools/robots-txt-tester/run",
  "api": {
    "id": "robots-txt-tester",
    "name": "Robots txt tester",
    "category": "web-tools",
    "description": "Check whether robots.txt rules allow a crawler to visit a URL.",
    "notes": "Paste robots.txt content; this tool does not fetch or change the remote file.",
    "human_path": "/en/web-tools/robots-txt-tester/",
    "method": "POST",
    "endpoint": "/api/v1/tools/robots-txt-tester/run",
    "schema_url": "/api/v1/tools/robots-txt-tester",
    "input_schema": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "input": {
          "type": "string",
          "maxLength": 1000000,
          "default": "",
          "description": "Plain text input. File tools use files instead unless otherwise documented."
        },
        "options": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "url": {
              "type": "string",
              "description": "URL to test",
              "default": "https://happytails.ai/path"
            },
            "agent": {
              "type": "string",
              "description": "Crawler user agent",
              "default": "Googlebot"
            }
          }
        }
      }
    },
    "output_schema": {
      "type": "object",
      "required": [
        "tool",
        "result"
      ],
      "properties": {
        "tool": {
          "type": "string"
        },
        "result": {
          "type": "object",
          "required": [
            "text",
            "files"
          ],
          "properties": {
            "text": {
              "type": [
                "string",
                "null"
              ]
            },
            "data": {
              "description": "Parsed JSON when the textual result is JSON."
            },
            "name": {
              "type": "string"
            },
            "mime": {
              "type": "string"
            },
            "files": {
              "type": "array",
              "items": {
                "type": "object",
                "required": [
                  "name",
                  "mime",
                  "bytes",
                  "base64"
                ],
                "properties": {
                  "name": {
                    "type": "string"
                  },
                  "mime": {
                    "type": "string"
                  },
                  "bytes": {
                    "type": "integer"
                  },
                  "base64": {
                    "type": "string",
                    "contentEncoding": "base64"
                  }
                }
              }
            }
          }
        }
      }
    },
    "processing": "Server; same transformation code as browser tools",
    "limits": {
      "request_bytes": 8388608,
      "input_characters": 1000000,
      "files_bytes": 5242880,
      "max_files": 0,
      "timeout_seconds": 12
    }
  },
  "agent_processing": "Server; same transformation code as browser tools",
  "browser_agent": {
    "supported": true,
    "name": "run_current_tool",
    "discovery": "WebMCP on the human page in a supporting browser",
    "verification": "See audit/API-AUDIT.md; availability is not verification"
  }
}

Page guide