Robots.txt & AI Crawler Checker
dev.workers.quietworklabs.robots-check/robots-check
MCP Bill of Materials
A machine-readable inventory of what this server exposes and depends on: tools with their effects, transports, packages, findings and provenance (schema version 1.0). Third-party text in it is escaped. Also available from the REST API at /api/v1/servers/2c758a8a-c3d4-422b-9dc0-c5511d29c0f2/mcbom.
MCP Bill of Materials
root{21}
schema_version: "1.0"
bom_format: "squatch-mcbom"
generator{3}
name: "squatch-mcp"
classifier_version: "rules_v1@2026-10-11.1"
unicode_version: "18.0.0"
server{7}
name: "dev.workers.quietworklabs.robots-check/robots-check"
title: "Robots.txt & AI Crawler Checker"
description: "robots.txt tester (RFC 9309), AI crawler access (GPTBot, ClaudeBot...) and sitemap check."
version: "1.0.0"
homepage_url: "https://robots-check.quietworklabs.workers.dev/"
source: "official_registry"
upstream_status: "active"
publisher{4}
namespace: "dev.workers.quietworklabs.robots-check"
display_name: "dev.workers.quietworklabs.robots-check"
type: "registry_namespace"
status: "unclaimed"
organization: null
repository: null
packages[0]
transports[1]
0{5}
kind: "remote"
type: "streamable-http"
endpoint_url: "https://robots-check.quietworklabs.workers.dev/mcp"
endpoint_host: "robots-check.quietworklabs.workers.dev"
auth_type: "none"
auth{3}
type: "none"
authorization_servers[0]
scopes[0]
protocol{4}
era: "legacy"
negotiated_version: "2025-11-25"
supported_versions[0]
transport_type: "streamable-http"
tools[3]
0{9}
name: "ai_crawler_access"
title: "Which AI crawlers a site allows or blocks"
effect: "READ"
effect_confidence: 0.32
capabilities[1]
0{2}
capability: "EXTERNAL_COMMUNICATION"
confidence: 0.76
risk_severity: "INFO"
fingerprint: "5833e227300b273e87927258c63ae620d0f8a7eff5be7f92e54963553e0f93cd"
input_schema_fingerprint: "b1d595108df4a93bf6a0a40e93015e4d58b2e9e5ad47597a0faf8412afbb9a8e"
annotations{5}
title: null
readOnlyHint: true
destructiveHint: null
idempotentHint: null
openWorldHint: true
1{9}
name: "robots_check"
title: "May this user-agent crawl this URL? (RFC 9309)"
effect: "READ"
effect_confidence: 0.777
capabilities[2]
0{2}
capability: "EXTERNAL_COMMUNICATION"
confidence: 0.718
1{2}
capability: "PERSONAL_DATA"
confidence: 0.746
risk_severity: "LOW"
fingerprint: "ab2142b4c89a3464db1556b99c572e064468fff0d48b05bf7041cc098b9ee930"
input_schema_fingerprint: "4f8e73d2f99a650a0b599fbe9f473a5ed7cdd2dafe41c1da1880282a902dd2f0"
annotations{5}
title: null
readOnlyHint: true
destructiveHint: null
idempotentHint: null
openWorldHint: true
2{9}
name: "sitemap_check"
title: "Find and validate an XML sitemap"
effect: "READ"
effect_confidence: 0.866
capabilities[1]
0{2}
capability: "EXTERNAL_COMMUNICATION"
confidence: 0.831
risk_severity: "INFO"
fingerprint: "376d54c4379e8d105949f27be860d6ae34578a02781e19a7b225dac7fcb717bd"
input_schema_fingerprint: "faf88624f7c346b219b4c8dc1ce6c3cc9fd44be2f2f09c5ad51c0ae9ecfab442"
annotations{5}
title: null
readOnlyHint: true
destructiveHint: null
idempotentHint: null
openWorldHint: true
resources[0]
prompts[0]
extensions[0]
data{6}
privacy_completeness: "not_provided"
data_categories[0]
ai_training: null
retention_policy: null
processing_regions[0]
subprocessors[0]
external_integrations[1]
0{3}
host: "robots-check.quietworklabs.workers.dev"
registrable_domain: "quietworklabs.workers.dev"
source: "transport"
findings[1]
0{6}
check_id: "SQ-REPO-MISSING"
category: "provenance"
severity: "LOW"
title: "No source repository linked"
subject_type: "server"
subject_name: null
trust{9}
registry_namespace: "verified_upstream"
publisher_identity: "unverified"
organization: "unverified"
domain: "unverified"
repository: "not_provided"
source_provenance: "unknown"
security_scan: "no_critical_findings"
privacy_transparency: "not_provided"
monitoring: "active"
provenance{4}
registry_name: "dev.workers.quietworklabs.robots-check/robots-check"
registry_content_sha256: "0b314c4e3f224ea56c843a911196a210d3bca2483a4d82fec0efaed081617eaf"
scan_snapshot_fingerprint: "8b13e14ae5e4a8e0e2720e2fc12b44fc7871e2fb11a24d3f6833fa0f4ab40fa8"
scanner_version: "1.0.0"
timestamps{4}
generated_at: "2026-11-15T04:53:46.708Z"
observed_at: "2026-11-12T10:46:34.538+00:00"
registry_published_at: "2026-10-18T03:32:52.238228+00:00"
registry_updated_at: null