Shodan

{
  "service_type": "rest_api",
  "base_url": "https://api.shodan.io",
  "auth_method": "query_param",
  "auth_config": {"header": null, "prefix": null},
  "endpoints": [
    {
      "path": "/shodan/host/{ip}",
      "method": "GET",
      "purpose": "Returns all services that have been found on the given host IP.",
      "params": {
        "key": {"type": "string", "required": true},
        "history": {"type": "boolean", "required": false},
        "minify": {"type": "boolean", "required": false},
        "ip": {"type": "string", "required": true}
      },
      "response_format": "json",
      "is_primary": true
    }
  ],
  "pricing_model": {
    "type": "tiered",
    "details": {},
    "free_tier": null,
    "paid_tiers": []
  },
  "rate_limits": {"rpm": null, "tpm": null, "daily": null, "concurrent": null},
  "capabilities": ["search", "host information"],
  "agent_readiness": {
    "supports_x402": false,
    "supports_streaming": false,
    "has_sandbox": false,
    "sdks": [],
    "agent_auth_methods": ["api_key"]
  }
}

```json
{
  "tests": [
    {
      "name": "host_information_happy_path",
      "endpoint": "/shodan/host/8.8.8.8",
      "method": "GET",
      "headers": {},
      "payload": {
        "key": "API_KEY_PLACEHOLDER"
      },
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns host information for 8.8.8.8 with ip_str, ports, and data array",
      "metrics": ["latency", "status_code", "response_size"],
      "validation": {"field": "ip_str", "type": "string", "min_length": 1}
    },
    {
      "name": "host_information_minified",
      "endpoint": "/shodan/host/8.8.8.8",
      "method": "GET",
      "headers": {},
      "payload": {
        "key": "API_KEY_PLACEHOLDER",
        "minify": "true"
      },
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns minified host info with ports list but no data banners",
      "metrics": ["latency", "status_code", "response_size"],
      "validation": {"field": "ports", "type": "array", "min_length": 1}
    },
    {
      "name": "host_information_invalid_ip",
      "endpoint": "/shodan/host/999.999.999.999",
      "method": "GET",
      "headers": {},
      "payload": {
        "key": "API_KEY_PLACEHOLDER"
      },
      "expected_status": 404,
      "intent": "validation",
      "expected_behavior": "Returns 404 with error message for invalid IP address",
      "metrics": ["status_code", "latency"],
      "validation": {"field": "error", "type": "string", "min_length": 1}
    },
    {
      "name": "host_information_no_key",
      "endpoint": "/shodan/host/8.8.8.8",
      "method": "GET",
      "headers": {},
      "payload": {},
      "expected_status": 401,
      "intent": "auth",
      "expected_behavior": "Returns 401 or error when API key is missing",
      "metrics": ["status_code", "latency"],
      "validation": {"field": "error", "type": "string", "min_length": 1}
    },
    {
      "name": "host_information_invalid_key",
      "endpoint": "/shodan/host/8.8.8.8",
      "method": "GET",
      "headers": {},
      "payload": {
        "key": "invalid_key_placeholder"
      },
      "expected_status": 401,
      "intent": "auth",
      "expected_behavior": "Returns 401 or error when API key is invalid",
      "metrics": ["status_code", "latency"],
      "validation": {"field": "error", "type": "string", "min_length": 1}
    },
    {
      "name": "api_info_plan_check",
      "endpoint": "/api-info",
      "method": "GET",
      "headers": {},
      "payload": {
        "key": "API_KEY_PLACEHOLDER"
      },
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns API plan information including query_credits and scan_credits",
      "metrics": ["latency", "status_code"],
      "validation": {"field": "plan", "type": "string", "min_length": 1}
    },
    {
      "name": "list_all_ports",
      "endpoint": "/shodan/ports",
      "method": "GET",
      "headers": {},
      "payload": {
        "key": "API_KEY_PLACEHOLDER"
      },
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns list of port numbers as array",
      "metrics": ["latency", "status_code"],
      "validation": {"field": "data", "type": "array", "min_length": 1}
    },
    {
      "name": "list_protocols",
      "endpoint": "/shodan/protocols",
      "method": "GET",
      "headers": {},
      "payload": {
        "key": "API_KEY_PLACEHOLDER"
      },
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns object of protocol names to descriptions",
      "metrics": ["latency", "status_code"],
      "validation": {"field": "data", "type": "object", "min_length": 1}
    },
    {
      "name": "dns_resolve_hostnames",
      "endpoint": "/dns/resolve",
      "method": "GET",
      "headers": {},
      "payload": {
        "key": "API_KEY_PLACEHOLDER",
        "hostnames": "google.com,facebook.com"
      },
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns DNS resolution mapping for provided hostnames",
      "metrics": ["latency", "status_code"],
      "validation": {"field": "google.com", "type": "string", "min_length": 1}
    },
    {
      "name": "dns_reverse_lookup",
      "endpoint": "/dns/reverse",
      "method": "GET",
      "headers": {},
      "payload": {
        "key": "API_KEY_PLACEHOLDER",
        "ips": "8.8.8.8,1.1.1.1"
      },
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns reverse DNS hostnames for provided IPs",
      "metrics": ["latency", "status_code"],
      "validation": {"field": "8.8.8.8", "type": "array", "min_length": 1}
    },
    {
      "name": "search_facets_list",
      "endpoint": "/shodan/host/search/facets",
      "method": "GET",
      "headers": {},
      "payload": {
        "key": "API_KEY_PLACEHOLDER"
      },
      "expected_status": 200,
      "intent": "happy

10/20 tests passed

{"model": "deepseek", "data": {"overall": 35, "dimensions": {"token_efficiency": 6.0, "first_try_success": 4.0, "response_parseability": 7.0, "error_clarity": 2.0, "doc_quality": 5.0, "auth_simplicity": 3.0, "latency": 9.0, "consistency": 8.0}, "pricing_normalized": {"cost_per_1k_requests": null, "cost_per_1m_input_tokens": null, "cost_per_1m_output_tokens": null, "free_tier_requests": null, "estimated_monthly_cost_light": null, "estimated_monthly_cost_heavy": null}, "issues": [{"severity": "high", "detail": "Authentication is inconsistent and non-functional: invalid/no API keys still return 200 with data (tests 'host_information_no_key' and 'host_information_invalid_key' expected 401 but got 200). Some endpoints return 401 with plain-text HTML-like error messages, while others return 200 without any key. This makes it impossible for an agent to reliably authenticate.", "endpoint": "/shodan/host/8.8.8.8"}, {"severity": "high", "detail": "Error responses for 401s are plain-text HTML ('401 Unauthorized...'), not structured JSON. Agents cannot programmatically parse the error reason or remediation.", "endpoint": "/shodan/host/999.999.999.999"}, {"severity": "medium", "detail": "Response content-type is inconsistent: host lookups return JSON (application/json), but 401 errors and /api-info return text/plain. The /tools/myip endpoint returns a raw IP string, not a JSON object.", "endpoint": "/tools/myip"}, {"severity": "medium", "detail": "Validation test for invalid IP (999.999.999.999) returned 401 instead of the expected 404, and the error body is unhelpful. The API does not provide field-level validation errors.", "endpoint": "/shodan/host/{ip}"}, {"severity": "medium", "detail": "Multiple endpoints (/shodan/protocols, /dns/resolve, /dns/reverse, /shodan/host/search, /shodan/host/count, /shodan/host/search/tokens, /account/profile, /notifier/provider) return 401 Unauthorized when called without authentication, which suggests a key is required but the benchmark found no way to obtain one. This creates a broken auth loop for agents.", "endpoint": "/shodan/protocols"}], "recommendations": ["Enforce consistent authentication: if an API key is required, reject requests without a valid key with 401 for ALL endpoints, and require it for all. Currently some endpoints work without keys, creating unpredictable behavior.", "Return structured JSON error bodies (e.g., {\"error\": {\"code\": 401, \"message\": \"Invalid API key\", \"details\": \"...\"}}) for all 4xx/5xx responses instead of plain-text HTML.", "Provide a clear onboarding path: document how to create an API key, and if possible add a self-service registration token endpoint or test key for evaluation.", "Standardize content-types across all endpoints. Return a consistent JSON envelope (e.g., {\"data\": [...]}) even for single values like IP addresses.", "Add OpenAPI/Swagger documentation with example requests and responses, including error scenarios, to help agents construct correct calls on the first attempt."]}}