Composio

{
  "service_type": "rest_api",
  "base_url": "https://backend.composio.dev",
  "auth_method": "api_key_header",
  "auth_config": {"header": "x-api-key", "prefix": ""},
  "endpoints": [
    {
      "path": "/api/v3.1/tools/execute",
      "method": "POST",
      "purpose": "Execute a tool with given parameters",
      "params": {
        "tool_slug": {"type": "string", "required": true},
        "arguments": {"type": "object", "required": false},
        "connected_account_id": {"type": "string", "required": false},
        "user_id": {"type": "string", "required": false}
      },
      "response_format": "json",
      "is_primary": true
    },
    {
      "path": "/api/v3.1/toolkits",
      "method": "GET",
      "purpose": "List available toolkits",
      "params": {},
      "response_format": "json",
      "is_primary": true
    },
    {
      "path": "/api/v3.1/tools",
      "method": "GET",
      "purpose": "List available tools",
      "params": {
        "toolkit_slug": {"type": "string", "required": false},
        "search": {"type": "string", "required": false}
      },
      "response_format": "json",
      "is_primary": true
    },
    {
      "path": "/api/v3.1/connected_accounts",
      "method": "GET",
      "purpose": "List connected accounts",
      "params": {
        "user_id": {"type": "string", "required": false}
      },
      "response_format": "json",
      "is_primary": false
    },
    {
      "path": "/api/v3.1/connected_accounts",
      "method": "POST",
      "purpose": "Initiate a connection for a user to a toolkit",
      "params": {
        "toolkit_slug": {"type": "string", "required": true},
        "user_id": {"type": "string", "required": true},
        "auth_config_id": {"type": "string", "required": false}
      },
      "response_format": "json",
      "is_primary": false
    },
    {
      "path": "/api/v3.1/auth_configs",
      "method": "GET",
      "purpose": "List auth configs",
      "params": {},
      "response_format": "json",
      "is_primary": false
    },
    {
      "path": "/api/v3.1/triggers",
      "method": "GET",
      "purpose": "List triggers",
      "params": {},
      "response_format": "json",
      "is_primary": false
    },
    {
      "path": "/api/v3.1/auth/me",
      "method": "GET",
      "purpose": "Retrieve authenticated account info",
      "params": {},
      "response_format": "json",
      "is_primary": false
    },
    {
      "path": "/api/v3.1/logs",
      "method": "GET",
      "purpose": "Retrieve execution logs",
      "params": {},
      "response_format": "json",
      "is_primary": false
    },
    {
      "path": "/api/v3.1/webhooks/subscriptions",
      "method": "GET",
      "purpose": "List webhook delivery subscriptions",
      "params": {},
      "response_format": "json",
      "is_primary": false
    }
  ],
  "pricing_model": {
    "type": "subscription",
    "details": {},
    "free_tier": null,
    "paid_tiers": []
  },
  "rate_limits": {"rpm": null, "tpm": null, "daily": null, "concurrent": null},
  "capabilities": [
    "tool_execution",
    "tool_discovery",
    "toolkits",
    "authentication_management",
    "connected_accounts",
    "triggers",
    "webhooks",
    "mcp_server_management",
    "logging",
    "file_management",
    "team_management",
    "payments",
    "billing"
  ],
  "agent_readiness": {
    "supports_x402": false,
    "supports_streaming": false,
    "has_sandbox": false,
    "sdks": ["python", "typescript"],
    "agent_auth_methods": ["api_key"]
  }
}

```json
{
  "tests": [
    {
      "name": "list_toolkits",
      "endpoint": "/api/v3.1/toolkits",
      "method": "GET",
      "headers": {"x-api-key": "{{API_KEY}}"},
      "payload": {},
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns paginated list of toolkits with items array",
      "metrics": ["latency", "status_code", "accuracy"],
      "validation": {"field": "items", "type": "array", "min_length": 1}
    },
    {
      "name": "list_tools",
      "endpoint": "/api/v3.1/tools",
      "method": "GET",
      "headers": {"x-api-key": "{{API_KEY}}"},
      "payload": {},
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns paginated list of tools with items array and total_items",
      "metrics": ["latency", "status_code", "accuracy"],
      "validation": {"field": "items", "type": "array", "min_length": 1}
    },
    {
      "name": "list_tools_filtered_by_toolkit",
      "endpoint": "/api/v3.1/tools?toolkit_slug=github",
      "method": "GET",
      "headers": {"x-api-key": "{{API_KEY}}"},
      "payload": {},
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns tools filtered to github toolkit",
      "metrics": ["latency", "status_code", "accuracy"],
      "validation": {"field": "items", "type": "array", "min_length": 1}
    },
    {
      "name": "get_tool_by_slug",
      "endpoint": "/api/v3.1/tools/GITHUB_CREATE_AN_ISSUE",
      "method": "GET",
      "headers": {"x-api-key": "{{API_KEY}}"},
      "payload": {},
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns tool metadata including slug, input_parameters, toolkit",
      "metrics": ["latency", "status_code", "accuracy"],
      "validation": {"field": "slug", "type": "string", "min_length": 1}
    },
    {
      "name": "list_connected_accounts",
      "endpoint": "/api/v3.1/connected_accounts",
      "method": "GET",
      "headers": {"x-api-key": "{{API_KEY}}"},
      "payload": {},
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns paginated list of connected accounts (may be empty)",
      "metrics": ["latency", "status_code"],
      "validation": {"field": "items", "type": "array"}
    },
    {
      "name": "list_auth_configs",
      "endpoint": "/api/v3.1/auth_configs",
      "method": "GET",
      "headers": {"x-api-key": "{{API_KEY}}"},
      "payload": {},
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns paginated list of auth configs",
      "metrics": ["latency", "status_code"],
      "validation": {"field": "items", "type": "array"}
    },
    {
      "name": "list_webhook_subscriptions",
      "endpoint": "/api/v3.1/webhook_subscriptions",
      "method": "GET",
      "headers": {"x-api-key": "{{API_KEY}}"},
      "payload": {},
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns paginated list of webhook subscriptions",
      "metrics": ["latency", "status_code"],
      "validation": {"field": "items", "type": "array"}
    },
    {
      "name": "execute_tool_happy_path",
      "endpoint": "/api/v3.1/tools/execute/GITHUB_CREATE_AN_ISSUE",
      "method": "POST",
      "headers": {"x-api-key": "{{API_KEY}}", "Content-Type": "application/json"},
      "payload": {
        "arguments": {
          "owner": "octocat",
          "repo": "Hello-World",
          "title": "Prowl benchmark test",
          "body": "Automated connectivity check"
        }
      },
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns data, error, successful fields; successful=true or error explains missing connection (402/403 also acceptable as connectivity proof)",
      "metrics": ["latency", "status_code", "accuracy"],
      "validation": {"field": "successful", "type": "boolean"}
    },
    {
      "name": "execute_tool_missing_required_slug",
      "endpoint": "/api/v3.1/tools/execute/",
      "method": "POST",
      "headers": {"x-api-key": "{{API_KEY}}", "Content-Type": "application/json"},
      "payload": {"arguments": {}},
      "expected_status": 400,
      "intent": "validation",
      "expected_behavior": "Rejects request with structured error body due to missing tool_slug path param",
      "metrics": ["status_code", "accuracy"],
      "validation": {"field": "error", "type": "object"}
    },
    {
      "name": "get_unknown_tool_slug",
      "endpoint": "/api/v3.1/tools/NONEXISTENT_TOOL_SLUG_XYZ",
      "method": "GET",
      "headers": {"x-api-key": "{{API_KEY}}"},
      "payload": {},
      "expected_status": 404,
      "intent": "validation",
      "expected_behavior": "Returns 404 with structured error for unknown tool slug",
      "metrics": ["status_code", "accuracy"],
      "validation": {"field": "error", "type": "object"}
    },
    {
      "name": "auth_missing_api_key",
    

0/12 tests passed

{"model": "deepseek", "data": {"overall": 38, "dimensions": {"token_efficiency": 6.0, "first_try_success": 2.0, "response_parseability": 9.0, "error_clarity": 10.0, "doc_quality": 5.0, "auth_simplicity": 3.0, "latency": 10.0, "consistency": 10.0}, "pricing_normalized": {"cost_per_1k_requests": null, "cost_per_1m_input_tokens": null, "cost_per_1m_output_tokens": null, "free_tier_requests": null, "estimated_monthly_cost_light": null, "estimated_monthly_cost_heavy": null}, "issues": [{"severity": "critical", "detail": "All happy-path tests returned 401 'Invalid API key: {{A**EY}}' \u2014 the benchmark embedded an unresolved API key template literal rather than a real credential. Nine tests expected 200 but got 401 solely due to this auth artifact, not an API defect.", "endpoint": "ALL"}, {"severity": "medium", "detail": "Validation test for missing required slug hit /api/v3.1/tools/execute/ (trailing slash) and received a 308 redirect instead of a 400/422 structured validation error, revealing the tool slug is a path param rather than a body field as the spec lists it. Planner/spec mismatch.", "endpoint": "/api/v3.1/tools/execute/"}, {"severity": "low", "detail": "No ONBOARD data provided; auth onboarding experience could not be evaluated. API key is required and no self-registration endpoint was observed.", "endpoint": "/api/v3.1/auth/me"}], "recommendations": ["Re-run the benchmark with a valid API key resolved from the vendor's dashboard \u2014 the {{A**EY}} placeholder indicates a config error, not an API failure.", "Vendor should reject/guide unauthenticated callers before deeper validation, or clearly document that auth is checked prior to path/param validation.", "Clarify in docs whether tool execution uses a path parameter (/tools/execute/{slug}) or request body field, to avoid the planner's trailing-slash 308.", "Provide a self-service registration endpoint for agents (email-based key issuance) to improve autonomous onboarding.", "Publish an OpenAPI spec with example responses so agents can construct calls without guessing."]}}