Debian Code Search

{
  "service_type": "rest_api",
  "base_url": "https://codesearch.debian.net/api/v1",
  "auth_method": "api_key_header",
  "auth_config": {"header": "x-dcs-apikey", "prefix": null},
  "endpoints": [
    {
      "path": "/search",
      "method": "GET",
      "purpose": "Searches through source code in the full Debian Code Search corpus, blocking until all results are available; results ordered by ranking",
      "params": {
        "query": {"type": "string", "required": true},
        "match_mode": {"type": "string", "required": false}
      },
      "response_format": "json",
      "is_primary": true
    },
    {
      "path": "/searchperpackage",
      "method": "GET",
      "purpose": "Like /search, but aggregates results per package",
      "params": {
        "query": {"type": "string", "required": true},
        "match_mode": {"type": "string", "required": false}
      },
      "response_format": "json",
      "is_primary": false
    }
  ],
  "pricing_model": {
    "type": "free_tier",
    "details": {},
    "free_tier": {"requests": null, "period": null},
    "paid_tiers": []
  },
  "rate_limits": {"rpm": null, "tpm": null, "daily": null, "concurrent": null},
  "capabilities": ["code_search", "regex_search", "literal_search", "per_package_aggregation", "debian_source_code"],
  "agent_readiness": {
    "supports_x402": false,
    "supports_streaming": false,
    "has_sandbox": false,
    "sdks": [],
    "agent_auth_methods": ["api_key", "email_signup"]
  }
}

```json
{
  "tests": [
    {
      "name": "auth_missing_header",
      "endpoint": "/search",
      "method": "GET",
      "headers": {},
      "payload": null,
      "query_params": {"query": "test", "match_mode": "literal"},
      "expected_status": 403,
      "intent": "auth",
      "expected_behavior": "Request without x-dcs-apikey header is rejected with 403 as documented",
      "metrics": ["status_code", "latency"],
      "validation": {"type": "rejection", "accept_status": [401, 403]}
    },
    {
      "name": "auth_invalid_key",
      "endpoint": "/search",
      "method": "GET",
      "headers": {"x-dcs-apikey": "invalid-key-00000000"},
      "payload": null,
      "query_params": {"query": "test", "match_mode": "literal"},
      "expected_status": 403,
      "intent": "auth",
      "expected_behavior": "Invalid API key is rejected with 403",
      "metrics": ["status_code", "latency"],
      "validation": {"type": "rejection", "accept_status": [401, 403]}
    },
    {
      "name": "search_literal_simple",
      "endpoint": "/search",
      "method": "GET",
      "headers": {"x-dcs-apikey": "${API_KEY}"},
      "payload": null,
      "query_params": {"query": "who knows", "match_mode": "literal"},
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns JSON array of SearchResult objects ordered by ranking",
      "metrics": ["latency", "status_code", "result_count"],
      "validation": {"field": "$", "type": "array"}
    },
    {
      "name": "search_regex_pattern",
      "endpoint": "/search",
      "method": "GET",
      "headers": {"x-dcs-apikey": "${API_KEY}"},
      "payload": null,
      "query_params": {"query": "load_font\\(", "match_mode": "regexp"},
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns JSON array of SearchResult objects matching RE2 regex",
      "metrics": ["latency", "status_code", "result_count"],
      "validation": {"field": "$", "type": "array"}
    },
    {
      "name": "search_default_match_mode",
      "endpoint": "/search",
      "method": "GET",
      "headers": {"x-dcs-apikey": "${API_KEY}"},
      "payload": null,
      "query_params": {"query": "main"},
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Omitting match_mode defaults to regexp and returns array",
      "metrics": ["latency", "status_code"],
      "validation": {"field": "$", "type": "array"}
    },
    {
      "name": "search_result_shape",
      "endpoint": "/search",
      "method": "GET",
      "headers": {"x-dcs-apikey": "${API_KEY}"},
      "payload": null,
      "query_params": {"query": "xcb_generate_id", "match_mode": "literal"},
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Each result contains required fields package, path, line, context",
      "metrics": ["latency", "accuracy", "status_code"],
      "validation": {"field": "0", "type": "object", "require_keys": ["package", "path", "line", "context"]}
    },
    {
      "name": "search_per_package_simple",
      "endpoint": "/searchperpackage",
      "method": "GET",
      "headers": {"x-dcs-apikey": "${API_KEY}"},
      "payload": null,
      "query_params": {"query": "load_font", "match_mode": "literal"},
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Returns array of PackageSearchResult with package + aggregated results (up to 2 per package)",
      "metrics": ["latency", "status_code", "result_count"],
      "validation": {"field": "$", "type": "array"}
    },
    {
      "name": "searchperpackage_result_shape",
      "endpoint": "/searchperpackage",
      "method": "GET",
      "headers": {"x-dcs-apikey": "${API_KEY}"},
      "payload": null,
      "query_params": {"query": "int main", "match_mode": "literal"},
      "expected_status": 200,
      "intent": "happy_path",
      "expected_behavior": "Each entry has package and results array",
      "metrics": ["latency", "accuracy", "status_code"],
      "validation": {"field": "0", "type": "object", "require_keys": ["package", "results"]}
    },
    {
      "name": "search_missing_query_param",
      "endpoint": "/search",
      "method": "GET",
      "headers": {"x-dcs-apikey": "${API_KEY}"},
      "payload": null,
      "query_params": {},
      "expected_status": 400,
      "intent": "validation",
      "expected_behavior": "Missing required query param is rejected (4xx) rather than returning results",
      "metrics": ["status_code"],
      "validation": {"type": "rejection", "accept_status": [400, 422]}
    },
    {
      "name": "search_invalid_match_mode",
      "endpoint": "/search",
      "method": "GET",
      "headers": {"x-dcs-apikey": "${API_KEY}"},
      "payload": null,
      "query_params": {"query": "test", "match_mode": "bogus"},
      "expected_status": 400,
      "intent": "validation",
      "expected_behavior": "Invalid enum value for match_mode is reje

0/13 tests passed

{"model": "deepseek", "data": {"overall": 18, "dimensions": {"token_efficiency": 7.0, "first_try_success": 2.0, "response_parseability": 3.0, "error_clarity": 2.0, "doc_quality": 4.0, "auth_simplicity": 3.0, "latency": 10.0, "consistency": 4.0}, "pricing_normalized": {"cost_per_1k_requests": 0, "cost_per_1m_input_tokens": null, "cost_per_1m_output_tokens": null, "free_tier_requests": null, "estimated_monthly_cost_light": 0, "estimated_monthly_cost_heavy": 0}, "issues": [{"severity": "critical", "detail": "All authenticated requests fail with 403 'invalid x-dcs-apikey header' \u2014 the benchmark had no valid API key, so no happy-path/validation behavior could be verified. Every search test returned 403 or 429 instead of the expected 200/400.", "endpoint": "/search"}, {"severity": "critical", "detail": "No self-registration or key-provisioning endpoint was found, so an autonomous agent cannot obtain the x-dcs-apikey credential required for any request.", "endpoint": "/search"}, {"severity": "high", "detail": "Repeated 429 responses with retry-after suggest strict rate limiting preventing agents from completing searches in a batch run.", "endpoint": "/search"}, {"severity": "medium", "detail": "Error responses are plain text ('invalid x-dcs-apikey header\\n') rather than structured JSON, with no machine-readable error code or field detail.", "endpoint": "/search"}, {"severity": "medium", "detail": "Validation errors cannot be confirmed \u2014 malformed/missing query params returned 403/429 rather than the expected 400 with a structured body.", "endpoint": "/search"}], "recommendations": ["Provision a valid x-dcs-apikey to the benchmark harness so happy-path and validation behavior can actually be tested; results are unusable without a working credential.", "Document an agent-friendly self-registration flow (e.g. email-based or Sign in with Notlogin) for obtaining an API key without human-in-the-loop.", "Return structured JSON error bodies with an error code and actionable message instead of plain text.", "Publish an OpenAPI spec or complete docs describing query syntax, match_mode values, pagination, rate limits, and response schema.", "Provide a free tier with published rate limits and generous per-minute quota so agents don't hit 429 on sequential searches.", "Expose a health/status endpoint so agents can verify connectivity before issuing queries."]}}