AI serving

Get AI serving deployment metrics

Returns latency quantile, capacity and throughput time-series per model for one serving deployment, sourced from the metrics store. Each series states its own unit because gateways disagree: vLLM publishes seconds and APISIX milliseconds, and a histogram cannot be rescaled after recording. Signals the deployment’s gateway does not publish are named in missing_signals rather than returned as zero. A range of 24h or less is answered from the per-replica series; a longer range is answered from the deployment-grain series, which carries no replica identity, so a replica filter beyond 24h is rejected. The response also states how the deployment’s pods map to accelerators, and never divides a card’s figure between pods. cost_per_mtok is the accelerator spend behind a million output tokens, at each point of the window; it is returned only where the deployment’s accelerator rate is known and the deployment is serving output tokens, and is named in missing_signals otherwise. gateway_connection says why a deployment has no series, since the three causes need three different actions: not_connected when no model server is connected for this namespace, selector_mismatch when one is but this deployment’s pods do not carry the labels it is read by, naming that selector, and connected when the deployment is being read and any silence is the gateway’s own. missing_signal_notes explains a missing signal the connected server only started publishing in a later release, so a request-stage breakdown an older build never emits reads as a version gap rather than as a broken scrape.

GET

api

tenant

deployments

{id}

metrics

Get AI serving deployment metrics

curl --request GET \
  --url https://api.costgraph.ai/api/v1/tenant/ai/deployments/{id}/metrics \
  --header 'Authorization: <api-key>' \
  --header 'X-CostGraph-Tenant-ID: <x-costgraph-tenant-id>'

import requests

url = "https://api.costgraph.ai/api/v1/tenant/ai/deployments/{id}/metrics"

headers = {
    "X-CostGraph-Tenant-ID": "<x-costgraph-tenant-id>",
    "Authorization": "<api-key>"
}

response = requests.get(url, headers=headers)

print(response.text)

const options = {
  method: 'GET',
  headers: {'X-CostGraph-Tenant-ID': '<x-costgraph-tenant-id>', Authorization: '<api-key>'}
};

fetch('https://api.costgraph.ai/api/v1/tenant/ai/deployments/{id}/metrics', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.costgraph.ai/api/v1/tenant/ai/deployments/{id}/metrics",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: <api-key>",
    "X-CostGraph-Tenant-ID: <x-costgraph-tenant-id>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.costgraph.ai/api/v1/tenant/ai/deployments/{id}/metrics"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("X-CostGraph-Tenant-ID", "<x-costgraph-tenant-id>")
	req.Header.Add("Authorization", "<api-key>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.get("https://api.costgraph.ai/api/v1/tenant/ai/deployments/{id}/metrics")
  .header("X-CostGraph-Tenant-ID", "<x-costgraph-tenant-id>")
  .header("Authorization", "<api-key>")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.costgraph.ai/api/v1/tenant/ai/deployments/{id}/metrics")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["X-CostGraph-Tenant-ID"] = '<x-costgraph-tenant-id>'
request["Authorization"] = '<api-key>'

response = http.request(request)
puts response.read_body

{
  "message": "some message",
  "status": "success",
  "data": {
    "end": "<string>",
    "gateway_connection": {
      "gateway": "<string>",
      "label_selector": "<string>",
      "reason": "<string>",
      "state": "<string>"
    },
    "gpu_attribution": {
      "devices": 123,
      "pods": 123,
      "reason": "<string>",
      "state": "<string>"
    },
    "missing_signal_notes": {},
    "missing_signals": [
      "<string>"
    ],
    "rate_window_seconds": 123,
    "series": [
      {
        "kind": "<string>",
        "model": "<string>",
        "points": [
          {
            "t": 123,
            "v": 123
          }
        ],
        "quantile": 123,
        "signal": "<string>",
        "unit": "<string>",
        "usage_kind": "<string>"
      }
    ],
    "start": "<string>",
    "step_seconds": 123
  }
}

{
  "message": "some message",
  "status": "error"
}

{
  "message": "some message",
  "status": "error"
}

{
  "message": "some message",
  "status": "error"
}

{
  "message": "some message",
  "status": "error"
}

{
  "message": "some message",
  "status": "error"
}

Authorizations

Authorization

string

header

required

Enter "Bearer {token}"

Headers

X-CostGraph-Tenant-ID

string

required

Tenant ID

Path Parameters

string

required

AI serving deployment ID

Query Parameters

range

string

Time window (e.g. 1h, 6h, 24h, 7d). Default 1h

step

string

Resolution step (e.g. 30s, 1m, 5m)

signals

string

Serving signals (comma-separated): all,time_to_first_token,inter_token,end_to_end,queue,prefill,decode,request_output_tokens,kv_cache_utilization,batch_size,queue_depth,preemptions,concurrency,tokens_per_second,cost_per_mtok. Default all

quantiles

string

Quantiles between 0 and 1 (comma-separated). Default 0.5,0.95,0.99

model

string

Restrict to one model

replica

string

Restrict to one replica, as namespace/pod

Response

message

string

required

Example:

"some message"

status

string

required

Example:

"success"

data

object

Show child attributes

Get AI serving deployment metrics

curl --request GET \
  --url https://api.costgraph.ai/api/v1/tenant/ai/deployments/{id}/metrics \
  --header 'Authorization: <api-key>' \
  --header 'X-CostGraph-Tenant-ID: <x-costgraph-tenant-id>'

import requests

url = "https://api.costgraph.ai/api/v1/tenant/ai/deployments/{id}/metrics"

headers = {
    "X-CostGraph-Tenant-ID": "<x-costgraph-tenant-id>",
    "Authorization": "<api-key>"
}

response = requests.get(url, headers=headers)

print(response.text)

const options = {
  method: 'GET',
  headers: {'X-CostGraph-Tenant-ID': '<x-costgraph-tenant-id>', Authorization: '<api-key>'}
};

fetch('https://api.costgraph.ai/api/v1/tenant/ai/deployments/{id}/metrics', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.costgraph.ai/api/v1/tenant/ai/deployments/{id}/metrics",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: <api-key>",
    "X-CostGraph-Tenant-ID: <x-costgraph-tenant-id>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.costgraph.ai/api/v1/tenant/ai/deployments/{id}/metrics"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("X-CostGraph-Tenant-ID", "<x-costgraph-tenant-id>")
	req.Header.Add("Authorization", "<api-key>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.get("https://api.costgraph.ai/api/v1/tenant/ai/deployments/{id}/metrics")
  .header("X-CostGraph-Tenant-ID", "<x-costgraph-tenant-id>")
  .header("Authorization", "<api-key>")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.costgraph.ai/api/v1/tenant/ai/deployments/{id}/metrics")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["X-CostGraph-Tenant-ID"] = '<x-costgraph-tenant-id>'
request["Authorization"] = '<api-key>'

response = http.request(request)
puts response.read_body

{
  "message": "some message",
  "status": "success",
  "data": {
    "end": "<string>",
    "gateway_connection": {
      "gateway": "<string>",
      "label_selector": "<string>",
      "reason": "<string>",
      "state": "<string>"
    },
    "gpu_attribution": {
      "devices": 123,
      "pods": 123,
      "reason": "<string>",
      "state": "<string>"
    },
    "missing_signal_notes": {},
    "missing_signals": [
      "<string>"
    ],
    "rate_window_seconds": 123,
    "series": [
      {
        "kind": "<string>",
        "model": "<string>",
        "points": [
          {
            "t": 123,
            "v": 123
          }
        ],
        "quantile": 123,
        "signal": "<string>",
        "unit": "<string>",
        "usage_kind": "<string>"
      }
    ],
    "start": "<string>",
    "step_seconds": 123
  }
}

{
  "message": "some message",
  "status": "error"
}

{
  "message": "some message",
  "status": "error"
}

{
  "message": "some message",
  "status": "error"
}

{
  "message": "some message",
  "status": "error"
}

{
  "message": "some message",
  "status": "error"
}

Getting started

Endpoints

Get AI serving deployment metrics

Authorizations

Headers

Path Parameters

Query Parameters

Response