TGI流式推理接口

接口功能

提供流式推理处理功能。

接口格式

操作类型：POST

URL：https://{ip}:{port}/generate_stream

请求参数

使用样例

请求样例：

POST https://{ip}:{port}/generate_stream

请求消息体：

• 单模态文本模型：

  {
      "inputs": "My name is Olivier and I",
      "parameters": {
          "decoder_input_details": false,
          "details": true,
          "do_sample": true,
          "max_new_tokens": 20,
          "repetition_penalty": 1.03,
          "return_full_text": false,
          "seed": null,
          "temperature": 0.5,
          "top_k": 10,
          "top_p": 0.95,
          "truncate": null,
          "typical_p": 0.5,
          "watermark": false,
          "stop": null,
          "adapter_id": "None"
      }
  }

• 多模态模型：
  

  "image_url"参数的取值请根据实际情况进行修改。

  {
      "inputs": [
          {"type": "text", "text": "My name is Olivier and I"},
          {
              "type": "image_url",
              "image_url": "/xxxx/test.png"
          }
      ],
      "parameters": {
          "decoder_input_details": false,
          "details": true,
          "do_sample": true,
          "max_new_tokens": 20,
          "repetition_penalty": 1.03,
          "return_full_text": false,
          "seed": null,
          "temperature": 0.5,
          "top_k": 10,
          "top_p": 0.95,
          "truncate": null,
          "typical_p": 0.5,
          "watermark": false,
          "stop": null,
          "adapter_id": "None"
      }
  }

响应样例：

• 响应样例1（使用sse格式返回）：

  data: {"token":{"id":13,"text":"\n","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":26626,"text":"Jan","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":300,"text":"et","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":3732,"text":" makes","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":395,"text":" $","logprob":null,"special":null},"generated_text":null,"details":null}
  
  ……
  
  data: {"token":{"id":395,"text":" $","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":29896,"text":"1","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":29896,"text":"1","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":29947,"text":"8","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":29889,"text":".","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":2,"text":"","logprob":null,"special":null},"generated_text":"\nJanet makes $104 a day at the farmers' market.\nThe number of eggs that Janet sells at the farmers' market each day is 16 - 3 - 4 = 7.\nShe makes $2 for each egg that she sells.\nThe total amount that she makes is $2 * 7 = $14.\nThe total amount that she makes is $14 + $104 = $118.\nThe total amount that she makes is $118.","details":{"prompt_tokens":74,"finish_reason":"eos_token","generated_tokens":116,"seed":8756727004129248931}}
  

• 响应样例2（配置项“fullTextEnabled”=true，使用sse格式返回）：

  data: {"token":{"id":198,"text":"\n","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":9707,"text":"\nHello","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":0,"text":"\nHello!","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":2585,"text":"\nHello! How","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":646,"text":"\nHello! How can","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":358,"text":"\nHello! How can I","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":7789,"text":"\nHello! How can I assist","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":498,"text":"\nHello! How can I assist you","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":3351,"text":"\nHello! How can I assist you today","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":30,"text":"\nHello! How can I assist you today?","logprob":null,"special":null},"generated_text":null,"details":null}
  
  data: {"token":{"id":151645,"text":"\nHello! How can I assist you today?","logprob":null,"special":null},"generated_text":"\nHello! How can I assist you today?","details":{"prompt_tokens":1,"finish_reason":"eos_token","generated_tokens":11,"seed":2296576927}}

输出说明