简介
数据类型LLVMReq定义在“${install_path}/latest/runtime/include/ge/llm_engine_types.h”。其中${install_path}为runtime安装路径,root用户的默认路径是“/usr/local/Ascend”。具体信息如下。
class LLMReq { public: LLMReq() = default; ~LLMReq() = default; //设置请求的id,不同请求的id不能重复 void SetReqId(const uint64_t req_id) { req_id_ = req_id; } //获取请求id uint64_t GetReqId() const { return req_id_; } //设置全量kv的长度,当前暂没用到 void SetPromptLength(const uint64_t prompt_length) { prompt_length_ = prompt_length; } //获取kv长度 uint64_t GetPromptLength() const { return prompt_length_; } //设置请求所在的全量集群id void SetPromptClusterId(const uint64_t prompt_cluster_id) { prompt_cluster_id_ = prompt_cluster_id; } //获取请求所在的全量集群id uint64_t GetPromptClusterId() const { return prompt_cluster_id_; } //设置请求所在的增量集群id void SetDecoderClusterId(const uint64_t decoder_cluster_id) { decoder_cluster_id_ = decoder_cluster_id; } //获取请求所在的增量集群id uint64_t GetDecoderClusterId() const { return decoder_cluster_id_; } //预加载公共前缀 void SetPrefixId(const uint64_t prefix_id) { prefix_id_ = prefix_id; } //释放公共前缀 uint64_t GetPrefixId() const { return prefix_id_; } void SetSequenceLen(const uint64_t sequence_length) { sequence_length_ = sequence_length; } uint64_t GetSequenceLen() const { return sequence_length_; } private: uint64_t req_id_{kInvalidReqId}; // 请求prompt的句子长度,做完padding的值, 用于申请prompt的KVCache uint64_t prompt_length_{0UL}; uint64_t prompt_cluster_id_{0UL}; uint64_t decoder_cluster_id_{0UL}; uint64_t prefix_id_{kInvalidPrefixId}; uint64_t sequence_length_{0UL}; int8_t reserved_[120]; };
父主题: LLMReq