vart_runner_factory.hpp

Go to the documentation of this file.

/*
 * Copyright (C) 2025-2026 Advanced Micro Devices, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
#pragma once
 
#include <any>
#include <chrono>
#include <cstdint>
#include <functional>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
 
#include "vart_npu_tensor.hpp"
 
namespace vart {
 
enum class RunnerType {
  VAIML,  
};
 
enum class RoundingMode {
  UNKNOWN,                
  ROUND_TO_NEAREST_EVEN,  
  ROUND_TOWARD_ZERO,      
};
 
struct QuantParameters {
  double scale = 0.0;
  int32_t zero_point = 0;
  RoundingMode rounding_mode = RoundingMode::UNKNOWN;
};
 
enum class StatusCode {
  SUCCESS = 0,          
  FAILURE,              
  INVALID_INPUT,        
  INVALID_OUTPUT,       
  OUT_OF_MEMORY,        
  RUNTIME_ERROR,        
  JOB_PENDING,          
  INVALID_JOB_ID,       
  RESOURCE_UNAVAILABLE  
};
 
struct JobHandle {
  StatusCode status;  // Submission status of the job.
  uint32_t job_id;    // Unique identifier for the job.
 
  JobHandle(StatusCode stat, uint32_t id) : status(stat), job_id(id) {}
 
  JobHandle() : status(StatusCode::FAILURE), job_id(0) {}
};
 
class Runner {
 protected:
  explicit Runner(const std::string& model_path, const std::unordered_map<std::string, std::any>& options = {}) {}
 
 public:
  virtual ~Runner() = default;
 
  virtual const std::vector<NpuTensorInfo>& get_tensors_info(TensorDirection direction, TensorType type) const = 0;
 
  virtual const NpuTensorInfo& get_tensor_info_by_name(const std::string& tensor_name, TensorType type) const = 0;
 
  virtual const QuantParameters& get_quant_parameters(const std::string& tensor_name) const = 0;
 
  virtual size_t get_num_input_tensors() const = 0;
 
  virtual size_t get_num_output_tensors() const = 0;
 
  virtual size_t get_batch_size() const = 0;
 
  virtual StatusCode execute(const std::vector<std::vector<NpuTensor>>& inputs,
                             std::vector<std::vector<NpuTensor>>& outputs) noexcept = 0;
 
  virtual JobHandle execute_async(const std::vector<std::vector<NpuTensor>>& inputs,
                                  std::vector<std::vector<NpuTensor>>& outputs) noexcept = 0;
 
  virtual StatusCode wait(const JobHandle& job_handle, std::chrono::milliseconds timeout) noexcept = 0;
 
  using ExecuteAsyncCallback = std::function<void(const JobHandle&)>;
 
  virtual JobHandle execute_async(const std::vector<std::vector<NpuTensor>>& inputs,
                                  std::vector<std::vector<NpuTensor>>& outputs,
                                  ExecuteAsyncCallback cb) noexcept = 0;
 
  virtual NpuTensor allocate_npu_tensor(const NpuTensorInfo& info) const = 0;
 
  virtual NpuTensor allocate_sub_tensor(const NpuTensor& parent, const NpuTensorInfo& info, size_t offset) const = 0;
};
 
class RunnerFactory {
 public:
  static std::shared_ptr<Runner> create_runner(RunnerType runner_type,
                                               const std::string& model_path,
                                               const std::unordered_map<std::string, std::any>& options = {});
};
 
}  // namespace vart