qc_compiler_spec.fbs

//============================================================================
//
// Copyright (c) Qualcomm Innovation Center, Inc.
// All rights reserved
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
//
//============================================================================

namespace qnn_delegate;

/// Defines the HTP hardware architecture available for HTP backend.
enum HtpArch: int {
  NONE = 0,
  V68 = 68,
  V69 = 69,
  V73 = 73,
  V75 = 75,
  V79 = 79,
  V81 = 81,
}

table HtpInfo {
  /// Represent the HTP hardware architecture
  htp_arch:HtpArch;

  /// Represent the vtcm size to use for graphs. VTCM size is provided in MB.
  vtcm_size_in_mb:uint;
}

/// Defines the LPAI hardware architecture available for LPAI backend.
enum LpaiHardwareVersion: int {
  NONE = 0,
  V6 = 6,
  V7 = 7,
}

table LpaiInfo {
  /// Represent the LPAI hardware version
  lpai_hardware_version:LpaiHardwareVersion;
}

/// You could refer to Qualcomm AI Engine Direct SDK
/// to get SoC Model in supported snapdragon devices
enum QcomChipset: int {
  UNKNOWN_SM = 0,
  SA8295 = 39,
  SA8797 = 72,
  SC8380XP = 60,
  SC8480XP = 88,
  SM7675 = 70,
  SM8350 = 30,
  SM8450 = 36,
  SM8475 = 42,
  SM8550 = 43,
  SM8635 = 68,
  SM8650 = 57,
  SM8750 = 69,
  SM8850 = 87,
  SSG2115P = 46,
  SSG2125P = 58,
  SXR1230P = 45,
  SXR2230P = 53,
  SXR2330P = 75,
  QCS9100 = 77,
  SAR2230P = 95,
  SA8255 = 52,
  SW6100 = 96,
  QCM6490 = 93,
  SM8845 = 97,
  SA8540 = 62,
}

/// Indicate the information of the specified SoC.
table SocInfo {
  /// Identifies SOC model.
  soc_model:QcomChipset;

  /// Identifies the htp information of the specified SoC.
  htp_info:HtpInfo;

  /// Identifies the lpai information of the specified SoC.
  lpai_info:LpaiInfo;
}

/// Defines performance modes available for GPU backend.
enum QnnExecuTorchGpuPerformanceMode: int {
  kGpuPerfHintHigh = 0,
  kGpuPerfHintNormal,
  kGpuPerfHintLow,
}

/// Defines the optimization levels of the graph tensors that are not input nor
/// output tensors. This enum controls the trade-off between performance and
/// accuracy.
enum QnnExecuTorchGpuPrecision: int {
  kGpuPrecisionFp32 = 0,
  kGpuPrecisionFp16,
  kGpuPrecisionHybrid,
  kGpuPrecisionUserProvided,
}

/// Specifies the backend options for the GPU backend.
table QnnExecuTorchGpuBackendOptions {
  /// kGpuPerfHintHigh - best inference latency at the expense of power consumption.
  /// kGpuPerfHintNormal - balanced performance dependent upon power management.
  /// kGpuPerfHintLow - lowest power consumption at the expense of inference latency.
  performance_mode:QnnExecuTorchGpuPerformanceMode;

  /// kGpuPrecisionFp32 - best accuracy at the expense of performance.
  /// kGpuPrecisionFp16 - best performance at the expense of accuracy.
  /// kGpuPrecisionHybrid - good trade-off between performance and accuracy.
  /// kGpuPrecisionUserProvided - backend will not optimize NATIVE tensor data types.
  precision:QnnExecuTorchGpuPrecision;

  /// Backend will share NATIVE tensor memory based upon analysis of the network topology.
  use_memory_optimizations:bool;

  /// Backend will fuse compatible operations into one operation to improve performance.
  use_node_optimizations:bool;

  /// Backend will use queue recording to improve performance.
  use_queue_recording:bool;

  /// When multiple graphs appear inside the same context,
  /// weights could be reused across all graphs.
  use_weight_sharing:bool;
}

/// Defines performance modes available for HTP backend.
enum QnnExecuTorchHtpPerformanceMode: int {
  kHtpDefault = 0,
  kHtpSustainedHighPerformance,
  kHtpBurst,
  kHtpHighPerformance,
  kHtpPowerSaver,
  kHtpLowPowerSaver,
  kHtpHighPowerSaver,
  kHtpLowBalanced,
  kHtpBalanced,
}

/// Defines the optimization levels of the graph tensors that are not input nor
/// output tensors. This enum controls the trade-off between performance and
/// accuracy.
enum QnnExecuTorchHtpPrecision: int {
  kHtpQuantized = 0,
  kHtpFp16,
}

/// The QNN backend used to delegate the model's nodes. Each backend has
/// its own set of supported ops and tensor types.
enum QnnExecuTorchBackendType: int {
  kUndefinedBackend = 0,
  kGpuBackend,
  kHtpBackend,
  kDspBackend,
  kLpaiBackend,
}

/// Defines pd sessions available for HTP backend.
enum QnnExecuTorchHtpPdSession: int {
  kHtpUnsignedPd = 0,
  kHtpSignedPd,
}

/// Specifies the backend options for the HTP backend.
table QnnExecuTorchHtpBackendOptions {
  /// Max spill-fill buffer across contexts.
  max_sf_buf_size:int;

  /// The default performance mode sets no configurations on the HTP.
  performance_mode:QnnExecuTorchHtpPerformanceMode;

  /// The default precision mode supports quantized networks. Other precision
  /// modes may only be supported on certain SoCs.
  precision:QnnExecuTorchHtpPrecision;

  /// Signed or unsigned HTP PD session. The default PD session is unsigned.
  pd_session:QnnExecuTorchHtpPdSession;

  /// With using conv hmx with short depths, we might have better performance,
  /// but convolution that have short depth and/or weights that are not
  /// symmetric could exhibit inaccurate results.
  use_conv_hmx:bool;

  /// Deep Learning Bandwidth Compression allows inputs to be
  /// compressed, such that the processing bandwidth can be lowered.
  use_dlbc:bool;

  /// With using fold relu, we might have better performance, this optimization
  /// is correct when quantization ranges for convolution are equal or subset of
  /// the Relu operation.
  use_fold_relu:bool;

  /// When multiple contexts are generated inside the same
  /// pte, it is possible to reserve a single spill-fill allocation that
  /// could be re-used across all the splits.
  use_multi_contexts:bool;

  /// When multiple graphs appear inside the same context,
  /// weights could be reused across all graphs.
  use_weight_sharing:bool;

  /// Allows user to enable the usage of the System Level Cache Allocator for a given graph. 
  /// It will help the by reducing overall bandwith on the use case.
  /// The feature is only supported by specific SOCs.
  use_slc_allocator:bool;

  /// When enabled, the compiled graph is split based on its structure and
  /// each part is compiled as an independent subgraph.
  use_graph_splitting:bool;
}

/// Real-time: Indicates that the model is intended for real-time use cases, where a specific performance threshold must be met.
/// Non-real-time: Refers to models without strict performance requirements.
enum QnnExecuTorchLpaiClientPerf: int {
  /// this is the limit of old version flatbuffer
  kUndefined = 0,
  kRealTime,
  kNonRealTime,
}

/// Offloaded Ops shall be executed on core with affinity specified.
enum QnnExecuTorchLpaiCoreAffinity: int {
  /// this is the limit of old version flatbuffer
  kUndefined = 0,
  kSoft,
  kHard,
}

/// Target environment for model execution.
enum QnnExecuTorchLpaiTargetEnv: int {
  kX86 = 0,
  kArm,
  kAdsp,
}

/// Specifies the backend options for the LPAI backend.
table QnnExecuTorchLpaiBackendOptions {
  /// Specifies how frequently inference must be completed.
  fps:int;

  /// Determines the hardware configuration to meet the latency requirement for inference.
  ftrt_ratio:int;

  /// Refers to models w/wo strict performance requirements.
  client_perf_type:QnnExecuTorchLpaiClientPerf;

  /// Offloaded Ops shall be executed on core with affinity specified.
  affinity:QnnExecuTorchLpaiCoreAffinity;

  /// Select which core for execution
  core_selection:uint;

  /// Specifies the target environment for model execution.
  target_env:QnnExecuTorchLpaiTargetEnv;
}

/// Determines the hardware configuration to meet the latency requirement for inference.
/// To ensure inference completes within this reduced time window, the eNPU must be boosted.
enum QnnExecuTorchLogLevel: int {
  kLogOff = 0,
  kLogLevelError,
  kLogLevelWarn,
  kLogLevelInfo,
  kLogLevelVerbose,
  kLogLevelDebug,
}

/// Profiling level of the delegate and QNN backend.
enum QnnExecuTorchProfileLevel: int {
  kProfileOff = 0,
  kProfileBasic,
  kProfileDetailed,
  kProfileOptrace,
}

/// The target of the op package library.
enum QnnExecuTorchOpPackageTarget: int {
  UNKNOWN = 0,
  CPU,
  HTP,
  LPAI,
}

/// The platform of the op package library.
enum QnnExecuTorchOpPackagePlatform: int {
  UNKNOWN = 0,
  X86_64,
  AARCH64_ANDROID,
  HEXAGON,
}

table QnnExecuTorchOpPackageInfo {
  /// The name of the op package.
  op_package_name:string;

  /// The path on disk to the op package library.
  op_package_path:string;

  /// The name of a function in the op package library which satisfies the
  /// QnnOpPackage_InterfaceProvider_t interface.
  interface_provider:string;

  /// The target which this op package library was compiled for.
  target:QnnExecuTorchOpPackageTarget;

  /// The name of torch operator.
  custom_op_name:string;

  /// The corresponding op type name defined in the op package.
  qnn_op_type_name:string;

  /// The platform which this op package library was compiled for.
  platform:QnnExecuTorchOpPackagePlatform;
}

table QnnExecuTorchOpPackageOptions {
  /// An array of QnnExecuTorchOpPackageInfo structures.
  op_package_infos:[QnnExecuTorchOpPackageInfo];
}

/// QNN backends currently supported
table QnnExecuTorchBackendOptions {
  /// The backend QNN library to open and execute the graph with. This is a
  /// required argument and will error out if kUndefinedBackend is supplied.
  backend_type:QnnExecuTorchBackendType;

  htp_options:QnnExecuTorchHtpBackendOptions;

  gpu_options:QnnExecuTorchGpuBackendOptions;

  lpai_options:QnnExecuTorchLpaiBackendOptions;
}

/// Backend configurations to use for one SoC target.
/// Currently we support exactly one backend_option per target.
table QnnExecuTorchTarget {
  soc_info:SocInfo;
  backend_options:[QnnExecuTorchBackendOptions];
}

/// target options (support multi-SoC).
table QnnExecuTorchTargetOptions {
  /// One backend-configuration group per SoC.
  targets:[QnnExecuTorchTarget];

  /// Enable host-AOT reference-weight sharing while appending FCB contexts.
  fcb_reference_weight_sharing:bool = true;
}

table QnnExecuTorchOptions {
  /// Specify SoC to compile or execute for.
  soc_info:SocInfo;

  /// Optional backend specific options for the HTP backend.
  backend_options:QnnExecuTorchBackendOptions;

  /// Optional parameter to override the QNN backend library.
  library_path:string;

  /// Logging level of the delegate and the backend. Default is off.
  log_level:QnnExecuTorchLogLevel;

  /// Check if on-device graph construction. Default is false.
  online_prepare:bool;

  /// If tensor dump is enabled, all intermediate tensors output will be dumped.
  /// This option exists for debugging accuracy issues. Default is off.
  dump_intermediate_outputs:bool;

  /// Profiling level of the delegate and the backend. Default is off.
  profile_level:QnnExecuTorchProfileLevel;

  /// Enables usage of shared buffer between application and backend for graph I/O.
  shared_buffer:bool;

  /// Is model from qnn context binary
  is_from_context_binary:bool;

  /// Enable this option to record all QNN API calls for debugging purpose
  saver:bool;

  /// Path to saver output folder
  saver_output_dir:string;

  /// Optional structure to specify op packages loaded and used by the backend.
  op_package_options:QnnExecuTorchOpPackageOptions;

  /// This experimental parameter is used to decide whether to enable multi-head attention to single-head attention pass, aiming to reduce time consumption in AOT and improve performance on HTP.
  use_mha2sha:bool;

  /// Emit a multi-SoC DLC instead of one device context binary.
  target_options:QnnExecuTorchTargetOptions;
}

root_type QnnExecuTorchOptions;