Qualcomm AI Engine Direct - Support Hybrid Mode for Llama3.2

Differential Revision: D66895016

Pull Request resolved: #7175

Author: winskuo-quic

backends/qualcomm/CMakeLists.txt

@@ -76,7 +76,6 @@ include_directories(
 
 set(_qnn_schema__srcs
   backends/qualcomm/serialization/qc_compiler_spec.fbs
-  backends/qualcomm/serialization/qc_binary_info.fbs
 )
 set(_qnn_schema__include_dir "${CMAKE_BINARY_DIR}/schema/include")
 # Paths to headers generated from the .fbs files.
@@ -116,6 +115,7 @@ add_library(qcir_utils STATIC)
 add_library(qnn_backend STATIC)
 add_library(qnn_backend_cache STATIC)
 add_library(qnn_context STATIC)
+add_library(qnn_custom_protocol STATIC)
 add_library(qnn_device STATIC)
 add_library(qnn_executorch_backend SHARED)
 add_library(qnn_executorch_header INTERFACE)
@@ -155,6 +155,7 @@ target_link_libraries(qnn_executorch_logging PRIVATE qnn_schema)
 target_link_libraries(qnn_profiler PRIVATE qnn_executorch_logging)
 target_link_libraries(qnn_logger PRIVATE qnn_implementation ${android_log})
 target_link_libraries(qnn_backend PRIVATE qnn_implementation qnn_logger)
+target_link_libraries(qnn_custom_protocol PRIVATE qcir_utils)
 target_link_libraries(
   qnn_device PRIVATE qnn_executorch_logging qnn_implementation qnn_logger
 )
@@ -177,7 +178,7 @@ target_link_libraries(
   qnn_factory
   PUBLIC qnn_header
   PRIVATE qnn_schema qnn_backend qnn_device qnn_context qnn_graph
-          qnn_mem_manager
+          qnn_mem_manager qnn_custom_protocol
 )
 target_link_libraries(
   qnn_manager PRIVATE qnn_factory wrappers qnn_schema utils shared_buffer

backends/qualcomm/aot/ir/qcir.fbs

@@ -80,17 +80,18 @@ table Tensor {
     type: TensorType;
     dtype: DataType;
     qparam: QuantizeParam;
-    data: [ubyte];
+    size: uint;
+    offset: ulong;
 }
 
 table Operator {
     name: string;
     package_name: string;
     type_name: string;
     // keep only tensor indexes
-    inputs: [int];
-    outputs: [int];
-    params: [int];
+    inputs: [uint];
+    outputs: [uint];
+    params: [uint];
 }
 
 table Graph {

backends/qualcomm/aot/ir/qcir_utils.cpp

@@ -235,11 +235,8 @@ Qnn_QuantizeParams_t ToQuantizeParam(const tensor_type& tensor) {
 
 flatbuffers::Offset<qcir::Tensor> ToTensor(
     const Qnn_Tensor_t& tensor,
+    const uint64_t data_offset,
     flatbuffers::FlatBufferBuilder* builder) {
-  std::vector<uint8_t> buffer(
-      static_cast<uint8_t*>(QNN_VER_PTR(tensor)->clientBuf.data),
-      static_cast<uint8_t*>(QNN_VER_PTR(tensor)->clientBuf.data) +
-          QNN_VER_PTR(tensor)->clientBuf.dataSize);
   std::vector<uint32_t> shape(
       QNN_VER_PTR(tensor)->dimensions,
       QNN_VER_PTR(tensor)->dimensions + QNN_VER_PTR(tensor)->rank);
@@ -251,10 +248,11 @@ flatbuffers::Offset<qcir::Tensor> ToTensor(
       ToTensorType(QNN_VER_PTR(tensor)->type),
       ToDataType(QNN_VER_PTR(tensor)->dataType),
       ToQuantizeParam(tensor, builder),
-      &buffer);
+      QNN_VER_PTR(tensor)->clientBuf.dataSize,
+      data_offset);
 }
 
-Qnn_Tensor_t ToTensor(const tensor_type& tensor) {
+Qnn_Tensor_t ToTensor(const tensor_type& tensor, const uint8_t* data_ptr) {
   auto is_io_tensor = [](Qnn_TensorType_t type) {
     return type < QNN_TENSOR_TYPE_STATIC;
   };
@@ -266,10 +264,10 @@ Qnn_Tensor_t ToTensor(const tensor_type& tensor) {
   QNN_VER_PTR(t)->quantizeParams = ToQuantizeParam(tensor);
   QNN_VER_PTR(t)->rank = tensor->shape()->size();
   QNN_VER_PTR(t)->dimensions = const_cast<uint32_t*>(tensor->shape()->data());
-  QNN_VER_PTR(t)->clientBuf.dataSize = tensor->data()->size();
+  QNN_VER_PTR(t)->clientBuf.dataSize = tensor->size();
   QNN_VER_PTR(t)->clientBuf.data = is_io_tensor(QNN_VER_PTR(t)->type)
       ? nullptr
-      : static_cast<void*>(const_cast<uint8_t*>(tensor->data()->Data()));
+      : static_cast<void*>(const_cast<uint8_t*>(data_ptr));
   return t;
 }
 

backends/qualcomm/aot/ir/qcir_utils.h

@@ -32,8 +32,9 @@ Qnn_QuantizeParams_t ToQuantizeParam(const tensor_type& tensor);
 
 flatbuffers::Offset<qcir::Tensor> ToTensor(
     const Qnn_Tensor_t& tensor,
+    const uint64_t data_offset,
     flatbuffers::FlatBufferBuilder* builder);
-Qnn_Tensor_t ToTensor(const tensor_type& tensor);
+Qnn_Tensor_t ToTensor(const tensor_type& tensor, const uint8_t* data_ptr);
 
 } // namespace qnn
 } // namespace backends