From 7aad6cbda69ba673f0b200cfa03cb5a07d0e178f Mon Sep 17 00:00:00 2001 From: matt23654 Date: Tue, 31 Dec 2024 21:56:51 +0000 Subject: [PATCH 1/7] Added init tensor calling code --- ggml/src/ggml-rpc/ggml-rpc.cpp | 64 ++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index 43108242639a3..4d87e80dcd8f8 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -93,9 +93,18 @@ enum rpc_cmd { RPC_CMD_COPY_TENSOR, RPC_CMD_GRAPH_COMPUTE, RPC_CMD_GET_DEVICE_MEMORY, + RPC_CMD_INIT_TENSOR, RPC_CMD_COUNT, }; +struct rpc_msg_init_tensor_req { + rpc_tensor tensor; +}; + +struct rpc_msg_init_tensor_rsp { + uint8_t result; // success/failure +}; + struct rpc_msg_alloc_buffer_req { uint64_t size; }; @@ -461,10 +470,18 @@ static rpc_tensor serialize_tensor(const ggml_tensor * tensor) { } static void ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { - UNUSED(buffer); + //UNUSED(buffer); + ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + if (ggml_is_quantized(tensor->type)) { // TODO: this check is due to MATRIX_ROW_PADDING in CUDA and should be generalized - GGML_ASSERT(tensor->ne[0] % 512 == 0 && "unsupported quantized tensor"); + //GGML_ASSERT(tensor->ne[0] % 512 == 0 && "unsupported quantized tensor"); + rpc_msg_init_tensor_req request; + request.tensor = serialize_tensor(tensor); + + //rpc_msg_init_tensor_rsp response; + bool status = send_rpc_cmd(ctx->sock, RPC_CMD_INIT_TENSOR, &request, sizeof(request), nullptr, 0); + GGML_ASSERT(status); } } @@ -757,6 +774,7 @@ class rpc_server { bool get_tensor(const rpc_msg_get_tensor_req & request, std::vector & response); bool copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_copy_tensor_rsp & response); bool graph_compute(const std::vector & input, rpc_msg_graph_compute_rsp & response); + bool init_tensor(const rpc_msg_init_tensor_req & request); private: ggml_tensor * deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor); @@ -905,6 +923,35 @@ bool rpc_server::set_tensor(const std::vector & input) { return true; } +bool rpc_server::init_tensor(const rpc_msg_init_tensor_req & request) { + struct ggml_init_params params { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + struct ggml_context * ctx = ggml_init(params); + ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); + if (tensor == nullptr) { + printf("Null tensor\n"); + ggml_free(ctx); + return false; + } + + printf("about to call buffer\n"); + + //ggml_backend_init_tensor + + // Call the backend's buffer_init_tensor function + ggml_backend_buffer_t buffer = tensor->buffer; + if (buffer && buffer->iface.init_tensor) { + printf("Calling buffer iface function\n"); + buffer->iface.init_tensor(buffer, tensor); + } + + ggml_free(ctx); + return true; +} + bool rpc_server::get_tensor(const rpc_msg_get_tensor_req & request, std::vector & response) { struct ggml_init_params params { /*.mem_size =*/ ggml_tensor_overhead(), @@ -1133,6 +1180,19 @@ static void rpc_serve_client(ggml_backend_t backend, sockfd_t sockfd, size_t fre } break; } + case RPC_CMD_INIT_TENSOR: { + rpc_msg_init_tensor_req request; + if (!recv_msg(sockfd, &request,sizeof(request))) { + return; + } + if (!server.init_tensor(request)) { + return; + } + if (!send_msg(sockfd, nullptr, 0)) { + return; + } + break; + } case RPC_CMD_GET_TENSOR: { rpc_msg_get_tensor_req request; if (!recv_msg(sockfd, &request, sizeof(request))) { From c47dc70b58f7f2ceabff4a0705196cec6ef7edcd Mon Sep 17 00:00:00 2001 From: matt23654 Date: Wed, 1 Jan 2025 03:37:10 +0000 Subject: [PATCH 2/7] Added get_alloc_size forwarding --- ggml/src/ggml-rpc/ggml-rpc.cpp | 82 ++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index 4d87e80dcd8f8..e8ba98ce78716 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -94,9 +94,18 @@ enum rpc_cmd { RPC_CMD_GRAPH_COMPUTE, RPC_CMD_GET_DEVICE_MEMORY, RPC_CMD_INIT_TENSOR, + RPC_CMD_GET_ALLOC_SIZE, RPC_CMD_COUNT, }; +struct rpc_msg_get_alloc_size_req { + rpc_tensor tensor; +}; + +struct rpc_msg_get_alloc_size_rsp { + uint64_t alloc_size; +}; + struct rpc_msg_init_tensor_req { rpc_tensor tensor; }; @@ -473,7 +482,7 @@ static void ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_t buffer, gg //UNUSED(buffer); ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; - if (ggml_is_quantized(tensor->type)) { + if (ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0)) { // TODO: this check is due to MATRIX_ROW_PADDING in CUDA and should be generalized //GGML_ASSERT(tensor->ne[0] % 512 == 0 && "unsupported quantized tensor"); rpc_msg_init_tensor_req request; @@ -594,8 +603,21 @@ static size_t ggml_backend_rpc_get_max_size(ggml_backend_buffer_type_t buft) { } static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { - UNUSED(buft); - return ggml_nbytes(tensor); + if (ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0)) { + ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context; + auto sock = get_socket(buft_ctx->endpoint); + + rpc_msg_get_alloc_size_req request; + request.tensor = serialize_tensor(tensor); + + rpc_msg_get_alloc_size_rsp response; + bool status = send_rpc_cmd(sock, RPC_CMD_GET_ALLOC_SIZE, &request, sizeof(request), &response, sizeof(response)); + GGML_ASSERT(status); + + return response.alloc_size; + } else { + return ggml_nbytes(tensor); + } } static ggml_backend_buffer_type_i ggml_backend_rpc_buffer_type_interface = { @@ -775,6 +797,7 @@ class rpc_server { bool copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_copy_tensor_rsp & response); bool graph_compute(const std::vector & input, rpc_msg_graph_compute_rsp & response); bool init_tensor(const rpc_msg_init_tensor_req & request); + bool get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_msg_get_alloc_size_rsp & response); private: ggml_tensor * deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor); @@ -788,6 +811,47 @@ class rpc_server { std::unordered_set buffers; }; +bool rpc_server::get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_msg_get_alloc_size_rsp & response) { + ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backend); + struct ggml_init_params params { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + struct ggml_context * ctx = ggml_init(params); + ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); + if (tensor == nullptr) { + printf("Got nullptr\n"); + ggml_free(ctx); + return false; + } + + printf("Getting buft\n"); + + //ggml_backend_buffer_get_alloc_size(tensor->buffer,tensor) + + //if (tensor->buffer == nullptr) { + // printf("Got null buffer\n"); + // response.alloc_size = 0; + // ggml_free(ctx); + // return true; + //} + + response.alloc_size = ggml_backend_buft_get_alloc_size(buft,tensor); + // Call the backend's buffer_type_get_alloc_size function + //ggml_backend_buffer_type_t buft = tensor->buffer->buft; + //if (buft && buft->iface.get_alloc_size) { + // printf("Called buffer type get alloc size\n"); + // response.alloc_size = buft->iface.get_alloc_size(buft, tensor); + //} else { + // printf("Called ggml_nbytes"); + // response.alloc_size = ggml_nbytes(tensor); + //} + + ggml_free(ctx); + return true; +} + void rpc_server::alloc_buffer(const rpc_msg_alloc_buffer_req & request, rpc_msg_alloc_buffer_rsp & response) { ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backend); ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, request.size); @@ -1105,6 +1169,18 @@ static void rpc_serve_client(ggml_backend_t backend, sockfd_t sockfd, size_t fre } break; } + case RPC_CMD_GET_ALLOC_SIZE: { + rpc_msg_get_alloc_size_req request; + if (!recv_msg(sockfd, &request, sizeof(request))) { + return; + } + rpc_msg_get_alloc_size_rsp response; + server.get_alloc_size(request, response); + if (!send_msg(sockfd, &response, sizeof(response))) { + return; + } + break; + } case RPC_CMD_GET_ALIGNMENT: { if (!recv_msg(sockfd, nullptr, 0)) { return; From 1948ae84911e9d12c6276835ca2668f97442e3fa Mon Sep 17 00:00:00 2001 From: matt23654 Date: Thu, 2 Jan 2025 15:53:50 +0000 Subject: [PATCH 3/7] Cleaned up and improved type/error handling. --- ggml/src/ggml-rpc/ggml-rpc.cpp | 55 +++++++++++++--------------------- 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index e8ba98ce78716..addd3050e77e9 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -110,9 +110,9 @@ struct rpc_msg_init_tensor_req { rpc_tensor tensor; }; -struct rpc_msg_init_tensor_rsp { - uint8_t result; // success/failure -}; +//struct rpc_msg_init_tensor_rsp { +// uint8_t result; // success/failure +//}; struct rpc_msg_alloc_buffer_req { uint64_t size; @@ -479,16 +479,15 @@ static rpc_tensor serialize_tensor(const ggml_tensor * tensor) { } static void ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { - //UNUSED(buffer); ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + // CUDA backend on the server pads everything to 512 due to CUDA limitations. + // Due to bandwidth constraints, we only call the server init tensor functions if necessary. if (ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0)) { - // TODO: this check is due to MATRIX_ROW_PADDING in CUDA and should be generalized - //GGML_ASSERT(tensor->ne[0] % 512 == 0 && "unsupported quantized tensor"); rpc_msg_init_tensor_req request; + request.tensor = serialize_tensor(tensor); - //rpc_msg_init_tensor_rsp response; bool status = send_rpc_cmd(ctx->sock, RPC_CMD_INIT_TENSOR, &request, sizeof(request), nullptr, 0); GGML_ASSERT(status); } @@ -603,11 +602,13 @@ static size_t ggml_backend_rpc_get_max_size(ggml_backend_buffer_type_t buft) { } static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { + // See comments in init_tensor. if (ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0)) { ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context; auto sock = get_socket(buft_ctx->endpoint); rpc_msg_get_alloc_size_req request; + request.tensor = serialize_tensor(tensor); rpc_msg_get_alloc_size_rsp response; @@ -812,41 +813,30 @@ class rpc_server { }; bool rpc_server::get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_msg_get_alloc_size_rsp & response) { - ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backend); + ggml_backend_buffer_type_t buft; struct ggml_init_params params { /*.mem_size =*/ ggml_tensor_overhead(), /*.mem_buffer =*/ NULL, /*.no_alloc =*/ true, }; + struct ggml_context * ctx = ggml_init(params); ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); + if (tensor == nullptr) { - printf("Got nullptr\n"); + fprintf(stderr,"Null tensor pointer passed to server get_alloc_size function.\n"); ggml_free(ctx); return false; } - printf("Getting buft\n"); - - //ggml_backend_buffer_get_alloc_size(tensor->buffer,tensor) - - //if (tensor->buffer == nullptr) { - // printf("Got null buffer\n"); - // response.alloc_size = 0; - // ggml_free(ctx); - // return true; - //} + if (tensor->buffer == nullptr) { + //No buffer allocated. + buft = ggml_backend_get_default_buffer_type(backend); + } else { + buft = tensor->buffer->buft; + } response.alloc_size = ggml_backend_buft_get_alloc_size(buft,tensor); - // Call the backend's buffer_type_get_alloc_size function - //ggml_backend_buffer_type_t buft = tensor->buffer->buft; - //if (buft && buft->iface.get_alloc_size) { - // printf("Called buffer type get alloc size\n"); - // response.alloc_size = buft->iface.get_alloc_size(buft, tensor); - //} else { - // printf("Called ggml_nbytes"); - // response.alloc_size = ggml_nbytes(tensor); - //} ggml_free(ctx); return true; @@ -996,20 +986,17 @@ bool rpc_server::init_tensor(const rpc_msg_init_tensor_req & request) { struct ggml_context * ctx = ggml_init(params); ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); if (tensor == nullptr) { - printf("Null tensor\n"); + fprintf(stderr,"Null tensor pointer passed to server init_tensor function.\n"); ggml_free(ctx); return false; } - printf("about to call buffer\n"); - - //ggml_backend_init_tensor - // Call the backend's buffer_init_tensor function ggml_backend_buffer_t buffer = tensor->buffer; if (buffer && buffer->iface.init_tensor) { - printf("Calling buffer iface function\n"); buffer->iface.init_tensor(buffer, tensor); + } else { + fprintf(stderr,"Null buffer for tensor passed to init_tensor function\n"); } ggml_free(ctx); From 840594f4012339ad58f2962567e37f7060b02583 Mon Sep 17 00:00:00 2001 From: matt23654 Date: Thu, 2 Jan 2025 23:28:32 +0000 Subject: [PATCH 4/7] fix: remove trailing whitespaces. --- ggml/src/ggml-rpc/ggml-rpc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index addd3050e77e9..273cf3eb8fe1a 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -813,7 +813,7 @@ class rpc_server { }; bool rpc_server::get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_msg_get_alloc_size_rsp & response) { - ggml_backend_buffer_type_t buft; + ggml_backend_buffer_type_t buft; struct ggml_init_params params { /*.mem_size =*/ ggml_tensor_overhead(), /*.mem_buffer =*/ NULL, @@ -822,7 +822,7 @@ bool rpc_server::get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_ struct ggml_context * ctx = ggml_init(params); ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); - + if (tensor == nullptr) { fprintf(stderr,"Null tensor pointer passed to server get_alloc_size function.\n"); ggml_free(ctx); From b66e91b1b23178c97fe8ad4f0f104bb6f4c217bf Mon Sep 17 00:00:00 2001 From: matt23654 Date: Fri, 3 Jan 2025 14:39:32 +0000 Subject: [PATCH 5/7] Cleanup and use GGML error logging functions. --- ggml/src/ggml-rpc/ggml-rpc.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index 273cf3eb8fe1a..424c97733fb15 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -110,10 +110,6 @@ struct rpc_msg_init_tensor_req { rpc_tensor tensor; }; -//struct rpc_msg_init_tensor_rsp { -// uint8_t result; // success/failure -//}; - struct rpc_msg_alloc_buffer_req { uint64_t size; }; @@ -824,7 +820,7 @@ bool rpc_server::get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_ ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); if (tensor == nullptr) { - fprintf(stderr,"Null tensor pointer passed to server get_alloc_size function.\n"); + GGML_LOG_ERROR("Null tensor pointer passed to server get_alloc_size function.\n"); ggml_free(ctx); return false; } @@ -986,7 +982,7 @@ bool rpc_server::init_tensor(const rpc_msg_init_tensor_req & request) { struct ggml_context * ctx = ggml_init(params); ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); if (tensor == nullptr) { - fprintf(stderr,"Null tensor pointer passed to server init_tensor function.\n"); + GGML_LOG_ERROR("Null tensor pointer passed to server init_tensor function.\n"); ggml_free(ctx); return false; } @@ -996,7 +992,7 @@ bool rpc_server::init_tensor(const rpc_msg_init_tensor_req & request) { if (buffer && buffer->iface.init_tensor) { buffer->iface.init_tensor(buffer, tensor); } else { - fprintf(stderr,"Null buffer for tensor passed to init_tensor function\n"); + GGML_LOG_ERROR("Null buffer for tensor passed to init_tensor function\n"); } ggml_free(ctx); From c111e8a5b2084584560de5f446e4d28a9beb6a6d Mon Sep 17 00:00:00 2001 From: matt23654 Date: Fri, 3 Jan 2025 22:14:21 +0000 Subject: [PATCH 6/7] Handle potentially dangerous edge cases. --- ggml/src/ggml-rpc/ggml-rpc.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index 424c97733fb15..030f027912c7e 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -479,7 +479,8 @@ static void ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_t buffer, gg // CUDA backend on the server pads everything to 512 due to CUDA limitations. // Due to bandwidth constraints, we only call the server init tensor functions if necessary. - if (ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0)) { + // In particular, this is tensors with padding that needs to be cleared, so base tensors only and only misaligned. + if (ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0) && (tensor->view_src == nullptr)) { rpc_msg_init_tensor_req request; request.tensor = serialize_tensor(tensor); @@ -995,6 +996,14 @@ bool rpc_server::init_tensor(const rpc_msg_init_tensor_req & request) { GGML_LOG_ERROR("Null buffer for tensor passed to init_tensor function\n"); } + if(tensor->extra != nullptr) { + // This pointer can either be passed around client/server, or probably better stored server-side and kept track of. + // Currently unimplemented. + GGML_LOG_ERROR("tensor->extra populated by the backend, this is currently unsupported.\n"); + ggml_free(ctx); + return false; + } + ggml_free(ctx); return true; } From 4973a298b659d16aa27c1b89f59037c0783c19e3 Mon Sep 17 00:00:00 2001 From: matt23654 Date: Fri, 3 Jan 2025 22:46:35 +0000 Subject: [PATCH 7/7] Apply suggestions from code review Co-authored-by: Diego Devesa --- ggml/src/ggml-rpc/ggml-rpc.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index 030f027912c7e..2213aba9f121a 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -479,7 +479,7 @@ static void ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_t buffer, gg // CUDA backend on the server pads everything to 512 due to CUDA limitations. // Due to bandwidth constraints, we only call the server init tensor functions if necessary. - // In particular, this is tensors with padding that needs to be cleared, so base tensors only and only misaligned. + // In particular, only quantized tensors need padding if (ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0) && (tensor->view_src == nullptr)) { rpc_msg_init_tensor_req request; @@ -600,7 +600,7 @@ static size_t ggml_backend_rpc_get_max_size(ggml_backend_buffer_type_t buft) { static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { // See comments in init_tensor. - if (ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0)) { + if (ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0) && (tensor->view_src == nullptr)) { ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context; auto sock = get_socket(buft_ctx->endpoint); @@ -996,7 +996,7 @@ bool rpc_server::init_tensor(const rpc_msg_init_tensor_req & request) { GGML_LOG_ERROR("Null buffer for tensor passed to init_tensor function\n"); } - if(tensor->extra != nullptr) { + if (tensor->extra != nullptr) { // This pointer can either be passed around client/server, or probably better stored server-side and kept track of. // Currently unimplemented. GGML_LOG_ERROR("tensor->extra populated by the backend, this is currently unsupported.\n");