Skip to content

Just for DIniesh, Ganesh to follow, work in progress, not for Review #51

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Just for DIniesh, Ganesh to follow, work in progress, not for Review #51

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 62 additions & 14 deletions examples/simple/simple-backend-tsi.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ float test_input_1[GGML_TSAVORITE_KERNEL_TYPE_COUNT][NUM_ELEMENTS] = {
{1.1, -4.4, 10, -5, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, -23, 24, 25, -26, 27, -28, 29, -30, 31, -32.6},
//SIN Kernel
{1.1, 4.4, 10, 5, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 20, 20, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32.6},
//RMS_NORM Kernel
{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32},
//SIGMOID Kernel need to fix not tested
{1.1, 4.4, 10, 5, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 20, 20, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32.6},
//SILU Kernel
Expand All @@ -64,6 +66,8 @@ float test_input_2[GGML_TSAVORITE_KERNEL_TYPE_COUNT][NUM_ELEMENTS] = {
{1.1, 2.2, 5, 10, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32},
//SIN Kernel input not used
{1.1, 2.2, 5, 10, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32},
//RMS_NORM Kernel input is not used
{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32},
//SIGMOID Kernel not used
{1.1, 4.4, 10, 5, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 20, 20, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32.6},
//SILU Kernel not used
Expand All @@ -89,11 +93,13 @@ float test_result[GGML_TSAVORITE_KERNEL_TYPE_COUNT][NUM_ELEMENTS] = {
{1.1, 4.4, 10, 5, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32.6},
//SIN Kernel
{0.891207, -0.951602, -0.544021, -0.958924, -0.958924, -0.279416, 0.656987, 0.989358, 0.412118, -0.544021, -0.999990, -0.536573, 0.420167, 0.990607, 0.650288, -0.287903, -0.961398, -0.750987, 0.149877, 0.912945, 0.912945, 0.912945, -0.846220, -0.905578, -0.132352, 0.762559, 0.956376, 0.270906, -0.663634, -0.988032, -0.404039, 0.926149},
//RMS_NORM Kernel
{0.052888, 0.105776, 0.158664, 0.211552, 0.264440, 0.317328, 0.370216, 0.423104, 0.475992, 0.528880, 0.581768, 0.634656, 0.687544, 0.740432, 0.793320, 0.846208, 0.899096, 0.951984, 1.004872, 1.057760, 1.110648, 1.163536, 1.216424, 1.269312, 1.322200, 1.375088, 1.427976, 1.480864, 1.533752, 1.586640, 1.639528, 1.692416},
//SIGMOID Kernel not tested
{0.891207, -0.951602, -0.544021, -0.958924, -0.958924, -0.279416, 0.656987, 0.989358, 0.412118, -0.544021, -0.999990, -0.536573, 0.420167, 0.990607, 0.650288, -0.287903, -0.961398, -0.750987, 0.149877, 0.912945, 0.912945, 0.912945, -0.846220, -0.905578, -0.132352, 0.762559, 0.956376, 0.270906, -0.663634, -0.988032, -0.404039, 0.926149},
// SILU Kernel
{-0.000002, -0.000005, -0.000012, -0.000029, -0.000074, -0.000184, -0.000454, -0.001111, -0.002683, -0.006377, -0.014836, -0.033464, -0.071945, -0.142278, -0.238406, -0.268941, 0.000000, 0.731059, 1.761594, 2.857722, 3.928055, 4.966536, 5.985164, 6.993623, 7.997317, 8.998889, 9.999546, 10.999816, 11.999926, 12.999971, 13.999988, 14.999995}

};

float test_input_scale_1[GGML_TSAVORITE_KERNEL_TYPE_COUNT][NUM_ELEMENTS_SCALE] = {
Expand Down Expand Up @@ -151,6 +157,12 @@ float test_input_scale_1[GGML_TSAVORITE_KERNEL_TYPE_COUNT][NUM_ELEMENTS_SCALE] =
-16, 25, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
//RMS_NORM Kernel
{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25},
//SIGMOID KERNEL need to fix input data
{-1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-9, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
Expand Down Expand Up @@ -217,6 +229,12 @@ float test_input_scale_2[GGML_TSAVORITE_KERNEL_TYPE_COUNT][NUM_ELEMENTS_SCALE] =
-16, 25, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
//RMS_NORM Kernel input not used
{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25},
//SIGMOID KERNEL input not used
{-1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-9, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
Expand Down Expand Up @@ -291,6 +309,24 @@ float test_result_scale[GGML_TSAVORITE_KERNEL_TYPE_COUNT][NUM_ELEMENTS_SCALE] =
-0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471,
0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471,
0.841471, 0.841471, 0.841471},
//RMS_NORM Kernel
{
0.054620, 0.109240, 0.163860, 0.218479, 0.273099, 0.327719, 0.382339, 0.436959, 0.491579, 0.546199,
0.600818, 0.655438, 0.710058, 0.764678, 0.819298, 0.873918, 0.928537, 0.983157, 1.037777, 1.092397,
1.147017, 1.201637, 1.256257, 1.310876, 1.365496, 1.420116, 1.474736, 1.529356, 1.583976, 1.638596,
1.693215, 1.747835, 0.054620, 0.109240, 0.163860, 0.218479, 0.273099, 0.327719, 0.382339, 0.436959,
0.491579, 0.546199, 0.600818, 0.655438, 0.710058, 0.764678, 0.819298, 0.873918, 0.928537, 0.983157,
1.037777, 1.092397, 1.147017, 1.201637, 1.256257, 1.310876, 1.365496, 1.420116, 1.474736, 1.529356,
1.583976, 1.638596, 1.693215, 1.747835, 0.054620, 0.109240, 0.163860, 0.218479, 0.273099, 0.327719,
0.382339, 0.436959, 0.491579, 0.546199, 0.600818, 0.655438, 0.710058, 0.764678, 0.819298, 0.873918,
0.928537, 0.983157, 1.037777, 1.092397, 1.147017, 1.201637, 1.256257, 1.310876, 1.365496, 1.420116,
1.474736, 1.529356, 1.583976, 1.638596, 1.693215, 1.747835, 0.054620, 0.109240, 0.163860, 0.218479,
0.273099, 0.327719, 0.382339, 0.436959, 0.491579, 0.546199, 0.600818, 0.655438, 0.710058, 0.764678,
0.819298, 0.873918, 0.928537, 0.983157, 1.037777, 1.092397, 1.147017, 1.201637, 1.256257, 1.310876,
1.365496, 1.420116, 1.474736, 1.529356, 1.583976, 1.638596, 1.693215, 1.747835, 0.054620, 0.109240,
0.163860, 0.218479, 0.273099, 0.327719, 0.382339, 0.436959, 0.491579, 0.546199, 0.600818, 0.655438,
0.710058, 0.764678, 0.819298, 0.873918, 0.928537, 0.983157, 1.037777, 1.092397, 1.147017, 1.201637,
1.256257, 1.310876, 1.365496},
// SIGMOID KERNEL, result need to change
{-0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471,
0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471, 0.841471,
Expand Down Expand Up @@ -335,14 +371,15 @@ static void ggml_log_callback_default(ggml_log_level level, const char * text, v
}


// --- FLOAT COMPARATOR
// --- FLOAT COMPARATOR
static bool ggml_tsi_compare_two_float(float a, float b) {
// For very small values, use absolute error
if (fabsf(a) < 1e-2f && fabsf(b) < 1e-2f) {
return fabsf(a - b) < 1e-6f; // Accept up to 1e-6 difference for small values
}
// For larger values, use relative error
const float epsilon = 1e-4f;
// For larger values, use relative error with increased tolerance
// Increased to 1e-3 (0.1%) to handle floating-point precision differences
const float epsilon = 1e-3f; // Changed from 1e-4f to 1e-3f
float diff = fabsf(a - b);
float max_val = fmaxf(fabsf(a), fabsf(b));
return diff < epsilon * max_val;
Expand Down Expand Up @@ -376,7 +413,7 @@ static bool load_model(simple_model & model, float * a, float * b, enum ggml_typ
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ true,
};
fprintf(stderr, "\n Calculating mem_size %ld %d and creating ggml context \n", ggml_tensor_overhead(), num_tensors);
fprintf(stderr, "\n Calculating mem_size %ld %d and creating ggml context \n", ggml_tensor_overhead(), num_tensors);

// create context
model.ctx = ggml_init(params);
Expand Down Expand Up @@ -475,6 +512,11 @@ static struct ggml_cgraph * build_graph(const simple_model& model, enum ggml_tsa
case GGML_TSAVORITE_KERNEL_TYPE_SIN:
result = ggml_sin(ctx0, model.a);
break;
case GGML_TSAVORITE_KERNEL_TYPE_RMS_NORM:
printf("\n ANOOP CALLINF RMS_NORM\n");
//result = ggml_rms_norm(ctx0, model.a, 1e-6f);
result = ggml_rms_norm(ctx0, model.a, 1e-5);
break;
case GGML_TSAVORITE_KERNEL_TYPE_SIGMOID:
result = ggml_sigmoid(ctx0, model.a);
break;
Expand All @@ -500,11 +542,11 @@ static struct ggml_tensor * compute(const simple_model & model, ggml_gallocr_t a

fprintf(stderr, "\n Under Test case for compute API creating build_graph \n");
struct ggml_cgraph * gf = build_graph(model, ops_type);
if (!gf) {
if (!gf) {
fprintf(stderr, "\ncompute failed\n");
return NULL;
}

// allocate tensors
ggml_gallocr_alloc_graph(allocr, gf);

Expand Down Expand Up @@ -533,6 +575,8 @@ enum ggml_tsavorite_kernel_type convert_testcase_to_ops_type (const char *testCa
return GGML_TSAVORITE_KERNEL_TYPE_ABS;
else if (!strcmp(testCase,"sin"))
return GGML_TSAVORITE_KERNEL_TYPE_SIN;
else if (!strcmp(testCase,"rms_norm"))
return GGML_TSAVORITE_KERNEL_TYPE_RMS_NORM;
else if (!strcmp(testCase,"sigmoid"))
return GGML_TSAVORITE_KERNEL_TYPE_SIGMOID;
else if (!strcmp(testCase,"silu"))
Expand Down Expand Up @@ -561,7 +605,10 @@ const char* convert_ops_type_to_testcase(enum ggml_tsavorite_kernel_type ops_typ
return "neg";
case GGML_TSAVORITE_KERNEL_TYPE_ABS:
return "abs";
case GGML_TSAVORITE_KERNEL_TYPE_SIN:
case GGML_TSAVORITE_KERNEL_TYPE_SIN:
return "sin";
case GGML_TSAVORITE_KERNEL_TYPE_RMS_NORM:
return "rms_norm";
return "sin";
case GGML_TSAVORITE_KERNEL_TYPE_SIGMOID:
return "sigmoid";
Expand Down Expand Up @@ -601,26 +648,27 @@ int main(int argc, char *argv[]) {
ops_type == GGML_TSAVORITE_KERNEL_TYPE_NEG ||
ops_type == GGML_TSAVORITE_KERNEL_TYPE_ABS ||
ops_type == GGML_TSAVORITE_KERNEL_TYPE_SIN ||
ops_type == GGML_TSAVORITE_KERNEL_TYPE_RMS_NORM ||
ops_type == GGML_TSAVORITE_KERNEL_TYPE_SIGMOID ||
ops_type == GGML_TSAVORITE_KERNEL_TYPE_SILU)
num_of_input_tensors = NUM_INPUT_URINARY_TENSORS;
else
else
num_of_input_tensors = NUM_INPUT_TENSORS;

if (data_scale) {
input1[ops_type] = test_input_scale_1[ops_type];
elements_A = NUM_ELEMENTS_SCALE;
elements_A = NUM_ELEMENTS_SCALE;
if (num_of_input_tensors != NUM_INPUT_URINARY_TENSORS) {
input2[ops_type] = test_input_scale_2[ops_type];
elements_B = NUM_ELEMENTS_SCALE;
elements_B = NUM_ELEMENTS_SCALE;
}
result_data[ops_type] = test_result_scale[ops_type];
} else {
input1[ops_type] = test_input_1[ops_type];
elements_A = NUM_ELEMENTS;
elements_A = NUM_ELEMENTS;
if (num_of_input_tensors != NUM_INPUT_URINARY_TENSORS) {
input2[ops_type] = test_input_2[ops_type];
elements_B = NUM_ELEMENTS;
elements_B = NUM_ELEMENTS;
}
result_data[ops_type] = test_result[ops_type];
}
Expand Down Expand Up @@ -676,7 +724,7 @@ int main(int argc, char *argv[]) {
uint32_t bits_expected, bits_actual;
memcpy(&bits_expected, &result_data[ops_type][i], sizeof(float));
memcpy(&bits_actual, &out_data[i], sizeof(float));
fprintf(stderr, "Index %d: expected bits %08x, actual bits %08x\n", i, bits_expected, bits_actual);
//fprintf(stderr, "Index %d: expected bits %08x, actual bits %08x\n", i, bits_expected, bits_actual);
#endif
if (ggml_tsi_compare_two_float(out_data[i], result_data[ops_type][i])) {
continue;
Expand Down
10 changes: 9 additions & 1 deletion ggml/include/ggml-tsavorite.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,10 @@ enum ggml_tsavorite_kernel_type {
GGML_TSAVORITE_KERNEL_TYPE_NEG,
GGML_TSAVORITE_KERNEL_TYPE_ABS,
GGML_TSAVORITE_KERNEL_TYPE_SIN,
GGML_TSAVORITE_KERNEL_TYPE_RMS_NORM,
GGML_TSAVORITE_KERNEL_TYPE_SIGMOID,
GGML_TSAVORITE_KERNEL_TYPE_SILU,
GGML_TSAVORITE_KERNEL_TYPE_MUL_MAT,

GGML_TSAVORITE_KERNEL_TYPE_COUNT
};
Expand Down Expand Up @@ -162,10 +164,16 @@ extern void _mlir_ciface_txe_abs_host(void *a, void *res);
extern void _mlir_ciface_txe_sin_host(void *a, void *res);
extern void _mlir_ciface_txe_sigmoid_host(void *a, void *res);
extern void _mlir_ciface_txe_silu_host(void *a, void *res);
extern void _mlir_ciface_txe_mul_mat_host(void *a, void *b, void *res, void *pre_mask);
extern void _mlir_ciface_txe_rms_norm_host(void *a, void *res, void *buf);
extern void _mlir_ciface_txe_rms_norm_6_host(void *a, void *res, void *buf);
extern void _mlir_ciface_txe_rms_norm_512_host(void *a, void *res, void *buf);

extern void ggml_tsi_log_tensor_data(tensor_log log_data);

#define NUM_OF_TXES 1
#define MEM_REF_DESCRIPTOR_RANK 1
#define MEM_REF_DESCRIPTOR_RANK 4
#define TSI_TVU_LOAD_SIZE 32

//
// backend API
Expand Down
46 changes: 45 additions & 1 deletion ggml/src/ggml-backend.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -861,6 +861,10 @@ static void ggml_backend_sched_set_if_supported(ggml_backend_sched_t sched, stru
}
}

static void anoop_backend()
{
return;
}
// assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend
static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {
// reset splits
Expand All @@ -875,6 +879,7 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg
};

ggml_free(sched->ctx);
//printf("\n\n ANOOP ggml_backend_sched_split_graph is called\n\n");

sched->ctx = ggml_init(params);
if (sched->ctx == NULL) {
Expand Down Expand Up @@ -932,6 +937,13 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg
continue;
}
int * node_backend_id = &tensor_backend_id(node);

if (node && node->op == GGML_OP_RMS_NORM) {
if ((node->ne[1] == 1 || node->ne[1] == 6 || node->ne[1] == 512) && node->ne[2] == 1 && (node->ne[3] == 1)) {
ggml_backend_sched_set_if_supported(sched, node, 0, node_backend_id);
//anoop_backend();
}
}
if (*node_backend_id != -1) {
if (*node_backend_id == sched->n_backends - 1) {
// skip cpu (lowest prio backend)
Expand All @@ -942,6 +954,8 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg
// Below Code is Optimization which i am disabling for now since we have not implemented other
// Operation at tsavorite
} else {
//if (node && node->op == GGML_OP_RMS_NORM)
// printf("\n ANOOP RMS COUNT -First STEP");
ggml_backend_sched_set_if_supported(sched, node, 0, node_backend_id);
}
}
Expand All @@ -955,14 +969,24 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg
continue;
}
int * node_backend_id = &tensor_backend_id(node);
#if 0
if (node && node->op == GGML_OP_RMS_NORM) {
if ((node->ne[1] == 1 || node->ne[1] == 512) && node->ne[2] == 1 && (node->ne[3] == 1)) {
ggml_backend_sched_set_if_supported(sched, node, 0, node_backend_id);
//anoop_backend();
}
}
#endif
if (*node_backend_id != -1) {
if (*node_backend_id == sched->n_backends - 1) {
// skip cpu (lowest prio backend)
cur_backend_id = -1;
} else {
cur_backend_id = *node_backend_id;
}
} else if (cur_backend_id != -1) {
} else if (cur_backend_id != -1) {
if (cur_backend_id != 0)
printf("\n AT GRAPH SPLIT expand gpu up");
ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);
}
}
Expand All @@ -976,9 +1000,20 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg
continue;
}
int * node_backend_id = &tensor_backend_id(node);

#if 0
if (node && node->op == GGML_OP_RMS_NORM) {
if ((node->ne[1] == 1 || node->ne[1] == 512) && node->ne[2] == 1 && (node->ne[3] == 1)) {
ggml_backend_sched_set_if_supported(sched, node, 0, node_backend_id);
//anoop_backend();
}
}
#endif
if (*node_backend_id != -1) {
cur_backend_id = *node_backend_id;
} else if (cur_backend_id != -1) {
//if (cur_backend_id != 0)
// printf("\n AT GRAPH SPLIT expand rest down");
ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);
}
}
Expand All @@ -992,9 +1027,18 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg
continue;
}
int * node_backend_id = &tensor_backend_id(node);

if (node && node->op == GGML_OP_RMS_NORM) {
if ((node->ne[1] == 1 || node->ne[1] == 512 || node->ne[1] == 6) && node->ne[2] == 1 && (node->ne[3] == 1)) {
ggml_backend_sched_set_if_supported(sched, node, 0, node_backend_id);
anoop_backend();
}
}
if (*node_backend_id != -1) {
cur_backend_id = *node_backend_id;
} else if (cur_backend_id != -1) {
if (cur_backend_id != 0)
printf("\n AT GRAPH SPLIT expand rest up");
ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);
}
}
Expand Down
2 changes: 2 additions & 0 deletions ggml/src/ggml-cpu/ggml-cpu.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1297,6 +1297,7 @@ static void ggml_compute_forward_mul_mat(
const bool src1_cont = ggml_is_contiguous(src1);

if (src1_cont) {
//printf("\n ANOOP GGML IS CONTIGIOUS\n");
for (int64_t i13 = 0; i13 < ne13; i13++)
for (int64_t i12 = 0; i12 < ne12; i12++)
if (!llamafile_sgemm(params,
Expand Down Expand Up @@ -1813,6 +1814,7 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
} break;
case GGML_OP_RMS_NORM:
{
//printf("\n under CPU GGML_OP_RMS_NORM 1\n");
ggml_compute_forward_rms_norm(params, tensor);
} break;
case GGML_OP_RMS_NORM_BACK:
Expand Down
Loading