q4_0c: quantize support

Author: unbounded

examples/quantize/quantize.cpp

@@ -13,6 +13,7 @@ int main(int argc, char ** argv) {
     if (argc != 4) {
         fprintf(stderr, "usage: %s model-f32.bin model-quant.bin type\n", argv[0]);
         fprintf(stderr, "  type = %d - q4_0\n", LLAMA_FTYPE_MOSTLY_Q4_0);
+        fprintf(stderr, "  type = %d - q4_0c\n", LLAMA_FTYPE_MOSTLY_Q4_0C);
         fprintf(stderr, "  type = %d - q4_1\n", LLAMA_FTYPE_MOSTLY_Q4_1);
         fprintf(stderr, "  type = %d - q4_2\n", LLAMA_FTYPE_MOSTLY_Q4_2);
         return 1;

ggml.c

@@ -12023,6 +12023,27 @@ size_t ggml_quantize_q4_0(const float * src, void * dst, int n, int k, int64_t *
     return (n/QK4_0*sizeof(block_q4_0));
 }
 
+size_t ggml_quantize_q4_0c(const float * src, void * dst, int n, int k, int64_t * hist) {
+    assert(k % QK4_0C == 0);
+    const int nb = k / QK4_0;
+
+    for (int j = 0; j < n; j += k) {
+        uint8_t * restrict y = (uint8_t *)dst + sizeof(block_q4_0)*j/QK4_0;
+
+        quantize_row_q4_0c_reference(src + j, y, k);
+
+        for (int i = 0; i < nb*QK4_0/2; i++) {
+            const uint8_t vi0 = y[i] & 0xF;
+            const uint8_t vi1 = y[i] >> 4;
+
+            hist[vi0]++;
+            hist[vi1]++;
+        }
+    }
+
+    return (n/QK4_0*sizeof(block_q4_0));
+}
+
 size_t ggml_quantize_q4_1(const float * src, void * dst, int n, int k, int64_t * hist) {
     assert(k % QK4_1 == 0);
     const int nb = k / QK4_1;

ggml.h

@@ -808,6 +808,7 @@ enum ggml_opt_result ggml_opt(
 //
 
 size_t ggml_quantize_q4_0(const float * src, void * dst, int n, int k, int64_t * hist);
+size_t ggml_quantize_q4_0c(const float * src, void * dst, int n, int k, int64_t * hist);
 size_t ggml_quantize_q4_1(const float * src, void * dst, int n, int k, int64_t * hist);
 size_t ggml_quantize_q4_2(const float * src, void * dst, int n, int k, int64_t * hist);
 

llama.cpp

@@ -477,6 +477,7 @@ struct llama_file_loader {
                 case GGML_TYPE_F32:
                 case GGML_TYPE_F16:
                 case GGML_TYPE_Q4_0:
+                case GGML_TYPE_Q4_0C:
                 case GGML_TYPE_Q4_1:
                 case GGML_TYPE_Q4_2:
                     break;
@@ -550,6 +551,7 @@ struct llama_file_saver {
             case GGML_TYPE_F32:
             case GGML_TYPE_F16:
             case GGML_TYPE_Q4_0:
+            case GGML_TYPE_Q4_0C:
             case GGML_TYPE_Q4_1:
             case GGML_TYPE_Q4_2:
                 break;
@@ -837,6 +839,7 @@ static const char *llama_ftype_name(enum llama_ftype ftype) {
         case LLAMA_FTYPE_ALL_F32:     return "all F32";
         case LLAMA_FTYPE_MOSTLY_F16:  return "mostly F16";
         case LLAMA_FTYPE_MOSTLY_Q4_0: return "mostly Q4_0";
+        case LLAMA_FTYPE_MOSTLY_Q4_0C: return "mostly Q4_1C";
         case LLAMA_FTYPE_MOSTLY_Q4_1: return "mostly Q4_1";
         case LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16:
                                       return "mostly Q4_1, some F16";
@@ -1573,6 +1576,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
     ggml_type quantized_type;
     switch (ftype) {
         case LLAMA_FTYPE_MOSTLY_Q4_0: quantized_type = GGML_TYPE_Q4_0; break;
+        case LLAMA_FTYPE_MOSTLY_Q4_0C: quantized_type = GGML_TYPE_Q4_0C; break;
         case LLAMA_FTYPE_MOSTLY_Q4_1: quantized_type = GGML_TYPE_Q4_1; break;
         case LLAMA_FTYPE_MOSTLY_Q4_2: quantized_type = GGML_TYPE_Q4_2; break;
         default: throw format("invalid output file type %d\n", ftype);
@@ -1644,6 +1648,10 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
                     {
                         new_size = ggml_quantize_q4_0(f32_data, new_data, nelements, (int) tensor.ne.at(0), hist_cur.data());
                     } break;
+                case GGML_TYPE_Q4_0C:
+                    {
+                        new_size = ggml_quantize_q4_0c(f32_data, new_data, nelements, (int) tensor.ne.at(0), hist_cur.data());
+                    } break;
                 case GGML_TYPE_Q4_1:
                     {
                         new_size = ggml_quantize_q4_1(f32_data, new_data, nelements, (int) tensor.ne.at(0), hist_cur.data());

llama.h

@@ -73,6 +73,7 @@ extern "C" {
         LLAMA_FTYPE_MOSTLY_Q4_1 = 3,  // except 1d tensors
         LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
         LLAMA_FTYPE_MOSTLY_Q4_2 = 5,  // except 1d tensors
+        LLAMA_FTYPE_MOSTLY_Q4_0C = 6,  // except 1d tensors
     };
 
     LLAMA_API struct llama_context_params llama_context_default_params();