Avoid allocate buffer for offloaded tensor when using no-mmap

Author: howard0su

llama.cpp

@@ -668,13 +668,21 @@ struct llama_model_loader {
 
     struct ggml_tensor * get_tensor_for(llama_load_tensor & lt, ggml_backend backend) {
         struct ggml_tensor * tensor;
+
+        if (backend != GGML_BACKEND_CPU) {
+            ggml_set_no_alloc(ggml_ctx, true);
+        }
         if (lt.ne.size() == 2) {
             tensor = ggml_new_tensor_2d(ggml_ctx, lt.type, lt.ne.at(0), lt.ne.at(1));
         } else {
             LLAMA_ASSERT(lt.ne.size() == 1);
             tensor = ggml_new_tensor_1d(ggml_ctx, lt.type, lt.ne.at(0));
         }
         ggml_set_name(tensor, lt.name.c_str());
+
+        if (backend != GGML_BACKEND_CPU) {
+            ggml_set_no_alloc(ggml_ctx, use_mmap);
+        }
         LLAMA_ASSERT(lt.ggml_tensor == NULL); // if this fails, we called get_tensor twice on the same tensor
         tensor->backend = backend;
         lt.ggml_tensor = tensor;
@@ -713,6 +721,11 @@ struct llama_model_loader {
             }
             LLAMA_ASSERT(lt.ggml_tensor); // unused tensors should have been caught by load_data already
             lt.data = (uint8_t *) lt.ggml_tensor->data;
+            // allocate temp buffer if not using mmap
+            if (!use_mmap && lt.data == NULL) {
+                lt.data = (uint8_t*)malloc(ggml_nbytes(lt.ggml_tensor));
+            }
+
             load_data_for(lt);
             switch(lt.ggml_tensor->backend) {
                 case GGML_BACKEND_CPU:
@@ -726,11 +739,17 @@ struct llama_model_loader {
 #ifdef GGML_USE_CUBLAS
                 case GGML_BACKEND_CUDA:
                     ggml_cuda_load_data(lt.data, lt.ggml_tensor);
+                    if (!use_mmap) {
+                        free(lt.data);
+                    }
                     break;
 #endif
 #ifdef GGML_USE_CLBLAST
                 case GGML_BACKEND_CL:
                     ggml_cl_transform_tensor(lt.data, lt.ggml_tensor);
+                    if (!use_mmap) {
+                        free(lt.data);
+                    }
                     break;
 #endif
                 default: