Skip to content

Commit bd9f7b4

Browse files
slarenslaren
committed
refactor cmake build
use MODULE target type for dl backend set backend output directory to the runtime directory ggml_backend_load_all searches backends in the system path first, then in the executable directory ggml-ci
1 parent 402a0e9 commit bd9f7b4

File tree

15 files changed

+220
-158
lines changed

15 files changed

+220
-158
lines changed

‎ggml/src/CMakeLists.txt‎

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -202,8 +202,8 @@ endif()
202202

203203
# ggml
204204

205-
if (GGML_BACKEND_DL)
206-
add_compile_definitions(GGML_BACKEND_DL)
205+
if (GGML_BACKEND_DL AND NOT BUILD_SHARED_LIBS)
206+
message(FATAL_ERROR "GGML_BACKEND_DL requires BUILD_SHARED_LIBS")
207207
endif()
208208

209209
add_library(ggml-base
@@ -234,6 +234,27 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux")
234234
target_link_libraries(ggml PRIVATE dl)
235235
endif()
236236

237+
function(ggml_add_backend_library backend)
238+
if (GGML_BACKEND_DL)
239+
add_library(${backend} MODULE ${ARGN})
240+
# write the shared library to the output directory
241+
set_target_properties(${backend} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
242+
target_compile_definitions(${backend} PRIVATE GGML_BACKEND_DL)
243+
else()
244+
add_library(${backend} ${ARGN})
245+
target_link_libraries(ggml PUBLIC ${backend})
246+
install(TARGETS ${backend} LIBRARY)
247+
endif()
248+
249+
target_link_libraries(${backend} PRIVATE ggml-base)
250+
target_include_directories(${backend} PRIVATE ..)
251+
252+
if (${BUILD_SHARED_LIBS})
253+
target_compile_definitions(${backend} PRIVATE GGML_BACKEND_BUILD)
254+
target_compile_definitions(${backend} PUBLIC GGML_BACKEND_SHARED)
255+
endif()
256+
endfunction()
257+
237258
function(ggml_add_backend backend)
238259
string(TOUPPER "GGML_${backend}" backend_id)
239260
if (${backend_id})
@@ -244,18 +265,7 @@ function(ggml_add_backend backend)
244265
# however, currently it is necessary for AMX, since it is enabled by default on llama.cpp
245266
if (${backend_id})
246267
message(STATUS "Including ${backend} backend")
247-
if (${BUILD_SHARED_LIBS})
248-
target_compile_definitions(${backend_target} PRIVATE GGML_BACKEND_BUILD)
249-
target_compile_definitions(${backend_target} PUBLIC GGML_BACKEND_SHARED)
250-
if (GGML_BACKEND_DL)
251-
target_compile_definitions(${backend_target} PRIVATE GGML_BACKEND_DL)
252-
endif()
253-
endif()
254-
if (GGML_BACKEND_DL)
255-
install(TARGETS ${backend_target} RUNTIME)
256-
else()
257-
install(TARGETS ${backend_target} LIBRARY)
258-
target_link_libraries(ggml PUBLIC ${backend_target})
268+
if (NOT GGML_BACKEND_DL)
259269
string(TOUPPER "GGML_USE_${backend}" backend_use)
260270
target_compile_definitions(ggml PUBLIC ${backend_use})
261271
endif()
@@ -271,10 +281,10 @@ ggml_add_backend(CUDA)
271281
ggml_add_backend(HIP)
272282
ggml_add_backend(Kompute)
273283
ggml_add_backend(METAL)
284+
ggml_add_backend(MUSA)
274285
ggml_add_backend(RPC)
275286
ggml_add_backend(SYCL)
276287
ggml_add_backend(Vulkan)
277-
ggml_add_backend(MUSA)
278288

279289
foreach (target ggml-base ggml)
280290
target_include_directories(${target} PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include> $<INSTALL_INTERFACE:include>)

‎ggml/src/ggml-amx/CMakeLists.txt‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,10 @@ if (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR CMAKE_GENERATOR_PLATFORM_LWR MA
99

1010
file(GLOB GGML_SOURCES_AMX "*.cpp")
1111

12-
add_library(ggml-amx
13-
${GGML_HEADERS_AMX}
14-
${GGML_SOURCES_AMX})
15-
16-
target_link_libraries(ggml-amx PRIVATE ggml-base)
17-
target_include_directories(ggml-amx PRIVATE . ..)
12+
ggml_add_backend_library(ggml-amx
13+
${GGML_HEADERS_AMX}
14+
${GGML_SOURCES_AMX}
15+
)
1816

1917
# this is duplicated from the CPU backend, since the AMX backend also depends on the architecture flags
2018
# TODO: integrate AMX backend into the CPU backend

‎ggml/src/ggml-backend-impl.h‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -63,20 +63,20 @@ extern "C" {
6363
enum ggml_backend_buffer_usage usage;
6464
};
6565

66-
ggml_backend_buffer_t ggml_backend_buffer_init(
66+
GGML_API ggml_backend_buffer_t ggml_backend_buffer_init(
6767
ggml_backend_buffer_type_t buft,
6868
struct ggml_backend_buffer_i iface,
6969
void * context,
7070
size_t size);
7171

7272
// do not use directly, use ggml_backend_tensor_copy instead
73-
bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);
73+
GGML_API bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);
7474

7575
// multi-buffer
7676
// buffer that contains a collection of buffers
77-
ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);
78-
bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);
79-
void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
77+
GGML_API ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);
78+
GGML_API bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);
79+
GGML_API void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
8080

8181
//
8282
// Backend (stream)
@@ -205,10 +205,12 @@ extern "C" {
205205
};
206206

207207
// Internal backend registry API
208-
void ggml_backend_register(ggml_backend_reg_t reg);
209-
void ggml_backend_device_register(ggml_backend_dev_t device);
208+
GGML_API void ggml_backend_register(ggml_backend_reg_t reg);
209+
GGML_API void ggml_backend_device_register(ggml_backend_dev_t device);
210210

211211
// Add backend dynamic loading support to the backend
212+
typedef ggml_backend_reg_t (*ggml_backend_init_t)(void);
213+
212214
#ifdef GGML_BACKEND_DL
213215
#ifdef __cplusplus
214216
# define GGML_BACKEND_DL_IMPL(reg_fn) \

‎ggml/src/ggml-backend-reg.cpp‎

Lines changed: 134 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include "ggml-impl.h"
44
#include <algorithm>
55
#include <cstring>
6+
#include <string>
67
#include <vector>
78

89
#ifdef _WIN32
@@ -11,11 +12,14 @@
1112
# define NOMINMAX
1213
# endif
1314
# include <windows.h>
15+
#elif defined(__APPLE__)
16+
# include <mach-o/dyld.h>
17+
# include <dlfcn.h>
1418
#else
1519
# include <dlfcn.h>
20+
# include <unistd.h>
1621
#endif
1722

18-
1923
// Backend registry
2024
#ifdef GGML_USE_CPU
2125
#include "ggml-cpu.h"
@@ -128,10 +132,59 @@ struct ggml_backend_registry {
128132
devices.push_back(device);
129133
}
130134

131-
void unload_backend(ggml_backend_reg_t reg, bool silent) {
132-
if (!silent) {
133-
GGML_LOG_INFO("%s: unloading %s backend\n", __func__, ggml_backend_reg_name(reg));
135+
ggml_backend_reg_t load_backend(const char * path, bool silent) {
136+
#ifdef _WIN32
137+
HMODULE handle = LoadLibraryA(path);
138+
if (!handle) {
139+
if (!silent) {
140+
GGML_LOG_ERROR("%s: failed to load %s: %lu\n", __func__, path, GetLastError());
141+
}
142+
return nullptr;
143+
}
144+
ggml_backend_init_t backend_init = (ggml_backend_init_t) GetProcAddress(handle, "ggml_backend_init");
145+
if (!backend_init) {
146+
if (!silent) {
147+
GGML_LOG_ERROR("%s: failed to find ggml_backend_init in %s: %lu\n", __func__, path, GetLastError());
148+
}
149+
FreeLibrary(handle);
150+
return nullptr;
151+
}
152+
#else
153+
void * handle = dlopen(path, RTLD_NOW | RTLD_LOCAL);
154+
if (!handle) {
155+
if (!silent) {
156+
GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, path, dlerror());
157+
}
158+
return nullptr;
159+
}
160+
auto * backend_init = (ggml_backend_init_t) dlsym(handle, "ggml_backend_init");
161+
if (!backend_init) {
162+
if (!silent) {
163+
GGML_LOG_ERROR("%s: failed to find ggml_backend_init in %s: %s\n", __func__, path, dlerror());
164+
}
165+
dlclose(handle);
166+
return nullptr;
167+
}
168+
#endif
169+
ggml_backend_reg_t reg = backend_init();
170+
if (!reg) {
171+
if (!silent) {
172+
GGML_LOG_ERROR("%s: failed to initialize backend from %s\n", __func__, path);
173+
}
174+
#ifdef _WIN32
175+
FreeLibrary(handle);
176+
#else
177+
dlclose(handle);
178+
#endif
179+
return nullptr;
134180
}
181+
182+
GGML_LOG_INFO("%s: loaded %s backend from %s\n", __func__, ggml_backend_reg_name(reg), path);
183+
register_backend(reg, handle);
184+
return reg;
185+
}
186+
187+
void unload_backend(ggml_backend_reg_t reg, bool silent) {
135188
auto it = std::find_if(backends.begin(), backends.end(),
136189
[reg](ggml_backend_reg_entry entry) { return entry.reg == reg; });
137190

@@ -258,75 +311,92 @@ ggml_backend_t ggml_backend_init_best(void) {
258311
return ggml_backend_dev_init(dev, nullptr);
259312
}
260313

261-
typedef ggml_backend_reg_t (*ggml_backend_init_t)(void);
262-
314+
// Dynamic loading
263315
ggml_backend_reg_t ggml_backend_load(const char * path) {
264-
#ifdef _WIN32
265-
HMODULE handle = LoadLibraryA(path);
266-
if (!handle) {
267-
GGML_LOG_ERROR("%s: failed to load %s: %lu\n", __func__, path, GetLastError());
268-
return nullptr;
269-
}
270-
ggml_backend_init_t backend_init = (ggml_backend_init_t) GetProcAddress(handle, "ggml_backend_init");
271-
if (!backend_init) {
272-
GGML_LOG_ERROR("%s: failed to find ggml_backend_init in %s: %lu\n", __func__, path, GetLastError());
273-
FreeLibrary(handle);
274-
return nullptr;
275-
}
276-
#else
277-
void * handle = dlopen(path, RTLD_NOW | RTLD_LOCAL);
278-
if (!handle) {
279-
GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, path, dlerror());
280-
return nullptr;
281-
}
282-
auto * backend_init = (ggml_backend_init_t) dlsym(handle, "ggml_backend_init");
283-
if (!backend_init) {
284-
GGML_LOG_ERROR("%s: failed to find ggml_backend_init in %s: %s\n", __func__, path, dlerror());
285-
dlclose(handle);
286-
return nullptr;
287-
}
288-
#endif
289-
ggml_backend_reg_t reg = backend_init();
290-
if (!reg) {
291-
GGML_LOG_ERROR("%s: failed to initialize backend from %s\n", __func__, path);
292-
#ifdef _WIN32
293-
FreeLibrary(handle);
294-
#else
295-
dlclose(handle);
296-
#endif
297-
return nullptr;
298-
}
299-
300-
GGML_LOG_DEBUG("%s: loaded %s backend from %s\n", __func__, ggml_backend_reg_name(reg), path);
301-
get_reg().register_backend(reg, handle);
302-
return reg;
316+
return get_reg().load_backend(path, false);
303317
}
304318

305319
void ggml_backend_unload(ggml_backend_reg_t reg) {
306320
get_reg().unload_backend(reg, true);
307321
}
308322

309323
void ggml_backend_load_all() {
324+
std::vector<std::string> search_prefix;
325+
326+
// add the executable directory to the search path
327+
// FIXME: this is convenient for development, but it should probably be disabled in production
328+
329+
#if defined(__APPLE__)
330+
// get executable path
331+
std::vector<char> path;
332+
uint32_t size;
333+
while (true) {
334+
size = path.size();
335+
if (_NSGetExecutablePath(path.data(), &size) == 0) {
336+
break;
337+
}
338+
path.resize(size);
339+
}
340+
std::string base_path(path.data(), size);
341+
// remove executable name
342+
auto last_slash = base_path.find_last_of('/');
343+
if (last_slash != std::string::npos) {
344+
base_path = base_path.substr(0, last_slash);
345+
}
346+
search_prefix.push_back(base_path + "/");
347+
#elif defined(__linux__)
348+
std::string base_path = ".";
349+
std::vector<char> path(1024);
350+
while (true) {
351+
// get executable path
352+
ssize_t len = readlink("/proc/self/exe", path.data(), path.size());
353+
if (len == -1) {
354+
break;
355+
}
356+
if (len < (ssize_t) path.size()) {
357+
base_path = std::string(path.data(), len);
358+
// remove executable name
359+
auto last_slash = base_path.find_last_of('/');
360+
if (last_slash != std::string::npos) {
361+
base_path = base_path.substr(0, last_slash);
362+
}
363+
break;
364+
}
365+
path.resize(path.size() * 2);
366+
}
367+
368+
search_prefix.push_back(base_path + "/");
369+
#endif
370+
371+
auto & reg = get_reg();
372+
373+
auto try_load = [&](const std::string & name) {
374+
std::string os_name;
310375
#ifdef _WIN32
311-
#define GGML_BACKEND_PATH(backend) "ggml-" backend ".dll"
312-
#elif defined(__APPLE__)
313-
// path is hardcoded to the cmake build directory for now
314-
// FIXME: should also search default system paths
315-
#define GGML_BACKEND_PATH(backend) "build/ggml/src/ggml-" backend "/libggml-" backend ".dylib"
376+
os_name = "ggml-" + name + ".dll";
316377
#else
317-
#define GGML_BACKEND_PATH(backend) "build/ggml/src/ggml-" backend "/libggml-" backend ".so"
378+
os_name = "libggml-" + name + ".so";
318379
#endif
319-
320-
ggml_backend_load(GGML_BACKEND_PATH("amx"));
321-
ggml_backend_load(GGML_BACKEND_PATH("blas"));
322-
ggml_backend_load(GGML_BACKEND_PATH("cann"));
323-
ggml_backend_load(GGML_BACKEND_PATH("cuda"));
324-
ggml_backend_load(GGML_BACKEND_PATH("hip"));
325-
ggml_backend_load(GGML_BACKEND_PATH("kompute"));
326-
ggml_backend_load(GGML_BACKEND_PATH("metal"));
327-
ggml_backend_load(GGML_BACKEND_PATH("rpc"));
328-
ggml_backend_load(GGML_BACKEND_PATH("sycl"));
329-
ggml_backend_load(GGML_BACKEND_PATH("vulkan"));
330-
ggml_backend_load(GGML_BACKEND_PATH("musa"));
331-
ggml_backend_load(GGML_BACKEND_PATH("cpu"));
380+
if (reg.load_backend(os_name.c_str(), true)) {
381+
return;
382+
}
383+
for (const auto & prefix : search_prefix) {
384+
if (reg.load_backend((prefix + os_name).c_str(), true)) {
385+
return;
386+
}
387+
}
388+
};
389+
390+
try_load("amx");
391+
try_load("blas");
392+
try_load("cann");
393+
try_load("cuda");
394+
try_load("hip");
395+
try_load("kompute");
396+
try_load("metal");
397+
try_load("rpc");
398+
try_load("sycl");
399+
try_load("vulkan");
400+
try_load("musa");
401+
try_load("cpu");
332402
}

‎ggml/src/ggml-blas/CMakeLists.txt‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,9 @@ find_package(BLAS)
1111
if (BLAS_FOUND)
1212
message(STATUS "BLAS found, Libraries: ${BLAS_LIBRARIES}")
1313

14-
add_library(ggml-blas
15-
ggml-blas.cpp
16-
)
17-
18-
target_link_libraries(ggml-blas PRIVATE ggml-base)
19-
target_include_directories(ggml-blas PRIVATE . ..)
14+
ggml_add_backend_library(ggml-blas
15+
ggml-blas.cpp
16+
)
2017

2118
if (${GGML_BLAS_VENDOR} MATCHES "Apple")
2219
add_compile_definitions(ACCELERATE_NEW_LAPACK)

‎ggml/src/ggml-cann/CMakeLists.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,9 @@ if (CANN_INSTALL_DIR)
6161

6262
file(GLOB GGML_SOURCES_CANN "*.cpp")
6363

64-
add_library(ggml-cann ${GGML_SOURCES_CANN})
65-
target_link_libraries(ggml-cann PRIVATE ggml-base ${CANN_LIBRARIES})
66-
target_include_directories(ggml-cann PRIVATE . .. ${CANN_INCLUDE_DIRS})
64+
ggml_add_backend_library(ggml-cann ${GGML_SOURCES_CANN})
65+
target_link_libraries(ggml-cann PRIVATE ${CANN_LIBRARIES})
66+
target_include_directories(ggml-cann PRIVATE ${CANN_INCLUDE_DIRS})
6767
target_link_directories(ggml-cann PRIVATE ${CANN_INSTALL_DIR}/lib64)
6868

6969
target_compile_definitions(ggml-cann PRIVATE "-D${SOC_TYPE_COMPILE_OPTION}")

0 commit comments

Comments
 (0)