Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions offload/languages/cuda/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
endif()

target_include_directories(LLVMcudart PUBLIC
${CMAKE_CURRENT_BINARY_DIR}/../include
${CMAKE_CURRENT_SOURCE_DIR}/../include/cuda
${CMAKE_CURRENT_SOURCE_DIR}/../include/kernel
${CMAKE_CURRENT_SOURCE_DIR}/include
${CMAKE_CURRENT_SOURCE_DIR}/../kernel/include
${CMAKE_CURRENT_SOURCE_DIR}/../../liboffload/include
Expand All @@ -36,4 +37,4 @@ set_target_properties(LLVMcudart PROPERTIES
BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
install(TARGETS LLVMcudart LIBRARY COMPONENT LLVMcudart DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")

install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/cuda_runtime.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload/cuda/)
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../include/cuda/cuda_runtime.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload/cuda/)
6 changes: 3 additions & 3 deletions offload/languages/cuda/src/cuda_runtime.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@

#define LANGUAGE cuda

#include "LanguageRuntime.cpp"
#include "../../kernel/src/LanguageRuntime.cpp"

#include "LanguageRegistration.cpp"
#include "../../kernel/src/LanguageRegistration.cpp"

#include "LanguageLaunch.cpp"
#include "../../kernel/src/LanguageLaunch.cpp"

// Must be last as it introduces alises for some definitions from above.
#include "LanguageAliases.h"
5 changes: 3 additions & 2 deletions offload/languages/hip/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
endif()

target_include_directories(LLVMhip64 PUBLIC
${CMAKE_CURRENT_BINARY_DIR}/../include
${CMAKE_CURRENT_SOURCE_DIR}/../include/hip
${CMAKE_CURRENT_SOURCE_DIR}/../include/kernel
${CMAKE_CURRENT_SOURCE_DIR}/include
${CMAKE_CURRENT_SOURCE_DIR}/../kernel/include
${CMAKE_CURRENT_SOURCE_DIR}/../../liboffload/include
Expand All @@ -36,4 +37,4 @@ set_target_properties(LLVMhip64 PROPERTIES
BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
install(TARGETS LLVMhip64 LIBRARY COMPONENT LLVMhip64 DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")

install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/hip_runtime.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload/hip/)
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../include/hip/hip_runtime.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload/hip/)
1 change: 0 additions & 1 deletion offload/languages/hip/include/hip/hip_runtime.h

This file was deleted.

6 changes: 3 additions & 3 deletions offload/languages/hip/src/hip_runtime.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@

#define LANGUAGE hip

#include "LanguageRuntime.cpp"
#include "../../kernel/src/LanguageRuntime.cpp"

#include "LanguageRegistration.cpp"
#include "../../kernel/src/LanguageRegistration.cpp"

#include "LanguageLaunch.cpp"
#include "../../kernel/src/LanguageLaunch.cpp"

// Must be last as it introduces alises for some definitions from above.
#include "LanguageAliases.h"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@

#define LANGUAGE cuda

#include "../../kernel/include/DefineLanguageNames.inc"
#include "../kernel/DefineLanguageNames.inc"

#include "../../kernel/include/LanguageRuntime.h"
#include "../kernel/LanguageRuntime.h"

#include "../../kernel/include/UndefineLanguageNames.inc"
#include "../kernel/UndefineLanguageNames.inc"

#undef LANGUAGE

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@

#define LANGUAGE hip

#include "../../kernel/include/DefineLanguageNames.inc"
#include "../kernel/DefineLanguageNames.inc"

#include "../../kernel/include/LanguageRuntime.h"
#include "../kernel/LanguageRuntime.h"

#include "../../kernel/include/UndefineLanguageNames.inc"
#include "../kernel/UndefineLanguageNames.inc"

#undef LANGUAGE

Expand Down
4 changes: 3 additions & 1 deletion offload/languages/kernel/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,6 @@ set_target_properties(LLVMOffloadKernel PROPERTIES
BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
install(TARGETS LLVMOffloadKernel LIBRARY COMPONENT LLVMOffloadKernel DESTINATION "${OFFLOAD_INSTALL_LIBDIR}")

install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/cuda_runtime.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload/cuda/)
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../include/kernel/DefineLanguageNames.inc DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload/kernel/)
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../include/kernel/LanguageRuntime.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload/kernel/)
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../include/kernel/UndefineLanguageNames.inc DESTINATION ${CMAKE_INSTALL_PREFIX}/include/offload/kernel/)
59 changes: 59 additions & 0 deletions offload/languages/kernel/include/LanguageLaunch.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
//===------ LanguageLaunch.h - Header for LanguageLaunch.cpp ------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
//===----------------------------------------------------------------------===//

#ifndef LLVM_LANGUAGE_LAUNCH_H
#define LLVM_LANGUAGE_LAUNCH_H

#include "ExportedAPI.h"
#include "Types.h"
#include "OffloadAPI.h"

#include <cstdint>
#include <cstddef>
#include <algorithm> // for std::max

extern "C" {

struct LLVMOffloadKernelArgsTy {
size_t Size;
void *Args;
void *_; // Reserved for future use
};

/// Push call configuration for kernel launch
unsigned llvmPushCallConfiguration(dim3 __grid_size, dim3 __block_size,
size_t __shared_memory, void *__stream);

/// Pop call configuration for kernel launch
unsigned llvmPopCallConfiguration(dim3 *__grid_size, dim3 *__block_size,
size_t *__shared_memory, void *__stream);

/// Internal kernel launch implementation
ol_result_t llvmLaunchKernelImpl(const char *KernelID, dim3 GridDim,
dim3 BlockDim, void *KernelArgsPtr,
size_t DynamicSharedMem, void *Stream,
LLVMOffloadKernelArgsTy *LOKA);

/// LLVM-style kernel launch entry points
unsigned __llvmLaunchKernel(const char *KernelID, dim3 GridDim, dim3 BlockDim,
void *KernelArgsPtr, size_t DynamicSharedMem,
void *Stream);

unsigned __llvmLaunchKernel_spt(const char *KernelID, dim3 GridDim,
dim3 BlockDim, void *KernelArgsPtr,
size_t DynamicSharedMem, void *Stream);

unsigned __llvmLaunchKernel_ptsz(const char *KernelID, dim3 GridDim,
dim3 BlockDim, void *KernelArgsPtr,
size_t DynamicSharedMem, void *Stream);

}

#endif // LLVM_LANGUAGE_LAUNCH_H
66 changes: 66 additions & 0 deletions offload/languages/kernel/include/LanguageRegistration.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
//===---- LanguageRegistration.h - Language (CUDA/HIP) registration api ---===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
//===----------------------------------------------------------------------===//

#include "ExportedAPI.h"

#include "OffloadAPI.h"

#include <cstdint>
#include <iterator>

#define HIP_FATBIN_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
constexpr auto HIP_FATBIN_MAGIC_STR_LEN = sizeof(HIP_FATBIN_MAGIC_STR) - 1;

namespace {
struct FatbinWrapperTy {
int Magic;
int Version;
const char *Data;
const char *DataEnd;
};

template <typename T> T readAndAdvance(const char *&Ptr) {
auto V = *reinterpret_cast<const T *>(Ptr);
std::advance(Ptr, sizeof(T));
return V;
}

} // namespace

static void readTUFatbin(const char *Binary, const FatbinWrapperTy *FW);

static void readHIPFatbinEntries(const char *Binary, const char *HIPFatbinPtr);

/// Hidden, but exported, Registration API
///{
extern "C" {

void llvmRegisterFunction(const char *Binary, const char *KernelID,
char *KernelName, const char *KernelName1, int,
uint3 *, uint3 *, dim3 *, dim3 *, int *);

const char *llvmRegisterFatBinary(const char *Binary);

void llvmUnregisterFatBinary(void *Handle);

void llvmRegisterVar(void **, char *, char *, const char *, int, int, int,
int);

void llvmRegisterManagedVar(void **, char *, char *, const char *, size_t,
unsigned);

void llvmRegisterSurface(void **, const struct surfaceReference *,
const void **, const char *, int, int);

void llvmRegisterTexture(void **, const struct textureReference *,
const void **, const char *, int, int, int);

}
///}
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,11 @@
//
//===----------------------------------------------------------------------===//

#include "ExportedAPI.h"
#include "Types.h"
#include "LanguageLaunch.h"

#include "OffloadAPI.h"

#include <cstdint>
#include <cstdio>

namespace {
struct LLVMOffloadKernelArgsTy {
size_t Size;
void *Args;
void *_;
};
extern "C" {

struct CallConfigurationTy {
dim3 GridSize;
Expand All @@ -31,12 +22,8 @@ struct CallConfigurationTy {
};

static thread_local CallConfigurationTy CC = {};
} // namespace

/// Hidden, but exported, Launch API
///{
extern "C" {

/// Push call configuration for kernel launch
unsigned llvmPushCallConfiguration(dim3 __grid_size, dim3 __block_size,
size_t __shared_memory, void *__stream) {
CallConfigurationTy &Kernel = CC;
Expand All @@ -47,6 +34,7 @@ unsigned llvmPushCallConfiguration(dim3 __grid_size, dim3 __block_size,
return 0;
}

/// Pop call configuration for kernel launch
unsigned llvmPopCallConfiguration(dim3 *__grid_size, dim3 *__block_size,
size_t *__shared_memory, void *__stream) {
CallConfigurationTy &Kernel = CC;
Expand All @@ -57,6 +45,7 @@ unsigned llvmPopCallConfiguration(dim3 *__grid_size, dim3 *__block_size,
return 0;
}

/// Internal kernel launch implementation
ol_result_t llvmLaunchKernelImpl(const char *KernelID, dim3 GridDim,
dim3 BlockDim, void *KernelArgsPtr,
size_t DynamicSharedMem, void *Stream,
Expand All @@ -74,16 +63,11 @@ ol_result_t llvmLaunchKernelImpl(const char *KernelID, dim3 GridDim,
LaunchSizeArgs.DynSharedMemory = DynamicSharedMem;
LaunchSizeArgs.Dimensions =
1 + !!(GridDim.y * BlockDim.y > 1) + !!(GridDim.z * BlockDim.z > 1);

ol_queue_handle_t Queue = Stream ? reinterpret_cast<ol_queue_handle_t>(Stream)
: olKGetDefaultQueue();

ol_result_t Result;
/// If LOKA is set, the kernel argument layout is known and already enforced
/// in LOKA->Args. Otherwise, indicate the plugins have to organize the
/// arguments themselves, as KernelArgsPtr is only an array of pointers to
/// arguments.
/// TODO: We should include APITypes.h and use
/// KernelLaunchParamsTy::UnknownSize instead of -1 below.
if (LOKA)
Result = olLaunchKernel(Queue, Device, Kernel, LOKA->Args, LOKA->Size,
&LaunchSizeArgs, nullptr);
Expand All @@ -95,18 +79,18 @@ ol_result_t llvmLaunchKernelImpl(const char *KernelID, dim3 GridDim,
}

#define LLVM_STYLE_LAUNCH(SUFFIX, PER_THREAD_STREAM) \
unsigned __llvmLaunchKernel##SUFFIX(const char *KernelID, dim3 GridDim, \
dim3 BlockDim, void *KernelArgsPtr, \
size_t DynamicSharedMem, void *Stream) { \
auto *LOKA = reinterpret_cast<LLVMOffloadKernelArgsTy *>(KernelArgsPtr); \
ol_result_t Result = \
llvmLaunchKernelImpl(KernelID, GridDim, BlockDim, KernelArgsPtr, \
DynamicSharedMem, Stream, LOKA); \
return Result ? Result->Code : 0; \
}
unsigned __llvmLaunchKernel##SUFFIX(const char *KernelID, dim3 GridDim, \
dim3 BlockDim, void *KernelArgsPtr, \
size_t DynamicSharedMem, void *Stream) {\
auto *LOKA = reinterpret_cast<LLVMOffloadKernelArgsTy *>(KernelArgsPtr); \
ol_result_t Result = \
llvmLaunchKernelImpl(KernelID, GridDim, BlockDim, KernelArgsPtr, \
DynamicSharedMem, Stream, LOKA); \
return Result ? Result->Code : 0; \
}

LLVM_STYLE_LAUNCH(, false);
LLVM_STYLE_LAUNCH(_spt, true);
LLVM_STYLE_LAUNCH(_ptsz, true);
}
///}

} // extern "C"
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//===---- LanguageRegistration.cpp - Language (CUDA/HIP) registration api -===//
//===---- LanguageRegistration.h - Language (CUDA/HIP) registration api ---===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
Expand All @@ -8,31 +8,7 @@
//
//===----------------------------------------------------------------------===//

#include "ExportedAPI.h"

#include "OffloadAPI.h"

#include <cstdint>
#include <iterator>

#define HIP_FATBIN_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
constexpr auto HIP_FATBIN_MAGIC_STR_LEN = sizeof(HIP_FATBIN_MAGIC_STR) - 1;

namespace {
struct FatbinWrapperTy {
int Magic;
int Version;
const char *Data;
const char *DataEnd;
};

template <typename T> T readAndAdvance(const char *&Ptr) {
auto V = *reinterpret_cast<const T *>(Ptr);
std::advance(Ptr, sizeof(T));
return V;
}

} // namespace
#include "LanguageRegistration.h"

static void readTUFatbin(const char *Binary, const FatbinWrapperTy *FW) {
ol_device_handle_t Device = olKGetDefaultDevice();
Expand Down Expand Up @@ -141,13 +117,16 @@ void llvmRegisterManagedVar(void **, char *, char *, const char *, size_t,
unsigned) {
fprintf(stderr, "RegisterManagedVar is not implemented!");
}

void llvmRegisterSurface(void **, const struct surfaceReference *,
const void **, const char *, int, int) {
fprintf(stderr, "RegisterSurface is not implemented!");
}

void llvmRegisterTexture(void **, const struct textureReference *,
const void **, const char *, int, int, int) {
fprintf(stderr, "RegisterTexture is not implemented!");
}

}
///}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
//
//===----------------------------------------------------------------------===//

#include "LanguageRuntime.h"

#ifndef LANGUAGE
#error This file should be included, or used, with a LANGUAGE macro set.
#endif
Expand Down