Skip to content

GPU Sanitizer support for shared memory #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Oct 10, 2024
Merged

GPU Sanitizer support for shared memory #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
c794896
Refactor GEP operand change
EthanLuisMcDonough Jul 25, 2024
44fdee6
Coverage for globals in shared memory
EthanLuisMcDonough Sep 7, 2024
a9e806e
Detect shadow global type
EthanLuisMcDonough Sep 12, 2024
a050d26
Instrument AtomicRMWInst
EthanLuisMcDonough Sep 23, 2024
25af6e2
Fix test formatting
EthanLuisMcDonough Sep 30, 2024
9708f8b
Undo inttoptr labeling
EthanLuisMcDonough Sep 30, 2024
0eeadf7
Replace user globals in GEP
EthanLuisMcDonough Oct 1, 2024
225d0b2
Add additional tests
EthanLuisMcDonough Oct 1, 2024
8703c6d
Replace constants in PHI and return instructions
EthanLuisMcDonough Oct 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
433 changes: 320 additions & 113 deletions llvm/lib/Transforms/Instrumentation/GPUSan.cpp
View file
Open in desktop

Large diffs are not rendered by default.

166 changes: 131 additions & 35 deletions offload/DeviceRTL/src/Sanitizer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ struct AllocationInfoGlobalTy {
uint64_t Length;
uint32_t Tag;
};
struct AllocationInfoSharedTy {
_AS_PTR(void, AllocationKind::SHARED) Start;
uint64_t Length;
uint32_t Tag;
};

template <AllocationKind AK> struct AllocationInfoTy {};
template <> struct AllocationInfoTy<AllocationKind::GLOBAL> {
Expand All @@ -41,6 +46,9 @@ template <> struct AllocationInfoTy<AllocationKind::GLOBAL> {
template <> struct AllocationInfoTy<AllocationKind::LOCAL> {
using ASVoidPtrTy = AllocationInfoLocalTy;
};
template <> struct AllocationInfoTy<AllocationKind::SHARED> {
using ASVoidPtrTy = AllocationInfoSharedTy;
};

template <>
AllocationPtrTy<AllocationKind::LOCAL>
Expand Down Expand Up @@ -88,7 +96,7 @@ template <AllocationKind AK> struct AllocationTracker {
// Reserve the 0 element for the null pointer in global space.
auto &AllocArr = getAllocationArray<AK>();
auto &Cnt = AllocArr.Cnt;
if constexpr (AK == AllocationKind::LOCAL)
if constexpr (AK == AllocationKind::LOCAL || AK == AllocationKind::SHARED)
Slot = ++Cnt;
if (Slot == -1)
Slot = ++Cnt;
Expand Down Expand Up @@ -155,10 +163,14 @@ template <AllocationKind AK> struct AllocationTracker {
if constexpr (AK == AllocationKind::LOCAL)
if (Length == 0)
Length = getAllocation<AK>(AP, AccessId, PC).Length;
if constexpr (AK == AllocationKind::GLOBAL)
if (AP.Magic != SanitizerConfig<AllocationKind::GLOBAL>::MAGIC)
if constexpr (AK == AllocationKind::GLOBAL ||
AK == AllocationKind::SHARED) {
if (AP.Magic != SanitizerConfig<AllocationKind::GLOBAL>::MAGIC) {
__sanitizer_trap_info_ptr->garbagePointer<AK>(AP, (void *)P, SourceId,
PC);
}
}

int64_t Offset = AP.Offset;
if (OMP_UNLIKELY(
Offset > Length - Size ||
Expand Down Expand Up @@ -212,29 +224,40 @@ template <AllocationKind AK> struct AllocationTracker {
__sanitizer_trap_info_ptr->memoryLeak<AK>(A, Slot);
}
}

[[clang::disable_sanitizer_instrumentation]] static bool
checkPtr(void *P, int64_t SourceId, uint64_t PC) {
auto AP = AllocationPtrTy<AK>::get(P);
if ((AllocationKind)AP.Kind != AK)
return false;
if (AP.Magic != SanitizerConfig<AK>::MAGIC)
__sanitizer_trap_info_ptr->garbagePointer<AK>(AP, P, SourceId, PC);
return true;
}
};

template <AllocationKind AK>
AllocationArrayTy<AK>
Allocations<AK>::Arr[SanitizerConfig<AK>::NUM_ALLOCATION_ARRAYS];

static void checkForMagic(bool IsGlobal, void *P, int64_t SourceId,
uint64_t PC) {
if (IsGlobal) {
auto AP = AllocationPtrTy<AllocationKind::GLOBAL>::get(P);
if (AP.Magic != SanitizerConfig<AllocationKind::GLOBAL>::MAGIC)
__sanitizer_trap_info_ptr->garbagePointer<AllocationKind::GLOBAL>(
AP, P, SourceId, PC);
} else {
auto AP = AllocationPtrTy<AllocationKind::LOCAL>::get(P);
if (AP.Magic != SanitizerConfig<AllocationKind::LOCAL>::MAGIC)
__sanitizer_trap_info_ptr->garbagePointer<AllocationKind::LOCAL>(
AP, P, SourceId, PC);
}
[[clang::disable_sanitizer_instrumentation,
gnu::always_inline]] static AllocationKind
getFakePtrType(void *P, int64_t SourceId, uint64_t PC) {
if (AllocationTracker<AllocationKind::SHARED>::checkPtr(P, SourceId, PC))
return AllocationKind::SHARED;
if (AllocationTracker<AllocationKind::GLOBAL>::checkPtr(P, SourceId, PC))
return AllocationKind::GLOBAL;
if (AllocationTracker<AllocationKind::LOCAL>::checkPtr(P, SourceId, PC))
return AllocationKind::LOCAL;

// Couldn't determine type
__sanitizer_trap_info_ptr->garbagePointer<AllocationKind::LOCAL>(
AllocationPtrTy<AllocationKind::LOCAL>::get(P), P, SourceId, PC);
}

extern "C" {

#define REAL_PTR_IS_SHARED(PTR) (isSharedMemPtr(PTR))
#define REAL_PTR_IS_LOCAL(PTR) (isThreadLocalMemPtr(PTR))
#define IS_GLOBAL(PTR) ((uintptr_t)PTR & (1UL << 63))

Expand All @@ -253,6 +276,14 @@ extern "C" {
return AllocationTracker<AllocationKind::GLOBAL>::create(
Start, Length, AllocationId, -1, SourceId, PC);
}
[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
gnu::used, gnu::retain]] _AS_PTR(void, AllocationKind::SHARED)
ompx_new_shared(_AS_PTR(void, AllocationKind::SHARED) Start,
uint64_t Length, int64_t AllocationId, int64_t SourceId,
uint64_t PC) {
return AllocationTracker<AllocationKind::SHARED>::create(
Start, Length, AllocationId, 0, SourceId, PC);
}
[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
gnu::used, gnu::retain]] void
__sanitizer_register_host(_AS_PTR(void, AllocationKind::GLOBAL) Start,
Expand All @@ -264,6 +295,9 @@ __sanitizer_register_host(_AS_PTR(void, AllocationKind::GLOBAL) Start,
gnu::used, gnu::retain]] void *
ompx_new(void *Start, uint64_t Length, int64_t AllocationId, int64_t SourceId,
uint64_t PC) {
if (REAL_PTR_IS_SHARED(Start))
return (void *)ompx_new_shared((_AS_PTR(void, AllocationKind::SHARED))Start,
Length, AllocationId, SourceId, PC);
if (REAL_PTR_IS_LOCAL(Start))
return (void *)ompx_new_local((_AS_PTR(void, AllocationKind::LOCAL))Start,
Length, AllocationId, SourceId, PC);
Expand All @@ -290,14 +324,23 @@ ompx_free_local(_AS_PTR(void, AllocationKind::LOCAL) P, int64_t SourceId) {
ompx_free_global(_AS_PTR(void, AllocationKind::GLOBAL) P, int64_t SourceId) {
return AllocationTracker<AllocationKind::GLOBAL>::remove(P, SourceId);
}
[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
gnu::used, gnu::retain]] void
ompx_free_shared(_AS_PTR(void, AllocationKind::SHARED) P, int64_t SourceId) {
return AllocationTracker<AllocationKind::SHARED>::remove(P, SourceId);
}
[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
gnu::used, gnu::retain]] void
ompx_free(void *P, int64_t SourceId, uint64_t PC) {
bool IsGlobal = IS_GLOBAL(P);
checkForMagic(IsGlobal, P, SourceId, PC);
if (IsGlobal)
auto PtrKind = getFakePtrType(P, SourceId, PC);
switch (PtrKind) {
case AllocationKind::GLOBAL:
return ompx_free_global((_AS_PTR(void, AllocationKind::GLOBAL))P, SourceId);
return ompx_free_local((_AS_PTR(void, AllocationKind::LOCAL))P, SourceId);
case AllocationKind::LOCAL:
return ompx_free_local((_AS_PTR(void, AllocationKind::LOCAL))P, SourceId);
case AllocationKind::SHARED:
return ompx_free_shared((_AS_PTR(void, AllocationKind::SHARED))P, SourceId);
}
}

[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
Expand All @@ -313,16 +356,28 @@ ompx_free(void *P, int64_t SourceId, uint64_t PC) {
return AllocationTracker<AllocationKind::GLOBAL>::advance(P, Offset,
SourceId);
}
[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
gnu::used, gnu::retain]] _AS_PTR(void, AllocationKind::SHARED)
ompx_gep_shared(_AS_PTR(void, AllocationKind::SHARED) P, uint64_t Offset,
int64_t SourceId) {
return AllocationTracker<AllocationKind::SHARED>::advance(P, Offset,
SourceId);
}
[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
gnu::used, gnu::retain]] void *
ompx_gep(void *P, uint64_t Offset, int64_t SourceId) {
bool IsGlobal = IS_GLOBAL(P);
checkForMagic(IsGlobal, P, SourceId, /*PC=*/0);
if (IsGlobal)
auto PtrKind = getFakePtrType(P, SourceId, 0);
switch (PtrKind) {
case AllocationKind::GLOBAL:
return (void *)ompx_gep_global((_AS_PTR(void, AllocationKind::GLOBAL))P,
Offset, SourceId);
return (void *)ompx_gep_local((_AS_PTR(void, AllocationKind::LOCAL))P, Offset,
SourceId);
case AllocationKind::LOCAL:
return (void *)ompx_gep_local((_AS_PTR(void, AllocationKind::LOCAL))P,
Offset, SourceId);
case AllocationKind::SHARED:
return (void *)ompx_gep_shared((_AS_PTR(void, AllocationKind::SHARED))P,
Offset, SourceId);
}
}

[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
Expand All @@ -339,17 +394,29 @@ ompx_gep(void *P, uint64_t Offset, int64_t SourceId) {
return AllocationTracker<AllocationKind::GLOBAL>::check(P, Size, AccessId,
SourceId, PC);
}
[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
gnu::used, gnu::retain]] _AS_PTR(void, AllocationKind::SHARED)
ompx_check_shared(_AS_PTR(void, AllocationKind::SHARED) P, uint64_t Size,
uint64_t AccessId, int64_t SourceId, uint64_t PC) {
return AllocationTracker<AllocationKind::SHARED>::check(P, Size, AccessId,
SourceId, PC);
}
[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
gnu::used, gnu::retain]] void *
ompx_check(void *P, uint64_t Size, uint64_t AccessId, int64_t SourceId,
uint64_t PC) {
bool IsGlobal = IS_GLOBAL(P);
checkForMagic(IsGlobal, P, SourceId, PC);
if (IsGlobal)
auto PtrKind = getFakePtrType(P, SourceId, PC);
switch (PtrKind) {
case AllocationKind::GLOBAL:
return (void *)ompx_check_global((_AS_PTR(void, AllocationKind::GLOBAL))P,
Size, AccessId, SourceId, PC);
return (void *)ompx_check_local((_AS_PTR(void, AllocationKind::LOCAL))P, Size,
AccessId, SourceId, PC);
case AllocationKind::LOCAL:
return (void *)ompx_check_local((_AS_PTR(void, AllocationKind::LOCAL))P,
Size, AccessId, SourceId, PC);
case AllocationKind::SHARED:
return (void *)ompx_check_shared((_AS_PTR(void, AllocationKind::SHARED))P,
Size, AccessId, SourceId, PC);
}
}

[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
Expand All @@ -374,6 +441,17 @@ ompx_check(void *P, uint64_t Size, uint64_t AccessId, int64_t SourceId,
P, Start, Length, Tag, Size, AccessId, SourceId, PC);
}

[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
gnu::used, gnu::retain]] _AS_PTR(void, AllocationKind::SHARED)
ompx_check_with_base_shared(_AS_PTR(void, AllocationKind::SHARED) P,
_AS_PTR(void, AllocationKind::SHARED) Start,
uint64_t Length, uint32_t Tag, uint64_t Size,
uint64_t AccessId, int64_t SourceId,
uint64_t PC) {
return AllocationTracker<AllocationKind::SHARED>::checkWithBase(
P, Start, Length, Tag, Size, AccessId, SourceId, PC);
}

[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
gnu::used, gnu::retain]] _AS_PTR(void, AllocationKind::LOCAL)
ompx_unpack_local(_AS_PTR(void, AllocationKind::LOCAL) P,
Expand All @@ -388,16 +466,29 @@ ompx_check(void *P, uint64_t Size, uint64_t AccessId, int64_t SourceId,
return AllocationTracker<AllocationKind::GLOBAL>::unpack(P, SourceId,
/*PC=*/0);
}
[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
gnu::used, gnu::retain]] _AS_PTR(void, AllocationKind::SHARED)
ompx_unpack_shared(_AS_PTR(void, AllocationKind::SHARED) P,
int64_t SourceId) {
return AllocationTracker<AllocationKind::SHARED>::unpack(P, SourceId,
/*PC=*/0);
}
[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
gnu::used, gnu::retain]] void *
ompx_unpack(void *P, int64_t SourceId) {
bool IsGlobal = IS_GLOBAL(P);
checkForMagic(IsGlobal, P, SourceId, /*PC=*/0);
if (IsGlobal)
printf("UNPACK GENERIC %p\n", P);
auto PtrKind = getFakePtrType(P, SourceId, 0);
switch (PtrKind) {
case AllocationKind::GLOBAL:
return (void *)ompx_unpack_global((_AS_PTR(void, AllocationKind::GLOBAL))P,
SourceId);
return (void *)ompx_unpack_local((_AS_PTR(void, AllocationKind::LOCAL))P,
SourceId);
case AllocationKind::LOCAL:
return (void *)ompx_unpack_local((_AS_PTR(void, AllocationKind::LOCAL))P,
SourceId);
case AllocationKind::SHARED:
return (void *)ompx_unpack_shared((_AS_PTR(void, AllocationKind::SHARED))P,
SourceId);
}
}

[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
Expand All @@ -421,6 +512,11 @@ ompx_get_allocation_info_local(_AS_PTR(void, AllocationKind::LOCAL) P) {
ompx_get_allocation_info_global(_AS_PTR(void, AllocationKind::GLOBAL) P) {
return AllocationTracker<AllocationKind::GLOBAL>::getAllocationInfo(P);
}
[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
gnu::used, gnu::retain]] struct AllocationInfoSharedTy
ompx_get_allocation_info_shared(_AS_PTR(void, AllocationKind::SHARED) P) {
return AllocationTracker<AllocationKind::SHARED>::getAllocationInfo(P);
}

[[clang::disable_sanitizer_instrumentation, gnu::flatten, gnu::always_inline,
gnu::used, gnu::retain]] void
Expand Down
26 changes: 21 additions & 5 deletions offload/include/Shared/Sanitizer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ int64_t __san_get_location_value();
#define INLINE gnu::always_inline
#define NOINLINE gnu::noinline

enum class AllocationKind { LOCAL, GLOBAL, LAST = GLOBAL };
enum class AllocationKind { LOCAL, GLOBAL, SHARED, LAST = SHARED };
constexpr uint32_t FAKE_PTR_KIND_BITS = 2;

template <AllocationKind AK> struct ASTypes {
using INT_TY = uint64_t;
Expand All @@ -33,22 +34,36 @@ template <AllocationKind AK> struct ASTypes {
template <> struct ASTypes<AllocationKind::LOCAL> {
using INT_TY = uint32_t;
};
template <> struct ASTypes<AllocationKind::SHARED> {
using INT_TY = uint32_t;
};
#pragma omp end declare variant

template <AllocationKind AK> struct ASAddrSpace {};
template <> struct ASAddrSpace<AllocationKind::GLOBAL> {
static constexpr uint32_t ADDR_SPACE = 0;
};
template <> struct ASAddrSpace<AllocationKind::LOCAL> {
static constexpr uint32_t ADDR_SPACE = 5;
};
template <> struct ASAddrSpace<AllocationKind::SHARED> {
static constexpr uint32_t ADDR_SPACE = 3;
};

template <AllocationKind AK> struct SanitizerConfig {
static constexpr uint32_t ADDR_SPACE = AK == AllocationKind::GLOBAL ? 0 : 5;
static constexpr uint32_t ADDR_SPACE = ASAddrSpace<AK>::ADDR_SPACE;
static constexpr uint32_t ADDR_SPACE_PTR_SIZE =
sizeof(typename ASTypes<AK>::INT_TY) * 8;

static constexpr uint32_t NUM_ALLOCATION_ARRAYS =
AK == AllocationKind::GLOBAL ? 1 : (1024 * 1024 * 2);
static constexpr uint32_t TAG_BITS = AK == AllocationKind::GLOBAL ? 1 : 8;
AK == AllocationKind::LOCAL ? (1024 * 1024 * 2) : 1;
static constexpr uint32_t TAG_BITS = AK == AllocationKind::LOCAL ? 8 : 1;
static constexpr uint32_t MAGIC_BITS = 3;
static constexpr uint32_t MAGIC = 0b101;

static constexpr uint32_t OBJECT_BITS = AK == AllocationKind::GLOBAL ? 10 : 7;
static constexpr uint32_t SLOTS = (1 << (OBJECT_BITS));
static constexpr uint32_t KIND_BITS = 1;
static constexpr uint32_t KIND_BITS = FAKE_PTR_KIND_BITS;
static constexpr uint32_t Id_BITS = 9 - KIND_BITS;

static constexpr uint32_t LENGTH_BITS =
Expand Down Expand Up @@ -104,6 +119,7 @@ template <AllocationKind AK> struct AllocationPtrTy {
};
#pragma omp begin declare variant match(device = {arch(amdgcn)})
static_assert(sizeof(AllocationPtrTy<AllocationKind::LOCAL>) * 8 == 32);
static_assert(sizeof(AllocationPtrTy<AllocationKind::SHARED>) * 8 == 32);
#pragma omp end declare variant

union TypePunUnion {
Expand Down
25 changes: 22 additions & 3 deletions offload/plugins-nextgen/common/include/PluginInterface.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -678,8 +678,20 @@ struct GPUSanTy {
void addGPUSanNewFn(GenericKernelTy &GK) { NewFns.push_back(&GK); }
void addGPUSanFreeFn(GenericKernelTy &GK) { FreeFns.push_back(&GK); }
void checkAndReportError();

bool hasSharedShadow(const char *GlobalName,
SmallVector<DeviceImageTy *> &Images);

Error transferFakePtrToDevice(const char *GlobalName, void *FakeHstPtr,
SmallVector<DeviceImageTy *> &Images);
Error readFakePtrFromDevice(const char *GlobalName, void *&FakeHstPtr,
SmallVector<DeviceImageTy *> &Images);

static std::string getShadowName(const char *GlobalName) {
std::string ShadowName("__san.global.");
ShadowName.append(GlobalName);
return ShadowName;
}

private:
uint32_t SlotCnt = SanitizerConfig<AllocationKind::GLOBAL>::SLOTS - 1;
Expand Down Expand Up @@ -825,13 +837,20 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
return PinnedAllocs.unlockUnmappedHostBuffer(HstPtr);
}

/// Transfers a fake pointer to its respective shadow variable to prevent
/// double initializing GPUSan shadow constants. Only runs if GPUSan is
/// enabled
/// Transfers a fake pointer to its respective shadow variable. Only run
/// if the shadow variable was not read from the device already. Only runs
/// if GPUSan is enabled
Error transferFakePtrToDevice(const char *GlobalName, void *FakeHstPtr) {
return GPUSan.transferFakePtrToDevice(GlobalName, FakeHstPtr, LoadedImages);
}

/// Attempts to read a fake pointer to its respective shadow variable to
/// prevent double initializing GPUSan shadow constants. FakeHstPtr will be
/// null if no corresponding shadow global is found
Error readFakePtrFromDevice(const char *GlobalName, void *&FakeHstPtr) {
return GPUSan.readFakePtrFromDevice(GlobalName, FakeHstPtr, LoadedImages);
}

/// Check whether the host buffer with address \p HstPtr is pinned by the
/// underlying vendor-specific runtime (if any). Retrieve the host pointer,
/// the device accessible pointer and the size of the original pinned buffer.
Expand Down
Loading