summaryrefslogtreecommitdiff
path: root/compiler-rt/lib/xray
diff options
context:
space:
mode:
Diffstat (limited to 'compiler-rt/lib/xray')
-rw-r--r--compiler-rt/lib/xray/xray_AArch64.cpp2
-rw-r--r--compiler-rt/lib/xray/xray_arm.cpp2
-rw-r--r--compiler-rt/lib/xray/xray_init.cpp20
-rw-r--r--compiler-rt/lib/xray/xray_interface.cpp84
-rw-r--r--compiler-rt/lib/xray/xray_interface_internal.h24
-rw-r--r--compiler-rt/lib/xray/xray_powerpc64.cpp16
-rw-r--r--compiler-rt/lib/xray/xray_trampoline_AArch64.S41
-rw-r--r--compiler-rt/lib/xray/xray_trampoline_arm.S21
-rw-r--r--compiler-rt/lib/xray/xray_trampoline_x86_64.S6
-rw-r--r--compiler-rt/lib/xray/xray_utils.cpp4
-rw-r--r--compiler-rt/lib/xray/xray_x86_64.cpp66
11 files changed, 202 insertions, 84 deletions
diff --git a/compiler-rt/lib/xray/xray_AArch64.cpp b/compiler-rt/lib/xray/xray_AArch64.cpp
index 081941b70375..00105d30b4db 100644
--- a/compiler-rt/lib/xray/xray_AArch64.cpp
+++ b/compiler-rt/lib/xray/xray_AArch64.cpp
@@ -61,7 +61,7 @@ inline static bool patchSled(const bool Enable, const uint32_t FuncId,
// When |Enable|==false, we set back the first instruction in the sled to be
// B #32
- uint32_t *FirstAddress = reinterpret_cast<uint32_t *>(Sled.Address);
+ uint32_t *FirstAddress = reinterpret_cast<uint32_t *>(Sled.address());
uint32_t *CurAddress = FirstAddress + 1;
if (Enable) {
*CurAddress = uint32_t(PatchOpcodes::PO_LdrW0_12);
diff --git a/compiler-rt/lib/xray/xray_arm.cpp b/compiler-rt/lib/xray/xray_arm.cpp
index 9ad8065eb886..e1818555906c 100644
--- a/compiler-rt/lib/xray/xray_arm.cpp
+++ b/compiler-rt/lib/xray/xray_arm.cpp
@@ -102,7 +102,7 @@ inline static bool patchSled(const bool Enable, const uint32_t FuncId,
// When |Enable|==false, we set back the first instruction in the sled to be
// B #20
- uint32_t *FirstAddress = reinterpret_cast<uint32_t *>(Sled.Address);
+ uint32_t *FirstAddress = reinterpret_cast<uint32_t *>(Sled.address());
uint32_t *CurAddress = FirstAddress + 1;
if (Enable) {
CurAddress =
diff --git a/compiler-rt/lib/xray/xray_init.cpp b/compiler-rt/lib/xray/xray_init.cpp
index 408396477975..00ba5fe4a52b 100644
--- a/compiler-rt/lib/xray/xray_init.cpp
+++ b/compiler-rt/lib/xray/xray_init.cpp
@@ -84,8 +84,24 @@ void __xray_init() XRAY_NEVER_INSTRUMENT {
SpinMutexLock Guard(&XRayInstrMapMutex);
XRayInstrMap.Sleds = __start_xray_instr_map;
XRayInstrMap.Entries = __stop_xray_instr_map - __start_xray_instr_map;
- XRayInstrMap.SledsIndex = __start_xray_fn_idx;
- XRayInstrMap.Functions = __stop_xray_fn_idx - __start_xray_fn_idx;
+ if (__start_xray_fn_idx != nullptr) {
+ XRayInstrMap.SledsIndex = __start_xray_fn_idx;
+ XRayInstrMap.Functions = __stop_xray_fn_idx - __start_xray_fn_idx;
+ } else {
+ size_t CountFunctions = 0;
+ uint64_t LastFnAddr = 0;
+
+ for (std::size_t I = 0; I < XRayInstrMap.Entries; I++) {
+ const auto &Sled = XRayInstrMap.Sleds[I];
+ const auto Function = Sled.function();
+ if (Function != LastFnAddr) {
+ CountFunctions++;
+ LastFnAddr = Function;
+ }
+ }
+
+ XRayInstrMap.Functions = CountFunctions;
+ }
}
atomic_store(&XRayInitialized, true, memory_order_release);
diff --git a/compiler-rt/lib/xray/xray_interface.cpp b/compiler-rt/lib/xray/xray_interface.cpp
index 0d22893eb30f..7669b9ab82be 100644
--- a/compiler-rt/lib/xray/xray_interface.cpp
+++ b/compiler-rt/lib/xray/xray_interface.cpp
@@ -175,6 +175,33 @@ bool patchSled(const XRaySledEntry &Sled, bool Enable,
return Success;
}
+const XRayFunctionSledIndex
+findFunctionSleds(int32_t FuncId,
+ const XRaySledMap &InstrMap) XRAY_NEVER_INSTRUMENT {
+ int32_t CurFn = 0;
+ uint64_t LastFnAddr = 0;
+ XRayFunctionSledIndex Index = {nullptr, nullptr};
+
+ for (std::size_t I = 0; I < InstrMap.Entries && CurFn <= FuncId; I++) {
+ const auto &Sled = InstrMap.Sleds[I];
+ const auto Function = Sled.function();
+ if (Function != LastFnAddr) {
+ CurFn++;
+ LastFnAddr = Function;
+ }
+
+ if (CurFn == FuncId) {
+ if (Index.Begin == nullptr)
+ Index.Begin = &Sled;
+ Index.End = &Sled;
+ }
+ }
+
+ Index.End += 1;
+
+ return Index;
+}
+
XRayPatchingStatus patchFunction(int32_t FuncId,
bool Enable) XRAY_NEVER_INSTRUMENT {
if (!atomic_load(&XRayInitialized,
@@ -205,10 +232,10 @@ XRayPatchingStatus patchFunction(int32_t FuncId,
}
// Now we patch ths sleds for this specific function.
- auto SledRange = InstrMap.SledsIndex[FuncId - 1];
+ auto SledRange = InstrMap.SledsIndex ? InstrMap.SledsIndex[FuncId - 1]
+ : findFunctionSleds(FuncId, InstrMap);
auto *f = SledRange.Begin;
auto *e = SledRange.End;
-
bool SucceedOnce = false;
while (f != e)
SucceedOnce |= patchSled(*f++, Enable, FuncId);
@@ -264,14 +291,14 @@ XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
// now we're assuming we can mprotect the whole section of text between the
// minimum sled address and the maximum sled address (+ the largest sled
// size).
- auto MinSled = InstrMap.Sleds[0];
- auto MaxSled = InstrMap.Sleds[InstrMap.Entries - 1];
+ auto *MinSled = &InstrMap.Sleds[0];
+ auto *MaxSled = &InstrMap.Sleds[InstrMap.Entries - 1];
for (std::size_t I = 0; I < InstrMap.Entries; I++) {
const auto &Sled = InstrMap.Sleds[I];
- if (Sled.Address < MinSled.Address)
- MinSled = Sled;
- if (Sled.Address > MaxSled.Address)
- MaxSled = Sled;
+ if (Sled.address() < MinSled->address())
+ MinSled = &Sled;
+ if (Sled.address() > MaxSled->address())
+ MaxSled = &Sled;
}
const size_t PageSize = flags()->xray_page_size_override > 0
@@ -283,9 +310,10 @@ XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
}
void *PageAlignedAddr =
- reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1));
+ reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1));
size_t MProtectLen =
- (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength;
+ (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) +
+ cSledLength;
MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
if (Protector.MakeWriteable() == -1) {
Report("Failed mprotect: %d\n", errno);
@@ -294,7 +322,7 @@ XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
for (std::size_t I = 0; I < InstrMap.Entries; ++I) {
auto &Sled = InstrMap.Sleds[I];
- auto F = Sled.Function;
+ auto F = Sled.function();
if (CurFun == 0)
CurFun = F;
if (F != CurFun) {
@@ -334,23 +362,25 @@ XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId,
// Here we compute the minumum sled and maximum sled associated with a
// particular function ID.
- auto SledRange = InstrMap.SledsIndex[FuncId - 1];
+ auto SledRange = InstrMap.SledsIndex ? InstrMap.SledsIndex[FuncId - 1]
+ : findFunctionSleds(FuncId, InstrMap);
auto *f = SledRange.Begin;
auto *e = SledRange.End;
- auto MinSled = *f;
- auto MaxSled = *(SledRange.End - 1);
+ auto *MinSled = f;
+ auto *MaxSled = (SledRange.End - 1);
while (f != e) {
- if (f->Address < MinSled.Address)
- MinSled = *f;
- if (f->Address > MaxSled.Address)
- MaxSled = *f;
+ if (f->address() < MinSled->address())
+ MinSled = f;
+ if (f->address() > MaxSled->address())
+ MaxSled = f;
++f;
}
void *PageAlignedAddr =
- reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1));
+ reinterpret_cast<void *>(MinSled->address() & ~(PageSize - 1));
size_t MProtectLen =
- (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength;
+ (MaxSled->address() - reinterpret_cast<uptr>(PageAlignedAddr)) +
+ cSledLength;
MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize);
if (Protector.MakeWriteable() == -1) {
Report("Failed mprotect: %d\n", errno);
@@ -461,10 +491,18 @@ int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) {
int __xray_remove_handler_arg1() { return __xray_set_handler_arg1(nullptr); }
uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
- SpinMutexLock Guard(&XRayInstrMapMutex);
- if (FuncId <= 0 || static_cast<size_t>(FuncId) > XRayInstrMap.Functions)
+ XRaySledMap InstrMap;
+ {
+ SpinMutexLock Guard(&XRayInstrMapMutex);
+ InstrMap = XRayInstrMap;
+ }
+
+ if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions)
return 0;
- return XRayInstrMap.SledsIndex[FuncId - 1].Begin->Function
+ const XRaySledEntry *Sled = InstrMap.SledsIndex
+ ? InstrMap.SledsIndex[FuncId - 1].Begin
+ : findFunctionSleds(FuncId, InstrMap).Begin;
+ return Sled->function()
// On PPC, function entries are always aligned to 16 bytes. The beginning of a
// sled might be a local entry, which is always +8 based on the global entry.
// Always return the global entry.
diff --git a/compiler-rt/lib/xray/xray_interface_internal.h b/compiler-rt/lib/xray/xray_interface_internal.h
index 0fea6377648d..390f389b1dca 100644
--- a/compiler-rt/lib/xray/xray_interface_internal.h
+++ b/compiler-rt/lib/xray/xray_interface_internal.h
@@ -29,6 +29,18 @@ struct XRaySledEntry {
unsigned char AlwaysInstrument;
unsigned char Version;
unsigned char Padding[13]; // Need 32 bytes
+ uint64_t function() const {
+ if (Version < 2)
+ return Function;
+ // The target address is relative to the location of the Function variable.
+ return reinterpret_cast<uint64_t>(&Function) + Function;
+ }
+ uint64_t address() const {
+ if (Version < 2)
+ return Address;
+ // The target address is relative to the location of the Address variable.
+ return reinterpret_cast<uint64_t>(&Address) + Address;
+ }
#elif SANITIZER_WORDSIZE == 32
uint32_t Address;
uint32_t Function;
@@ -36,6 +48,18 @@ struct XRaySledEntry {
unsigned char AlwaysInstrument;
unsigned char Version;
unsigned char Padding[5]; // Need 16 bytes
+ uint32_t function() const {
+ if (Version < 2)
+ return Function;
+ // The target address is relative to the location of the Function variable.
+ return reinterpret_cast<uint32_t>(&Function) + Function;
+ }
+ uint32_t address() const {
+ if (Version < 2)
+ return Address;
+ // The target address is relative to the location of the Address variable.
+ return reinterpret_cast<uint32_t>(&Address) + Address;
+ }
#else
#error "Unsupported word size."
#endif
diff --git a/compiler-rt/lib/xray/xray_powerpc64.cpp b/compiler-rt/lib/xray/xray_powerpc64.cpp
index b41f1bce6f21..c3553d848313 100644
--- a/compiler-rt/lib/xray/xray_powerpc64.cpp
+++ b/compiler-rt/lib/xray/xray_powerpc64.cpp
@@ -52,35 +52,37 @@ namespace __xray {
bool patchFunctionEntry(const bool Enable, uint32_t FuncId,
const XRaySledEntry &Sled,
void (*Trampoline)()) XRAY_NEVER_INSTRUMENT {
+ const uint64_t Address = Sled.address();
if (Enable) {
// lis 0, FuncId[16..32]
// li 0, FuncId[0..15]
- *reinterpret_cast<uint64_t *>(Sled.Address) =
+ *reinterpret_cast<uint64_t *>(Address) =
(0x3c000000ull + (FuncId >> 16)) +
((0x60000000ull + (FuncId & 0xffff)) << 32);
} else {
// b +JumpOverInstNum instructions.
- *reinterpret_cast<uint32_t *>(Sled.Address) =
+ *reinterpret_cast<uint32_t *>(Address) =
0x48000000ull + (JumpOverInstNum << 2);
}
- clearCache(reinterpret_cast<void *>(Sled.Address), 8);
+ clearCache(reinterpret_cast<void *>(Address), 8);
return true;
}
bool patchFunctionExit(const bool Enable, uint32_t FuncId,
const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+ const uint64_t Address = Sled.address();
if (Enable) {
// lis 0, FuncId[16..32]
// li 0, FuncId[0..15]
- *reinterpret_cast<uint64_t *>(Sled.Address) =
+ *reinterpret_cast<uint64_t *>(Address) =
(0x3c000000ull + (FuncId >> 16)) +
((0x60000000ull + (FuncId & 0xffff)) << 32);
} else {
// Copy the blr/b instruction after JumpOverInstNum instructions.
- *reinterpret_cast<uint32_t *>(Sled.Address) =
- *(reinterpret_cast<uint32_t *>(Sled.Address) + JumpOverInstNum);
+ *reinterpret_cast<uint32_t *>(Address) =
+ *(reinterpret_cast<uint32_t *>(Address) + JumpOverInstNum);
}
- clearCache(reinterpret_cast<void *>(Sled.Address), 8);
+ clearCache(reinterpret_cast<void *>(Address), 8);
return true;
}
diff --git a/compiler-rt/lib/xray/xray_trampoline_AArch64.S b/compiler-rt/lib/xray/xray_trampoline_AArch64.S
index 4d1b04fb7d90..3bf52cef60fe 100644
--- a/compiler-rt/lib/xray/xray_trampoline_AArch64.S
+++ b/compiler-rt/lib/xray/xray_trampoline_AArch64.S
@@ -7,6 +7,7 @@
.p2align 2
/* Let C/C++ see the symbol */
.global __xray_FunctionEntry
+ .hidden __xray_FunctionEntry
.type __xray_FunctionEntry, %function
/* In C++ it is void extern "C" __xray_FunctionEntry(uint32_t FuncId) with
FuncId passed in W0 register. */
@@ -26,10 +27,14 @@ __xray_FunctionEntry:
STP Q2, Q3, [SP, #-32]!
STP Q4, Q5, [SP, #-32]!
STP Q6, Q7, [SP, #-32]!
- /* Load the address of _ZN6__xray19XRayPatchedFunctionE into X1 */
- LDR X1, =_ZN6__xray19XRayPatchedFunctionE
+ /* X8 is the indirect result register and needs to be preserved for the body
+ of the function to use */
+ STP X8, X0, [SP, #-16]!
+
+ /* Load the page address of _ZN6__xray19XRayPatchedFunctionE into X1 */
+ ADRP X1, _ZN6__xray19XRayPatchedFunctionE
/* Load the handler function pointer into X2 */
- LDR X2, [X1]
+ LDR X2, [X1, #:lo12:_ZN6__xray19XRayPatchedFunctionE]
/* Handler address is nullptr if handler is not set */
CMP X2, #0
BEQ FunctionEntry_restore
@@ -40,6 +45,7 @@ __xray_FunctionEntry:
BLR X2
FunctionEntry_restore:
/* Pop the saved registers */
+ LDP X8, X0, [SP], #16
LDP Q6, Q7, [SP], #32
LDP Q4, Q5, [SP], #32
LDP Q2, Q3, [SP], #32
@@ -54,6 +60,7 @@ FunctionEntry_restore:
.p2align 2
/* Let C/C++ see the symbol */
.global __xray_FunctionExit
+ .hidden __xray_FunctionExit
.type __xray_FunctionExit, %function
/* In C++ it is void extern "C" __xray_FunctionExit(uint32_t FuncId) with
FuncId passed in W0 register. */
@@ -69,11 +76,18 @@ __xray_FunctionExit:
STP X3, X4, [SP, #-16]!
STP X5, X6, [SP, #-16]!
STP X7, X30, [SP, #-16]!
- STR Q0, [SP, #-16]!
- /* Load the address of _ZN6__xray19XRayPatchedFunctionE into X1 */
- LDR X1, =_ZN6__xray19XRayPatchedFunctionE
+ STP Q0, Q1, [SP, #-32]!
+ STP Q2, Q3, [SP, #-32]!
+ STP Q4, Q5, [SP, #-32]!
+ STP Q6, Q7, [SP, #-32]!
+ /* X8 is the indirect result register and needs to be preserved for the body
+ of the function to use */
+ STP X8, X0, [SP, #-16]!
+
+ /* Load the page address of _ZN6__xray19XRayPatchedFunctionE into X1 */
+ ADRP X1, _ZN6__xray19XRayPatchedFunctionE
/* Load the handler function pointer into X2 */
- LDR X2, [X1]
+ LDR X2, [X1, #:lo12:_ZN6__xray19XRayPatchedFunctionE]
/* Handler address is nullptr if handler is not set */
CMP X2, #0
BEQ FunctionExit_restore
@@ -83,7 +97,11 @@ __xray_FunctionExit:
/* Call the handler with 2 parameters in W0 and X1 */
BLR X2
FunctionExit_restore:
- LDR Q0, [SP], #16
+ LDP X8, X0, [SP], #16
+ LDP Q6, Q7, [SP], #32
+ LDP Q4, Q5, [SP], #32
+ LDP Q2, Q3, [SP], #32
+ LDP Q0, Q1, [SP], #32
LDP X7, X30, [SP], #16
LDP X5, X6, [SP], #16
LDP X3, X4, [SP], #16
@@ -94,6 +112,7 @@ FunctionExit_restore:
.p2align 2
/* Let C/C++ see the symbol */
.global __xray_FunctionTailExit
+ .hidden __xray_FunctionTailExit
.type __xray_FunctionTailExit, %function
/* In C++ it is void extern "C" __xray_FunctionTailExit(uint32_t FuncId)
with FuncId passed in W0 register. */
@@ -114,10 +133,10 @@ __xray_FunctionTailExit:
STP Q2, Q3, [SP, #-32]!
STP Q4, Q5, [SP, #-32]!
STP Q6, Q7, [SP, #-32]!
- /* Load the address of _ZN6__xray19XRayPatchedFunctionE into X1 */
- LDR X1, =_ZN6__xray19XRayPatchedFunctionE
+ /* Load the page address of _ZN6__xray19XRayPatchedFunctionE into X1 */
+ ADRP X1, _ZN6__xray19XRayPatchedFunctionE
/* Load the handler function pointer into X2 */
- LDR X2, [X1]
+ LDR X2, [X1, #:lo12:_ZN6__xray19XRayPatchedFunctionE]
/* Handler address is nullptr if handler is not set */
CMP X2, #0
BEQ FunctionTailExit_restore
diff --git a/compiler-rt/lib/xray/xray_trampoline_arm.S b/compiler-rt/lib/xray/xray_trampoline_arm.S
index 71dbee65d825..3ffc1e443761 100644
--- a/compiler-rt/lib/xray/xray_trampoline_arm.S
+++ b/compiler-rt/lib/xray/xray_trampoline_arm.S
@@ -10,6 +10,7 @@
.p2align 2
@ Let C/C++ see the symbol
.global __xray_FunctionEntry
+ .hidden __xray_FunctionEntry
@ It preserves all registers except r0, r12(ip), r14(lr) and r15(pc)
@ Assume that "q" part of the floating-point registers is not used
@ for passing parameters to C/C++ functions.
@@ -20,9 +21,9 @@ __xray_FunctionEntry:
PUSH {r1-r3,lr}
@ Save floating-point parameters of the instrumented function
VPUSH {d0-d7}
- MOVW r1,#:lower16:_ZN6__xray19XRayPatchedFunctionE
- MOVT r1,#:upper16:_ZN6__xray19XRayPatchedFunctionE
- LDR r2, [r1]
+ MOVW r1, #:lower16:_ZN6__xray19XRayPatchedFunctionE - (. + 16)
+ MOVT r1, #:upper16:_ZN6__xray19XRayPatchedFunctionE - (. + 12)
+ LDR r2, [pc, r1]
@ Handler address is nullptr if handler is not set
CMP r2, #0
BEQ FunctionEntry_restore
@@ -40,6 +41,7 @@ FunctionEntry_restore:
.p2align 2
@ Let C/C++ see the symbol
.global __xray_FunctionExit
+ .hidden __xray_FunctionExit
@ Assume that d1-d7 are not used for the return value.
@ Assume that "q" part of the floating-point registers is not used for the
@ return value in C/C++.
@@ -51,9 +53,9 @@ __xray_FunctionExit:
@ Save the floating-point return value of the instrumented function
VPUSH {d0}
@ Load the handler address
- MOVW r1,#:lower16:_ZN6__xray19XRayPatchedFunctionE
- MOVT r1,#:upper16:_ZN6__xray19XRayPatchedFunctionE
- LDR r2, [r1]
+ MOVW r1, #:lower16:_ZN6__xray19XRayPatchedFunctionE - (. + 16)
+ MOVT r1, #:upper16:_ZN6__xray19XRayPatchedFunctionE - (. + 12)
+ LDR r2, [pc, r1]
@ Handler address is nullptr if handler is not set
CMP r2, #0
BEQ FunctionExit_restore
@@ -71,6 +73,7 @@ FunctionExit_restore:
.p2align 2
@ Let C/C++ see the symbol
.global __xray_FunctionTailExit
+ .hidden __xray_FunctionTailExit
@ It preserves all registers except r0, r12(ip), r14(lr) and r15(pc)
@ Assume that "q" part of the floating-point registers is not used
@ for passing parameters to C/C++ functions.
@@ -81,9 +84,9 @@ __xray_FunctionTailExit:
PUSH {r1-r3,lr}
@ Save floating-point parameters of the instrumented function
VPUSH {d0-d7}
- MOVW r1,#:lower16:_ZN6__xray19XRayPatchedFunctionE
- MOVT r1,#:upper16:_ZN6__xray19XRayPatchedFunctionE
- LDR r2, [r1]
+ MOVW r1, #:lower16:_ZN6__xray19XRayPatchedFunctionE - (. + 16)
+ MOVT r1, #:upper16:_ZN6__xray19XRayPatchedFunctionE - (. + 12)
+ LDR r2, [pc, r1]
@ Handler address is nullptr if handler is not set
CMP r2, #0
BEQ FunctionTailExit_restore
diff --git a/compiler-rt/lib/xray/xray_trampoline_x86_64.S b/compiler-rt/lib/xray/xray_trampoline_x86_64.S
index 1e58362cdc80..12c5a6ccd9a4 100644
--- a/compiler-rt/lib/xray/xray_trampoline_x86_64.S
+++ b/compiler-rt/lib/xray/xray_trampoline_x86_64.S
@@ -98,6 +98,7 @@
//===----------------------------------------------------------------------===//
.globl ASM_SYMBOL(__xray_FunctionEntry)
+ ASM_HIDDEN(__xray_FunctionEntry)
.align 16, 0x90
ASM_TYPE_FUNCTION(__xray_FunctionEntry)
# LLVM-MCA-BEGIN __xray_FunctionEntry
@@ -126,6 +127,7 @@ ASM_SYMBOL(__xray_FunctionEntry):
//===----------------------------------------------------------------------===//
.globl ASM_SYMBOL(__xray_FunctionExit)
+ ASM_HIDDEN(__xray_FunctionExit)
.align 16, 0x90
ASM_TYPE_FUNCTION(__xray_FunctionExit)
# LLVM-MCA-BEGIN __xray_FunctionExit
@@ -166,6 +168,7 @@ ASM_SYMBOL(__xray_FunctionExit):
//===----------------------------------------------------------------------===//
.globl ASM_SYMBOL(__xray_FunctionTailExit)
+ ASM_HIDDEN(__xray_FunctionTailExit)
.align 16, 0x90
ASM_TYPE_FUNCTION(__xray_FunctionTailExit)
# LLVM-MCA-BEGIN __xray_FunctionTailExit
@@ -192,6 +195,7 @@ ASM_SYMBOL(__xray_FunctionTailExit):
//===----------------------------------------------------------------------===//
.globl ASM_SYMBOL(__xray_ArgLoggerEntry)
+ ASM_HIDDEN(__xray_ArgLoggerEntry)
.align 16, 0x90
ASM_TYPE_FUNCTION(__xray_ArgLoggerEntry)
# LLVM-MCA-BEGIN __xray_ArgLoggerEntry
@@ -231,6 +235,7 @@ ASM_SYMBOL(__xray_ArgLoggerEntry):
//===----------------------------------------------------------------------===//
.global ASM_SYMBOL(__xray_CustomEvent)
+ ASM_HIDDEN(__xray_CustomEvent)
.align 16, 0x90
ASM_TYPE_FUNCTION(__xray_CustomEvent)
# LLVM-MCA-BEGIN __xray_CustomEvent
@@ -256,6 +261,7 @@ ASM_SYMBOL(__xray_CustomEvent):
//===----------------------------------------------------------------------===//
.global ASM_SYMBOL(__xray_TypedEvent)
+ ASM_HIDDEN(__xray_TypedEvent)
.align 16, 0x90
ASM_TYPE_FUNCTION(__xray_TypedEvent)
# LLVM-MCA-BEGIN __xray_TypedEvent
diff --git a/compiler-rt/lib/xray/xray_utils.cpp b/compiler-rt/lib/xray/xray_utils.cpp
index 1036d17a7725..4c8ad5b92be7 100644
--- a/compiler-rt/lib/xray/xray_utils.cpp
+++ b/compiler-rt/lib/xray/xray_utils.cpp
@@ -69,6 +69,10 @@ void LogWriter::WriteAll(const char *Begin, const char *End) XRAY_NEVER_INSTRUME
return;
}
Offset += TotalBytes;
+
+ // Record the data size as a property of the VMO.
+ _zx_object_set_property(Vmo, ZX_PROP_VMO_CONTENT_SIZE,
+ &Offset, sizeof(Offset));
}
void LogWriter::Flush() XRAY_NEVER_INSTRUMENT {
diff --git a/compiler-rt/lib/xray/xray_x86_64.cpp b/compiler-rt/lib/xray/xray_x86_64.cpp
index e63ee1b3bd02..f3742ac71290 100644
--- a/compiler-rt/lib/xray/xray_x86_64.cpp
+++ b/compiler-rt/lib/xray/xray_x86_64.cpp
@@ -151,23 +151,24 @@ bool patchFunctionEntry(const bool Enable, const uint32_t FuncId,
// opcode and first operand.
//
// Prerequisite is to compute the relative offset to the trampoline's address.
+ const uint64_t Address = Sled.address();
int64_t TrampolineOffset = reinterpret_cast<int64_t>(Trampoline) -
- (static_cast<int64_t>(Sled.Address) + 11);
+ (static_cast<int64_t>(Address) + 11);
if (TrampolineOffset < MinOffset || TrampolineOffset > MaxOffset) {
- Report("XRay Entry trampoline (%p) too far from sled (%p)\n",
- Trampoline, reinterpret_cast<void *>(Sled.Address));
+ Report("XRay Entry trampoline (%p) too far from sled (%p)\n", Trampoline,
+ reinterpret_cast<void *>(Address));
return false;
}
if (Enable) {
- *reinterpret_cast<uint32_t *>(Sled.Address + 2) = FuncId;
- *reinterpret_cast<uint8_t *>(Sled.Address + 6) = CallOpCode;
- *reinterpret_cast<uint32_t *>(Sled.Address + 7) = TrampolineOffset;
+ *reinterpret_cast<uint32_t *>(Address + 2) = FuncId;
+ *reinterpret_cast<uint8_t *>(Address + 6) = CallOpCode;
+ *reinterpret_cast<uint32_t *>(Address + 7) = TrampolineOffset;
std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), MovR10Seq,
+ reinterpret_cast<std::atomic<uint16_t> *>(Address), MovR10Seq,
std::memory_order_release);
} else {
std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), Jmp9Seq,
+ reinterpret_cast<std::atomic<uint16_t> *>(Address), Jmp9Seq,
std::memory_order_release);
// FIXME: Write out the nops still?
}
@@ -196,23 +197,24 @@ bool patchFunctionExit(const bool Enable, const uint32_t FuncId,
//
// Prerequisite is to compute the relative offset fo the
// __xray_FunctionExit function's address.
+ const uint64_t Address = Sled.address();
int64_t TrampolineOffset = reinterpret_cast<int64_t>(__xray_FunctionExit) -
- (static_cast<int64_t>(Sled.Address) + 11);
+ (static_cast<int64_t>(Address) + 11);
if (TrampolineOffset < MinOffset || TrampolineOffset > MaxOffset) {
Report("XRay Exit trampoline (%p) too far from sled (%p)\n",
- __xray_FunctionExit, reinterpret_cast<void *>(Sled.Address));
+ __xray_FunctionExit, reinterpret_cast<void *>(Address));
return false;
}
if (Enable) {
- *reinterpret_cast<uint32_t *>(Sled.Address + 2) = FuncId;
- *reinterpret_cast<uint8_t *>(Sled.Address + 6) = JmpOpCode;
- *reinterpret_cast<uint32_t *>(Sled.Address + 7) = TrampolineOffset;
+ *reinterpret_cast<uint32_t *>(Address + 2) = FuncId;
+ *reinterpret_cast<uint8_t *>(Address + 6) = JmpOpCode;
+ *reinterpret_cast<uint32_t *>(Address + 7) = TrampolineOffset;
std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), MovR10Seq,
+ reinterpret_cast<std::atomic<uint16_t> *>(Address), MovR10Seq,
std::memory_order_release);
} else {
std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint8_t> *>(Sled.Address), RetOpCode,
+ reinterpret_cast<std::atomic<uint8_t> *>(Address), RetOpCode,
std::memory_order_release);
// FIXME: Write out the nops still?
}
@@ -223,24 +225,25 @@ bool patchFunctionTailExit(const bool Enable, const uint32_t FuncId,
const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
// Here we do the dance of replacing the tail call sled with a similar
// sequence as the entry sled, but calls the tail exit sled instead.
+ const uint64_t Address = Sled.address();
int64_t TrampolineOffset =
reinterpret_cast<int64_t>(__xray_FunctionTailExit) -
- (static_cast<int64_t>(Sled.Address) + 11);
+ (static_cast<int64_t>(Address) + 11);
if (TrampolineOffset < MinOffset || TrampolineOffset > MaxOffset) {
Report("XRay Tail Exit trampoline (%p) too far from sled (%p)\n",
- __xray_FunctionTailExit, reinterpret_cast<void *>(Sled.Address));
+ __xray_FunctionTailExit, reinterpret_cast<void *>(Address));
return false;
}
if (Enable) {
- *reinterpret_cast<uint32_t *>(Sled.Address + 2) = FuncId;
- *reinterpret_cast<uint8_t *>(Sled.Address + 6) = CallOpCode;
- *reinterpret_cast<uint32_t *>(Sled.Address + 7) = TrampolineOffset;
+ *reinterpret_cast<uint32_t *>(Address + 2) = FuncId;
+ *reinterpret_cast<uint8_t *>(Address + 6) = CallOpCode;
+ *reinterpret_cast<uint32_t *>(Address + 7) = TrampolineOffset;
std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), MovR10Seq,
+ reinterpret_cast<std::atomic<uint16_t> *>(Address), MovR10Seq,
std::memory_order_release);
} else {
std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), Jmp9Seq,
+ reinterpret_cast<std::atomic<uint16_t> *>(Address), Jmp9Seq,
std::memory_order_release);
// FIXME: Write out the nops still?
}
@@ -267,26 +270,28 @@ bool patchCustomEvent(const bool Enable, const uint32_t FuncId,
//
// ---
//
- // In Version 1:
+ // In Version 1 or 2:
//
// The jump offset is now 15 bytes (0x0f), so when restoring the nopw back
// to a jmp, use 15 bytes instead.
//
+ const uint64_t Address = Sled.address();
if (Enable) {
std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), NopwSeq,
+ reinterpret_cast<std::atomic<uint16_t> *>(Address), NopwSeq,
std::memory_order_release);
} else {
switch (Sled.Version) {
case 1:
+ case 2:
std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), Jmp15Seq,
+ reinterpret_cast<std::atomic<uint16_t> *>(Address), Jmp15Seq,
std::memory_order_release);
break;
case 0:
default:
std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), Jmp20Seq,
+ reinterpret_cast<std::atomic<uint16_t> *>(Address), Jmp20Seq,
std::memory_order_release);
break;
}
@@ -313,14 +318,15 @@ bool patchTypedEvent(const bool Enable, const uint32_t FuncId,
// unstashes the registers and returns. If the arguments are already in
// the correct registers, the stashing and unstashing become equivalently
// sized nops.
+ const uint64_t Address = Sled.address();
if (Enable) {
std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), NopwSeq,
+ reinterpret_cast<std::atomic<uint16_t> *>(Address), NopwSeq,
std::memory_order_release);
} else {
- std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), Jmp20Seq,
- std::memory_order_release);
+ std::atomic_store_explicit(
+ reinterpret_cast<std::atomic<uint16_t> *>(Address), Jmp20Seq,
+ std::memory_order_release);
}
return false;
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-04 10:51:11 +0000