diff options
Diffstat (limited to 'contrib/llvm-project/compiler-rt/lib/profile')
25 files changed, 5820 insertions, 0 deletions
diff --git a/contrib/llvm-project/compiler-rt/lib/profile/GCDAProfiling.c b/contrib/llvm-project/compiler-rt/lib/profile/GCDAProfiling.c new file mode 100644 index 000000000000..4f46fd2839b9 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/GCDAProfiling.c @@ -0,0 +1,637 @@ +/*===- GCDAProfiling.c - Support library for GCDA file emission -----------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +|*===----------------------------------------------------------------------===*| +|* +|* This file implements the call back routines for the gcov profiling +|* instrumentation pass. Link against this library when running code through +|* the -insert-gcov-profiling LLVM pass. +|* +|* We emit files in a corrupt version of GCOV's "gcda" file format. These files +|* are only close enough that LCOV will happily parse them. Anything that lcov +|* ignores is missing. +|* +|* TODO: gcov is multi-process safe by having each exit open the existing file +|* and append to it. We'd like to achieve that and be thread-safe too. +|* +\*===----------------------------------------------------------------------===*/ + +#if !defined(__Fuchsia__) + +#include <errno.h> +#include <fcntl.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include "WindowsMMap.h" +#else +#include <sys/file.h> +#include <sys/mman.h> +#include <sys/types.h> +#include <unistd.h> +#endif + +#include "InstrProfiling.h" +#include "InstrProfilingUtil.h" + +/* #define DEBUG_GCDAPROFILING */ + +enum { + GCOV_DATA_MAGIC = 0x67636461, // "gcda" + + GCOV_TAG_FUNCTION = 0x01000000, + GCOV_TAG_COUNTER_ARCS = 0x01a10000, + // GCOV_TAG_OBJECT_SUMMARY superseded GCOV_TAG_PROGRAM_SUMMARY in GCC 9. + GCOV_TAG_OBJECT_SUMMARY = 0xa1000000, + GCOV_TAG_PROGRAM_SUMMARY = 0xa3000000, +}; + +/* + * --- GCOV file format I/O primitives --- + */ + +/* + * The current file name we're outputting. Used primarily for error logging. + */ +static char *filename = NULL; + +/* + * The current file we're outputting. + */ +static FILE *output_file = NULL; + +/* + * Buffer that we write things into. + */ +#define WRITE_BUFFER_SIZE (128 * 1024) +static unsigned char *write_buffer = NULL; +static uint64_t cur_buffer_size = 0; +static uint64_t cur_pos = 0; +static uint64_t file_size = 0; +static int new_file = 0; +static int gcov_version; +#if defined(_WIN32) +static HANDLE mmap_handle = NULL; +#endif +static int fd = -1; + +typedef void (*fn_ptr)(void); + +typedef void* dynamic_object_id; +// The address of this variable identifies a given dynamic object. +static dynamic_object_id current_id; +#define CURRENT_ID (¤t_id) + +struct fn_node { + dynamic_object_id id; + fn_ptr fn; + struct fn_node* next; +}; + +struct fn_list { + struct fn_node *head, *tail; +}; + +/* + * A list of functions to write out the data, shared between all dynamic objects. + */ +struct fn_list writeout_fn_list; + +/* + * A list of reset functions, shared between all dynamic objects. + */ +struct fn_list reset_fn_list; + +static void fn_list_insert(struct fn_list* list, fn_ptr fn) { + struct fn_node* new_node = malloc(sizeof(struct fn_node)); + new_node->fn = fn; + new_node->next = NULL; + new_node->id = CURRENT_ID; + + if (!list->head) { + list->head = list->tail = new_node; + } else { + list->tail->next = new_node; + list->tail = new_node; + } +} + +static void fn_list_remove(struct fn_list* list) { + struct fn_node* curr = list->head; + struct fn_node* prev = NULL; + struct fn_node* next = NULL; + + while (curr) { + next = curr->next; + + if (curr->id == CURRENT_ID) { + if (curr == list->head) { + list->head = next; + } + + if (curr == list->tail) { + list->tail = prev; + } + + if (prev) { + prev->next = next; + } + + free(curr); + } else { + prev = curr; + } + + curr = next; + } +} + +static void resize_write_buffer(uint64_t size) { + if (!new_file) return; + size += cur_pos; + if (size <= cur_buffer_size) return; + size = (size - 1) / WRITE_BUFFER_SIZE + 1; + size *= WRITE_BUFFER_SIZE; + write_buffer = realloc(write_buffer, size); + cur_buffer_size = size; +} + +static void write_bytes(const char *s, size_t len) { + resize_write_buffer(len); + memcpy(&write_buffer[cur_pos], s, len); + cur_pos += len; +} + +static void write_32bit_value(uint32_t i) { + write_bytes((char*)&i, 4); +} + +static void write_64bit_value(uint64_t i) { + // GCOV uses a lo-/hi-word format even on big-endian systems. + // See also GCOVBuffer::readInt64 in LLVM. + uint32_t lo = (uint32_t) i; + uint32_t hi = (uint32_t) (i >> 32); + write_32bit_value(lo); + write_32bit_value(hi); +} + +static uint32_t read_32bit_value(void) { + uint32_t val; + + if (new_file) + return (uint32_t)-1; + + val = *(uint32_t*)&write_buffer[cur_pos]; + cur_pos += 4; + return val; +} + +static uint64_t read_64bit_value(void) { + // GCOV uses a lo-/hi-word format even on big-endian systems. + // See also GCOVBuffer::readInt64 in LLVM. + uint32_t lo = read_32bit_value(); + uint32_t hi = read_32bit_value(); + return ((uint64_t)hi << 32) | ((uint64_t)lo); +} + +static char *mangle_filename(const char *orig_filename) { + char *new_filename; + size_t prefix_len; + int prefix_strip; + const char *prefix = lprofGetPathPrefix(&prefix_strip, &prefix_len); + + if (prefix == NULL) + return strdup(orig_filename); + + new_filename = malloc(prefix_len + 1 + strlen(orig_filename) + 1); + lprofApplyPathPrefix(new_filename, orig_filename, prefix, prefix_len, + prefix_strip); + + return new_filename; +} + +static int map_file(void) { + fseek(output_file, 0L, SEEK_END); + file_size = ftell(output_file); + + /* A size of 0 means the file has been created just now (possibly by another + * process in lock-after-open race condition). No need to mmap. */ + if (file_size == 0) + return -1; + +#if defined(_WIN32) + HANDLE mmap_fd; + if (fd == -1) + mmap_fd = INVALID_HANDLE_VALUE; + else + mmap_fd = (HANDLE)_get_osfhandle(fd); + + mmap_handle = CreateFileMapping(mmap_fd, NULL, PAGE_READWRITE, DWORD_HI(file_size), DWORD_LO(file_size), NULL); + if (mmap_handle == NULL) { + fprintf(stderr, "profiling: %s: cannot create file mapping: %lu\n", + filename, GetLastError()); + return -1; + } + + write_buffer = MapViewOfFile(mmap_handle, FILE_MAP_WRITE, 0, 0, file_size); + if (write_buffer == NULL) { + fprintf(stderr, "profiling: %s: cannot map: %lu\n", filename, + GetLastError()); + CloseHandle(mmap_handle); + return -1; + } +#else + write_buffer = mmap(0, file_size, PROT_READ | PROT_WRITE, + MAP_FILE | MAP_SHARED, fd, 0); + if (write_buffer == (void *)-1) { + int errnum = errno; + fprintf(stderr, "profiling: %s: cannot map: %s\n", filename, + strerror(errnum)); + return -1; + } +#endif + + return 0; +} + +static void unmap_file(void) { +#if defined(_WIN32) + if (!UnmapViewOfFile(write_buffer)) { + fprintf(stderr, "profiling: %s: cannot unmap mapped view: %lu\n", filename, + GetLastError()); + } + + if (!CloseHandle(mmap_handle)) { + fprintf(stderr, "profiling: %s: cannot close file mapping handle: %lu\n", + filename, GetLastError()); + } + + mmap_handle = NULL; +#else + if (munmap(write_buffer, file_size) == -1) { + int errnum = errno; + fprintf(stderr, "profiling: %s: cannot munmap: %s\n", filename, + strerror(errnum)); + } +#endif + + write_buffer = NULL; + file_size = 0; +} + +/* + * --- LLVM line counter API --- + */ + +/* A file in this case is a translation unit. Each .o file built with line + * profiling enabled will emit to a different file. Only one file may be + * started at a time. + */ +COMPILER_RT_VISIBILITY +void llvm_gcda_start_file(const char *orig_filename, uint32_t version, + uint32_t checksum) { + const char *mode = "r+b"; + filename = mangle_filename(orig_filename); + + /* Try just opening the file. */ + fd = open(filename, O_RDWR | O_BINARY); + + if (fd == -1) { + /* Try creating the file. */ + fd = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644); + if (fd != -1) { + mode = "w+b"; + } else { + /* Try creating the directories first then opening the file. */ + __llvm_profile_recursive_mkdir(filename); + fd = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644); + if (fd != -1) { + mode = "w+b"; + } else { + /* Another process may have created the file just now. + * Try opening it without O_CREAT and O_EXCL. */ + fd = open(filename, O_RDWR | O_BINARY); + if (fd == -1) { + /* Bah! It's hopeless. */ + int errnum = errno; + fprintf(stderr, "profiling: %s: cannot open: %s\n", filename, + strerror(errnum)); + return; + } + } + } + } + + /* Try to flock the file to serialize concurrent processes writing out to the + * same GCDA. This can fail if the filesystem doesn't support it, but in that + * case we'll just carry on with the old racy behaviour and hope for the best. + */ + lprofLockFd(fd); + output_file = fdopen(fd, mode); + + /* Initialize the write buffer. */ + new_file = 0; + write_buffer = NULL; + cur_buffer_size = 0; + cur_pos = 0; + + if (map_file() == -1) { + /* The file has been created just now (file_size == 0) or mmap failed + * unexpectedly. In the latter case, try to recover by clobbering. */ + new_file = 1; + write_buffer = NULL; + resize_write_buffer(WRITE_BUFFER_SIZE); + memset(write_buffer, 0, WRITE_BUFFER_SIZE); + } + + /* gcda file, version, stamp checksum. */ + { + uint8_t c3 = version >> 24; + uint8_t c2 = (version >> 16) & 255; + uint8_t c1 = (version >> 8) & 255; + gcov_version = c3 >= 'A' ? (c3 - 'A') * 100 + (c2 - '0') * 10 + c1 - '0' + : (c3 - '0') * 10 + c1 - '0'; + } + write_32bit_value(GCOV_DATA_MAGIC); + write_32bit_value(version); + write_32bit_value(checksum); + +#ifdef DEBUG_GCDAPROFILING + fprintf(stderr, "llvmgcda: [%s]\n", orig_filename); +#endif +} + +COMPILER_RT_VISIBILITY +void llvm_gcda_emit_function(uint32_t ident, uint32_t func_checksum, + uint32_t cfg_checksum) { + uint32_t len = 2; + int use_extra_checksum = gcov_version >= 47; + + if (use_extra_checksum) + len++; +#ifdef DEBUG_GCDAPROFILING + fprintf(stderr, "llvmgcda: function id=0x%08x\n", ident); +#endif + if (!output_file) return; + + /* function tag */ + write_32bit_value(GCOV_TAG_FUNCTION); + write_32bit_value(len); + write_32bit_value(ident); + write_32bit_value(func_checksum); + if (use_extra_checksum) + write_32bit_value(cfg_checksum); +} + +COMPILER_RT_VISIBILITY +void llvm_gcda_emit_arcs(uint32_t num_counters, uint64_t *counters) { + uint32_t i; + uint64_t *old_ctrs = NULL; + uint32_t val = 0; + uint64_t save_cur_pos = cur_pos; + + if (!output_file) return; + + val = read_32bit_value(); + + if (val != (uint32_t)-1) { + /* There are counters present in the file. Merge them. */ + if (val != GCOV_TAG_COUNTER_ARCS) { + fprintf(stderr, "profiling: %s: cannot merge previous GCDA file: " + "corrupt arc tag (0x%08x)\n", + filename, val); + return; + } + + val = read_32bit_value(); + if (val == (uint32_t)-1 || val / 2 != num_counters) { + fprintf(stderr, "profiling: %s: cannot merge previous GCDA file: " + "mismatched number of counters (%d)\n", + filename, val); + return; + } + + old_ctrs = malloc(sizeof(uint64_t) * num_counters); + for (i = 0; i < num_counters; ++i) + old_ctrs[i] = read_64bit_value(); + } + + cur_pos = save_cur_pos; + + /* Counter #1 (arcs) tag */ + write_32bit_value(GCOV_TAG_COUNTER_ARCS); + write_32bit_value(num_counters * 2); + for (i = 0; i < num_counters; ++i) { + counters[i] += (old_ctrs ? old_ctrs[i] : 0); + write_64bit_value(counters[i]); + } + + free(old_ctrs); + +#ifdef DEBUG_GCDAPROFILING + fprintf(stderr, "llvmgcda: %u arcs\n", num_counters); + for (i = 0; i < num_counters; ++i) + fprintf(stderr, "llvmgcda: %llu\n", (unsigned long long)counters[i]); +#endif +} + +COMPILER_RT_VISIBILITY +void llvm_gcda_summary_info(void) { + uint32_t runs = 1; + static uint32_t run_counted = 0; // We only want to increase the run count once. + uint32_t val = 0; + uint64_t save_cur_pos = cur_pos; + + if (!output_file) return; + + val = read_32bit_value(); + + if (val != (uint32_t)-1) { + /* There are counters present in the file. Merge them. */ + uint32_t gcov_tag = + gcov_version >= 90 ? GCOV_TAG_OBJECT_SUMMARY : GCOV_TAG_PROGRAM_SUMMARY; + if (val != gcov_tag) { + fprintf(stderr, + "profiling: %s: cannot merge previous run count: " + "corrupt object tag (0x%08x)\n", + filename, val); + return; + } + + val = read_32bit_value(); /* length */ + uint32_t prev_runs; + if (gcov_version < 90) { + read_32bit_value(); + read_32bit_value(); + prev_runs = read_32bit_value(); + } else { + prev_runs = read_32bit_value(); + read_32bit_value(); + } + for (uint32_t i = gcov_version < 90 ? 3 : 2; i < val; ++i) + read_32bit_value(); + /* Add previous run count to new counter, if not already counted before. */ + runs = run_counted ? prev_runs : prev_runs + 1; + } + + cur_pos = save_cur_pos; + + if (gcov_version >= 90) { + write_32bit_value(GCOV_TAG_OBJECT_SUMMARY); + write_32bit_value(2); + write_32bit_value(runs); + write_32bit_value(0); // sum_max + } else { + // Before gcov 4.8 (r190952), GCOV_TAG_SUMMARY_LENGTH was 9. r190952 set + // GCOV_TAG_SUMMARY_LENGTH to 22. We simply use the smallest length which + // can make gcov read "Runs:". + write_32bit_value(GCOV_TAG_PROGRAM_SUMMARY); + write_32bit_value(3); + write_32bit_value(0); + write_32bit_value(0); + write_32bit_value(runs); + } + + run_counted = 1; + +#ifdef DEBUG_GCDAPROFILING + fprintf(stderr, "llvmgcda: %u runs\n", runs); +#endif +} + +COMPILER_RT_VISIBILITY +void llvm_gcda_end_file(void) { + /* Write out EOF record. */ + if (output_file) { + write_bytes("\0\0\0\0\0\0\0\0", 8); + + if (new_file) { + fwrite(write_buffer, cur_pos, 1, output_file); + free(write_buffer); + } else { + unmap_file(); + } + + fflush(output_file); + lprofUnlockFd(fd); + fclose(output_file); + output_file = NULL; + write_buffer = NULL; + } + free(filename); + +#ifdef DEBUG_GCDAPROFILING + fprintf(stderr, "llvmgcda: -----\n"); +#endif +} + +COMPILER_RT_VISIBILITY +void llvm_register_writeout_function(fn_ptr fn) { + fn_list_insert(&writeout_fn_list, fn); +} + +COMPILER_RT_VISIBILITY +void llvm_writeout_files(void) { + struct fn_node *curr = writeout_fn_list.head; + + while (curr) { + if (curr->id == CURRENT_ID) { + curr->fn(); + } + curr = curr->next; + } +} + +#ifndef _WIN32 +// __attribute__((destructor)) and destructors whose priorities are greater than +// 100 run before this function and can thus be tracked. The priority is +// compatible with GCC 7 onwards. +#if __GNUC__ >= 9 +#pragma GCC diagnostic ignored "-Wprio-ctor-dtor" +#endif +__attribute__((destructor(100))) +#endif +static void llvm_writeout_and_clear(void) { + llvm_writeout_files(); + fn_list_remove(&writeout_fn_list); +} + +COMPILER_RT_VISIBILITY +void llvm_register_reset_function(fn_ptr fn) { + fn_list_insert(&reset_fn_list, fn); +} + +COMPILER_RT_VISIBILITY +void llvm_delete_reset_function_list(void) { fn_list_remove(&reset_fn_list); } + +COMPILER_RT_VISIBILITY +void llvm_reset_counters(void) { + struct fn_node *curr = reset_fn_list.head; + + while (curr) { + if (curr->id == CURRENT_ID) { + curr->fn(); + } + curr = curr->next; + } +} + +#if !defined(_WIN32) +COMPILER_RT_VISIBILITY +pid_t __gcov_fork() { + pid_t parent_pid = getpid(); + pid_t pid = fork(); + + if (pid == 0) { + pid_t child_pid = getpid(); + if (child_pid != parent_pid) { + // The pid changed so we've a fork (one could have its own fork function) + // Just reset the counters for this child process + // threads. + llvm_reset_counters(); + } + } + return pid; +} +#endif + +COMPILER_RT_VISIBILITY +void llvm_gcov_init(fn_ptr wfn, fn_ptr rfn) { + static int atexit_ran = 0; + + if (wfn) + llvm_register_writeout_function(wfn); + + if (rfn) + llvm_register_reset_function(rfn); + + if (atexit_ran == 0) { + atexit_ran = 1; + + /* Make sure we write out the data and delete the data structures. */ + atexit(llvm_delete_reset_function_list); +#ifdef _WIN32 + atexit(llvm_writeout_and_clear); +#endif + } +} + +void __gcov_dump(void) { + for (struct fn_node *f = writeout_fn_list.head; f; f = f->next) + f->fn(); +} + +void __gcov_reset(void) { + for (struct fn_node *f = reset_fn_list.head; f; f = f->next) + f->fn(); +} + +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfiling.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfiling.c new file mode 100644 index 000000000000..da04d8ebdec9 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfiling.c @@ -0,0 +1,91 @@ +/*===- InstrProfiling.c - Support library for PGO instrumentation ---------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +// Note: This is linked into the Darwin kernel, and must remain compatible +// with freestanding compilation. See `darwin_add_builtin_libraries`. + +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" + +#define INSTR_PROF_VALUE_PROF_DATA +#include "profile/InstrProfData.inc" + +static uint32_t __llvm_profile_global_timestamp = 1; + +COMPILER_RT_VISIBILITY +void INSTR_PROF_PROFILE_SET_TIMESTAMP(uint64_t *Probe) { + if (*Probe == 0 || *Probe == (uint64_t)-1) + *Probe = __llvm_profile_global_timestamp++; +} + +COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_magic(void) { + return sizeof(void *) == sizeof(uint64_t) ? (INSTR_PROF_RAW_MAGIC_64) + : (INSTR_PROF_RAW_MAGIC_32); +} + +COMPILER_RT_VISIBILITY void __llvm_profile_set_dumped(void) { + lprofSetProfileDumped(1); +} + +/* Return the number of bytes needed to add to SizeInBytes to make it + * the result a multiple of 8. + */ +COMPILER_RT_VISIBILITY uint8_t +__llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes) { + return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t)); +} + +COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_version(void) { + return INSTR_PROF_RAW_VERSION_VAR; +} + +COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) { + if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF) + __llvm_profile_global_timestamp = 1; + + char *I = __llvm_profile_begin_counters(); + char *E = __llvm_profile_end_counters(); + + char ResetValue = + (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) ? 0xFF : 0; + memset(I, ResetValue, E - I); + + I = __llvm_profile_begin_bitmap(); + E = __llvm_profile_end_bitmap(); + memset(I, 0x0, E - I); + + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const __llvm_profile_data *DI; + for (DI = DataBegin; DI < DataEnd; ++DI) { + uint64_t CurrentVSiteCount = 0; + uint32_t VKI, i; + if (!DI->Values) + continue; + + ValueProfNode **ValueCounters = (ValueProfNode **)DI->Values; + + for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI) + CurrentVSiteCount += DI->NumValueSites[VKI]; + + for (i = 0; i < CurrentVSiteCount; ++i) { + ValueProfNode *CurrVNode = ValueCounters[i]; + + while (CurrVNode) { + CurrVNode->Count = 0; + CurrVNode = CurrVNode->Next; + } + } + } + lprofSetProfileDumped(0); +} diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfiling.h b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfiling.h new file mode 100644 index 000000000000..6906d52eacaf --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfiling.h @@ -0,0 +1,327 @@ +/*===- InstrProfiling.h- Support library for PGO instrumentation ----------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#ifndef PROFILE_INSTRPROFILING_H_ +#define PROFILE_INSTRPROFILING_H_ + +#include "InstrProfilingPort.h" +#include <stdio.h> + +// Make sure __LLVM_INSTR_PROFILE_GENERATE is always defined before +// including instr_prof_interface.h so the interface functions are +// declared correctly for the runtime. +// __LLVM_INSTR_PROFILE_GENERATE is always `#undef`ed after the header, +// because compiler-rt does not support profiling the profiling runtime itself. +#ifndef __LLVM_INSTR_PROFILE_GENERATE +#define __LLVM_INSTR_PROFILE_GENERATE +#endif +#include "profile/instr_prof_interface.h" +#undef __LLVM_INSTR_PROFILE_GENERATE + +#define INSTR_PROF_VISIBILITY COMPILER_RT_VISIBILITY +#include "profile/InstrProfData.inc" + +enum ValueKind { +#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value, +#include "profile/InstrProfData.inc" +}; + +typedef void *IntPtrT; +typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT) + __llvm_profile_data { +#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) Type Name; +#include "profile/InstrProfData.inc" +} __llvm_profile_data; + +typedef struct __llvm_profile_header { +#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) Type Name; +#include "profile/InstrProfData.inc" +} __llvm_profile_header; + +typedef struct ValueProfNode * PtrToNodeT; +typedef struct ValueProfNode { +#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) Type Name; +#include "profile/InstrProfData.inc" +} ValueProfNode; + +typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT) VTableProfData { +#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) Type Name; +#include "profile/InstrProfData.inc" +} VTableProfData; + +/*! + * \brief Return 1 if profile counters are continuously synced to the raw + * profile via an mmap(). This is in contrast to the default mode, in which + * the raw profile is written out at program exit time. + */ +int __llvm_profile_is_continuous_mode_enabled(void); + +/*! + * \brief Enable continuous mode. + * + * See \ref __llvm_profile_is_continuous_mode_enabled. The behavior is undefined + * if continuous mode is already enabled, or if it cannot be enable due to + * conflicting options. + */ +void __llvm_profile_enable_continuous_mode(void); + +/*! + * \brief Disable continuous mode. + * + */ +void __llvm_profile_disable_continuous_mode(void); + +/*! + * \brief Set the page size. + * + * This is a pre-requisite for enabling continuous mode. The buffer size + * calculation code inside of libprofile cannot simply call getpagesize(), as + * it is not allowed to depend on libc. + */ +void __llvm_profile_set_page_size(unsigned PageSize); + +/*! + * \brief Get number of bytes necessary to pad the argument to eight + * byte boundary. + */ +uint8_t __llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes); + +/*! + * \brief Get required size for profile buffer. + */ +uint64_t __llvm_profile_get_size_for_buffer(void); + +/*! + * \brief Write instrumentation data to the given buffer. + * + * \pre \c Buffer is the start of a buffer at least as big as \a + * __llvm_profile_get_size_for_buffer(). + */ +int __llvm_profile_write_buffer(char *Buffer); + +const __llvm_profile_data *__llvm_profile_begin_data(void); +const __llvm_profile_data *__llvm_profile_end_data(void); +const char *__llvm_profile_begin_names(void); +const char *__llvm_profile_end_names(void); +const char *__llvm_profile_begin_vtabnames(void); +const char *__llvm_profile_end_vtabnames(void); +char *__llvm_profile_begin_counters(void); +char *__llvm_profile_end_counters(void); +char *__llvm_profile_begin_bitmap(void); +char *__llvm_profile_end_bitmap(void); +ValueProfNode *__llvm_profile_begin_vnodes(); +ValueProfNode *__llvm_profile_end_vnodes(); +const VTableProfData *__llvm_profile_begin_vtables(); +const VTableProfData *__llvm_profile_end_vtables(); +uint32_t *__llvm_profile_begin_orderfile(); + +/*! + * \brief Merge profile data from buffer. + * + * Read profile data from buffer \p Profile and merge with in-process profile + * counters and bitmaps. The client is expected to have checked or already + * know the profile data in the buffer matches the in-process counter + * structure before calling it. Returns 0 (success) if the profile data is + * valid. Upon reading invalid/corrupted profile data, returns 1 (failure). + */ +int __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size); + +/*! \brief Check if profile in buffer matches the current binary. + * + * Returns 0 (success) if the profile data in buffer \p Profile with size + * \p Size was generated by the same binary and therefore matches + * structurally the in-process counters and bitmaps. If the profile data in + * buffer is not compatible, the interface returns 1 (failure). + */ +int __llvm_profile_check_compatibility(const char *Profile, + uint64_t Size); + +/*! + * \brief Counts the number of times a target value is seen. + * + * Records the target value for the CounterIndex if not seen before. Otherwise, + * increments the counter associated w/ the target value. + * void __llvm_profile_instrument_target(uint64_t TargetValue, void *Data, + * uint32_t CounterIndex); + */ +void INSTR_PROF_VALUE_PROF_FUNC( +#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) ArgType ArgName +#include "profile/InstrProfData.inc" + ); + +void __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data, + uint32_t CounterIndex, + uint64_t CounterValue); + +/*! + * \brief Write instrumentation data to the current file. + * + * Writes to the file with the last name given to \a * + * __llvm_profile_set_filename(), + * or if it hasn't been called, the \c LLVM_PROFILE_FILE environment variable, + * or if that's not set, the last name set to INSTR_PROF_PROFILE_NAME_VAR, + * or if that's not set, \c "default.profraw". + */ +int __llvm_profile_write_file(void); + +int __llvm_orderfile_write_file(void); + +/*! + * \brief Set the FILE object for writing instrumentation data. Return 0 if set + * successfully or return 1 if failed. + * + * Sets the FILE object to be used for subsequent calls to + * \a __llvm_profile_write_file(). The profile file name set by environment + * variable, command-line option, or calls to \a __llvm_profile_set_filename + * will be ignored. + * + * \c File will not be closed after a call to \a __llvm_profile_write_file() but + * it may be flushed. Passing NULL restores default behavior. + * + * If \c EnableMerge is nonzero, the runtime will always merge profiling data + * with the contents of the profiling file. If EnableMerge is zero, the runtime + * may still merge the data if it would have merged for another reason (for + * example, because of a %m specifier in the file name). + * + * Note: There may be multiple copies of the profile runtime (one for each + * instrumented image/DSO). This API only modifies the file object within the + * copy of the runtime available to the calling image. + * + * Warning: This is a no-op if EnableMerge is 0 in continuous mode (\ref + * __llvm_profile_is_continuous_mode_enabled), because disable merging requires + * copying the old profile file to new profile file and this function is usually + * used when the proess doesn't have permission to open file. + */ +int __llvm_profile_set_file_object(FILE *File, int EnableMerge); + +/*! \brief Register to write instrumentation data to file at exit. */ +int __llvm_profile_register_write_file_atexit(void); + +/*! \brief Initialize file handling. */ +void __llvm_profile_initialize_file(void); + +/*! \brief Initialize the profile runtime. */ +void __llvm_profile_initialize(void); + +/*! + * \brief Return path prefix (excluding the base filename) of the profile data. + * This is useful for users using \c -fprofile-generate=./path_prefix who do + * not care about the default raw profile name. It is also useful to collect + * more than more profile data files dumped in the same directory (Online + * merge mode is turned on for instrumented programs with shared libs). + * Side-effect: this API call will invoke malloc with dynamic memory allocation. + */ +const char *__llvm_profile_get_path_prefix(); + +/*! + * \brief Return filename (including path) of the profile data. Note that if the + * user calls __llvm_profile_set_filename later after invoking this interface, + * the actual file name may differ from what is returned here. + * Side-effect: this API call will invoke malloc with dynamic memory allocation + * (the returned pointer must be passed to `free` to avoid a leak). + * + * Note: There may be multiple copies of the profile runtime (one for each + * instrumented image/DSO). This API only retrieves the filename from the copy + * of the runtime available to the calling image. + */ +const char *__llvm_profile_get_filename(); + +/*! \brief Get the magic token for the file format. */ +uint64_t __llvm_profile_get_magic(void); + +/*! \brief Get the version of the file format. */ +uint64_t __llvm_profile_get_version(void); + +/*! \brief Get the number of entries in the profile data section. */ +uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, + const __llvm_profile_data *End); + +/*! \brief Get the size of the profile data section in bytes. */ +uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, + const __llvm_profile_data *End); + +/*! \brief Get the size in bytes of a single counter entry. */ +size_t __llvm_profile_counter_entry_size(void); + +/*! \brief Get the number of entries in the profile counters section. */ +uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End); + +/*! \brief Get the size of the profile counters section in bytes. */ +uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End); + +/*! \brief Get the number of bytes in the profile bitmap section. */ +uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, + const char *End); + +/*! \brief Get the size of the profile name section in bytes. */ +uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End); + +/*! \brief Get the number of virtual table profile data entries */ +uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin, + const VTableProfData *End); + +/*! \brief Get the size of virtual table profile data in bytes. */ +uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin, + const VTableProfData *End); + +/* ! \brief Given the sizes of the data and counter information, computes the + * number of padding bytes before and after the counter section, as well as the + * number of padding bytes after other setions in the raw profile. + * Returns -1 upon errors and 0 upon success. Output parameters should be used + * iff return value is 0. + * + * Note: When mmap() mode is disabled, no padding bytes before/after counters + * are needed. However, in mmap() mode, the counter section in the raw profile + * must be page-aligned: this API computes the number of padding bytes + * needed to achieve that. + */ +int __llvm_profile_get_padding_sizes_for_counters( + uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, + uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize, + uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, + uint64_t *PaddingBytesAfterBitmap, uint64_t *PaddingBytesAfterNames, + uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVNames); + +/*! + * \brief Set the flag that profile data has been dumped to the file. + * This is useful for users to disable dumping profile data to the file for + * certain processes in case the processes don't have permission to write to + * the disks, and trying to do so would result in side effects such as crashes. + */ +void __llvm_profile_set_dumped(); + +/*! + * This variable is defined in InstrProfilingRuntime.cpp as a hidden + * symbol. Its main purpose is to enable profile runtime user to + * bypass runtime initialization code -- if the client code explicitly + * define this variable, then InstProfileRuntime.o won't be linked in. + * Note that this variable's visibility needs to be hidden so that the + * definition of this variable in an instrumented shared library won't + * affect runtime initialization decision of the main program. + * __llvm_profile_profile_runtime. */ +COMPILER_RT_VISIBILITY extern int INSTR_PROF_PROFILE_RUNTIME_VAR; + +/*! + * This variable is defined in InstrProfilingVersionVar.c as a hidden symbol + * (except on Apple platforms where this symbol is checked by TAPI). Its main + * purpose is to encode the raw profile version value and other format related + * information such as whether the profile is from IR based instrumentation. The + * variable is defined as weak so that compiler can emit an overriding + * definition depending on user option. + */ +COMPILER_RT_VISIBILITY extern uint64_t + INSTR_PROF_RAW_VERSION_VAR; /* __llvm_profile_raw_version */ + +/*! + * This variable is a weak symbol defined in InstrProfiling.c. It allows + * compiler instrumentation to provide overriding definition with value + * from compiler command line. This variable has default visibility. + */ +extern char INSTR_PROF_PROFILE_NAME_VAR[1]; /* __llvm_profile_filename. */ + +#endif /* PROFILE_INSTRPROFILING_H_ */ diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingBuffer.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingBuffer.c new file mode 100644 index 000000000000..1c451d7ec756 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -0,0 +1,256 @@ +/*===- InstrProfilingBuffer.c - Write instrumentation to a memory buffer --===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +// Note: This is linked into the Darwin kernel, and must remain compatible +// with freestanding compilation. See `darwin_add_builtin_libraries`. + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingPort.h" + +/* When continuous mode is enabled (%c), this parameter is set to 1. + * + * This parameter is defined here in InstrProfilingBuffer.o, instead of in + * InstrProfilingFile.o, to sequester all libc-dependent code in + * InstrProfilingFile.o. The test `instrprof-without-libc` will break if this + * layering is violated. */ +static int ContinuouslySyncProfile = 0; + +/* The system page size. Only valid when non-zero. If 0, the page size is + * unavailable. */ +static unsigned PageSize = 0; + +COMPILER_RT_VISIBILITY int __llvm_profile_is_continuous_mode_enabled(void) { + return ContinuouslySyncProfile && PageSize; +} + +COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) { + ContinuouslySyncProfile = 1; +} + +COMPILER_RT_VISIBILITY void __llvm_profile_disable_continuous_mode(void) { + ContinuouslySyncProfile = 0; +} + +COMPILER_RT_VISIBILITY void __llvm_profile_set_page_size(unsigned PS) { + PageSize = PS; +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_size_for_buffer(void) { + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const char *CountersBegin = __llvm_profile_begin_counters(); + const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); + const char *NamesBegin = __llvm_profile_begin_names(); + const char *NamesEnd = __llvm_profile_end_names(); + const VTableProfData *VTableBegin = __llvm_profile_begin_vtables(); + const VTableProfData *VTableEnd = __llvm_profile_end_vtables(); + const char *VNamesBegin = __llvm_profile_begin_vtabnames(); + const char *VNamesEnd = __llvm_profile_end_vtabnames(); + + return __llvm_profile_get_size_for_buffer_internal( + DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd, + NamesBegin, NamesEnd, VTableBegin, VTableEnd, VNamesBegin, VNamesEnd); +} + +// NOTE: Caller should guarantee that `Begin` and `End` specifies a half-open +// interval [Begin, End). Namely, `End` is one-byte past the end of the array. +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, + const __llvm_profile_data *End) { + intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End; + return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) / + sizeof(__llvm_profile_data); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, + const __llvm_profile_data *End) { + return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data); +} + +// Counts the number of `VTableProfData` elements within the range of [Begin, +// End). Caller should guarantee that End points to one byte past the inclusive +// range. +// FIXME: Add a compiler-rt test to make sure the number of vtables in the +// raw profile is the same as the number of vtable elements in the instrumented +// binary. +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin, + const VTableProfData *End) { + // Convert pointers to intptr_t to use integer arithmetic. + intptr_t EndI = (intptr_t)End, BeginI = (intptr_t)Begin; + return (EndI - BeginI) / sizeof(VTableProfData); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin, + const VTableProfData *End) { + return (intptr_t)(End) - (intptr_t)(Begin); +} + +COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) { + if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) + return sizeof(uint8_t); + return sizeof(uint64_t); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End) { + intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End; + return ((EndI + __llvm_profile_counter_entry_size() - 1) - BeginI) / + __llvm_profile_counter_entry_size(); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End) { + return __llvm_profile_get_num_counters(Begin, End) * + __llvm_profile_counter_entry_size(); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, + const char *End) { + return (End - Begin); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End) { + return End - Begin; +} + +/// Calculate the number of padding bytes needed to add to \p Offset in order +/// for (\p Offset + Padding) to be page-aligned. +static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) { + uint64_t OffsetModPage = Offset % PageSize; + if (OffsetModPage > 0) + return PageSize - OffsetModPage; + return 0; +} + +static int needsCounterPadding(void) { +#if defined(__APPLE__) + return __llvm_profile_is_continuous_mode_enabled(); +#else + return 0; +#endif +} + +COMPILER_RT_VISIBILITY +int __llvm_profile_get_padding_sizes_for_counters( + uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, + uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize, + uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, + uint64_t *PaddingBytesAfterBitmapBytes, uint64_t *PaddingBytesAfterNames, + uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVName) { + // Counter padding is needed only if continuous mode is enabled. + if (!needsCounterPadding()) { + *PaddingBytesBeforeCounters = 0; + *PaddingBytesAfterCounters = + __llvm_profile_get_num_padding_bytes(CountersSize); + *PaddingBytesAfterBitmapBytes = + __llvm_profile_get_num_padding_bytes(NumBitmapBytes); + *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize); + if (PaddingBytesAfterVTable != NULL) + *PaddingBytesAfterVTable = + __llvm_profile_get_num_padding_bytes(VTableSize); + if (PaddingBytesAfterVName != NULL) + *PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(VNameSize); + return 0; + } + + // Value profiling not supported in continuous mode at profile-write time. + // Return -1 to alert the incompatibility. + if (VTableSize != 0 || VNameSize != 0) + return -1; + + // In continuous mode, the file offsets for headers and for the start of + // counter sections need to be page-aligned. + *PaddingBytesBeforeCounters = + calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize); + *PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize); + *PaddingBytesAfterBitmapBytes = + calculateBytesNeededToPageAlign(NumBitmapBytes); + *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize); + // Set these two variables to zero to avoid uninitialized variables + // even if VTableSize and VNameSize are known to be zero. + if (PaddingBytesAfterVTable != NULL) + *PaddingBytesAfterVTable = 0; + if (PaddingBytesAfterVName != NULL) + *PaddingBytesAfterVName = 0; + return 0; +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_size_for_buffer_internal( + const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, + const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, + const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd, + const VTableProfData *VTableBegin, const VTableProfData *VTableEnd, + const char *VNamesBegin, const char *VNamesEnd) { + /* Match logic in __llvm_profile_write_buffer(). */ + const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); + uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + uint64_t CountersSize = + __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + const uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); + const uint64_t VTableSize = + __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd); + const uint64_t VNameSize = + __llvm_profile_get_name_size(VNamesBegin, VNamesEnd); + + /* Determine how much padding is needed before/after the counters and after + * the names. */ + uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes, + PaddingBytesAfterVTable, PaddingBytesAfterVNames; + __llvm_profile_get_padding_sizes_for_counters( + DataSize, CountersSize, NumBitmapBytes, NamesSize, 0 /* VTableSize */, + 0 /* VNameSize */, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes, + &PaddingBytesAfterNames, &PaddingBytesAfterVTable, + &PaddingBytesAfterVNames); + + return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + + DataSize + PaddingBytesBeforeCounters + CountersSize + + PaddingBytesAfterCounters + NumBitmapBytes + + PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames + + VTableSize + PaddingBytesAfterVTable + VNameSize + + PaddingBytesAfterVNames; +} + +COMPILER_RT_VISIBILITY +void initBufferWriter(ProfDataWriter *BufferWriter, char *Buffer) { + BufferWriter->Write = lprofBufferWriter; + BufferWriter->WriterCtx = Buffer; +} + +COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer(char *Buffer) { + ProfDataWriter BufferWriter; + initBufferWriter(&BufferWriter, Buffer); + return lprofWriteData(&BufferWriter, 0, 0); +} + +COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal( + char *Buffer, const __llvm_profile_data *DataBegin, + const __llvm_profile_data *DataEnd, const char *CountersBegin, + const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, + const char *NamesBegin, const char *NamesEnd) { + ProfDataWriter BufferWriter; + initBufferWriter(&BufferWriter, Buffer); + // Set virtual table arguments to NULL since they are not supported yet. + return lprofWriteDataImpl( + &BufferWriter, DataBegin, DataEnd, CountersBegin, CountersEnd, + BitmapBegin, BitmapEnd, /*VPDataReader=*/0, NamesBegin, NamesEnd, + /*VTableBegin=*/NULL, /*VTableEnd=*/NULL, /*VNamesBegin=*/NULL, + /*VNamesEnd=*/NULL, /*SkipNameDataWrite=*/0); +} diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingFile.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingFile.c new file mode 100644 index 000000000000..1c58584d2d4f --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingFile.c @@ -0,0 +1,1278 @@ +/*===- InstrProfilingFile.c - Write instrumentation to a file -------------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#if !defined(__Fuchsia__) + +#include <assert.h> +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#ifdef _MSC_VER +/* For _alloca. */ +#include <malloc.h> +#endif +#if defined(_WIN32) +#include "WindowsMMap.h" +/* For _chsize_s */ +#include <io.h> +#include <process.h> +#else +#include <sys/file.h> +#include <sys/mman.h> +#include <unistd.h> +#if defined(__linux__) +#include <sys/types.h> +#endif +#endif + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingPort.h" +#include "InstrProfilingUtil.h" + +/* From where is profile name specified. + * The order the enumerators define their + * precedence. Re-order them may lead to + * runtime behavior change. */ +typedef enum ProfileNameSpecifier { + PNS_unknown = 0, + PNS_default, + PNS_command_line, + PNS_environment, + PNS_runtime_api +} ProfileNameSpecifier; + +static const char *getPNSStr(ProfileNameSpecifier PNS) { + switch (PNS) { + case PNS_default: + return "default setting"; + case PNS_command_line: + return "command line"; + case PNS_environment: + return "environment variable"; + case PNS_runtime_api: + return "runtime API"; + default: + return "Unknown"; + } +} + +#define MAX_PID_SIZE 16 +/* Data structure holding the result of parsed filename pattern. */ +typedef struct lprofFilename { + /* File name string possibly with %p or %h specifiers. */ + const char *FilenamePat; + /* A flag indicating if FilenamePat's memory is allocated + * by runtime. */ + unsigned OwnsFilenamePat; + const char *ProfilePathPrefix; + char PidChars[MAX_PID_SIZE]; + char *TmpDir; + char Hostname[COMPILER_RT_MAX_HOSTLEN]; + unsigned NumPids; + unsigned NumHosts; + /* When in-process merging is enabled, this parameter specifies + * the total number of profile data files shared by all the processes + * spawned from the same binary. By default the value is 1. If merging + * is not enabled, its value should be 0. This parameter is specified + * by the %[0-9]m specifier. For instance %2m enables merging using + * 2 profile data files. %1m is equivalent to %m. Also %m specifier + * can only appear once at the end of the name pattern. */ + unsigned MergePoolSize; + ProfileNameSpecifier PNS; +} lprofFilename; + +static lprofFilename lprofCurFilename = {0, 0, 0, {0}, NULL, + {0}, 0, 0, 0, PNS_unknown}; + +static int ProfileMergeRequested = 0; +static int getProfileFileSizeForMerging(FILE *ProfileFile, + uint64_t *ProfileFileSize); + +#if defined(__APPLE__) +static const int ContinuousModeSupported = 1; +static const int UseBiasVar = 0; +static const char *FileOpenMode = "a+b"; +static void *BiasAddr = NULL; +static void *BiasDefaultAddr = NULL; +static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { + /* Get the sizes of various profile data sections. Taken from + * __llvm_profile_get_size_for_buffer(). */ + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const char *CountersBegin = __llvm_profile_begin_counters(); + const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); + const char *NamesBegin = __llvm_profile_begin_names(); + const char *NamesEnd = __llvm_profile_end_names(); + const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); + uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + uint64_t CountersSize = + __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); + + /* Check that the counter, bitmap, and data sections in this image are + * page-aligned. */ + unsigned PageSize = getpagesize(); + if ((intptr_t)CountersBegin % PageSize != 0) { + PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n", + CountersBegin, PageSize); + return 1; + } + if ((intptr_t)BitmapBegin % PageSize != 0) { + PROF_ERR("Bitmap section not page-aligned (start = %p, pagesz = %u).\n", + BitmapBegin, PageSize); + return 1; + } + if ((intptr_t)DataBegin % PageSize != 0) { + PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n", + DataBegin, PageSize); + return 1; + } + + int Fileno = fileno(File); + /* Determine how much padding is needed before/after the counters and + * after the names. */ + uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes, + PaddingBytesAfterVTable, PaddingBytesAfterVNames; + __llvm_profile_get_padding_sizes_for_counters( + DataSize, CountersSize, NumBitmapBytes, NamesSize, /*VTableSize=*/0, + /*VNameSize=*/0, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, + &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames, + &PaddingBytesAfterVTable, &PaddingBytesAfterVNames); + + uint64_t PageAlignedCountersLength = CountersSize + PaddingBytesAfterCounters; + uint64_t FileOffsetToCounters = CurrentFileOffset + + sizeof(__llvm_profile_header) + DataSize + + PaddingBytesBeforeCounters; + void *CounterMmap = mmap((void *)CountersBegin, PageAlignedCountersLength, + PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED, + Fileno, FileOffsetToCounters); + if (CounterMmap != CountersBegin) { + PROF_ERR( + "Continuous counter sync mode is enabled, but mmap() failed (%s).\n" + " - CountersBegin: %p\n" + " - PageAlignedCountersLength: %" PRIu64 "\n" + " - Fileno: %d\n" + " - FileOffsetToCounters: %" PRIu64 "\n", + strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno, + FileOffsetToCounters); + return 1; + } + + /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap() + * will fail with EINVAL. */ + if (NumBitmapBytes == 0) + return 0; + + uint64_t PageAlignedBitmapLength = + NumBitmapBytes + PaddingBytesAfterBitmapBytes; + uint64_t FileOffsetToBitmap = + FileOffsetToCounters + CountersSize + PaddingBytesAfterCounters; + void *BitmapMmap = + mmap((void *)BitmapBegin, PageAlignedBitmapLength, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToBitmap); + if (BitmapMmap != BitmapBegin) { + PROF_ERR( + "Continuous counter sync mode is enabled, but mmap() failed (%s).\n" + " - BitmapBegin: %p\n" + " - PageAlignedBitmapLength: %" PRIu64 "\n" + " - Fileno: %d\n" + " - FileOffsetToBitmap: %" PRIu64 "\n", + strerror(errno), BitmapBegin, PageAlignedBitmapLength, Fileno, + FileOffsetToBitmap); + return 1; + } + return 0; +} +#elif defined(__ELF__) || defined(_WIN32) + +#define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \ + INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR, _default) +COMPILER_RT_VISIBILITY intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0; + +/* This variable is a weak external reference which could be used to detect + * whether or not the compiler defined this symbol. */ +#if defined(_MSC_VER) +COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; +#if defined(_M_IX86) || defined(__i386__) +#define WIN_SYM_PREFIX "_" +#else +#define WIN_SYM_PREFIX +#endif +#pragma comment( \ + linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \ + INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" WIN_SYM_PREFIX \ + INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR)) +#else +COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR + __attribute__((weak, alias(INSTR_PROF_QUOTE( + INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR)))); +#endif +static const int ContinuousModeSupported = 1; +static const int UseBiasVar = 1; +/* TODO: If there are two DSOs, the second DSO initilization will truncate the + * first profile file. */ +static const char *FileOpenMode = "w+b"; +/* This symbol is defined by the compiler when runtime counter relocation is + * used and runtime provides a weak alias so we can check if it's defined. */ +static void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; +static void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR; +static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { + /* Get the sizes of various profile data sections. Taken from + * __llvm_profile_get_size_for_buffer(). */ + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const char *CountersBegin = __llvm_profile_begin_counters(); + const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); + uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + /* Get the file size. */ + uint64_t FileSize = 0; + if (getProfileFileSizeForMerging(File, &FileSize)) + return 1; + + int Fileno = fileno(File); + uint64_t FileOffsetToCounters = + sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize; + + /* Map the profile. */ + char *Profile = (char *)mmap(NULL, FileSize, PROT_READ | PROT_WRITE, + MAP_SHARED, Fileno, 0); + if (Profile == MAP_FAILED) { + PROF_ERR("Unable to mmap profile: %s\n", strerror(errno)); + return 1; + } + /* Update the profile fields based on the current mapping. */ + INSTR_PROF_PROFILE_COUNTER_BIAS_VAR = + (intptr_t)Profile - (uintptr_t)CountersBegin + FileOffsetToCounters; + + /* Return the memory allocated for counters to OS. */ + lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd); + + /* BIAS MODE not supported yet for Bitmap (MCDC). */ + + /* Return the memory allocated for counters to OS. */ + lprofReleaseMemoryPagesToOS((uintptr_t)BitmapBegin, (uintptr_t)BitmapEnd); + return 0; +} +#else +static const int ContinuousModeSupported = 0; +static const int UseBiasVar = 0; +static const char *FileOpenMode = "a+b"; +static void *BiasAddr = NULL; +static void *BiasDefaultAddr = NULL; +static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { + return 0; +} +#endif + +static int isProfileMergeRequested(void) { return ProfileMergeRequested; } +static void setProfileMergeRequested(int EnableMerge) { + ProfileMergeRequested = EnableMerge; +} + +static FILE *ProfileFile = NULL; +static FILE *getProfileFile(void) { return ProfileFile; } +static void setProfileFile(FILE *File) { ProfileFile = File; } + +static int getCurFilenameLength(void); +static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf); +static unsigned doMerging(void) { + return lprofCurFilename.MergePoolSize || isProfileMergeRequested(); +} + +/* Return 1 if there is an error, otherwise return 0. */ +static uint32_t fileWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, + uint32_t NumIOVecs) { + uint32_t I; + FILE *File = (FILE *)This->WriterCtx; + char Zeroes[sizeof(uint64_t)] = {0}; + for (I = 0; I < NumIOVecs; I++) { + if (IOVecs[I].Data) { + if (fwrite(IOVecs[I].Data, IOVecs[I].ElmSize, IOVecs[I].NumElm, File) != + IOVecs[I].NumElm) + return 1; + } else if (IOVecs[I].UseZeroPadding) { + size_t BytesToWrite = IOVecs[I].ElmSize * IOVecs[I].NumElm; + while (BytesToWrite > 0) { + size_t PartialWriteLen = + (sizeof(uint64_t) > BytesToWrite) ? BytesToWrite : sizeof(uint64_t); + if (fwrite(Zeroes, sizeof(uint8_t), PartialWriteLen, File) != + PartialWriteLen) { + return 1; + } + BytesToWrite -= PartialWriteLen; + } + } else { + if (fseek(File, IOVecs[I].ElmSize * IOVecs[I].NumElm, SEEK_CUR) == -1) + return 1; + } + } + return 0; +} + +/* TODO: make buffer size controllable by an internal option, and compiler can pass the size + to runtime via a variable. */ +static uint32_t orderFileWriter(FILE *File, const uint32_t *DataStart) { + if (fwrite(DataStart, sizeof(uint32_t), INSTR_ORDER_FILE_BUFFER_SIZE, File) != + INSTR_ORDER_FILE_BUFFER_SIZE) + return 1; + return 0; +} + +static void initFileWriter(ProfDataWriter *This, FILE *File) { + This->Write = fileWriter; + This->WriterCtx = File; +} + +COMPILER_RT_VISIBILITY ProfBufferIO * +lprofCreateBufferIOInternal(void *File, uint32_t BufferSz) { + FreeHook = &free; + DynamicBufferIOBuffer = (uint8_t *)calloc(1, BufferSz); + VPBufferSize = BufferSz; + ProfDataWriter *fileWriter = + (ProfDataWriter *)calloc(1, sizeof(ProfDataWriter)); + initFileWriter(fileWriter, File); + ProfBufferIO *IO = lprofCreateBufferIO(fileWriter); + IO->OwnFileWriter = 1; + return IO; +} + +static void setupIOBuffer(void) { + const char *BufferSzStr = 0; + BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE"); + if (BufferSzStr && BufferSzStr[0]) { + VPBufferSize = atoi(BufferSzStr); + DynamicBufferIOBuffer = (uint8_t *)calloc(VPBufferSize, 1); + } +} + +/* Get the size of the profile file. If there are any errors, print the + * message under the assumption that the profile is being read for merging + * purposes, and return -1. Otherwise return the file size in the inout param + * \p ProfileFileSize. */ +static int getProfileFileSizeForMerging(FILE *ProfileFile, + uint64_t *ProfileFileSize) { + if (fseek(ProfileFile, 0L, SEEK_END) == -1) { + PROF_ERR("Unable to merge profile data, unable to get size: %s\n", + strerror(errno)); + return -1; + } + *ProfileFileSize = ftell(ProfileFile); + + /* Restore file offset. */ + if (fseek(ProfileFile, 0L, SEEK_SET) == -1) { + PROF_ERR("Unable to merge profile data, unable to rewind: %s\n", + strerror(errno)); + return -1; + } + + if (*ProfileFileSize > 0 && + *ProfileFileSize < sizeof(__llvm_profile_header)) { + PROF_WARN("Unable to merge profile data: %s\n", + "source profile file is too small."); + return -1; + } + return 0; +} + +/* mmap() \p ProfileFile for profile merging purposes, assuming that an + * exclusive lock is held on the file and that \p ProfileFileSize is the + * length of the file. Return the mmap'd buffer in the inout variable + * \p ProfileBuffer. Returns -1 on failure. On success, the caller is + * responsible for unmapping the mmap'd buffer in \p ProfileBuffer. */ +static int mmapProfileForMerging(FILE *ProfileFile, uint64_t ProfileFileSize, + char **ProfileBuffer) { + *ProfileBuffer = mmap(NULL, ProfileFileSize, PROT_READ, MAP_SHARED | MAP_FILE, + fileno(ProfileFile), 0); + if (*ProfileBuffer == MAP_FAILED) { + PROF_ERR("Unable to merge profile data, mmap failed: %s\n", + strerror(errno)); + return -1; + } + + if (__llvm_profile_check_compatibility(*ProfileBuffer, ProfileFileSize)) { + (void)munmap(*ProfileBuffer, ProfileFileSize); + PROF_WARN("Unable to merge profile data: %s\n", + "source profile file is not compatible."); + return -1; + } + return 0; +} + +/* Read profile data in \c ProfileFile and merge with in-memory + profile counters. Returns -1 if there is fatal error, otheriwse + 0 is returned. Returning 0 does not mean merge is actually + performed. If merge is actually done, *MergeDone is set to 1. +*/ +static int doProfileMerging(FILE *ProfileFile, int *MergeDone) { + uint64_t ProfileFileSize; + char *ProfileBuffer; + + /* Get the size of the profile on disk. */ + if (getProfileFileSizeForMerging(ProfileFile, &ProfileFileSize) == -1) + return -1; + + /* Nothing to merge. */ + if (!ProfileFileSize) + return 0; + + /* mmap() the profile and check that it is compatible with the data in + * the current image. */ + if (mmapProfileForMerging(ProfileFile, ProfileFileSize, &ProfileBuffer) == -1) + return -1; + + /* Now start merging */ + if (__llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize)) { + PROF_ERR("%s\n", "Invalid profile data to merge"); + (void)munmap(ProfileBuffer, ProfileFileSize); + return -1; + } + + // Truncate the file in case merging of value profile did not happen to + // prevent from leaving garbage data at the end of the profile file. + (void)COMPILER_RT_FTRUNCATE(ProfileFile, + __llvm_profile_get_size_for_buffer()); + + (void)munmap(ProfileBuffer, ProfileFileSize); + *MergeDone = 1; + + return 0; +} + +/* Create the directory holding the file, if needed. */ +static void createProfileDir(const char *Filename) { + size_t Length = strlen(Filename); + if (lprofFindFirstDirSeparator(Filename)) { + char *Copy = (char *)COMPILER_RT_ALLOCA(Length + 1); + strncpy(Copy, Filename, Length + 1); + __llvm_profile_recursive_mkdir(Copy); + } +} + +/* Open the profile data for merging. It opens the file in r+b mode with + * file locking. If the file has content which is compatible with the + * current process, it also reads in the profile data in the file and merge + * it with in-memory counters. After the profile data is merged in memory, + * the original profile data is truncated and gets ready for the profile + * dumper. With profile merging enabled, each executable as well as any of + * its instrumented shared libraries dump profile data into their own data file. +*/ +static FILE *openFileForMerging(const char *ProfileFileName, int *MergeDone) { + FILE *ProfileFile = getProfileFile(); + int rc; + // initializeProfileForContinuousMode will lock the profile, but if + // ProfileFile is set by user via __llvm_profile_set_file_object, it's assumed + // unlocked at this point. + if (ProfileFile && !__llvm_profile_is_continuous_mode_enabled()) { + lprofLockFileHandle(ProfileFile); + } + if (!ProfileFile) { + createProfileDir(ProfileFileName); + ProfileFile = lprofOpenFileEx(ProfileFileName); + } + if (!ProfileFile) + return NULL; + + rc = doProfileMerging(ProfileFile, MergeDone); + if (rc || (!*MergeDone && COMPILER_RT_FTRUNCATE(ProfileFile, 0L)) || + fseek(ProfileFile, 0L, SEEK_SET) == -1) { + PROF_ERR("Profile Merging of file %s failed: %s\n", ProfileFileName, + strerror(errno)); + fclose(ProfileFile); + return NULL; + } + return ProfileFile; +} + +static FILE *getFileObject(const char *OutputName) { + FILE *File; + File = getProfileFile(); + if (File != NULL) { + return File; + } + + return fopen(OutputName, "ab"); +} + +/* Write profile data to file \c OutputName. */ +static int writeFile(const char *OutputName) { + int RetVal; + FILE *OutputFile; + + int MergeDone = 0; + VPMergeHook = &lprofMergeValueProfData; + if (doMerging()) + OutputFile = openFileForMerging(OutputName, &MergeDone); + else + OutputFile = getFileObject(OutputName); + + if (!OutputFile) + return -1; + + FreeHook = &free; + setupIOBuffer(); + ProfDataWriter fileWriter; + initFileWriter(&fileWriter, OutputFile); + RetVal = lprofWriteData(&fileWriter, lprofGetVPDataReader(), MergeDone); + + if (OutputFile == getProfileFile()) { + fflush(OutputFile); + if (doMerging() && !__llvm_profile_is_continuous_mode_enabled()) { + lprofUnlockFileHandle(OutputFile); + } + } else { + fclose(OutputFile); + } + + return RetVal; +} + +/* Write order data to file \c OutputName. */ +static int writeOrderFile(const char *OutputName) { + int RetVal; + FILE *OutputFile; + + OutputFile = fopen(OutputName, "w"); + + if (!OutputFile) { + PROF_WARN("can't open file with mode ab: %s\n", OutputName); + return -1; + } + + FreeHook = &free; + setupIOBuffer(); + const uint32_t *DataBegin = __llvm_profile_begin_orderfile(); + RetVal = orderFileWriter(OutputFile, DataBegin); + + fclose(OutputFile); + return RetVal; +} + +#define LPROF_INIT_ONCE_ENV "__LLVM_PROFILE_RT_INIT_ONCE" + +static void truncateCurrentFile(void) { + const char *Filename; + char *FilenameBuf; + FILE *File; + int Length; + + Length = getCurFilenameLength(); + FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + Filename = getCurFilename(FilenameBuf, 0); + if (!Filename) + return; + + /* Only create the profile directory and truncate an existing profile once. + * In continuous mode, this is necessary, as the profile is written-to by the + * runtime initializer. */ + int initialized = getenv(LPROF_INIT_ONCE_ENV) != NULL; + if (initialized) + return; +#if defined(_WIN32) + _putenv(LPROF_INIT_ONCE_ENV "=" LPROF_INIT_ONCE_ENV); +#else + setenv(LPROF_INIT_ONCE_ENV, LPROF_INIT_ONCE_ENV, 1); +#endif + + /* Create the profile dir (even if online merging is enabled), so that + * the profile file can be set up if continuous mode is enabled. */ + createProfileDir(Filename); + + /* By pass file truncation to allow online raw profile merging. */ + if (lprofCurFilename.MergePoolSize) + return; + + /* Truncate the file. Later we'll reopen and append. */ + File = fopen(Filename, "w"); + if (!File) + return; + fclose(File); +} + +/* Write a partial profile to \p Filename, which is required to be backed by + * the open file object \p File. */ +static int writeProfileWithFileObject(const char *Filename, FILE *File) { + setProfileFile(File); + int rc = writeFile(Filename); + if (rc) + PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno)); + setProfileFile(NULL); + return rc; +} + +static void initializeProfileForContinuousMode(void) { + if (!__llvm_profile_is_continuous_mode_enabled()) + return; + if (!ContinuousModeSupported) { + PROF_ERR("%s\n", "continuous mode is unsupported on this platform"); + return; + } + if (UseBiasVar && BiasAddr == BiasDefaultAddr) { + PROF_ERR("%s\n", "__llvm_profile_counter_bias is undefined"); + return; + } + + /* Get the sizes of counter section. */ + uint64_t CountersSize = __llvm_profile_get_counters_size( + __llvm_profile_begin_counters(), __llvm_profile_end_counters()); + + int Length = getCurFilenameLength(); + char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + const char *Filename = getCurFilename(FilenameBuf, 0); + if (!Filename) + return; + + FILE *File = NULL; + uint64_t CurrentFileOffset = 0; + if (doMerging()) { + /* We are merging profiles. Map the counter section as shared memory into + * the profile, i.e. into each participating process. An increment in one + * process should be visible to every other process with the same counter + * section mapped. */ + File = lprofOpenFileEx(Filename); + if (!File) + return; + + uint64_t ProfileFileSize = 0; + if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) { + lprofUnlockFileHandle(File); + fclose(File); + return; + } + if (ProfileFileSize == 0) { + /* Grow the profile so that mmap() can succeed. Leak the file handle, as + * the file should stay open. */ + if (writeProfileWithFileObject(Filename, File) != 0) { + lprofUnlockFileHandle(File); + fclose(File); + return; + } + } else { + /* The merged profile has a non-zero length. Check that it is compatible + * with the data in this process. */ + char *ProfileBuffer; + if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1) { + lprofUnlockFileHandle(File); + fclose(File); + return; + } + (void)munmap(ProfileBuffer, ProfileFileSize); + } + } else { + File = fopen(Filename, FileOpenMode); + if (!File) + return; + /* Check that the offset within the file is page-aligned. */ + CurrentFileOffset = ftell(File); + unsigned PageSize = getpagesize(); + if (CurrentFileOffset % PageSize != 0) { + PROF_ERR("Continuous counter sync mode is enabled, but raw profile is not" + "page-aligned. CurrentFileOffset = %" PRIu64 ", pagesz = %u.\n", + (uint64_t)CurrentFileOffset, PageSize); + fclose(File); + return; + } + if (writeProfileWithFileObject(Filename, File) != 0) { + fclose(File); + return; + } + } + + /* mmap() the profile counters so long as there is at least one counter. + * If there aren't any counters, mmap() would fail with EINVAL. */ + if (CountersSize > 0) + mmapForContinuousMode(CurrentFileOffset, File); + + if (doMerging()) { + lprofUnlockFileHandle(File); + } + if (File != NULL) { + fclose(File); + } +} + +static const char *DefaultProfileName = "default.profraw"; +static void resetFilenameToDefault(void) { + if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) { +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +#endif + free((void *)lprofCurFilename.FilenamePat); +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop +#endif + } + memset(&lprofCurFilename, 0, sizeof(lprofCurFilename)); + lprofCurFilename.FilenamePat = DefaultProfileName; + lprofCurFilename.PNS = PNS_default; +} + +static unsigned getMergePoolSize(const char *FilenamePat, int *I) { + unsigned J = 0, Num = 0; + for (;; ++J) { + char C = FilenamePat[*I + J]; + if (C == 'm') { + *I += J; + return Num ? Num : 1; + } + if (C < '0' || C > '9') + break; + Num = Num * 10 + C - '0'; + + /* If FilenamePat[*I+J] is between '0' and '9', the next byte is guaranteed + * to be in-bound as the string is null terminated. */ + } + return 0; +} + +/* Assert that Idx does index past a string null terminator. Return the + * result of the check. */ +static int checkBounds(int Idx, int Strlen) { + assert(Idx <= Strlen && "Indexing past string null terminator"); + return Idx <= Strlen; +} + +/* Parses the pattern string \p FilenamePat and stores the result to + * lprofcurFilename structure. */ +static int parseFilenamePattern(const char *FilenamePat, + unsigned CopyFilenamePat) { + int NumPids = 0, NumHosts = 0, I; + char *PidChars = &lprofCurFilename.PidChars[0]; + char *Hostname = &lprofCurFilename.Hostname[0]; + int MergingEnabled = 0; + int FilenamePatLen = strlen(FilenamePat); + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +#endif + /* Clean up cached prefix and filename. */ + if (lprofCurFilename.ProfilePathPrefix) + free((void *)lprofCurFilename.ProfilePathPrefix); + + if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) { + free((void *)lprofCurFilename.FilenamePat); + } +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop +#endif + + memset(&lprofCurFilename, 0, sizeof(lprofCurFilename)); + + if (!CopyFilenamePat) + lprofCurFilename.FilenamePat = FilenamePat; + else { + lprofCurFilename.FilenamePat = strdup(FilenamePat); + lprofCurFilename.OwnsFilenamePat = 1; + } + /* Check the filename for "%p", which indicates a pid-substitution. */ + for (I = 0; checkBounds(I, FilenamePatLen) && FilenamePat[I]; ++I) { + if (FilenamePat[I] == '%') { + ++I; /* Advance to the next character. */ + if (!checkBounds(I, FilenamePatLen)) + break; + if (FilenamePat[I] == 'p') { + if (!NumPids++) { + if (snprintf(PidChars, MAX_PID_SIZE, "%ld", (long)getpid()) <= 0) { + PROF_WARN("Unable to get pid for filename pattern %s. Using the " + "default name.", + FilenamePat); + return -1; + } + } + } else if (FilenamePat[I] == 'h') { + if (!NumHosts++) + if (COMPILER_RT_GETHOSTNAME(Hostname, COMPILER_RT_MAX_HOSTLEN)) { + PROF_WARN("Unable to get hostname for filename pattern %s. Using " + "the default name.", + FilenamePat); + return -1; + } + } else if (FilenamePat[I] == 't') { + lprofCurFilename.TmpDir = getenv("TMPDIR"); + if (!lprofCurFilename.TmpDir) { + PROF_WARN("Unable to get the TMPDIR environment variable, referenced " + "in %s. Using the default path.", + FilenamePat); + return -1; + } + } else if (FilenamePat[I] == 'c') { + if (__llvm_profile_is_continuous_mode_enabled()) { + PROF_WARN("%%c specifier can only be specified once in %s.\n", + FilenamePat); + __llvm_profile_disable_continuous_mode(); + return -1; + } +#if defined(__APPLE__) || defined(__ELF__) || defined(_WIN32) + __llvm_profile_set_page_size(getpagesize()); + __llvm_profile_enable_continuous_mode(); +#else + PROF_WARN("%s", "Continous mode is currently only supported for Mach-O," + " ELF and COFF formats."); + return -1; +#endif + } else { + unsigned MergePoolSize = getMergePoolSize(FilenamePat, &I); + if (!MergePoolSize) + continue; + if (MergingEnabled) { + PROF_WARN("%%m specifier can only be specified once in %s.\n", + FilenamePat); + return -1; + } + MergingEnabled = 1; + lprofCurFilename.MergePoolSize = MergePoolSize; + } + } + } + + lprofCurFilename.NumPids = NumPids; + lprofCurFilename.NumHosts = NumHosts; + return 0; +} + +static void parseAndSetFilename(const char *FilenamePat, + ProfileNameSpecifier PNS, + unsigned CopyFilenamePat) { + + const char *OldFilenamePat = lprofCurFilename.FilenamePat; + ProfileNameSpecifier OldPNS = lprofCurFilename.PNS; + + /* The old profile name specifier takes precedence over the old one. */ + if (PNS < OldPNS) + return; + + if (!FilenamePat) + FilenamePat = DefaultProfileName; + + if (OldFilenamePat && !strcmp(OldFilenamePat, FilenamePat)) { + lprofCurFilename.PNS = PNS; + return; + } + + /* When PNS >= OldPNS, the last one wins. */ + if (!FilenamePat || parseFilenamePattern(FilenamePat, CopyFilenamePat)) + resetFilenameToDefault(); + lprofCurFilename.PNS = PNS; + + if (!OldFilenamePat) { + if (getenv("LLVM_PROFILE_VERBOSE")) + PROF_NOTE("Set profile file path to \"%s\" via %s.\n", + lprofCurFilename.FilenamePat, getPNSStr(PNS)); + } else { + if (getenv("LLVM_PROFILE_VERBOSE")) + PROF_NOTE("Override old profile path \"%s\" via %s to \"%s\" via %s.\n", + OldFilenamePat, getPNSStr(OldPNS), lprofCurFilename.FilenamePat, + getPNSStr(PNS)); + } + + truncateCurrentFile(); + if (__llvm_profile_is_continuous_mode_enabled()) + initializeProfileForContinuousMode(); +} + +/* Return buffer length that is required to store the current profile + * filename with PID and hostname substitutions. */ +/* The length to hold uint64_t followed by 3 digits pool id including '_' */ +#define SIGLEN 24 +static int getCurFilenameLength(void) { + int Len; + if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0]) + return 0; + + if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts || + lprofCurFilename.TmpDir || lprofCurFilename.MergePoolSize)) + return strlen(lprofCurFilename.FilenamePat); + + Len = strlen(lprofCurFilename.FilenamePat) + + lprofCurFilename.NumPids * (strlen(lprofCurFilename.PidChars) - 2) + + lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname) - 2) + + (lprofCurFilename.TmpDir ? (strlen(lprofCurFilename.TmpDir) - 1) : 0); + if (lprofCurFilename.MergePoolSize) + Len += SIGLEN; + return Len; +} + +/* Return the pointer to the current profile file name (after substituting + * PIDs and Hostnames in filename pattern. \p FilenameBuf is the buffer + * to store the resulting filename. If no substitution is needed, the + * current filename pattern string is directly returned, unless ForceUseBuf + * is enabled. */ +static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) { + int I, J, PidLength, HostNameLength, TmpDirLength, FilenamePatLength; + const char *FilenamePat = lprofCurFilename.FilenamePat; + + if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0]) + return 0; + + if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts || + lprofCurFilename.TmpDir || lprofCurFilename.MergePoolSize || + __llvm_profile_is_continuous_mode_enabled())) { + if (!ForceUseBuf) + return lprofCurFilename.FilenamePat; + + FilenamePatLength = strlen(lprofCurFilename.FilenamePat); + memcpy(FilenameBuf, lprofCurFilename.FilenamePat, FilenamePatLength); + FilenameBuf[FilenamePatLength] = '\0'; + return FilenameBuf; + } + + PidLength = strlen(lprofCurFilename.PidChars); + HostNameLength = strlen(lprofCurFilename.Hostname); + TmpDirLength = lprofCurFilename.TmpDir ? strlen(lprofCurFilename.TmpDir) : 0; + /* Construct the new filename. */ + for (I = 0, J = 0; FilenamePat[I]; ++I) + if (FilenamePat[I] == '%') { + if (FilenamePat[++I] == 'p') { + memcpy(FilenameBuf + J, lprofCurFilename.PidChars, PidLength); + J += PidLength; + } else if (FilenamePat[I] == 'h') { + memcpy(FilenameBuf + J, lprofCurFilename.Hostname, HostNameLength); + J += HostNameLength; + } else if (FilenamePat[I] == 't') { + memcpy(FilenameBuf + J, lprofCurFilename.TmpDir, TmpDirLength); + FilenameBuf[J + TmpDirLength] = DIR_SEPARATOR; + J += TmpDirLength + 1; + } else { + if (!getMergePoolSize(FilenamePat, &I)) + continue; + char LoadModuleSignature[SIGLEN + 1]; + int S; + int ProfilePoolId = getpid() % lprofCurFilename.MergePoolSize; + S = snprintf(LoadModuleSignature, SIGLEN + 1, "%" PRIu64 "_%d", + lprofGetLoadModuleSignature(), ProfilePoolId); + if (S == -1 || S > SIGLEN) + S = SIGLEN; + memcpy(FilenameBuf + J, LoadModuleSignature, S); + J += S; + } + /* Drop any unknown substitutions. */ + } else + FilenameBuf[J++] = FilenamePat[I]; + FilenameBuf[J] = 0; + + return FilenameBuf; +} + +/* Returns the pointer to the environment variable + * string. Returns null if the env var is not set. */ +static const char *getFilenamePatFromEnv(void) { + const char *Filename = getenv("LLVM_PROFILE_FILE"); + if (!Filename || !Filename[0]) + return 0; + return Filename; +} + +COMPILER_RT_VISIBILITY +const char *__llvm_profile_get_path_prefix(void) { + int Length; + char *FilenameBuf, *Prefix; + const char *Filename, *PrefixEnd; + + if (lprofCurFilename.ProfilePathPrefix) + return lprofCurFilename.ProfilePathPrefix; + + Length = getCurFilenameLength(); + FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + Filename = getCurFilename(FilenameBuf, 0); + if (!Filename) + return "\0"; + + PrefixEnd = lprofFindLastDirSeparator(Filename); + if (!PrefixEnd) + return "\0"; + + Length = PrefixEnd - Filename + 1; + Prefix = (char *)malloc(Length + 1); + if (!Prefix) { + PROF_ERR("Failed to %s\n", "allocate memory."); + return "\0"; + } + memcpy(Prefix, Filename, Length); + Prefix[Length] = '\0'; + lprofCurFilename.ProfilePathPrefix = Prefix; + return Prefix; +} + +COMPILER_RT_VISIBILITY +const char *__llvm_profile_get_filename(void) { + int Length; + char *FilenameBuf; + const char *Filename; + + Length = getCurFilenameLength(); + FilenameBuf = (char *)malloc(Length + 1); + if (!FilenameBuf) { + PROF_ERR("Failed to %s\n", "allocate memory."); + return "\0"; + } + Filename = getCurFilename(FilenameBuf, 1); + if (!Filename) + return "\0"; + + return FilenameBuf; +} + +/* This API initializes the file handling, both user specified + * profile path via -fprofile-instr-generate= and LLVM_PROFILE_FILE + * environment variable can override this default value. + */ +COMPILER_RT_VISIBILITY +void __llvm_profile_initialize_file(void) { + const char *EnvFilenamePat; + const char *SelectedPat = NULL; + ProfileNameSpecifier PNS = PNS_unknown; + int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0); + + EnvFilenamePat = getFilenamePatFromEnv(); + if (EnvFilenamePat) { + /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid + at the moment when __llvm_profile_write_file() gets executed. */ + parseAndSetFilename(EnvFilenamePat, PNS_environment, 1); + return; + } else if (hasCommandLineOverrider) { + SelectedPat = INSTR_PROF_PROFILE_NAME_VAR; + PNS = PNS_command_line; + } else { + SelectedPat = NULL; + PNS = PNS_default; + } + + parseAndSetFilename(SelectedPat, PNS, 0); +} + +/* This method is invoked by the runtime initialization hook + * InstrProfilingRuntime.o if it is linked in. + */ +COMPILER_RT_VISIBILITY +void __llvm_profile_initialize(void) { + __llvm_profile_initialize_file(); + if (!__llvm_profile_is_continuous_mode_enabled()) + __llvm_profile_register_write_file_atexit(); +} + +/* This API is directly called by the user application code. It has the + * highest precedence compared with LLVM_PROFILE_FILE environment variable + * and command line option -fprofile-instr-generate=<profile_name>. + */ +COMPILER_RT_VISIBILITY +void __llvm_profile_set_filename(const char *FilenamePat) { + if (__llvm_profile_is_continuous_mode_enabled()) + return; + parseAndSetFilename(FilenamePat, PNS_runtime_api, 1); +} + +/* The public API for writing profile data into the file with name + * set by previous calls to __llvm_profile_set_filename or + * __llvm_profile_override_default_filename or + * __llvm_profile_initialize_file. */ +COMPILER_RT_VISIBILITY +int __llvm_profile_write_file(void) { + int rc, Length; + const char *Filename; + char *FilenameBuf; + + // Temporarily suspend getting SIGKILL when the parent exits. + int PDeathSig = lprofSuspendSigKill(); + + if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) { + PROF_NOTE("Profile data not written to file: %s.\n", "already written"); + if (PDeathSig == 1) + lprofRestoreSigKill(); + return 0; + } + + Length = getCurFilenameLength(); + FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + Filename = getCurFilename(FilenameBuf, 0); + + /* Check the filename. */ + if (!Filename) { + PROF_ERR("Failed to write file : %s\n", "Filename not set"); + if (PDeathSig == 1) + lprofRestoreSigKill(); + return -1; + } + + /* Check if there is llvm/runtime version mismatch. */ + if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) { + PROF_ERR("Runtime and instrumentation version mismatch : " + "expected %d, but get %d\n", + INSTR_PROF_RAW_VERSION, + (int)GET_VERSION(__llvm_profile_get_version())); + if (PDeathSig == 1) + lprofRestoreSigKill(); + return -1; + } + + /* Write profile data to the file. */ + rc = writeFile(Filename); + if (rc) + PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno)); + + // Restore SIGKILL. + if (PDeathSig == 1) + lprofRestoreSigKill(); + + return rc; +} + +COMPILER_RT_VISIBILITY +int __llvm_profile_dump(void) { + if (!doMerging()) + PROF_WARN("Later invocation of __llvm_profile_dump can lead to clobbering " + " of previously dumped profile data : %s. Either use %%m " + "in profile name or change profile name before dumping.\n", + "online profile merging is not on"); + int rc = __llvm_profile_write_file(); + lprofSetProfileDumped(1); + return rc; +} + +/* Order file data will be saved in a file with suffx .order. */ +static const char *OrderFileSuffix = ".order"; + +COMPILER_RT_VISIBILITY +int __llvm_orderfile_write_file(void) { + int rc, Length, LengthBeforeAppend, SuffixLength; + const char *Filename; + char *FilenameBuf; + + // Temporarily suspend getting SIGKILL when the parent exits. + int PDeathSig = lprofSuspendSigKill(); + + SuffixLength = strlen(OrderFileSuffix); + Length = getCurFilenameLength() + SuffixLength; + FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + Filename = getCurFilename(FilenameBuf, 1); + + /* Check the filename. */ + if (!Filename) { + PROF_ERR("Failed to write file : %s\n", "Filename not set"); + if (PDeathSig == 1) + lprofRestoreSigKill(); + return -1; + } + + /* Append order file suffix */ + LengthBeforeAppend = strlen(Filename); + memcpy(FilenameBuf + LengthBeforeAppend, OrderFileSuffix, SuffixLength); + FilenameBuf[LengthBeforeAppend + SuffixLength] = '\0'; + + /* Check if there is llvm/runtime version mismatch. */ + if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) { + PROF_ERR("Runtime and instrumentation version mismatch : " + "expected %d, but get %d\n", + INSTR_PROF_RAW_VERSION, + (int)GET_VERSION(__llvm_profile_get_version())); + if (PDeathSig == 1) + lprofRestoreSigKill(); + return -1; + } + + /* Write order data to the file. */ + rc = writeOrderFile(Filename); + if (rc) + PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno)); + + // Restore SIGKILL. + if (PDeathSig == 1) + lprofRestoreSigKill(); + + return rc; +} + +COMPILER_RT_VISIBILITY +int __llvm_orderfile_dump(void) { + int rc = __llvm_orderfile_write_file(); + return rc; +} + +static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); } + +COMPILER_RT_VISIBILITY +int __llvm_profile_register_write_file_atexit(void) { + static int HasBeenRegistered = 0; + + if (HasBeenRegistered) + return 0; + + lprofSetupValueProfiler(); + + HasBeenRegistered = 1; + return atexit(writeFileWithoutReturn); +} + +COMPILER_RT_VISIBILITY int __llvm_profile_set_file_object(FILE *File, + int EnableMerge) { + if (__llvm_profile_is_continuous_mode_enabled()) { + if (!EnableMerge) { + PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported in " + "continuous sync mode when merging is disabled\n", + fileno(File)); + return 1; + } + if (lprofLockFileHandle(File) != 0) { + PROF_WARN("Data may be corrupted during profile merging : %s\n", + "Fail to obtain file lock due to system limit."); + } + uint64_t ProfileFileSize = 0; + if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) { + lprofUnlockFileHandle(File); + return 1; + } + if (ProfileFileSize == 0) { + FreeHook = &free; + setupIOBuffer(); + ProfDataWriter fileWriter; + initFileWriter(&fileWriter, File); + if (lprofWriteData(&fileWriter, 0, 0)) { + lprofUnlockFileHandle(File); + PROF_ERR("Failed to write file \"%d\": %s\n", fileno(File), + strerror(errno)); + return 1; + } + fflush(File); + } else { + /* The merged profile has a non-zero length. Check that it is compatible + * with the data in this process. */ + char *ProfileBuffer; + if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1) { + lprofUnlockFileHandle(File); + return 1; + } + (void)munmap(ProfileBuffer, ProfileFileSize); + } + mmapForContinuousMode(0, File); + lprofUnlockFileHandle(File); + } else { + setProfileFile(File); + setProfileMergeRequested(EnableMerge); + } + return 0; +} + +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingInternal.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingInternal.c new file mode 100644 index 000000000000..3dd659f90510 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingInternal.c @@ -0,0 +1,26 @@ +/*===- InstrProfilingInternal.c - Support library for PGO instrumentation -===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +// Note: This is linked into the Darwin kernel, and must remain compatible +// with freestanding compilation. See `darwin_add_builtin_libraries`. + +#if !defined(__Fuchsia__) + +#include "InstrProfilingInternal.h" + +static unsigned ProfileDumped = 0; + +COMPILER_RT_VISIBILITY unsigned lprofProfileDumped(void) { + return ProfileDumped; +} + +COMPILER_RT_VISIBILITY void lprofSetProfileDumped(unsigned Value) { + ProfileDumped = Value; +} + +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingInternal.h b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingInternal.h new file mode 100644 index 000000000000..d5bd0e41fb12 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingInternal.h @@ -0,0 +1,215 @@ +/*===- InstrProfiling.h- Support library for PGO instrumentation ----------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#ifndef PROFILE_INSTRPROFILING_INTERNALH_ +#define PROFILE_INSTRPROFILING_INTERNALH_ + +#include <stddef.h> + +#include "InstrProfiling.h" + +/*! + * \brief Write instrumentation data to the given buffer, given explicit + * pointers to the live data in memory. This function is probably not what you + * want. Use __llvm_profile_get_size_for_buffer instead. Use this function if + * your program has a custom memory layout. + */ +uint64_t __llvm_profile_get_size_for_buffer_internal( + const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, + const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, + const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd, + const VTableProfData *VTableBegin, const VTableProfData *VTableEnd, + const char *VNamesBegin, const char *VNamesEnd); + +/*! + * \brief Write instrumentation data to the given buffer, given explicit + * pointers to the live data in memory. This function is probably not what you + * want. Use __llvm_profile_write_buffer instead. Use this function if your + * program has a custom memory layout. + * + * \pre \c Buffer is the start of a buffer at least as big as \a + * __llvm_profile_get_size_for_buffer_internal(). + */ +int __llvm_profile_write_buffer_internal( + char *Buffer, const __llvm_profile_data *DataBegin, + const __llvm_profile_data *DataEnd, const char *CountersBegin, + const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, + const char *NamesBegin, const char *NamesEnd); + +/*! + * The data structure describing the data to be written by the + * low level writer callback function. + * + * If \ref ProfDataIOVec.Data is null, and \ref ProfDataIOVec.UseZeroPadding is + * 0, the write is skipped (the writer simply advances ElmSize*NumElm bytes). + * + * If \ref ProfDataIOVec.Data is null, and \ref ProfDataIOVec.UseZeroPadding is + * nonzero, ElmSize*NumElm zero bytes are written. + */ +typedef struct ProfDataIOVec { + const void *Data; + size_t ElmSize; + size_t NumElm; + int UseZeroPadding; +} ProfDataIOVec; + +struct ProfDataWriter; +typedef uint32_t (*WriterCallback)(struct ProfDataWriter *This, ProfDataIOVec *, + uint32_t NumIOVecs); + +typedef struct ProfDataWriter { + WriterCallback Write; + void *WriterCtx; +} ProfDataWriter; + +/*! + * The data structure for buffered IO of profile data. + */ +typedef struct ProfBufferIO { + ProfDataWriter *FileWriter; + uint32_t OwnFileWriter; + /* The start of the buffer. */ + uint8_t *BufferStart; + /* Total size of the buffer. */ + uint32_t BufferSz; + /* Current byte offset from the start of the buffer. */ + uint32_t CurOffset; +} ProfBufferIO; + +/* The creator interface used by testing. */ +ProfBufferIO *lprofCreateBufferIOInternal(void *File, uint32_t BufferSz); + +/*! + * This is the interface to create a handle for buffered IO. + */ +ProfBufferIO *lprofCreateBufferIO(ProfDataWriter *FileWriter); + +/*! + * The interface to destroy the bufferIO handle and reclaim + * the memory. + */ +void lprofDeleteBufferIO(ProfBufferIO *BufferIO); + +/*! + * This is the interface to write \c Data of \c Size bytes through + * \c BufferIO. Returns 0 if successful, otherwise return -1. + */ +int lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, + uint32_t Size); +/*! + * The interface to flush the remaining data in the buffer. + * through the low level writer callback. + */ +int lprofBufferIOFlush(ProfBufferIO *BufferIO); + +/* The low level interface to write data into a buffer. It is used as the + * callback by other high level writer methods such as buffered IO writer + * and profile data writer. */ +uint32_t lprofBufferWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, + uint32_t NumIOVecs); +void initBufferWriter(ProfDataWriter *BufferWriter, char *Buffer); + +struct ValueProfData; +struct ValueProfRecord; +struct InstrProfValueData; +struct ValueProfNode; + +/*! + * The class that defines a set of methods to read value profile + * data for streaming/serialization from the instrumentation runtime. + */ +typedef struct VPDataReaderType { + uint32_t (*InitRTRecord)(const __llvm_profile_data *Data, + uint8_t *SiteCountArray[]); + /* Function pointer to getValueProfRecordHeader method. */ + uint32_t (*GetValueProfRecordHeaderSize)(uint32_t NumSites); + /* Function pointer to getFristValueProfRecord method. */ + struct ValueProfRecord *(*GetFirstValueProfRecord)(struct ValueProfData *); + /* Return the number of value data for site \p Site. */ + uint32_t (*GetNumValueDataForSite)(uint32_t VK, uint32_t Site); + /* Return the total size of the value profile data of the + * current function. */ + uint32_t (*GetValueProfDataSize)(void); + /*! + * Read the next \p N value data for site \p Site and store the data + * in \p Dst. \p StartNode is the first value node to start with if + * it is not null. The function returns the pointer to the value + * node pointer to be used as the \p StartNode of the next batch reading. + * If there is nothing left, it returns NULL. + */ + struct ValueProfNode *(*GetValueData)(uint32_t ValueKind, uint32_t Site, + struct InstrProfValueData *Dst, + struct ValueProfNode *StartNode, + uint32_t N); +} VPDataReaderType; + +/* Write profile data to destination. If SkipNameDataWrite is set to 1, + the name data is already in destination, we just skip over it. */ +int lprofWriteData(ProfDataWriter *Writer, VPDataReaderType *VPDataReader, + int SkipNameDataWrite); +int lprofWriteDataImpl(ProfDataWriter *Writer, + const __llvm_profile_data *DataBegin, + const __llvm_profile_data *DataEnd, + const char *CountersBegin, const char *CountersEnd, + const char *BitmapBegin, const char *BitmapEnd, + VPDataReaderType *VPDataReader, const char *NamesBegin, + const char *NamesEnd, const VTableProfData *VTableBegin, + const VTableProfData *VTableEnd, const char *VNamesBegin, + const char *VNamesEnd, int SkipNameDataWrite); + +/* Merge value profile data pointed to by SrcValueProfData into + * in-memory profile counters pointed by to DstData. */ +void lprofMergeValueProfData(struct ValueProfData *SrcValueProfData, + __llvm_profile_data *DstData); + +VPDataReaderType *lprofGetVPDataReader(); + +/* Internal interface used by test to reset the max number of + * tracked values per value site to be \p MaxVals. + */ +void lprofSetMaxValsPerSite(uint32_t MaxVals); +void lprofSetupValueProfiler(); + +/* Return the profile header 'signature' value associated with the current + * executable or shared library. The signature value can be used to for + * a profile name that is unique to this load module so that it does not + * collide with profiles from other binaries. It also allows shared libraries + * to dump merged profile data into its own profile file. */ +uint64_t lprofGetLoadModuleSignature(); + +/* + * Return non zero value if the profile data has already been + * dumped to the file. + */ +unsigned lprofProfileDumped(void); +void lprofSetProfileDumped(unsigned); + +COMPILER_RT_VISIBILITY extern void (*FreeHook)(void *); +COMPILER_RT_VISIBILITY extern uint8_t *DynamicBufferIOBuffer; +COMPILER_RT_VISIBILITY extern uint32_t VPBufferSize; +COMPILER_RT_VISIBILITY extern uint32_t VPMaxNumValsPerSite; +/* Pointer to the start of static value counters to be allocted. */ +COMPILER_RT_VISIBILITY extern ValueProfNode *CurrentVNode; +COMPILER_RT_VISIBILITY extern ValueProfNode *EndVNode; +extern void (*VPMergeHook)(struct ValueProfData *, __llvm_profile_data *); + +/* + * Write binary ids into profiles if writer is given. + * Return -1 if an error occurs, otherwise, return total size of binary ids. + */ +int __llvm_write_binary_ids(ProfDataWriter *Writer); + +/* + * Write binary id length and then its data, because binary id does not + * have a fixed length. + */ +int lprofWriteOneBinaryId(ProfDataWriter *Writer, uint64_t BinaryIdLen, + const uint8_t *BinaryIdData, + uint64_t BinaryIdPadding); + +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingMerge.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingMerge.c new file mode 100644 index 000000000000..c0706b73e166 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -0,0 +1,261 @@ +/*===- InstrProfilingMerge.c - Profile in-process Merging ---------------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +|*===----------------------------------------------------------------------===* +|* This file defines the API needed for in-process merging of profile data +|* stored in memory buffer. +\*===---------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingUtil.h" + +#define INSTR_PROF_VALUE_PROF_DATA +#include "profile/InstrProfData.inc" + +COMPILER_RT_VISIBILITY +void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *); + +COMPILER_RT_VISIBILITY +uint64_t lprofGetLoadModuleSignature(void) { + /* A very fast way to compute a module signature. */ + uint64_t Version = __llvm_profile_get_version(); + uint64_t NumCounters = __llvm_profile_get_num_counters( + __llvm_profile_begin_counters(), __llvm_profile_end_counters()); + uint64_t NumData = __llvm_profile_get_num_data(__llvm_profile_begin_data(), + __llvm_profile_end_data()); + uint64_t NamesSize = + (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names()); + uint64_t NumVnodes = + (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes()); + const __llvm_profile_data *FirstD = __llvm_profile_begin_data(); + + return (NamesSize << 40) + (NumCounters << 30) + (NumData << 20) + + (NumVnodes << 10) + (NumData > 0 ? FirstD->NameRef : 0) + Version + + __llvm_profile_get_magic(); +} + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +#endif + +/* Returns 1 if profile is not structurally compatible. */ +COMPILER_RT_VISIBILITY +int __llvm_profile_check_compatibility(const char *ProfileData, + uint64_t ProfileSize) { + __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData; + __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData; + SrcDataStart = + (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) + + Header->BinaryIdsSize); + SrcDataEnd = SrcDataStart + Header->NumData; + + if (ProfileSize < sizeof(__llvm_profile_header)) + return 1; + + /* Check the header first. */ + if (Header->Magic != __llvm_profile_get_magic() || + Header->Version != __llvm_profile_get_version() || + Header->NumData != + __llvm_profile_get_num_data(__llvm_profile_begin_data(), + __llvm_profile_end_data()) || + Header->NumCounters != + __llvm_profile_get_num_counters(__llvm_profile_begin_counters(), + __llvm_profile_end_counters()) || + Header->NumBitmapBytes != + __llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(), + __llvm_profile_end_bitmap()) || + Header->NamesSize != + __llvm_profile_get_name_size(__llvm_profile_begin_names(), + __llvm_profile_end_names()) || + Header->ValueKindLast != IPVK_Last) + return 1; + + if (ProfileSize < + sizeof(__llvm_profile_header) + Header->BinaryIdsSize + + Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize + + Header->NumCounters * __llvm_profile_counter_entry_size() + + Header->NumBitmapBytes) + return 1; + + for (SrcData = SrcDataStart, + DstData = (__llvm_profile_data *)__llvm_profile_begin_data(); + SrcData < SrcDataEnd; ++SrcData, ++DstData) { + if (SrcData->NameRef != DstData->NameRef || + SrcData->FuncHash != DstData->FuncHash || + SrcData->NumCounters != DstData->NumCounters || + SrcData->NumBitmapBytes != DstData->NumBitmapBytes) + return 1; + } + + /* Matched! */ + return 0; +} + +static uintptr_t signextIfWin64(void *V) { +#ifdef _WIN64 + return (uintptr_t)(int32_t)(uintptr_t)V; +#else + return (uintptr_t)V; +#endif +} + +// Skip names section, vtable profile data section and vtable names section +// for runtime profile merge. To merge runtime addresses from multiple +// profiles collected from the same instrumented binary, the binary should be +// loaded at fixed base address (e.g., build with -no-pie, or run with ASLR +// disabled). In this set-up these three sections remain unchanged. +static uint64_t +getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) { + const uint64_t VTableSectionSize = + Header->NumVTables * sizeof(VTableProfData); + const uint64_t PaddingBytesAfterVTableSection = + __llvm_profile_get_num_padding_bytes(VTableSectionSize); + const uint64_t VNamesSize = Header->VNamesSize; + const uint64_t PaddingBytesAfterVNamesSize = + __llvm_profile_get_num_padding_bytes(VNamesSize); + return Header->NamesSize + + __llvm_profile_get_num_padding_bytes(Header->NamesSize) + + VTableSectionSize + PaddingBytesAfterVTableSection + VNamesSize + + PaddingBytesAfterVNamesSize; +} + +COMPILER_RT_VISIBILITY +int __llvm_profile_merge_from_buffer(const char *ProfileData, + uint64_t ProfileSize) { + if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF) { + PROF_ERR("%s\n", + "Temporal profiles do not support profile merging at runtime. " + "Instead, merge raw profiles using the llvm-profdata tool."); + return 1; + } + + __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData; + __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData; + char *SrcCountersStart, *DstCounter; + const char *SrcCountersEnd, *SrcCounter; + const char *SrcBitmapStart; + const char *SrcNameStart; + const char *SrcValueProfDataStart, *SrcValueProfData; + uintptr_t CountersDelta = Header->CountersDelta; + uintptr_t BitmapDelta = Header->BitmapDelta; + + SrcDataStart = + (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) + + Header->BinaryIdsSize); + SrcDataEnd = SrcDataStart + Header->NumData; + SrcCountersStart = (char *)SrcDataEnd; + SrcCountersEnd = SrcCountersStart + + Header->NumCounters * __llvm_profile_counter_entry_size(); + SrcBitmapStart = SrcCountersEnd; + SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes; + SrcValueProfDataStart = + SrcNameStart + getDistanceFromCounterToValueProf(Header); + if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart) + return 1; + + // Merge counters by iterating the entire counter section when data section is + // empty due to correlation. + if (Header->NumData == 0) { + for (SrcCounter = SrcCountersStart, + DstCounter = __llvm_profile_begin_counters(); + SrcCounter < SrcCountersEnd;) { + if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) { + *DstCounter &= *SrcCounter; + } else { + *(uint64_t *)DstCounter += *(uint64_t *)SrcCounter; + } + SrcCounter += __llvm_profile_counter_entry_size(); + DstCounter += __llvm_profile_counter_entry_size(); + } + return 0; + } + + for (SrcData = SrcDataStart, + DstData = (__llvm_profile_data *)__llvm_profile_begin_data(), + SrcValueProfData = SrcValueProfDataStart; + SrcData < SrcDataEnd; ++SrcData, ++DstData) { + // For the in-memory destination, CounterPtr is the distance from the start + // address of the data to the start address of the counter. On WIN64, + // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign + // extend CounterPtr to get the original value. + char *DstCounters = + (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr)); + char *DstBitmap = + (char *)((uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr)); + unsigned NVK = 0; + + // SrcData is a serialized representation of the memory image. We need to + // compute the in-buffer counter offset from the in-memory address distance. + // The initial CountersDelta is the in-memory address difference + // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr - + // CountersDelta computes the offset into the in-buffer counter section. + // + // On WIN64, CountersDelta is truncated as well, so no need for signext. + char *SrcCounters = + SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta); + // CountersDelta needs to be decreased as we advance to the next data + // record. + CountersDelta -= sizeof(*SrcData); + unsigned NC = SrcData->NumCounters; + if (NC == 0) + return 1; + if (SrcCounters < SrcCountersStart || SrcCounters >= SrcNameStart || + (SrcCounters + __llvm_profile_counter_entry_size() * NC) > SrcNameStart) + return 1; + for (unsigned I = 0; I < NC; I++) { + if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) { + // A value of zero signifies the function is covered. + DstCounters[I] &= SrcCounters[I]; + } else { + ((uint64_t *)DstCounters)[I] += ((uint64_t *)SrcCounters)[I]; + } + } + + const char *SrcBitmap = + SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta); + // BitmapDelta also needs to be decreased as we advance to the next data + // record. + BitmapDelta -= sizeof(*SrcData); + unsigned NB = SrcData->NumBitmapBytes; + // NumBitmapBytes may legitimately be 0. Just keep going. + if (NB != 0) { + if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart) + return 1; + // Merge Src and Dst Bitmap bytes by simply ORing them together. + for (unsigned I = 0; I < NB; I++) + DstBitmap[I] |= SrcBitmap[I]; + } + + /* Now merge value profile data. */ + if (!VPMergeHook) + continue; + + for (unsigned I = 0; I <= IPVK_Last; I++) + NVK += (SrcData->NumValueSites[I] != 0); + + if (!NVK) + continue; + + if (SrcValueProfData >= ProfileData + ProfileSize) + return 1; + VPMergeHook((ValueProfData *)SrcValueProfData, DstData); + SrcValueProfData = + SrcValueProfData + ((ValueProfData *)SrcValueProfData)->TotalSize; + } + + return 0; +} + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingMergeFile.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingMergeFile.c new file mode 100644 index 000000000000..8923ba21cc58 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingMergeFile.c @@ -0,0 +1,45 @@ +/*===- InstrProfilingMergeFile.c - Profile in-process Merging ------------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +|*===----------------------------------------------------------------------=== +|* This file defines APIs needed to support in-process merging for profile data +|* stored in files. +\*===----------------------------------------------------------------------===*/ + +#if !defined(__Fuchsia__) + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingUtil.h" + +#define INSTR_PROF_VALUE_PROF_DATA +#include "profile/InstrProfData.inc" + +/* Merge value profile data pointed to by SrcValueProfData into + * in-memory profile counters pointed by to DstData. */ +COMPILER_RT_VISIBILITY +void lprofMergeValueProfData(ValueProfData *SrcValueProfData, + __llvm_profile_data *DstData) { + unsigned I, S, V, DstIndex = 0; + InstrProfValueData *VData; + ValueProfRecord *VR = getFirstValueProfRecord(SrcValueProfData); + for (I = 0; I < SrcValueProfData->NumValueKinds; I++) { + VData = getValueProfRecordValueData(VR); + unsigned SrcIndex = 0; + for (S = 0; S < VR->NumValueSites; S++) { + uint8_t NV = VR->SiteCountArray[S]; + for (V = 0; V < NV; V++) { + __llvm_profile_instrument_target_value(VData[SrcIndex].Value, DstData, + DstIndex, VData[SrcIndex].Count); + ++SrcIndex; + } + ++DstIndex; + } + VR = getValueProfRecordNext(VR); + } +} + +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingNameVar.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingNameVar.c new file mode 100644 index 000000000000..407272806ba3 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingNameVar.c @@ -0,0 +1,17 @@ +/*===- InstrProfilingNameVar.c - profile name variable setup -------------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" + +/* char __llvm_profile_filename[1] + * + * The runtime should only provide its own definition of this symbol when the + * user has not specified one. Set this up by moving the runtime's copy of this + * symbol to an object file within the archive. + */ +COMPILER_RT_WEAK COMPILER_RT_VISIBILITY char INSTR_PROF_PROFILE_NAME_VAR[1] = {0}; diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c new file mode 100644 index 000000000000..b9d51b698b41 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c @@ -0,0 +1,221 @@ +/*===- InstrProfilingPlatformAIX.c - Profile data AIX platform ------------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#if defined(_AIX) + +#ifdef __64BIT__ +#define __XCOFF64__ +#endif +#include <errno.h> +#include <stdlib.h> +#include <string.h> +#include <sys/ldr.h> +#include <xcoff.h> + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" + +#define BIN_ID_PREFIX "xcoff_binary_id:" + +// If found, write the build-id into the Result buffer. +static size_t FindBinaryId(char *Result, size_t Size) { + unsigned long EntryAddr = (unsigned long)__builtin_return_address(0); + + // Use loadquery to get information about loaded modules; loadquery writes + // its result into a buffer of unknown size. + char Buf[1024]; + size_t BufSize = sizeof(Buf); + char *BufPtr = Buf; + int RC = -1; + + errno = 0; + RC = loadquery(L_GETXINFO | L_IGNOREUNLOAD, BufPtr, (unsigned int)BufSize); + if (RC == -1 && errno == ENOMEM) { + BufSize = 64000; // should be plenty for any program. + BufPtr = malloc(BufSize); + if (BufPtr != 0) + RC = loadquery(L_GETXINFO | L_IGNOREUNLOAD, BufPtr, (unsigned int)BufSize); + } + + if (RC == -1) + goto done; + + // Locate the ld_xinfo corresponding to this module. + struct ld_xinfo *CurInfo = (struct ld_xinfo *)BufPtr; + while (1) { + unsigned long CurTextStart = (uint64_t)CurInfo->ldinfo_textorg; + unsigned long CurTextEnd = CurTextStart + CurInfo->ldinfo_textsize; + if (CurTextStart <= EntryAddr && EntryAddr < CurTextEnd) { + // Found my slot. Now search for the build-id. + char *p = (char *)CurInfo->ldinfo_textorg; + + FILHDR *f = (FILHDR *)p; + AOUTHDR *a = (AOUTHDR *)(p + FILHSZ); + SCNHDR *s = + (SCNHDR *)(p + FILHSZ + f->f_opthdr + SCNHSZ * (a->o_snloader - 1)); + LDHDR *ldhdr = (LDHDR *)(p + s->s_scnptr); + // This is the loader string table + char *lstr = (char *)ldhdr + ldhdr->l_stoff; + + // If the build-id exists, it's the first entry. + // Each entry is comprised of a 2-byte size component, followed by the + // data. + size_t len = *(short *)lstr; + char *str = (char *)(lstr + 2); + size_t PrefixLen = sizeof(BIN_ID_PREFIX) - 1; + if (len > PrefixLen && (len - PrefixLen) <= Size && + strncmp(str, BIN_ID_PREFIX, PrefixLen) == 0) { + memcpy(Result, str + PrefixLen, len - PrefixLen); + RC = len - PrefixLen; + goto done; + } + break; + } + if (CurInfo->ldinfo_next == 0u) + break; + CurInfo = (struct ld_xinfo *)((char *)CurInfo + CurInfo->ldinfo_next); + } +done: + if (BufSize != sizeof(Buf) && BufPtr != 0) + free(BufPtr); + return RC; +} + +static int StrToHexError = 0; +static uint8_t StrToHex(char c) { + if (c >= '0' && c <= '9') + return c - '0'; + if (c >= 'a' && c <= 'f') + return c - 'a' + 0xa; + if (c >= 'A' && c <= 'F') + return c - 'A' + 0xa; + StrToHexError = 1; + return 0; +} + +COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) { + // 200 bytes should be enough for the build-id hex string. + static char Buf[200]; + // Profile reading tools expect this to be 8-bytes long. + static int64_t BinaryIdLen = 0; + static uint8_t *BinaryIdData = 0; + + // -1 means we already checked for a BinaryId and didn't find one. + if (BinaryIdLen == -1) + return 0; + + // Are we being called for the first time? + if (BinaryIdLen == 0) { + if (getenv("LLVM_PROFILE_NO_BUILD_ID")) + goto fail; + + int BuildIdLen = FindBinaryId(Buf, sizeof(Buf)); + if (BuildIdLen <= 0) + goto fail; + + if (Buf[BuildIdLen - 1] == '\0') + BuildIdLen--; + + // assume even number of digits/chars, so 0xabc must be 0x0abc + if ((BuildIdLen % 2) != 0 || BuildIdLen == 0) + goto fail; + + // The numeric ID is represented as an ascii string in the loader section, + // so convert it to raw binary. + BinaryIdLen = BuildIdLen / 2; + BinaryIdData = (uint8_t *)Buf; + + // Skip "0x" prefix if it exists. + if (Buf[0] == '0' && Buf[1] == 'x') { + BinaryIdLen -= 1; + BinaryIdData += 2; + } + + StrToHexError = 0; + for (int i = 0; i < BinaryIdLen; i++) + BinaryIdData[i] = (StrToHex(BinaryIdData[2 * i]) << 4) + + StrToHex(BinaryIdData[2 * i + 1]); + + if (StrToHexError) + goto fail; + + if (getenv("LLVM_PROFILE_VERBOSE")) { + char *StrBuf = (char *)COMPILER_RT_ALLOCA(2 * BinaryIdLen + 1); + for (int i = 0; i < (int)BinaryIdLen; i++) + sprintf(&StrBuf[2 * i], "%02x", BinaryIdData[i]); + PROF_NOTE("Writing binary id: %s\n", StrBuf); + } + } + + uint8_t BinaryIdPadding = __llvm_profile_get_num_padding_bytes(BinaryIdLen); + if (Writer && lprofWriteOneBinaryId(Writer, BinaryIdLen, BinaryIdData, + BinaryIdPadding) == -1) + return -1; // Return -1 rather goto fail to match the NT_GNU_BUILD_ID path. + + return sizeof(BinaryIdLen) + BinaryIdLen + BinaryIdPadding; + +fail: + if (getenv("LLVM_PROFILE_VERBOSE")) + fprintf(stderr, "no or invalid binary id: %.*s\n", (int)sizeof(Buf), Buf); + BinaryIdLen = -1; + return 0; +} + +// Empty stubs to allow linking object files using the registration-based scheme +COMPILER_RT_VISIBILITY +void __llvm_profile_register_function(void *Data_) {} + +COMPILER_RT_VISIBILITY +void __llvm_profile_register_names_function(void *NamesStart, + uint64_t NamesSize) {} + +// The __start_SECNAME and __stop_SECNAME symbols (for SECNAME \in +// {"__llvm_prf_cnts", "__llvm_prf_data", "__llvm_prf_name", "__llvm_prf_vnds", +// "__llvm_prf_vns", "__llvm_prf_vtab"}) +// are always live when linking on AIX, regardless if the .o's being linked +// reference symbols from the profile library (for example when no files were +// compiled with -fprofile-generate). That's because these symbols are kept +// alive through references in constructor functions that are always live in the +// default linking model on AIX (-bcdtors:all). The __start_SECNAME and +// __stop_SECNAME symbols are only resolved by the linker when the SECNAME +// section exists. So for the scenario where the user objects have no such +// section (i.e. when they are compiled with -fno-profile-generate), we always +// define these zero length variables in each of the above 4 sections. +static int dummy_cnts[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_CNTS_SECT_NAME); +static int dummy_bits[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_BITS_SECT_NAME); +static int dummy_data[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_DATA_SECT_NAME); +static const int dummy_name[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_NAME_SECT_NAME); +static int dummy_vnds[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_VNODES_SECT_NAME); +static int dummy_orderfile[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_ORDERFILE_SECT_NAME); +static int dummy_vname[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_VNAME_SECT_NAME); +static int dummy_vtab[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_VTAB_SECT_NAME); + +// To avoid GC'ing of the dummy variables by the linker, reference them in an +// array and reference the array in the runtime registration code +// (InstrProfilingRuntime.cpp) +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif +COMPILER_RT_VISIBILITY +void *__llvm_profile_keep[] = {(void *)&dummy_cnts, (void *)&dummy_bits, + (void *)&dummy_data, (void *)&dummy_name, + (void *)&dummy_vnds, (void *)&dummy_orderfile, + (void *)&dummy_vname, (void *)&dummy_vtab}; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c new file mode 100644 index 000000000000..6adc7f328cbf --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c @@ -0,0 +1,107 @@ +/*===- InstrProfilingPlatformDarwin.c - Profile data on Darwin ------------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +// Note: This is linked into the Darwin kernel, and must remain compatible +// with freestanding compilation. See `darwin_add_builtin_libraries`. + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" + +#if defined(__APPLE__) +/* Use linker magic to find the bounds of the Data section. */ +COMPILER_RT_VISIBILITY +extern __llvm_profile_data + DataStart __asm("section$start$__DATA$" INSTR_PROF_DATA_SECT_NAME); +COMPILER_RT_VISIBILITY +extern __llvm_profile_data + DataEnd __asm("section$end$__DATA$" INSTR_PROF_DATA_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char + NamesStart __asm("section$start$__DATA$" INSTR_PROF_NAME_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char NamesEnd __asm("section$end$__DATA$" INSTR_PROF_NAME_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char + CountersStart __asm("section$start$__DATA$" INSTR_PROF_CNTS_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char + BitmapStart __asm("section$start$__DATA$" INSTR_PROF_BITS_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char BitmapEnd __asm("section$end$__DATA$" INSTR_PROF_BITS_SECT_NAME); +COMPILER_RT_VISIBILITY +extern VTableProfData + VTableProfStart __asm("section$start$__DATA$" INSTR_PROF_VTAB_SECT_NAME); +COMPILER_RT_VISIBILITY +extern VTableProfData + VTableProfEnd __asm("section$end$__DATA$" INSTR_PROF_VTAB_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char + VNameStart __asm("section$start$__DATA$" INSTR_PROF_VNAME_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char VNameEnd __asm("section$end$__DATA$" INSTR_PROF_VNAME_SECT_NAME); +COMPILER_RT_VISIBILITY +extern uint32_t + OrderFileStart __asm("section$start$__DATA$" INSTR_PROF_ORDERFILE_SECT_NAME); + +COMPILER_RT_VISIBILITY +extern ValueProfNode + VNodesStart __asm("section$start$__DATA$" INSTR_PROF_VNODES_SECT_NAME); +COMPILER_RT_VISIBILITY +extern ValueProfNode + VNodesEnd __asm("section$end$__DATA$" INSTR_PROF_VNODES_SECT_NAME); + +COMPILER_RT_VISIBILITY +const __llvm_profile_data *__llvm_profile_begin_data(void) { + return &DataStart; +} +COMPILER_RT_VISIBILITY +const __llvm_profile_data *__llvm_profile_end_data(void) { return &DataEnd; } +COMPILER_RT_VISIBILITY +const char *__llvm_profile_begin_names(void) { return &NamesStart; } +COMPILER_RT_VISIBILITY +const char *__llvm_profile_end_names(void) { return &NamesEnd; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_begin_counters(void) { return &CountersStart; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_end_counters(void) { return &CountersEnd; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_begin_bitmap(void) { return &BitmapStart; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_end_bitmap(void) { return &BitmapEnd; } +COMPILER_RT_VISIBILITY +const VTableProfData *__llvm_profile_begin_vtables(void) { + return &VTableProfStart; +} +COMPILER_RT_VISIBILITY +const VTableProfData *__llvm_profile_end_vtables(void) { + return &VTableProfEnd; +} +COMPILER_RT_VISIBILITY +const char *__llvm_profile_begin_vtabnames(void) { return &VNameStart; } +COMPILER_RT_VISIBILITY +const char *__llvm_profile_end_vtabnames(void) { return &VNameEnd; } +COMPILER_RT_VISIBILITY +uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } + +COMPILER_RT_VISIBILITY +ValueProfNode *__llvm_profile_begin_vnodes(void) { + return &VNodesStart; +} +COMPILER_RT_VISIBILITY +ValueProfNode *__llvm_profile_end_vnodes(void) { return &VNodesEnd; } + +COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = &VNodesStart; +COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = &VNodesEnd; + +COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) { + return 0; +} + +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c new file mode 100644 index 000000000000..fdcb82e4d72b --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c @@ -0,0 +1,190 @@ +/*===- InstrProfilingPlatformFuchsia.c - Profile data Fuchsia platform ----===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ +/* + * This file implements the profiling runtime for Fuchsia and defines the + * shared profile runtime interface. Each module (executable or DSO) statically + * links in the whole profile runtime to satisfy the calls from its + * instrumented code. Several modules in the same program might be separately + * compiled and even use different versions of the instrumentation ABI and data + * format. All they share in common is the VMO and the offset, which live in + * exported globals so that exactly one definition will be shared across all + * modules. Each module has its own independent runtime that registers its own + * atexit hook to append its own data into the shared VMO which is published + * via the data sink hook provided by Fuchsia's dynamic linker. + */ + +#if defined(__Fuchsia__) + +#include <inttypes.h> +#include <stdarg.h> +#include <stdbool.h> +#include <stdlib.h> + +#include <zircon/process.h> +#include <zircon/sanitizer.h> +#include <zircon/status.h> +#include <zircon/syscalls.h> + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingUtil.h" + +/* This variable is an external reference to symbol defined by the compiler. */ +COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; + +COMPILER_RT_VISIBILITY unsigned lprofProfileDumped(void) { + return 1; +} +COMPILER_RT_VISIBILITY void lprofSetProfileDumped(unsigned Value) {} + +static const char ProfileSinkName[] = "llvm-profile"; + +static inline void lprofWrite(const char *fmt, ...) { + char s[256]; + + va_list ap; + va_start(ap, fmt); + int ret = vsnprintf(s, sizeof(s), fmt, ap); + va_end(ap); + + __sanitizer_log_write(s, ret); +} + +struct lprofVMOWriterCtx { + /* VMO that contains the profile data for this module. */ + zx_handle_t Vmo; + /* Current offset within the VMO where data should be written next. */ + uint64_t Offset; +}; + +static uint32_t lprofVMOWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, + uint32_t NumIOVecs) { + struct lprofVMOWriterCtx *Ctx = (struct lprofVMOWriterCtx *)This->WriterCtx; + + /* Compute the total length of data to be written. */ + size_t Length = 0; + for (uint32_t I = 0; I < NumIOVecs; I++) + Length += IOVecs[I].ElmSize * IOVecs[I].NumElm; + + /* Resize the VMO to ensure there's sufficient space for the data. */ + zx_status_t Status = _zx_vmo_set_size(Ctx->Vmo, Ctx->Offset + Length); + if (Status != ZX_OK) + return -1; + + /* Copy the data into VMO. */ + for (uint32_t I = 0; I < NumIOVecs; I++) { + size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm; + if (IOVecs[I].Data) { + Status = _zx_vmo_write(Ctx->Vmo, IOVecs[I].Data, Ctx->Offset, Length); + if (Status != ZX_OK) + return -1; + } else if (IOVecs[I].UseZeroPadding) { + /* Resizing the VMO should zero fill. */ + } + Ctx->Offset += Length; + } + + /* Record the profile size as a property of the VMO. */ + _zx_object_set_property(Ctx->Vmo, ZX_PROP_VMO_CONTENT_SIZE, &Ctx->Offset, + sizeof(Ctx->Offset)); + + return 0; +} + +static void initVMOWriter(ProfDataWriter *This, struct lprofVMOWriterCtx *Ctx) { + This->Write = lprofVMOWriter; + This->WriterCtx = Ctx; +} + +/* This method is invoked by the runtime initialization hook + * InstrProfilingRuntime.o if it is linked in. */ +COMPILER_RT_VISIBILITY +void __llvm_profile_initialize(void) { + /* Check if there is llvm/runtime version mismatch. */ + if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) { + lprofWrite("LLVM Profile: runtime and instrumentation version mismatch: " + "expected %d, but got %d\n", + INSTR_PROF_RAW_VERSION, + (int)GET_VERSION(__llvm_profile_get_version())); + return; + } + + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const char *CountersBegin = __llvm_profile_begin_counters(); + const char *CountersEnd = __llvm_profile_end_counters(); + const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + const uint64_t CountersOffset = + sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize; + uint64_t CountersSize = + __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + + /* Don't publish a VMO if there are no counters. */ + if (!CountersSize) + return; + + zx_status_t Status; + + /* Create a VMO to hold the profile data. */ + zx_handle_t Vmo = ZX_HANDLE_INVALID; + Status = _zx_vmo_create(0, ZX_VMO_RESIZABLE, &Vmo); + if (Status != ZX_OK) { + lprofWrite("LLVM Profile: cannot create VMO: %s\n", + _zx_status_get_string(Status)); + return; + } + + /* Give the VMO a name that includes the module signature. */ + char VmoName[ZX_MAX_NAME_LEN]; + snprintf(VmoName, sizeof(VmoName), "%" PRIu64 ".profraw", + lprofGetLoadModuleSignature()); + _zx_object_set_property(Vmo, ZX_PROP_NAME, VmoName, strlen(VmoName)); + + /* Write the profile data into the mapped region. */ + ProfDataWriter VMOWriter; + struct lprofVMOWriterCtx Ctx = {.Vmo = Vmo, .Offset = 0}; + initVMOWriter(&VMOWriter, &Ctx); + if (lprofWriteData(&VMOWriter, 0, 0) != 0) { + lprofWrite("LLVM Profile: failed to write data\n"); + _zx_handle_close(Vmo); + return; + } + + uint64_t Len = 0; + Status = _zx_vmo_get_size(Vmo, &Len); + if (Status != ZX_OK) { + lprofWrite("LLVM Profile: failed to get the VMO size: %s\n", + _zx_status_get_string(Status)); + _zx_handle_close(Vmo); + return; + } + + uintptr_t Mapping; + Status = + _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, + Vmo, 0, Len, &Mapping); + if (Status != ZX_OK) { + lprofWrite("LLVM Profile: failed to map the VMO: %s\n", + _zx_status_get_string(Status)); + _zx_handle_close(Vmo); + return; + } + + /* Publish the VMO which contains profile data to the system. Note that this + * also consumes the VMO handle. */ + __sanitizer_publish_data(ProfileSinkName, Vmo); + + /* Update the profile fields based on the current mapping. */ + INSTR_PROF_PROFILE_COUNTER_BIAS_VAR = + (intptr_t)Mapping - (uintptr_t)CountersBegin + CountersOffset; + + /* Return the memory allocated for counters to OS. */ + lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd); +} + +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c new file mode 100644 index 000000000000..b766436497b7 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -0,0 +1,236 @@ +/*===- InstrProfilingPlatformLinux.c - Profile data Linux platform ------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \ + (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) || \ + defined(_AIX) + +#if !defined(_AIX) +#include <elf.h> +#include <link.h> +#endif +#include <stdlib.h> +#include <string.h> + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" + +#define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_COMMON) +#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_COMMON) +#define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_COMMON) +#define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON) +#define PROF_VNAME_START INSTR_PROF_SECT_START(INSTR_PROF_VNAME_COMMON) +#define PROF_VNAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNAME_COMMON) +#define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON) +#define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON) +#define PROF_VTABLE_START INSTR_PROF_SECT_START(INSTR_PROF_VTAB_COMMON) +#define PROF_VTABLE_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VTAB_COMMON) +#define PROF_BITS_START INSTR_PROF_SECT_START(INSTR_PROF_BITS_COMMON) +#define PROF_BITS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_BITS_COMMON) +#define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON) +#define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_COMMON) +#define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_COMMON) + +/* Declare section start and stop symbols for various sections + * generated by compiler instrumentation. + */ +extern __llvm_profile_data PROF_DATA_START COMPILER_RT_VISIBILITY + COMPILER_RT_WEAK; +extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY + COMPILER_RT_WEAK; +extern char PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern VTableProfData PROF_VTABLE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern VTableProfData PROF_VTABLE_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_VNAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_VNAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_BITS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_BITS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_NAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern ValueProfNode PROF_VNODES_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern ValueProfNode PROF_VNODES_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; + +COMPILER_RT_VISIBILITY const __llvm_profile_data * +__llvm_profile_begin_data(void) { + return &PROF_DATA_START; +} +COMPILER_RT_VISIBILITY const __llvm_profile_data * +__llvm_profile_end_data(void) { + return &PROF_DATA_STOP; +} +COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) { + return &PROF_NAME_START; +} +COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) { + return &PROF_NAME_STOP; +} +COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_vtabnames(void) { + return &PROF_VNAME_START; +} +COMPILER_RT_VISIBILITY const char *__llvm_profile_end_vtabnames(void) { + return &PROF_VNAME_STOP; +} +COMPILER_RT_VISIBILITY const VTableProfData * +__llvm_profile_begin_vtables(void) { + return &PROF_VTABLE_START; +} +COMPILER_RT_VISIBILITY const VTableProfData *__llvm_profile_end_vtables(void) { + return &PROF_VTABLE_STOP; +} +COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) { + return &PROF_CNTS_START; +} +COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { + return &PROF_CNTS_STOP; +} +COMPILER_RT_VISIBILITY char *__llvm_profile_begin_bitmap(void) { + return &PROF_BITS_START; +} +COMPILER_RT_VISIBILITY char *__llvm_profile_end_bitmap(void) { + return &PROF_BITS_STOP; +} +COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { + return &PROF_ORDERFILE_START; +} + +COMPILER_RT_VISIBILITY ValueProfNode * +__llvm_profile_begin_vnodes(void) { + return &PROF_VNODES_START; +} +COMPILER_RT_VISIBILITY ValueProfNode *__llvm_profile_end_vnodes(void) { + return &PROF_VNODES_STOP; +} +COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = &PROF_VNODES_START; +COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = &PROF_VNODES_STOP; + +#ifdef NT_GNU_BUILD_ID +static size_t RoundUp(size_t size, size_t align) { + return (size + align - 1) & ~(align - 1); +} + +/* + * Look for the note that has the name "GNU\0" and type NT_GNU_BUILD_ID + * that contains build id. If build id exists, write binary id. + * + * Each note in notes section starts with a struct which includes + * n_namesz, n_descsz, and n_type members. It is followed by the name + * (whose length is defined in n_namesz) and then by the descriptor + * (whose length is defined in n_descsz). + * + * Note sections like .note.ABI-tag and .note.gnu.build-id are aligned + * to 4 bytes, so round n_namesz and n_descsz to the nearest 4 bytes. + */ +static int WriteBinaryIdForNote(ProfDataWriter *Writer, + const ElfW(Nhdr) * Note) { + int BinaryIdSize = 0; + const char *NoteName = (const char *)Note + sizeof(ElfW(Nhdr)); + if (Note->n_type == NT_GNU_BUILD_ID && Note->n_namesz == 4 && + memcmp(NoteName, "GNU\0", 4) == 0) { + uint64_t BinaryIdLen = Note->n_descsz; + const uint8_t *BinaryIdData = + (const uint8_t *)(NoteName + RoundUp(Note->n_namesz, 4)); + uint8_t BinaryIdPadding = __llvm_profile_get_num_padding_bytes(BinaryIdLen); + if (Writer != NULL && + lprofWriteOneBinaryId(Writer, BinaryIdLen, BinaryIdData, + BinaryIdPadding) == -1) + return -1; + + BinaryIdSize = sizeof(BinaryIdLen) + BinaryIdLen + BinaryIdPadding; + } + + return BinaryIdSize; +} + +/* + * Helper function that iterates through notes section and find build ids. + * If writer is given, write binary ids into profiles. + * If an error happens while writing, return -1. + */ +static int WriteBinaryIds(ProfDataWriter *Writer, const ElfW(Nhdr) * Note, + const ElfW(Nhdr) * NotesEnd) { + int BinaryIdsSize = 0; + while (Note < NotesEnd) { + int OneBinaryIdSize = WriteBinaryIdForNote(Writer, Note); + if (OneBinaryIdSize == -1) + return -1; + BinaryIdsSize += OneBinaryIdSize; + + /* Calculate the offset of the next note in notes section. */ + size_t NoteOffset = sizeof(ElfW(Nhdr)) + RoundUp(Note->n_namesz, 4) + + RoundUp(Note->n_descsz, 4); + Note = (const ElfW(Nhdr) *)((const char *)(Note) + NoteOffset); + } + + return BinaryIdsSize; +} + +/* + * Write binary ids into profiles if writer is given. + * Return the total size of binary ids. + * If an error happens while writing, return -1. + */ +COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) { + extern const ElfW(Ehdr) __ehdr_start __attribute__((visibility("hidden"))); + const ElfW(Ehdr) *ElfHeader = &__ehdr_start; + const ElfW(Phdr) *ProgramHeader = + (const ElfW(Phdr) *)((uintptr_t)ElfHeader + ElfHeader->e_phoff); + + int TotalBinaryIdsSize = 0; + uint32_t I; + /* Iterate through entries in the program header. */ + for (I = 0; I < ElfHeader->e_phnum; I++) { + /* Look for the notes segment in program header entries. */ + if (ProgramHeader[I].p_type != PT_NOTE) + continue; + + /* There can be multiple notes segment, and examine each of them. */ + const ElfW(Nhdr) * Note; + const ElfW(Nhdr) * NotesEnd; + /* + * When examining notes in file, use p_offset, which is the offset within + * the elf file, to find the start of notes. + */ + if (ProgramHeader[I].p_memsz == 0 || + ProgramHeader[I].p_memsz == ProgramHeader[I].p_filesz) { + Note = (const ElfW(Nhdr) *)((uintptr_t)ElfHeader + + ProgramHeader[I].p_offset); + NotesEnd = (const ElfW(Nhdr) *)((const char *)(Note) + + ProgramHeader[I].p_filesz); + } else { + /* + * When examining notes in memory, use p_vaddr, which is the address of + * section after loaded to memory, to find the start of notes. + */ + Note = + (const ElfW(Nhdr) *)((uintptr_t)ElfHeader + ProgramHeader[I].p_vaddr); + NotesEnd = + (const ElfW(Nhdr) *)((const char *)(Note) + ProgramHeader[I].p_memsz); + } + + int BinaryIdsSize = WriteBinaryIds(Writer, Note, NotesEnd); + if (TotalBinaryIdsSize == -1) + return -1; + + TotalBinaryIdsSize += BinaryIdsSize; + } + + return TotalBinaryIdsSize; +} +#elif !defined(_AIX) /* !NT_GNU_BUILD_ID */ +/* + * Fallback implementation for targets that don't support the GNU + * extensions NT_GNU_BUILD_ID and __ehdr_start. + */ +COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) { + return 0; +} +#endif + +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformOther.c new file mode 100644 index 000000000000..aa79a5641cec --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -0,0 +1,128 @@ +/*===- InstrProfilingPlatformOther.c - Profile data default platform ------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \ + !defined(__Fuchsia__) && !(defined(__sun__) && defined(__svr4__)) && \ + !defined(__NetBSD__) && !defined(_WIN32) && !defined(_AIX) + +#include <stdlib.h> +#include <stdio.h> + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" + +static const __llvm_profile_data *DataFirst = NULL; +static const __llvm_profile_data *DataLast = NULL; +static const VTableProfData *VTableProfDataFirst = NULL; +static const VTableProfData *VTableProfDataLast = NULL; +static const char *NamesFirst = NULL; +static const char *NamesLast = NULL; +static const char *VNamesFirst = NULL; +static const char *VNamesLast = NULL; +static char *CountersFirst = NULL; +static char *CountersLast = NULL; +static uint32_t *OrderFileFirst = NULL; + +static const void *getMinAddr(const void *A1, const void *A2) { + return A1 < A2 ? A1 : A2; +} + +static const void *getMaxAddr(const void *A1, const void *A2) { + return A1 > A2 ? A1 : A2; +} + +/*! + * \brief Register an instrumented function. + * + * Calls to this are emitted by clang with -fprofile-instr-generate. Such + * calls are only required (and only emitted) on targets where we haven't + * implemented linker magic to find the bounds of the sections. + */ +COMPILER_RT_VISIBILITY +void __llvm_profile_register_function(void *Data_) { + /* TODO: Only emit this function if we can't use linker magic. */ + const __llvm_profile_data *Data = (__llvm_profile_data *)Data_; + if (!DataFirst) { + DataFirst = Data; + DataLast = Data + 1; + CountersFirst = (char *)((uintptr_t)Data_ + Data->CounterPtr); + CountersLast = + CountersFirst + Data->NumCounters * __llvm_profile_counter_entry_size(); + return; + } + + DataFirst = (const __llvm_profile_data *)getMinAddr(DataFirst, Data); + CountersFirst = (char *)getMinAddr( + CountersFirst, (char *)((uintptr_t)Data_ + Data->CounterPtr)); + + DataLast = (const __llvm_profile_data *)getMaxAddr(DataLast, Data + 1); + CountersLast = (char *)getMaxAddr( + CountersLast, + (char *)((uintptr_t)Data_ + Data->CounterPtr) + + Data->NumCounters * __llvm_profile_counter_entry_size()); +} + +COMPILER_RT_VISIBILITY +void __llvm_profile_register_names_function(void *NamesStart, + uint64_t NamesSize) { + if (!NamesFirst) { + NamesFirst = (const char *)NamesStart; + NamesLast = (const char *)NamesStart + NamesSize; + return; + } + NamesFirst = (const char *)getMinAddr(NamesFirst, NamesStart); + NamesLast = + (const char *)getMaxAddr(NamesLast, (const char *)NamesStart + NamesSize); +} + +COMPILER_RT_VISIBILITY +const __llvm_profile_data *__llvm_profile_begin_data(void) { return DataFirst; } +COMPILER_RT_VISIBILITY +const __llvm_profile_data *__llvm_profile_end_data(void) { return DataLast; } +COMPILER_RT_VISIBILITY const VTableProfData * +__llvm_profile_begin_vtables(void) { + return VTableProfDataFirst; +} +COMPILER_RT_VISIBILITY const VTableProfData *__llvm_profile_end_vtables(void) { + return VTableProfDataLast; +} +COMPILER_RT_VISIBILITY +const char *__llvm_profile_begin_names(void) { return NamesFirst; } +COMPILER_RT_VISIBILITY +const char *__llvm_profile_end_names(void) { return NamesLast; } +COMPILER_RT_VISIBILITY +const char *__llvm_profile_begin_vtabnames(void) { return VNamesFirst; } +COMPILER_RT_VISIBILITY +const char *__llvm_profile_end_vtabnames(void) { return VNamesLast; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_begin_counters(void) { return CountersFirst; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_end_counters(void) { return CountersLast; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_begin_bitmap(void) { return BitmapFirst; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_end_bitmap(void) { return BitmapLast; } +/* TODO: correctly set up OrderFileFirst. */ +COMPILER_RT_VISIBILITY +uint32_t *__llvm_profile_begin_orderfile(void) { return OrderFileFirst; } + +COMPILER_RT_VISIBILITY +ValueProfNode *__llvm_profile_begin_vnodes(void) { + return 0; +} +COMPILER_RT_VISIBILITY +ValueProfNode *__llvm_profile_end_vnodes(void) { return 0; } + +COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = 0; +COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = 0; + +COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) { + return 0; +} + +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c new file mode 100644 index 000000000000..741b01faada4 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -0,0 +1,111 @@ +/*===- InstrProfilingPlatformWindows.c - Profile data on Windows ----------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#include <stddef.h> + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" + +#if defined(_WIN32) + +#if defined(_MSC_VER) +/* Merge read-write sections into .data. */ +#pragma comment(linker, "/MERGE:.lprfb=.data") +#pragma comment(linker, "/MERGE:.lprfd=.data") +#pragma comment(linker, "/MERGE:.lprfv=.data") +#pragma comment(linker, "/MERGE:.lprfnd=.data") +/* Do *NOT* merge .lprfn and .lcovmap into .rdata. llvm-cov must be able to find + * after the fact. + * Do *NOT* merge .lprfc .rdata. When binary profile correlation is enabled, + * llvm-cov must be able to find after the fact. + */ + +/* Allocate read-only section bounds. */ +#pragma section(".lprfn$A", read) +#pragma section(".lprfn$Z", read) + +/* Allocate read-write section bounds. */ +#pragma section(".lprfd$A", read, write) +#pragma section(".lprfd$Z", read, write) +#pragma section(".lprfc$A", read, write) +#pragma section(".lprfc$Z", read, write) +#pragma section(".lprfb$A", read, write) +#pragma section(".lprfb$Z", read, write) +#pragma section(".lorderfile$A", read, write) +#pragma section(".lprfnd$A", read, write) +#pragma section(".lprfnd$Z", read, write) +#endif + +__llvm_profile_data COMPILER_RT_SECTION(".lprfd$A") DataStart = {0}; +__llvm_profile_data COMPILER_RT_SECTION(".lprfd$Z") DataEnd = {0}; + +const char COMPILER_RT_SECTION(".lprfn$A") NamesStart = '\0'; +const char COMPILER_RT_SECTION(".lprfn$Z") NamesEnd = '\0'; + +char COMPILER_RT_SECTION(".lprfc$A") CountersStart; +char COMPILER_RT_SECTION(".lprfc$Z") CountersEnd; +char COMPILER_RT_SECTION(".lprfb$A") BitmapStart; +char COMPILER_RT_SECTION(".lprfb$Z") BitmapEnd; +uint32_t COMPILER_RT_SECTION(".lorderfile$A") OrderFileStart; + +ValueProfNode COMPILER_RT_SECTION(".lprfnd$A") VNodesStart; +ValueProfNode COMPILER_RT_SECTION(".lprfnd$Z") VNodesEnd; + +const __llvm_profile_data *__llvm_profile_begin_data(void) { + return &DataStart + 1; +} +const __llvm_profile_data *__llvm_profile_end_data(void) { return &DataEnd; } + +// Type profiling isn't implemented under MSVC ABI, so return NULL (rather than +// implementing linker magic on Windows) to make it more explicit. To elaborate, +// the current type profiling implementation maps a profiled vtable address to a +// vtable variable through vtables mangled name. Under MSVC ABI, the variable +// name for vtables might not be the mangled name (see +// MicrosoftCXXABI::getAddrOfVTable in MicrosoftCXXABI.cpp for more details on +// how a vtable name is computed). Note the mangled name is still in the vtable +// IR (just not variable name) for mapping purpose, but more implementation work +// is required. +const VTableProfData *__llvm_profile_begin_vtables(void) { return NULL; } +const VTableProfData *__llvm_profile_end_vtables(void) { return NULL; } + +const char *__llvm_profile_begin_names(void) { return &NamesStart + 1; } +const char *__llvm_profile_end_names(void) { return &NamesEnd; } + +// Type profiling isn't supported on Windows, so return NULl to make it more +// explicit. +const char *__llvm_profile_begin_vtabnames(void) { return NULL; } +const char *__llvm_profile_end_vtabnames(void) { return NULL; } + +char *__llvm_profile_begin_counters(void) { return &CountersStart + 1; } +char *__llvm_profile_end_counters(void) { return &CountersEnd; } +char *__llvm_profile_begin_bitmap(void) { return &BitmapStart + 1; } +char *__llvm_profile_end_bitmap(void) { return &BitmapEnd; } +uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } + +ValueProfNode *__llvm_profile_begin_vnodes(void) { return &VNodesStart + 1; } +ValueProfNode *__llvm_profile_end_vnodes(void) { return &VNodesEnd; } + +ValueProfNode *CurrentVNode = &VNodesStart + 1; +ValueProfNode *EndVNode = &VNodesEnd; + +/* lld-link provides __buildid symbol which points to the 16 bytes build id when + * using /build-id flag. https://lld.llvm.org/windows_support.html#lld-flags */ +#define BUILD_ID_LEN 16 +COMPILER_RT_WEAK uint8_t __buildid[BUILD_ID_LEN] = {0}; +COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) { + static const uint8_t zeros[BUILD_ID_LEN] = {0}; + if (memcmp(__buildid, zeros, BUILD_ID_LEN) != 0) { + if (Writer && + lprofWriteOneBinaryId(Writer, BUILD_ID_LEN, __buildid, 0) == -1) + return -1; + return sizeof(uint64_t) + BUILD_ID_LEN; + } + return 0; +} + +#endif diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPort.h b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPort.h new file mode 100644 index 000000000000..ed0905cc5f20 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingPort.h @@ -0,0 +1,152 @@ +/*===- InstrProfilingPort.h- Support library for PGO instrumentation ------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +/* This header must be included after all others so it can provide fallback + definitions for stuff missing in system headers. */ + +#ifndef PROFILE_INSTRPROFILING_PORT_H_ +#define PROFILE_INSTRPROFILING_PORT_H_ + +#ifdef _MSC_VER +#define COMPILER_RT_ALIGNAS(x) __declspec(align(x)) +#define COMPILER_RT_VISIBILITY +/* FIXME: selectany does not have the same semantics as weak. */ +#define COMPILER_RT_WEAK __declspec(selectany) +/* Need to include <windows.h> */ +#define COMPILER_RT_ALLOCA _alloca +/* Need to include <stdio.h> and <io.h> */ +#define COMPILER_RT_FTRUNCATE(f,l) _chsize(_fileno(f),l) +#define COMPILER_RT_ALWAYS_INLINE __forceinline +#define COMPILER_RT_CLEANUP(x) +#define COMPILER_RT_USED +#elif __GNUC__ +#ifdef _WIN32 +#define COMPILER_RT_FTRUNCATE(f, l) _chsize(fileno(f), l) +#define COMPILER_RT_VISIBILITY +#define COMPILER_RT_WEAK __attribute__((selectany)) +#else +#define COMPILER_RT_FTRUNCATE(f, l) ftruncate(fileno(f), l) +#define COMPILER_RT_VISIBILITY __attribute__((visibility("hidden"))) +#define COMPILER_RT_WEAK __attribute__((weak)) +#endif +#define COMPILER_RT_ALIGNAS(x) __attribute__((aligned(x))) +#define COMPILER_RT_ALLOCA __builtin_alloca +#define COMPILER_RT_ALWAYS_INLINE inline __attribute((always_inline)) +#define COMPILER_RT_CLEANUP(x) __attribute__((cleanup(x))) +#define COMPILER_RT_USED __attribute__((used)) +#endif + +#if defined(__APPLE__) +#define COMPILER_RT_SEG "__DATA," +#else +#define COMPILER_RT_SEG "" +#endif + +#ifdef _MSC_VER +#define COMPILER_RT_SECTION(Sect) __declspec(allocate(Sect)) +#else +#define COMPILER_RT_SECTION(Sect) __attribute__((section(Sect))) +#endif + +#define COMPILER_RT_MAX_HOSTLEN 128 +#ifdef __ORBIS__ +#define COMPILER_RT_GETHOSTNAME(Name, Len) ((void)(Name), (void)(Len), (-1)) +#else +#define COMPILER_RT_GETHOSTNAME(Name, Len) lprofGetHostName(Name, Len) +#endif + +#if COMPILER_RT_HAS_ATOMICS == 1 +#ifdef _WIN32 +#include <windows.h> +#if defined(_MSC_VER) && _MSC_VER < 1900 +#define snprintf _snprintf +#endif +#if defined(_WIN64) +#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \ + (InterlockedCompareExchange64((LONGLONG volatile *)Ptr, (LONGLONG)NewV, \ + (LONGLONG)OldV) == (LONGLONG)OldV) +#define COMPILER_RT_PTR_FETCH_ADD(DomType, PtrVar, PtrIncr) \ + (DomType *)InterlockedExchangeAdd64((LONGLONG volatile *)&PtrVar, \ + (LONGLONG)sizeof(DomType) * PtrIncr) +#else /* !defined(_WIN64) */ +#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \ + (InterlockedCompareExchange((LONG volatile *)Ptr, (LONG)NewV, (LONG)OldV) == \ + (LONG)OldV) +#define COMPILER_RT_PTR_FETCH_ADD(DomType, PtrVar, PtrIncr) \ + (DomType *)InterlockedExchangeAdd((LONG volatile *)&PtrVar, \ + (LONG)sizeof(DomType) * PtrIncr) +#endif +#else /* !defined(_WIN32) */ +#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \ + __sync_bool_compare_and_swap(Ptr, OldV, NewV) +#define COMPILER_RT_PTR_FETCH_ADD(DomType, PtrVar, PtrIncr) \ + (DomType *)__sync_fetch_and_add((long *)&PtrVar, sizeof(DomType) * PtrIncr) +#endif +#else /* COMPILER_RT_HAS_ATOMICS != 1 */ +#include "InstrProfilingUtil.h" +#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \ + lprofBoolCmpXchg((void **)Ptr, OldV, NewV) +#define COMPILER_RT_PTR_FETCH_ADD(DomType, PtrVar, PtrIncr) \ + (DomType *)lprofPtrFetchAdd((void **)&PtrVar, sizeof(DomType) * PtrIncr) +#endif + +#if defined(_WIN32) +#define DIR_SEPARATOR '\\' +#define DIR_SEPARATOR_2 '/' +#else +#define DIR_SEPARATOR '/' +#endif + +#ifndef DIR_SEPARATOR_2 +#define IS_DIR_SEPARATOR(ch) ((ch) == DIR_SEPARATOR) +#else /* DIR_SEPARATOR_2 */ +#define IS_DIR_SEPARATOR(ch) \ + (((ch) == DIR_SEPARATOR) || ((ch) == DIR_SEPARATOR_2)) +#endif /* DIR_SEPARATOR_2 */ + +#if defined(_WIN32) +#include <windows.h> +static inline size_t getpagesize() { + SYSTEM_INFO S; + GetNativeSystemInfo(&S); + return S.dwPageSize; +} +#else /* defined(_WIN32) */ +#include <unistd.h> +#endif /* defined(_WIN32) */ + +#define PROF_ERR(Format, ...) \ + fprintf(stderr, "LLVM Profile Error: " Format, __VA_ARGS__); + +#define PROF_WARN(Format, ...) \ + fprintf(stderr, "LLVM Profile Warning: " Format, __VA_ARGS__); + +#define PROF_NOTE(Format, ...) \ + fprintf(stderr, "LLVM Profile Note: " Format, __VA_ARGS__); + +#ifndef MAP_FILE +#define MAP_FILE 0 +#endif + +#ifndef O_BINARY +#define O_BINARY 0 +#endif + +#if defined(__FreeBSD__) + +#include <inttypes.h> +#include <sys/types.h> + +#else /* defined(__FreeBSD__) */ + +#include <inttypes.h> +#include <stdint.h> + +#endif /* defined(__FreeBSD__) && defined(__i386__) */ + +#endif /* PROFILE_INSTRPROFILING_PORT_H_ */ diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingRuntime.cpp b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingRuntime.cpp new file mode 100644 index 000000000000..6b2ce9700173 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingRuntime.cpp @@ -0,0 +1,24 @@ +//===- InstrProfilingRuntime.cpp - PGO runtime initialization -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +extern "C" { + +#include "InstrProfiling.h" + +static int RegisterRuntime() { + __llvm_profile_initialize(); +#ifdef _AIX + extern COMPILER_RT_VISIBILITY void *__llvm_profile_keep[]; + (void)*(void *volatile *)__llvm_profile_keep; +#endif + return 0; +} + +/* int __llvm_profile_runtime */ +COMPILER_RT_VISIBILITY int INSTR_PROF_PROFILE_RUNTIME_VAR = RegisterRuntime(); +} diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingUtil.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingUtil.c new file mode 100644 index 000000000000..cd18cba3e268 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingUtil.c @@ -0,0 +1,375 @@ +/*===- InstrProfilingUtil.c - Support library for PGO instrumentation -----===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#ifdef _WIN32 +#include <direct.h> +#include <process.h> +#include <windows.h> +#include "WindowsMMap.h" +#else +#include <errno.h> +#include <fcntl.h> +#include <sys/file.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#endif + +#ifdef COMPILER_RT_HAS_UNAME +#include <sys/utsname.h> +#endif + +#include <stdlib.h> +#include <string.h> + +#if defined(__linux__) +#include <signal.h> +#include <sys/prctl.h> +#endif + +#if defined(__Fuchsia__) +#include <zircon/process.h> +#include <zircon/syscalls.h> +#endif + +#if defined(__FreeBSD__) +#include <signal.h> +#include <sys/procctl.h> +#endif + +#include "InstrProfiling.h" +#include "InstrProfilingUtil.h" + +COMPILER_RT_VISIBILITY unsigned lprofDirMode = 0755; + +COMPILER_RT_VISIBILITY +void __llvm_profile_recursive_mkdir(char *path) { + int i; + int start = 1; + +#if defined(__ANDROID__) && defined(__ANDROID_API__) && \ + defined(__ANDROID_API_FUTURE__) && \ + __ANDROID_API__ == __ANDROID_API_FUTURE__ + // Avoid spammy selinux denial messages in Android by not attempting to + // create directories in GCOV_PREFIX. These denials occur when creating (or + // even attempting to stat()) top-level directories like "/data". + // + // Do so by ignoring ${GCOV_PREFIX} when invoking mkdir(). + const char *gcov_prefix = getenv("GCOV_PREFIX"); + if (gcov_prefix != NULL) { + const int gcov_prefix_len = strlen(gcov_prefix); + if (strncmp(path, gcov_prefix, gcov_prefix_len) == 0) + start = gcov_prefix_len; + } +#endif + + for (i = start; path[i] != '\0'; ++i) { + char save = path[i]; + if (!IS_DIR_SEPARATOR(path[i])) + continue; + path[i] = '\0'; +#ifdef _WIN32 + _mkdir(path); +#else + /* Some of these will fail, ignore it. */ + mkdir(path, __llvm_profile_get_dir_mode()); +#endif + path[i] = save; + } +} + +COMPILER_RT_VISIBILITY +void __llvm_profile_set_dir_mode(unsigned Mode) { lprofDirMode = Mode; } + +COMPILER_RT_VISIBILITY +unsigned __llvm_profile_get_dir_mode(void) { return lprofDirMode; } + +#if COMPILER_RT_HAS_ATOMICS != 1 +COMPILER_RT_VISIBILITY +uint32_t lprofBoolCmpXchg(void **Ptr, void *OldV, void *NewV) { + void *R = *Ptr; + if (R == OldV) { + *Ptr = NewV; + return 1; + } + return 0; +} +COMPILER_RT_VISIBILITY +void *lprofPtrFetchAdd(void **Mem, long ByteIncr) { + void *Old = *Mem; + *((char **)Mem) += ByteIncr; + return Old; +} + +#endif + +#ifdef _WIN32 +COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) { + WCHAR Buffer[COMPILER_RT_MAX_HOSTLEN]; + DWORD BufferSize = sizeof(Buffer); + BOOL Result = + GetComputerNameExW(ComputerNameDnsFullyQualified, Buffer, &BufferSize); + if (!Result) + return -1; + if (WideCharToMultiByte(CP_UTF8, 0, Buffer, -1, Name, Len, NULL, NULL) == 0) + return -1; + return 0; +} +#elif defined(COMPILER_RT_HAS_UNAME) +COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) { + struct utsname N; + int R = uname(&N); + if (R >= 0) { + strncpy(Name, N.nodename, Len); + return 0; + } + return R; +} +#endif + +COMPILER_RT_VISIBILITY int lprofLockFd(int fd) { +#ifdef COMPILER_RT_HAS_FCNTL_LCK + struct flock s_flock; + + s_flock.l_whence = SEEK_SET; + s_flock.l_start = 0; + s_flock.l_len = 0; /* Until EOF. */ + s_flock.l_pid = getpid(); + s_flock.l_type = F_WRLCK; + + while (fcntl(fd, F_SETLKW, &s_flock) == -1) { + if (errno != EINTR) { + if (errno == ENOLCK) { + return -1; + } + break; + } + } + return 0; +#else + flock(fd, LOCK_EX); + return 0; +#endif +} + +COMPILER_RT_VISIBILITY int lprofUnlockFd(int fd) { +#ifdef COMPILER_RT_HAS_FCNTL_LCK + struct flock s_flock; + + s_flock.l_whence = SEEK_SET; + s_flock.l_start = 0; + s_flock.l_len = 0; /* Until EOF. */ + s_flock.l_pid = getpid(); + s_flock.l_type = F_UNLCK; + + while (fcntl(fd, F_SETLKW, &s_flock) == -1) { + if (errno != EINTR) { + if (errno == ENOLCK) { + return -1; + } + break; + } + } + return 0; +#else + flock(fd, LOCK_UN); + return 0; +#endif +} + +COMPILER_RT_VISIBILITY int lprofLockFileHandle(FILE *F) { + int fd; +#if defined(_WIN32) + fd = _fileno(F); +#else + fd = fileno(F); +#endif + return lprofLockFd(fd); +} + +COMPILER_RT_VISIBILITY int lprofUnlockFileHandle(FILE *F) { + int fd; +#if defined(_WIN32) + fd = _fileno(F); +#else + fd = fileno(F); +#endif + return lprofUnlockFd(fd); +} + +COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) { + FILE *f; + int fd; +#ifdef COMPILER_RT_HAS_FCNTL_LCK + fd = open(ProfileName, O_RDWR | O_CREAT, 0666); + if (fd < 0) + return NULL; + + if (lprofLockFd(fd) != 0) + PROF_WARN("Data may be corrupted during profile merging : %s\n", + "Fail to obtain file lock due to system limit."); + + f = fdopen(fd, "r+b"); +#elif defined(_WIN32) + // FIXME: Use the wide variants to handle Unicode filenames. + HANDLE h = CreateFileA(ProfileName, GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, 0); + if (h == INVALID_HANDLE_VALUE) + return NULL; + + fd = _open_osfhandle((intptr_t)h, 0); + if (fd == -1) { + CloseHandle(h); + return NULL; + } + + if (lprofLockFd(fd) != 0) + PROF_WARN("Data may be corrupted during profile merging : %s\n", + "Fail to obtain file lock due to system limit."); + + f = _fdopen(fd, "r+b"); + if (f == 0) { + CloseHandle(h); + return NULL; + } +#else + /* Worst case no locking applied. */ + PROF_WARN("Concurrent file access is not supported : %s\n", + "lack file locking"); + fd = open(ProfileName, O_RDWR | O_CREAT, 0666); + if (fd < 0) + return NULL; + f = fdopen(fd, "r+b"); +#endif + + return f; +} + +COMPILER_RT_VISIBILITY const char *lprofGetPathPrefix(int *PrefixStrip, + size_t *PrefixLen) { + const char *Prefix = getenv("GCOV_PREFIX"); + const char *PrefixStripStr = getenv("GCOV_PREFIX_STRIP"); + + *PrefixLen = 0; + *PrefixStrip = 0; + if (Prefix == NULL || Prefix[0] == '\0') + return NULL; + + if (PrefixStripStr) { + *PrefixStrip = atoi(PrefixStripStr); + + /* Negative GCOV_PREFIX_STRIP values are ignored */ + if (*PrefixStrip < 0) + *PrefixStrip = 0; + } else { + *PrefixStrip = 0; + } + *PrefixLen = strlen(Prefix); + + return Prefix; +} + +COMPILER_RT_VISIBILITY void +lprofApplyPathPrefix(char *Dest, const char *PathStr, const char *Prefix, + size_t PrefixLen, int PrefixStrip) { + + const char *Ptr; + int Level; + const char *StrippedPathStr = PathStr; + + for (Level = 0, Ptr = PathStr + 1; Level < PrefixStrip; ++Ptr) { + if (*Ptr == '\0') + break; + + if (!IS_DIR_SEPARATOR(*Ptr)) + continue; + + StrippedPathStr = Ptr; + ++Level; + } + + memcpy(Dest, Prefix, PrefixLen); + + if (!IS_DIR_SEPARATOR(Prefix[PrefixLen - 1])) + Dest[PrefixLen++] = DIR_SEPARATOR; + + memcpy(Dest + PrefixLen, StrippedPathStr, strlen(StrippedPathStr) + 1); +} + +COMPILER_RT_VISIBILITY const char * +lprofFindFirstDirSeparator(const char *Path) { + const char *Sep = strchr(Path, DIR_SEPARATOR); +#if defined(DIR_SEPARATOR_2) + const char *Sep2 = strchr(Path, DIR_SEPARATOR_2); + if (Sep2 && (!Sep || Sep2 < Sep)) + Sep = Sep2; +#endif + return Sep; +} + +COMPILER_RT_VISIBILITY const char *lprofFindLastDirSeparator(const char *Path) { + const char *Sep = strrchr(Path, DIR_SEPARATOR); +#if defined(DIR_SEPARATOR_2) + const char *Sep2 = strrchr(Path, DIR_SEPARATOR_2); + if (Sep2 && (!Sep || Sep2 > Sep)) + Sep = Sep2; +#endif + return Sep; +} + +COMPILER_RT_VISIBILITY int lprofSuspendSigKill(void) { +#if defined(__linux__) + int PDeachSig = 0; + /* Temporarily suspend getting SIGKILL upon exit of the parent process. */ + if (prctl(PR_GET_PDEATHSIG, &PDeachSig) == 0 && PDeachSig == SIGKILL) + prctl(PR_SET_PDEATHSIG, 0); + return (PDeachSig == SIGKILL); +#elif defined(__FreeBSD__) + int PDeachSig = 0, PDisableSig = 0; + if (procctl(P_PID, 0, PROC_PDEATHSIG_STATUS, &PDeachSig) == 0 && + PDeachSig == SIGKILL) + procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PDisableSig); + return (PDeachSig == SIGKILL); +#else + return 0; +#endif +} + +COMPILER_RT_VISIBILITY void lprofRestoreSigKill(void) { +#if defined(__linux__) + prctl(PR_SET_PDEATHSIG, SIGKILL); +#elif defined(__FreeBSD__) + int PEnableSig = SIGKILL; + procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PEnableSig); +#endif +} + +COMPILER_RT_VISIBILITY int lprofReleaseMemoryPagesToOS(uintptr_t Begin, + uintptr_t End) { +#if defined(__ve__) + // VE doesn't support madvise. + return 0; +#else + size_t PageSize = getpagesize(); + uintptr_t BeginAligned = lprofRoundUpTo((uintptr_t)Begin, PageSize); + uintptr_t EndAligned = lprofRoundDownTo((uintptr_t)End, PageSize); + if (BeginAligned < EndAligned) { +#if defined(__Fuchsia__) + return _zx_vmar_op_range(_zx_vmar_root_self(), ZX_VMAR_OP_DECOMMIT, + (zx_vaddr_t)BeginAligned, + EndAligned - BeginAligned, NULL, 0); +#else + return madvise((void *)BeginAligned, EndAligned - BeginAligned, + MADV_DONTNEED); +#endif + } + return 0; +#endif +} diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingUtil.h b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingUtil.h new file mode 100644 index 000000000000..4a88a0358094 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingUtil.h @@ -0,0 +1,87 @@ +/*===- InstrProfilingUtil.h - Support library for PGO instrumentation -----===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#ifndef PROFILE_INSTRPROFILINGUTIL_H +#define PROFILE_INSTRPROFILINGUTIL_H + +#include <inttypes.h> +#include <stddef.h> +#include <stdio.h> + +/*! \brief Create a directory tree. */ +void __llvm_profile_recursive_mkdir(char *Pathname); + +/*! Set the mode used when creating profile directories. */ +void __llvm_profile_set_dir_mode(unsigned Mode); + +/*! Return the directory creation mode. */ +unsigned __llvm_profile_get_dir_mode(void); + +int lprofLockFd(int fd); +int lprofUnlockFd(int fd); +int lprofLockFileHandle(FILE *F); +int lprofUnlockFileHandle(FILE *F); + +/*! Open file \c Filename for read+write with write + * lock for exclusive access. The caller will block + * if the lock is already held by another process. */ +FILE *lprofOpenFileEx(const char *Filename); +/* PS4 doesn't have setenv/getenv/fork. Define a shim. */ +#if __ORBIS__ +#include <sys/types.h> +static inline char *getenv(const char *name) { return NULL; } +static inline int setenv(const char *name, const char *value, int overwrite) +{ return 0; } +static pid_t fork() { return -1; } +#endif /* #if __ORBIS__ */ + +/* GCOV_PREFIX and GCOV_PREFIX_STRIP support */ +/* Return the path prefix specified by GCOV_PREFIX environment variable. + * If GCOV_PREFIX_STRIP is also specified, the strip level (integer value) + * is returned via \c *PrefixStrip. The prefix length is stored in *PrefixLen. + */ +const char *lprofGetPathPrefix(int *PrefixStrip, size_t *PrefixLen); +/* Apply the path prefix specified in \c Prefix to path string in \c PathStr, + * and store the result to buffer pointed to by \c Buffer. If \c PrefixStrip + * is not zero, path prefixes are stripped from \c PathStr (the level of + * stripping is specified by \c PrefixStrip) before \c Prefix is added. + */ +void lprofApplyPathPrefix(char *Dest, const char *PathStr, const char *Prefix, + size_t PrefixLen, int PrefixStrip); + +/* Returns a pointer to the first occurrence of \c DIR_SEPARATOR char in + * the string \c Path, or NULL if the char is not found. */ +const char *lprofFindFirstDirSeparator(const char *Path); +/* Returns a pointer to the last occurrence of \c DIR_SEPARATOR char in + * the string \c Path, or NULL if the char is not found. */ +const char *lprofFindLastDirSeparator(const char *Path); + +int lprofGetHostName(char *Name, int Len); + +unsigned lprofBoolCmpXchg(void **Ptr, void *OldV, void *NewV); +void *lprofPtrFetchAdd(void **Mem, long ByteIncr); + +/* Temporarily suspend SIGKILL. Return value of 1 means a restore is needed. + * Other return values mean no restore is needed. + */ +int lprofSuspendSigKill(); + +/* Restore previously suspended SIGKILL. */ +void lprofRestoreSigKill(); + +static inline size_t lprofRoundUpTo(size_t x, size_t boundary) { + return (x + boundary - 1) & ~(boundary - 1); +} + +static inline size_t lprofRoundDownTo(size_t x, size_t boundary) { + return x & ~(boundary - 1); +} + +int lprofReleaseMemoryPagesToOS(uintptr_t Begin, uintptr_t End); + +#endif /* PROFILE_INSTRPROFILINGUTIL_H */ diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingValue.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingValue.c new file mode 100644 index 000000000000..a608d41d39e7 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingValue.c @@ -0,0 +1,370 @@ +/*===- InstrProfilingValue.c - Support library for PGO instrumentation ----===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#include <assert.h> +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingUtil.h" + +#define INSTR_PROF_VALUE_PROF_DATA +#define INSTR_PROF_COMMON_API_IMPL +#define INSTR_PROF_VALUE_PROF_MEMOP_API +#include "profile/InstrProfData.inc" + +static int hasStaticCounters = 1; +static int OutOfNodesWarnings = 0; +static int hasNonDefaultValsPerSite = 0; +#define INSTR_PROF_MAX_VP_WARNS 10 +#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 24 +#define INSTR_PROF_VNODE_POOL_SIZE 1024 + +#ifndef _MSC_VER +/* A shared static pool in addition to the vnodes statically + * allocated by the compiler. */ +COMPILER_RT_VISIBILITY ValueProfNode + lprofValueProfNodes[INSTR_PROF_VNODE_POOL_SIZE] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_VNODES_SECT_NAME); +#endif + +COMPILER_RT_VISIBILITY uint32_t VPMaxNumValsPerSite = + INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE; + +COMPILER_RT_VISIBILITY void lprofSetupValueProfiler(void) { + const char *Str = 0; + Str = getenv("LLVM_VP_MAX_NUM_VALS_PER_SITE"); + if (Str && Str[0]) { + VPMaxNumValsPerSite = atoi(Str); + hasNonDefaultValsPerSite = 1; + } + if (VPMaxNumValsPerSite > INSTR_PROF_MAX_NUM_VAL_PER_SITE) + VPMaxNumValsPerSite = INSTR_PROF_MAX_NUM_VAL_PER_SITE; +} + +COMPILER_RT_VISIBILITY void lprofSetMaxValsPerSite(uint32_t MaxVals) { + VPMaxNumValsPerSite = MaxVals; + hasNonDefaultValsPerSite = 1; +} + +/* This method is only used in value profiler mock testing. */ +COMPILER_RT_VISIBILITY void +__llvm_profile_set_num_value_sites(__llvm_profile_data *Data, + uint32_t ValueKind, uint16_t NumValueSites) { +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +#endif + *((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop +#endif +} + +/* This method is only used in value profiler mock testing. */ +COMPILER_RT_VISIBILITY const __llvm_profile_data * +__llvm_profile_iterate_data(const __llvm_profile_data *Data) { + return Data + 1; +} + +/* This method is only used in value profiler mock testing. */ +COMPILER_RT_VISIBILITY void * +__llvm_get_function_addr(const __llvm_profile_data *Data) { + return Data->FunctionPointer; +} + +/* Allocate an array that holds the pointers to the linked lists of + * value profile counter nodes. The number of element of the array + * is the total number of value profile sites instrumented. Returns + * 0 if allocation fails. + */ + +static int allocateValueProfileCounters(__llvm_profile_data *Data) { + uint64_t NumVSites = 0; + uint32_t VKI; + + /* This function will never be called when value site array is allocated + statically at compile time. */ + hasStaticCounters = 0; + /* When dynamic allocation is enabled, allow tracking the max number of + * values allowd. */ + if (!hasNonDefaultValsPerSite) + VPMaxNumValsPerSite = INSTR_PROF_MAX_NUM_VAL_PER_SITE; + + for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI) + NumVSites += Data->NumValueSites[VKI]; + + // If NumVSites = 0, calloc is allowed to return a non-null pointer. + assert(NumVSites > 0 && "NumVSites can't be zero"); + ValueProfNode **Mem = + (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *)); + if (!Mem) + return 0; + if (!COMPILER_RT_BOOL_CMPXCHG(&Data->Values, 0, Mem)) { + free(Mem); + return 0; + } + return 1; +} + +static ValueProfNode *allocateOneNode(void) { + ValueProfNode *Node; + + if (!hasStaticCounters) + return (ValueProfNode *)calloc(1, sizeof(ValueProfNode)); + + /* Early check to avoid value wrapping around. */ + if (CurrentVNode + 1 > EndVNode) { + if (OutOfNodesWarnings++ < INSTR_PROF_MAX_VP_WARNS) { + PROF_WARN("Unable to track new values: %s. " + " Consider using option -mllvm -vp-counters-per-site=<n> to " + "allocate more" + " value profile counters at compile time. \n", + "Running out of static counters"); + } + return 0; + } + Node = COMPILER_RT_PTR_FETCH_ADD(ValueProfNode, CurrentVNode, 1); + /* Due to section padding, EndVNode point to a byte which is one pass + * an incomplete VNode, so we need to skip the last incomplete node. */ + if (Node + 1 > EndVNode) + return 0; + + return Node; +} + +static COMPILER_RT_ALWAYS_INLINE void +instrumentTargetValueImpl(uint64_t TargetValue, void *Data, + uint32_t CounterIndex, uint64_t CountValue) { + __llvm_profile_data *PData = (__llvm_profile_data *)Data; + if (!PData) + return; + if (!CountValue) + return; + if (!PData->Values) { + if (!allocateValueProfileCounters(PData)) + return; + } + + ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values; + ValueProfNode *PrevVNode = NULL; + ValueProfNode *MinCountVNode = NULL; + ValueProfNode *CurVNode = ValueCounters[CounterIndex]; + uint64_t MinCount = UINT64_MAX; + + uint8_t VDataCount = 0; + while (CurVNode) { + if (TargetValue == CurVNode->Value) { + CurVNode->Count += CountValue; + return; + } + if (CurVNode->Count < MinCount) { + MinCount = CurVNode->Count; + MinCountVNode = CurVNode; + } + PrevVNode = CurVNode; + CurVNode = CurVNode->Next; + ++VDataCount; + } + + if (VDataCount >= VPMaxNumValsPerSite) { + /* Bump down the min count node's count. If it reaches 0, + * evict it. This eviction/replacement policy makes hot + * targets more sticky while cold targets less so. In other + * words, it makes it less likely for the hot targets to be + * prematurally evicted during warmup/establishment period, + * when their counts are still low. In a special case when + * the number of values tracked is reduced to only one, this + * policy will guarantee that the dominating target with >50% + * total count will survive in the end. Note that this scheme + * allows the runtime to track the min count node in an adaptive + * manner. It can correct previous mistakes and eventually + * lock on a cold target that is alread in stable state. + * + * In very rare cases, this replacement scheme may still lead + * to target loss. For instance, out of \c N value slots, \c N-1 + * slots are occupied by luke warm targets during the warmup + * period and the remaining one slot is competed by two or more + * very hot targets. If those hot targets occur in an interleaved + * way, none of them will survive (gain enough weight to throw out + * other established entries) due to the ping-pong effect. + * To handle this situation, user can choose to increase the max + * number of tracked values per value site. Alternatively, a more + * expensive eviction mechanism can be implemented. It requires + * the runtime to track the total number of evictions per-site. + * When the total number of evictions reaches certain threshold, + * the runtime can wipe out more than one lowest count entries + * to give space for hot targets. + */ + if (MinCountVNode->Count <= CountValue) { + CurVNode = MinCountVNode; + CurVNode->Value = TargetValue; + CurVNode->Count = CountValue; + } else + MinCountVNode->Count -= CountValue; + + return; + } + + CurVNode = allocateOneNode(); + if (!CurVNode) + return; + CurVNode->Value = TargetValue; + CurVNode->Count += CountValue; + + uint32_t Success = 0; + if (!ValueCounters[CounterIndex]) + Success = + COMPILER_RT_BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurVNode); + else if (PrevVNode && !PrevVNode->Next) + Success = COMPILER_RT_BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurVNode); + + if (!Success && !hasStaticCounters) { + free(CurVNode); + return; + } +} + +COMPILER_RT_VISIBILITY void +__llvm_profile_instrument_target(uint64_t TargetValue, void *Data, + uint32_t CounterIndex) { + instrumentTargetValueImpl(TargetValue, Data, CounterIndex, 1); +} +COMPILER_RT_VISIBILITY void +__llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data, + uint32_t CounterIndex, + uint64_t CountValue) { + instrumentTargetValueImpl(TargetValue, Data, CounterIndex, CountValue); +} + +/* + * The target values are partitioned into multiple ranges. The range spec is + * defined in InstrProfData.inc. + */ +COMPILER_RT_VISIBILITY void +__llvm_profile_instrument_memop(uint64_t TargetValue, void *Data, + uint32_t CounterIndex) { + // Map the target value to the representative value of its range. + uint64_t RepValue = InstrProfGetRangeRepValue(TargetValue); + __llvm_profile_instrument_target(RepValue, Data, CounterIndex); +} + +/* + * A wrapper struct that represents value profile runtime data. + * Like InstrProfRecord class which is used by profiling host tools, + * ValueProfRuntimeRecord also implements the abstract interfaces defined in + * ValueProfRecordClosure so that the runtime data can be serialized using + * shared C implementation. + */ +typedef struct ValueProfRuntimeRecord { + const __llvm_profile_data *Data; + ValueProfNode **NodesKind[IPVK_Last + 1]; + uint8_t **SiteCountArray; +} ValueProfRuntimeRecord; + +/* ValueProfRecordClosure Interface implementation. */ + +static uint32_t getNumValueSitesRT(const void *R, uint32_t VK) { + return ((const ValueProfRuntimeRecord *)R)->Data->NumValueSites[VK]; +} + +static uint32_t getNumValueDataRT(const void *R, uint32_t VK) { + uint32_t S = 0, I; + const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; + if (Record->SiteCountArray[VK] == INSTR_PROF_NULLPTR) + return 0; + for (I = 0; I < Record->Data->NumValueSites[VK]; I++) + S += Record->SiteCountArray[VK][I]; + return S; +} + +static uint32_t getNumValueDataForSiteRT(const void *R, uint32_t VK, + uint32_t S) { + const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; + return Record->SiteCountArray[VK][S]; +} + +static ValueProfRuntimeRecord RTRecord; +static ValueProfRecordClosure RTRecordClosure = { + &RTRecord, INSTR_PROF_NULLPTR, /* GetNumValueKinds */ + getNumValueSitesRT, getNumValueDataRT, getNumValueDataForSiteRT, + INSTR_PROF_NULLPTR, /* RemapValueData */ + INSTR_PROF_NULLPTR, /* GetValueForSite, */ + INSTR_PROF_NULLPTR /* AllocValueProfData */ +}; + +static uint32_t +initializeValueProfRuntimeRecord(const __llvm_profile_data *Data, + uint8_t *SiteCountArray[]) { + unsigned I, J, S = 0, NumValueKinds = 0; + ValueProfNode **Nodes = (ValueProfNode **)Data->Values; + RTRecord.Data = Data; + RTRecord.SiteCountArray = SiteCountArray; + for (I = 0; I <= IPVK_Last; I++) { + uint16_t N = Data->NumValueSites[I]; + if (!N) + continue; + + NumValueKinds++; + + RTRecord.NodesKind[I] = Nodes ? &Nodes[S] : INSTR_PROF_NULLPTR; + for (J = 0; J < N; J++) { + /* Compute value count for each site. */ + uint32_t C = 0; + ValueProfNode *Site = + Nodes ? RTRecord.NodesKind[I][J] : INSTR_PROF_NULLPTR; + while (Site) { + C++; + Site = Site->Next; + } + if (C > UCHAR_MAX) + C = UCHAR_MAX; + RTRecord.SiteCountArray[I][J] = C; + } + S += N; + } + return NumValueKinds; +} + +static ValueProfNode *getNextNValueData(uint32_t VK, uint32_t Site, + InstrProfValueData *Dst, + ValueProfNode *StartNode, uint32_t N) { + unsigned I; + ValueProfNode *VNode = StartNode ? StartNode : RTRecord.NodesKind[VK][Site]; + for (I = 0; I < N; I++) { + Dst[I].Value = VNode->Value; + Dst[I].Count = VNode->Count; + VNode = VNode->Next; + } + return VNode; +} + +static uint32_t getValueProfDataSizeWrapper(void) { + return getValueProfDataSize(&RTRecordClosure); +} + +static uint32_t getNumValueDataForSiteWrapper(uint32_t VK, uint32_t S) { + return getNumValueDataForSiteRT(&RTRecord, VK, S); +} + +static VPDataReaderType TheVPDataReader = { + initializeValueProfRuntimeRecord, getValueProfRecordHeaderSize, + getFirstValueProfRecord, getNumValueDataForSiteWrapper, + getValueProfDataSizeWrapper, getNextNValueData}; + +COMPILER_RT_VISIBILITY VPDataReaderType *lprofGetVPDataReader(void) { + return &TheVPDataReader; +} diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingVersionVar.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingVersionVar.c new file mode 100644 index 000000000000..21400bfb2caa --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingVersionVar.c @@ -0,0 +1,18 @@ +/*===- InstrProfilingVersionVar.c - profile version variable setup -------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" + +/* uint64 __llvm_profile_raw_version + * + * The runtime should only provide its own definition of this symbol when the + * user has not specified one. Set this up by moving the runtime's copy of this + * symbol to an object file within the archive. + */ +COMPILER_RT_VISIBILITY COMPILER_RT_WEAK uint64_t INSTR_PROF_RAW_VERSION_VAR = + INSTR_PROF_RAW_VERSION; diff --git a/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingWriter.c b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingWriter.c new file mode 100644 index 000000000000..8816a7115551 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -0,0 +1,379 @@ +/*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +// Note: This is linked into the Darwin kernel, and must remain compatible +// with freestanding compilation. See `darwin_add_builtin_libraries`. + +#ifdef _MSC_VER +/* For _alloca */ +#include <malloc.h> +#endif +#include <string.h> + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingPort.h" + +#define INSTR_PROF_VALUE_PROF_DATA +#include "profile/InstrProfData.inc" + +COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL; +static ProfBufferIO TheBufferIO; +#define VP_BUFFER_SIZE 8 * 1024 +static uint8_t BufferIOBuffer[VP_BUFFER_SIZE]; +static InstrProfValueData VPDataArray[16]; +static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray); + +COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0; +COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0; + +/* The buffer writer is responsible in keeping writer state + * across the call. + */ +COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This, + ProfDataIOVec *IOVecs, + uint32_t NumIOVecs) { + uint32_t I; + char **Buffer = (char **)&This->WriterCtx; + for (I = 0; I < NumIOVecs; I++) { + size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm; + if (IOVecs[I].Data) + memcpy(*Buffer, IOVecs[I].Data, Length); + else if (IOVecs[I].UseZeroPadding) { + /* Allocating the buffer should zero fill. */ + } + *Buffer += Length; + } + return 0; +} + +static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter, + uint8_t *Buffer, uint32_t BufferSz) { + BufferIO->FileWriter = FileWriter; + BufferIO->OwnFileWriter = 0; + BufferIO->BufferStart = Buffer; + BufferIO->BufferSz = BufferSz; + BufferIO->CurOffset = 0; +} + +COMPILER_RT_VISIBILITY ProfBufferIO * +lprofCreateBufferIO(ProfDataWriter *FileWriter) { + uint8_t *Buffer = DynamicBufferIOBuffer; + uint32_t BufferSize = VPBufferSize; + if (!Buffer) { + Buffer = &BufferIOBuffer[0]; + BufferSize = sizeof(BufferIOBuffer); + } + llvmInitBufferIO(&TheBufferIO, FileWriter, Buffer, BufferSize); + return &TheBufferIO; +} + +COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) { + if (BufferIO->OwnFileWriter) + FreeHook(BufferIO->FileWriter); + if (DynamicBufferIOBuffer) { + FreeHook(DynamicBufferIOBuffer); + DynamicBufferIOBuffer = 0; + VPBufferSize = 0; + } +} + +COMPILER_RT_VISIBILITY int +lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) { + /* Buffer is not large enough, it is time to flush. */ + if (Size + BufferIO->CurOffset > BufferIO->BufferSz) { + if (lprofBufferIOFlush(BufferIO) != 0) + return -1; + } + /* Special case, bypass the buffer completely. */ + ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size, 0}}; + if (Size > BufferIO->BufferSz) { + if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1)) + return -1; + } else { + /* Write the data to buffer */ + uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset; + ProfDataWriter BufferWriter; + initBufferWriter(&BufferWriter, (char *)Buffer); + lprofBufferWriter(&BufferWriter, IO, 1); + BufferIO->CurOffset = + (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart; + } + return 0; +} + +COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) { + if (BufferIO->CurOffset) { + ProfDataIOVec IO[] = { + {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset, 0}}; + if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1)) + return -1; + BufferIO->CurOffset = 0; + } + return 0; +} + +/* Write out value profile data for function specified with \c Data. + * The implementation does not use the method \c serializeValueProfData + * which depends on dynamic memory allocation. In this implementation, + * value profile data is written out to \c BufferIO piecemeal. + */ +static int writeOneValueProfData(ProfBufferIO *BufferIO, + VPDataReaderType *VPDataReader, + const __llvm_profile_data *Data) { + unsigned I, NumValueKinds = 0; + ValueProfData VPHeader; + uint8_t *SiteCountArray[IPVK_Last + 1]; + + for (I = 0; I <= IPVK_Last; I++) { + if (!Data->NumValueSites[I]) + SiteCountArray[I] = 0; + else { + uint32_t Sz = + VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) - + offsetof(ValueProfRecord, SiteCountArray); + /* Only use alloca for this small byte array to avoid excessive + * stack growth. */ + SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz); + memset(SiteCountArray[I], 0, Sz); + } + } + + /* If NumValueKinds returned is 0, there is nothing to write, report + success and return. This should match the raw profile reader's behavior. */ + if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray))) + return 0; + + /* First write the header structure. */ + VPHeader.TotalSize = VPDataReader->GetValueProfDataSize(); + VPHeader.NumValueKinds = NumValueKinds; + if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPHeader, + sizeof(ValueProfData))) + return -1; + + /* Make sure nothing else needs to be written before value profile + * records. */ + if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) != + (void *)(&VPHeader + 1)) + return -1; + + /* Write out the value profile record for each value kind + * one by one. */ + for (I = 0; I <= IPVK_Last; I++) { + uint32_t J; + ValueProfRecord RecordHeader; + /* The size of the value prof record header without counting the + * site count array .*/ + uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray); + uint32_t SiteCountArraySize; + + if (!Data->NumValueSites[I]) + continue; + + /* Write out the record header. */ + RecordHeader.Kind = I; + RecordHeader.NumValueSites = Data->NumValueSites[I]; + if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&RecordHeader, + RecordHeaderSize)) + return -1; + + /* Write out the site value count array including padding space. */ + SiteCountArraySize = + VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) - + RecordHeaderSize; + if (lprofBufferIOWrite(BufferIO, SiteCountArray[I], SiteCountArraySize)) + return -1; + + /* Write out the value profile data for each value site. */ + for (J = 0; J < Data->NumValueSites[I]; J++) { + uint32_t NRead, NRemain; + ValueProfNode *NextStartNode = 0; + NRemain = VPDataReader->GetNumValueDataForSite(I, J); + if (!NRemain) + continue; + /* Read and write out value data in small chunks till it is done. */ + do { + NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain); + NextStartNode = + VPDataReader->GetValueData(I, /* ValueKind */ + J, /* Site */ + &VPDataArray[0], NextStartNode, NRead); + if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPDataArray[0], + NRead * sizeof(InstrProfValueData))) + return -1; + NRemain -= NRead; + } while (NRemain != 0); + } + } + /* All done report success. */ + return 0; +} + +static int writeValueProfData(ProfDataWriter *Writer, + VPDataReaderType *VPDataReader, + const __llvm_profile_data *DataBegin, + const __llvm_profile_data *DataEnd) { + ProfBufferIO *BufferIO; + const __llvm_profile_data *DI = 0; + + if (!VPDataReader) + return 0; + + BufferIO = lprofCreateBufferIO(Writer); + + for (DI = DataBegin; DI < DataEnd; DI++) { + if (writeOneValueProfData(BufferIO, VPDataReader, DI)) + return -1; + } + + if (lprofBufferIOFlush(BufferIO) != 0) + return -1; + lprofDeleteBufferIO(BufferIO); + + return 0; +} + +COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer, + VPDataReaderType *VPDataReader, + int SkipNameDataWrite) { + /* Match logic in __llvm_profile_write_buffer(). */ + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const char *CountersBegin = __llvm_profile_begin_counters(); + const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); + const char *NamesBegin = __llvm_profile_begin_names(); + const char *NamesEnd = __llvm_profile_end_names(); + const VTableProfData *VTableBegin = __llvm_profile_begin_vtables(); + const VTableProfData *VTableEnd = __llvm_profile_end_vtables(); + const char *VNamesBegin = __llvm_profile_begin_vtabnames(); + const char *VNamesEnd = __llvm_profile_end_vtabnames(); + return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin, + CountersEnd, BitmapBegin, BitmapEnd, VPDataReader, + NamesBegin, NamesEnd, VTableBegin, VTableEnd, + VNamesBegin, VNamesEnd, SkipNameDataWrite); +} + +COMPILER_RT_VISIBILITY int +lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, + const __llvm_profile_data *DataEnd, + const char *CountersBegin, const char *CountersEnd, + const char *BitmapBegin, const char *BitmapEnd, + VPDataReaderType *VPDataReader, const char *NamesBegin, + const char *NamesEnd, const VTableProfData *VTableBegin, + const VTableProfData *VTableEnd, const char *VNamesBegin, + const char *VNamesEnd, int SkipNameDataWrite) { + /* Calculate size of sections. */ + const uint64_t DataSectionSize = + __llvm_profile_get_data_size(DataBegin, DataEnd); + const uint64_t NumData = __llvm_profile_get_num_data(DataBegin, DataEnd); + const uint64_t CountersSectionSize = + __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + const uint64_t NumCounters = + __llvm_profile_get_num_counters(CountersBegin, CountersEnd); + const uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); + const uint64_t NamesSize = __llvm_profile_get_name_size(NamesBegin, NamesEnd); + const uint64_t NumVTables = + __llvm_profile_get_num_vtable(VTableBegin, VTableEnd); + const uint64_t VTableSectionSize = + __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd); + const uint64_t VNamesSize = + __llvm_profile_get_name_size(VNamesBegin, VNamesEnd); + + /* Create the header. */ + __llvm_profile_header Header; + + /* Determine how much padding is needed before/after the counters and after + * the names. */ + uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, + PaddingBytesAfterBitmapBytes, PaddingBytesAfterNames, + PaddingBytesAfterVTable, PaddingBytesAfterVNames; + if (__llvm_profile_get_padding_sizes_for_counters( + DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize, + VTableSectionSize, VNamesSize, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes, + &PaddingBytesAfterNames, &PaddingBytesAfterVTable, + &PaddingBytesAfterVNames) == -1) + return -1; + + { +/* Initialize header structure. */ +#define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init; +#include "profile/InstrProfData.inc" + } + + /* On WIN64, label differences are truncated 32-bit values. Truncate + * CountersDelta to match. */ +#ifdef _WIN64 + Header.CountersDelta = (uint32_t)Header.CountersDelta; + Header.BitmapDelta = (uint32_t)Header.BitmapDelta; +#endif + + /* The data and names sections are omitted in lightweight mode. */ + if (NumData == 0 && NamesSize == 0) { + Header.CountersDelta = 0; + Header.NamesDelta = 0; + } + + /* Write the profile header. */ + ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}}; + if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec))) + return -1; + + /* Write the binary id lengths and data. */ + if (__llvm_write_binary_ids(Writer) == -1) + return -1; + + /* Write the profile data. */ + ProfDataIOVec IOVecData[] = { + {DataBegin, sizeof(uint8_t), DataSectionSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, + {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, + {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1}, + {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}, + {VTableBegin, sizeof(uint8_t), VTableSectionSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterVTable, 1}, + {SkipNameDataWrite ? NULL : VNamesBegin, sizeof(uint8_t), VNamesSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterVNames, 1}}; + if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData))) + return -1; + + /* Value profiling is not yet supported in continuous mode and profile + * correlation mode. */ + if (__llvm_profile_is_continuous_mode_enabled() || + (NumData == 0 && NamesSize == 0)) + return 0; + + return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd); +} + +/* + * Write binary id length and then its data, because binary id does not + * have a fixed length. + */ +COMPILER_RT_VISIBILITY +int lprofWriteOneBinaryId(ProfDataWriter *Writer, uint64_t BinaryIdLen, + const uint8_t *BinaryIdData, + uint64_t BinaryIdPadding) { + ProfDataIOVec BinaryIdIOVec[] = { + {&BinaryIdLen, sizeof(uint64_t), 1, 0}, + {BinaryIdData, sizeof(uint8_t), BinaryIdLen, 0}, + {NULL, sizeof(uint8_t), BinaryIdPadding, 1}, + }; + if (Writer->Write(Writer, BinaryIdIOVec, + sizeof(BinaryIdIOVec) / sizeof(*BinaryIdIOVec))) + return -1; + + /* Successfully wrote binary id, report success. */ + return 0; +} diff --git a/contrib/llvm-project/compiler-rt/lib/profile/WindowsMMap.c b/contrib/llvm-project/compiler-rt/lib/profile/WindowsMMap.c new file mode 100644 index 000000000000..9d7da835b1ed --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/WindowsMMap.c @@ -0,0 +1,187 @@ +/* + * This code is derived from uClibc (original license follows). + * https://git.uclibc.org/uClibc/tree/utils/mmap-windows.c + */ + /* mmap() replacement for Windows + * + * Author: Mike Frysinger <vapier@gentoo.org> + * Placed into the public domain + */ + +/* References: + * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx + * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx + * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx + * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx + */ + +#if defined(_WIN32) + +#include "WindowsMMap.h" + +#define WIN32_LEAN_AND_MEAN +#include <windows.h> + +#include "InstrProfiling.h" + +COMPILER_RT_VISIBILITY +void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset) +{ + if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC)) + return MAP_FAILED; + if (fd == -1) { + if (!(flags & MAP_ANON) || offset) + return MAP_FAILED; + } else if (flags & MAP_ANON) + return MAP_FAILED; + + DWORD flProtect; + if (prot & PROT_WRITE) { + if (prot & PROT_EXEC) + flProtect = PAGE_EXECUTE_READWRITE; + else + flProtect = PAGE_READWRITE; + } else if (prot & PROT_EXEC) { + if (prot & PROT_READ) + flProtect = PAGE_EXECUTE_READ; + else if (prot & PROT_EXEC) + flProtect = PAGE_EXECUTE; + } else + flProtect = PAGE_READONLY; + + off_t end = length + offset; + HANDLE mmap_fd, h; + if (fd == -1) + mmap_fd = INVALID_HANDLE_VALUE; + else + mmap_fd = (HANDLE)_get_osfhandle(fd); + h = CreateFileMapping(mmap_fd, NULL, flProtect, DWORD_HI(end), DWORD_LO(end), NULL); + if (h == NULL) + return MAP_FAILED; + + DWORD dwDesiredAccess; + if (prot & PROT_WRITE) + dwDesiredAccess = FILE_MAP_WRITE; + else + dwDesiredAccess = FILE_MAP_READ; + if (prot & PROT_EXEC) + dwDesiredAccess |= FILE_MAP_EXECUTE; + if (flags & MAP_PRIVATE) + dwDesiredAccess |= FILE_MAP_COPY; + void *ret = MapViewOfFile(h, dwDesiredAccess, DWORD_HI(offset), DWORD_LO(offset), length); + if (ret == NULL) { + CloseHandle(h); + ret = MAP_FAILED; + } + return ret; +} + +COMPILER_RT_VISIBILITY +void munmap(void *addr, size_t length) +{ + UnmapViewOfFile(addr); + /* ruh-ro, we leaked handle from CreateFileMapping() ... */ +} + +COMPILER_RT_VISIBILITY +int msync(void *addr, size_t length, int flags) +{ + if (flags & MS_INVALIDATE) + return -1; /* Not supported. */ + + /* Exactly one of MS_ASYNC or MS_SYNC must be specified. */ + switch (flags & (MS_ASYNC | MS_SYNC)) { + case MS_SYNC: + case MS_ASYNC: + break; + default: + return -1; + } + + if (!FlushViewOfFile(addr, length)) + return -1; + + if (flags & MS_SYNC) { + /* FIXME: No longer have access to handle from CreateFileMapping(). */ + /* + * if (!FlushFileBuffers(h)) + * return -1; + */ + } + + return 0; +} + +COMPILER_RT_VISIBILITY +int madvise(void *addr, size_t length, int advice) +{ + if (advice != MADV_DONTNEED) + return -1; /* Not supported. */ + + if (!VirtualUnlock(addr, length)) + return -1; + + return 0; +} + +static int lock(HANDLE handle, DWORD lockType, BOOL blocking) { + DWORD flags = lockType; + if (!blocking) + flags |= LOCKFILE_FAIL_IMMEDIATELY; + + OVERLAPPED overlapped; + ZeroMemory(&overlapped, sizeof(OVERLAPPED)); + overlapped.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + BOOL result = LockFileEx(handle, flags, 0, MAXDWORD, MAXDWORD, &overlapped); + if (!result) { + DWORD dw = GetLastError(); + + // In non-blocking mode, return an error if the file is locked. + if (!blocking && dw == ERROR_LOCK_VIOLATION) + return -1; // EWOULDBLOCK + + // If the error is ERROR_IO_PENDING, we need to wait until the operation + // finishes. Otherwise, we return an error. + if (dw != ERROR_IO_PENDING) + return -1; + + DWORD dwNumBytes; + if (!GetOverlappedResult(handle, &overlapped, &dwNumBytes, TRUE)) + return -1; + } + + return 0; +} + +COMPILER_RT_VISIBILITY +int flock(int fd, int operation) { + HANDLE handle = (HANDLE)_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) + return -1; + + BOOL blocking = (operation & LOCK_NB) == 0; + int op = operation & ~LOCK_NB; + + switch (op) { + case LOCK_EX: + return lock(handle, LOCKFILE_EXCLUSIVE_LOCK, blocking); + + case LOCK_SH: + return lock(handle, 0, blocking); + + case LOCK_UN: + if (!UnlockFile(handle, 0, 0, MAXDWORD, MAXDWORD)) + return -1; + break; + + default: + return -1; + } + + return 0; +} + +#undef DWORD_HI +#undef DWORD_LO + +#endif /* _WIN32 */ diff --git a/contrib/llvm-project/compiler-rt/lib/profile/WindowsMMap.h b/contrib/llvm-project/compiler-rt/lib/profile/WindowsMMap.h new file mode 100644 index 000000000000..1df1a0be0b02 --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/profile/WindowsMMap.h @@ -0,0 +1,82 @@ +/*===- WindowsMMap.h - Support library for PGO instrumentation ------------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#ifndef PROFILE_INSTRPROFILING_WINDOWS_MMAP_H +#define PROFILE_INSTRPROFILING_WINDOWS_MMAP_H + +#if defined(_WIN32) + +#include <basetsd.h> +#include <io.h> +#include <sys/types.h> + +/* + * mmap() flags + */ +#define PROT_READ 0x1 +#define PROT_WRITE 0x2 +#define PROT_EXEC 0x0 + +#define MAP_FILE 0x00 +#define MAP_SHARED 0x01 +#define MAP_PRIVATE 0x02 +#define MAP_ANONYMOUS 0x20 +#define MAP_ANON MAP_ANONYMOUS +#define MAP_FAILED ((void *) -1) + +/* + * msync() flags + */ +#define MS_ASYNC 0x0001 /* return immediately */ +#define MS_INVALIDATE 0x0002 /* invalidate all cached data */ +#define MS_SYNC 0x0010 /* msync synchronously */ + +/* + * madvise() flags + */ + +#define MADV_NORMAL 0 /* no special treatment */ +#define MADV_WILLNEED 3 /* expect access in the near future */ +#define MADV_DONTNEED 4 /* do not expect access in the near future */ + +/* + * flock() operations + */ +#define LOCK_SH 1 /* shared lock */ +#define LOCK_EX 2 /* exclusive lock */ +#define LOCK_NB 4 /* don't block when locking */ +#define LOCK_UN 8 /* unlock */ + +#ifdef __USE_FILE_OFFSET64 +# define DWORD_HI(x) (x >> 32) +# define DWORD_LO(x) ((x) & 0xffffffff) +#else +# define DWORD_HI(x) (0) +# define DWORD_LO(x) (x) +#endif + +#define mmap __llvm_profile_mmap +#define munmap __llvm_profile_munmap +#define msync __llvm_profile_msync +#define madvise __llvm_profile_madvise +#define flock __llvm_profile_flock + +void *mmap(void *start, size_t length, int prot, int flags, int fd, + off_t offset); + +void munmap(void *addr, size_t length); + +int msync(void *addr, size_t length, int flags); + +int madvise(void *addr, size_t length, int advice); + +int flock(int fd, int operation); + +#endif /* _WIN32 */ + +#endif /* PROFILE_INSTRPROFILING_WINDOWS_MMAP_H */ |