aboutsummaryrefslogtreecommitdiff
path: root/contrib
diff options
context:
space:
mode:
authorDimitry Andric <dim@FreeBSD.org>2017-12-20 19:12:15 +0000
committerDimitry Andric <dim@FreeBSD.org>2017-12-20 19:12:15 +0000
commit8a51db9c42ac3e4a0d4c9bc2c4d61ccf0bd5f41a (patch)
treeee959e7d66bf923526989c0e1a43cdb34958ec67 /contrib
parentacac075be8bce18f411c310a849e242ec445dfc3 (diff)
parentcdf4f3055e964bb585f294cf77cb549ead82783f (diff)
Notes
Diffstat (limited to 'contrib')
-rw-r--r--contrib/compiler-rt/include/sanitizer/allocator_interface.h7
-rw-r--r--contrib/compiler-rt/include/sanitizer/asan_interface.h4
-rw-r--r--contrib/compiler-rt/include/sanitizer/coverage_interface.h7
-rw-r--r--contrib/compiler-rt/include/sanitizer/hwasan_interface.h33
-rw-r--r--contrib/compiler-rt/include/sanitizer/lsan_interface.h6
-rw-r--r--contrib/compiler-rt/include/sanitizer/scudo_interface.h34
-rw-r--r--contrib/compiler-rt/include/sanitizer/tsan_interface.h6
-rw-r--r--contrib/compiler-rt/include/xray/xray_interface.h8
-rw-r--r--contrib/compiler-rt/include/xray/xray_log_interface.h50
-rw-r--r--contrib/compiler-rt/include/xray/xray_records.h37
-rw-r--r--contrib/compiler-rt/lib/BlocksRuntime/Block.h2
-rw-r--r--contrib/compiler-rt/lib/BlocksRuntime/Block_private.h2
-rw-r--r--contrib/compiler-rt/lib/asan/asan_activation.cc7
-rw-r--r--contrib/compiler-rt/lib/asan/asan_allocator.cc84
-rw-r--r--contrib/compiler-rt/lib/asan/asan_allocator.h11
-rw-r--r--contrib/compiler-rt/lib/asan/asan_descriptions.cc39
-rw-r--r--contrib/compiler-rt/lib/asan/asan_descriptions.h11
-rw-r--r--contrib/compiler-rt/lib/asan/asan_errors.cc150
-rw-r--r--contrib/compiler-rt/lib/asan/asan_errors.h82
-rw-r--r--contrib/compiler-rt/lib/asan/asan_fake_stack.cc8
-rw-r--r--contrib/compiler-rt/lib/asan/asan_flags.cc7
-rw-r--r--contrib/compiler-rt/lib/asan/asan_flags.inc9
-rw-r--r--contrib/compiler-rt/lib/asan/asan_fuchsia.cc218
-rw-r--r--contrib/compiler-rt/lib/asan/asan_globals.cc7
-rw-r--r--contrib/compiler-rt/lib/asan/asan_init_version.h7
-rw-r--r--contrib/compiler-rt/lib/asan/asan_interceptors.cc191
-rw-r--r--contrib/compiler-rt/lib/asan/asan_interceptors.h54
-rw-r--r--contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cc44
-rw-r--r--contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h148
-rw-r--r--contrib/compiler-rt/lib/asan/asan_internal.h7
-rw-r--r--contrib/compiler-rt/lib/asan/asan_linux.cc81
-rw-r--r--contrib/compiler-rt/lib/asan/asan_malloc_linux.cc52
-rw-r--r--contrib/compiler-rt/lib/asan/asan_mapping.h32
-rw-r--r--contrib/compiler-rt/lib/asan/asan_memory_profile.cc3
-rw-r--r--contrib/compiler-rt/lib/asan/asan_new_delete.cc102
-rw-r--r--contrib/compiler-rt/lib/asan/asan_poisoning.cc2
-rw-r--r--contrib/compiler-rt/lib/asan/asan_poisoning.h7
-rw-r--r--contrib/compiler-rt/lib/asan/asan_posix.cc56
-rw-r--r--contrib/compiler-rt/lib/asan/asan_premap_shadow.cc79
-rw-r--r--contrib/compiler-rt/lib/asan/asan_premap_shadow.h30
-rw-r--r--contrib/compiler-rt/lib/asan/asan_report.cc151
-rw-r--r--contrib/compiler-rt/lib/asan/asan_report.h6
-rw-r--r--contrib/compiler-rt/lib/asan/asan_rtl.cc147
-rw-r--r--contrib/compiler-rt/lib/asan/asan_scariness_score.h2
-rw-r--r--contrib/compiler-rt/lib/asan/asan_shadow_setup.cc165
-rw-r--r--contrib/compiler-rt/lib/asan/asan_stack.h52
-rw-r--r--contrib/compiler-rt/lib/asan/asan_thread.cc58
-rw-r--r--contrib/compiler-rt/lib/asan/asan_thread.h35
-rw-r--r--contrib/compiler-rt/lib/asan/asan_win.cc4
-rw-r--r--contrib/compiler-rt/lib/builtins/adddf3.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/addsf3.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S9
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S9
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/aeabi_idivmod.S9
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/aeabi_memcmp.S6
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/aeabi_memcpy.S6
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/aeabi_memmove.S6
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/aeabi_memset.S13
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S12
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/bswapdi2.S8
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/bswapsi2.S8
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/clzdi2.S9
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/clzsi2.S8
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/comparesf2.S20
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/divmodsi4.S8
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/divsi3.S12
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/modsi3.S8
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/udivmodsi4.S8
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/udivsi3.S9
-rw-r--r--contrib/compiler-rt/lib/builtins/arm/umodsi3.S8
-rw-r--r--contrib/compiler-rt/lib/builtins/ashldi3.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/ashrdi3.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/assembly.h71
-rw-r--r--contrib/compiler-rt/lib/builtins/clear_cache.c31
-rw-r--r--contrib/compiler-rt/lib/builtins/comparedf2.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/comparesf2.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/cpu_model.c12
-rw-r--r--contrib/compiler-rt/lib/builtins/divdf3.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/divsf3.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/divsi3.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/emutls.c14
-rw-r--r--contrib/compiler-rt/lib/builtins/enable_execute_stack.c2
-rw-r--r--contrib/compiler-rt/lib/builtins/extendhfsf2.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/extendsfdf2.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/fixdfdi.c12
-rw-r--r--contrib/compiler-rt/lib/builtins/fixdfsi.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/fixsfdi.c12
-rw-r--r--contrib/compiler-rt/lib/builtins/fixsfsi.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/fixunsdfdi.c12
-rw-r--r--contrib/compiler-rt/lib/builtins/fixunsdfsi.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/fixunssfdi.c12
-rw-r--r--contrib/compiler-rt/lib/builtins/fixunssfsi.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/floatdidf.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/floatdisf.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/floatsidf.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/floatsisf.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/floatundidf.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/floatundisf.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/floatunsidf.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/floatunsisf.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/int_endianness.h4
-rw-r--r--contrib/compiler-rt/lib/builtins/int_lib.h4
-rw-r--r--contrib/compiler-rt/lib/builtins/lshrdi3.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/muldf3.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/muldi3.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/mulsf3.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/negdf2.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/negsf2.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/subdf3.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/subsf3.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/truncdfhf2.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/truncdfsf2.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/truncsfhf2.c5
-rw-r--r--contrib/compiler-rt/lib/builtins/udivsi3.c5
-rw-r--r--contrib/compiler-rt/lib/cfi/cfi.cc2
-rw-r--r--contrib/compiler-rt/lib/cfi/cfi_blacklist.txt16
-rw-r--r--contrib/compiler-rt/lib/dfsan/dfsan.cc1
-rw-r--r--contrib/compiler-rt/lib/esan/esan_sideline.h4
-rw-r--r--contrib/compiler-rt/lib/esan/esan_sideline_linux.cpp3
-rw-r--r--contrib/compiler-rt/lib/esan/working_set_posix.cpp7
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan.cc303
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan.h172
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan.syms.extra2
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc330
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan_allocator.h55
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan_blacklist.txt7
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan_flags.h30
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan_flags.inc29
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc483
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h97
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan_linux.cc251
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc66
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc36
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h25
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan_report.cc133
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan_thread.cc75
-rw-r--r--contrib/compiler-rt/lib/hwasan/hwasan_thread.h81
-rw-r--r--contrib/compiler-rt/lib/interception/interception.h62
-rw-r--r--contrib/compiler-rt/lib/interception/interception_linux.cc29
-rw-r--r--contrib/compiler-rt/lib/interception/interception_linux.h11
-rw-r--r--contrib/compiler-rt/lib/interception/interception_mac.cc5
-rw-r--r--contrib/compiler-rt/lib/interception/interception_mac.h4
-rw-r--r--contrib/compiler-rt/lib/interception/interception_type_test.cc9
-rw-r--r--contrib/compiler-rt/lib/interception/interception_win.cc19
-rw-r--r--contrib/compiler-rt/lib/interception/interception_win.h4
-rw-r--r--contrib/compiler-rt/lib/lsan/lsan.cc15
-rw-r--r--contrib/compiler-rt/lib/lsan/lsan.h40
-rw-r--r--contrib/compiler-rt/lib/lsan/lsan_allocator.h11
-rw-r--r--contrib/compiler-rt/lib/lsan/lsan_common.cc26
-rw-r--r--contrib/compiler-rt/lib/lsan/lsan_common.h11
-rw-r--r--contrib/compiler-rt/lib/lsan/lsan_common_linux.cc20
-rw-r--r--contrib/compiler-rt/lib/lsan/lsan_common_mac.cc21
-rw-r--r--contrib/compiler-rt/lib/lsan/lsan_interceptors.cc74
-rw-r--r--contrib/compiler-rt/lib/lsan/weak_symbols.txt1
-rw-r--r--contrib/compiler-rt/lib/msan/msan.cc17
-rw-r--r--contrib/compiler-rt/lib/msan/msan.h64
-rw-r--r--contrib/compiler-rt/lib/msan/msan_allocator.cc9
-rw-r--r--contrib/compiler-rt/lib/msan/msan_interceptors.cc336
-rw-r--r--contrib/compiler-rt/lib/msan/msan_linux.cc8
-rw-r--r--contrib/compiler-rt/lib/msan/msan_report.cc31
-rw-r--r--contrib/compiler-rt/lib/profile/GCDAProfiling.c19
-rw-r--r--contrib/compiler-rt/lib/profile/InstrProfData.inc4
-rw-r--r--contrib/compiler-rt/lib/profile/InstrProfiling.c6
-rw-r--r--contrib/compiler-rt/lib/profile/InstrProfilingFile.c16
-rw-r--r--contrib/compiler-rt/lib/profile/InstrProfilingInternal.h3
-rw-r--r--contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c6
-rw-r--r--contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c7
-rw-r--r--contrib/compiler-rt/lib/profile/InstrProfilingPort.h14
-rw-r--r--contrib/compiler-rt/lib/profile/InstrProfilingUtil.c68
-rw-r--r--contrib/compiler-rt/lib/profile/InstrProfilingUtil.h3
-rw-r--r--contrib/compiler-rt/lib/profile/InstrProfilingValue.c10
-rw-r--r--contrib/compiler-rt/lib/profile/InstrProfilingWriter.c5
-rw-r--r--contrib/compiler-rt/lib/profile/WindowsMMap.c61
-rw-r--r--contrib/compiler-rt/lib/safestack/safestack.cc2
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc9
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h13
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.cc23
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h15
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h4
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h3
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_local_cache.h96
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h73
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h527
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h30
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h8
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h4
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_bvgraph.h2
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc147
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h219
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc198
-rwxr-xr-xcontrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc2
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc1
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc180
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc240
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc7
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc32
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h4
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.cc177
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h110
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h2
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc18
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc539
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.h31
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h47
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h13
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h33
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc5
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc354
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h24
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc295
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc38
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h11
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc2
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h4
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h33
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h372
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc363
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h582
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc8
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h45
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc366
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h485
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc25
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h2
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc95
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_printf.cc35
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h74
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc76
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc42
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc68
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc163
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc57
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h33
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_report_decorator.h5
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc87
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc219
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h9
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc88
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc17
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc10
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc3
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cc4
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h4
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.cc107
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h1
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc3
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc72
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc14
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc35
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc3
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc141
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc3
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc69
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h5
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cc18
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc11
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_vector.h (renamed from contrib/compiler-rt/lib/tsan/rtl/tsan_vector.h)32
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc180
-rw-r--r--contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cc2
-rwxr-xr-xcontrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh4
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_allocator.cpp620
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_allocator.h51
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_allocator_combined.h14
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_allocator_secondary.h76
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_crc32.h2
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_flags.cpp59
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_flags.inc22
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_interface_internal.h22
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_new_delete.cpp2
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_platform.h80
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_tls.h47
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_tls_android.cpp95
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_tls_android.inc44
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_tls_context_android.inc54
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_tls_context_linux.inc29
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_tls_linux.inc48
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_tsd.h72
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp (renamed from contrib/compiler-rt/lib/scudo/scudo_tls_linux.cpp)30
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc46
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_tsd_shared.cpp87
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_tsd_shared.inc48
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_utils.cpp136
-rw-r--r--contrib/compiler-rt/lib/scudo/scudo_utils.h64
-rw-r--r--contrib/compiler-rt/lib/stats/stats.cc1
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_fd.cc6
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc558
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h12
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc20
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc6
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc15
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc4
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc91
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h15
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h144
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc9
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc6
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc44
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_report.h2
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc50
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h30
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S50
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S104
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc4
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc71
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc2
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_sync.h4
-rw-r--r--contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h2
-rw-r--r--contrib/compiler-rt/lib/ubsan/ubsan_checks.inc1
-rw-r--r--contrib/compiler-rt/lib/ubsan/ubsan_diag.cc37
-rw-r--r--contrib/compiler-rt/lib/ubsan/ubsan_diag.h11
-rw-r--r--contrib/compiler-rt/lib/ubsan/ubsan_diag_standalone.cc5
-rw-r--r--contrib/compiler-rt/lib/ubsan/ubsan_flags.cc6
-rw-r--r--contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc52
-rw-r--r--contrib/compiler-rt/lib/ubsan/ubsan_handlers.h22
-rw-r--r--contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc4
-rw-r--r--contrib/compiler-rt/lib/ubsan/ubsan_init.cc38
-rw-r--r--contrib/compiler-rt/lib/ubsan/ubsan_init_standalone.cc13
-rw-r--r--contrib/compiler-rt/lib/ubsan/ubsan_interface.inc5
-rw-r--r--contrib/compiler-rt/lib/ubsan/ubsan_platform.h11
-rw-r--r--contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cc53
-rw-r--r--contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.h25
-rw-r--r--contrib/compiler-rt/lib/ubsan_minimal/ubsan.syms.extra1
-rw-r--r--contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc104
-rw-r--r--contrib/compiler-rt/lib/xray/weak_symbols.txt4
-rw-r--r--contrib/compiler-rt/lib/xray/xray_buffer_queue.cc79
-rw-r--r--contrib/compiler-rt/lib/xray/xray_buffer_queue.h97
-rw-r--r--contrib/compiler-rt/lib/xray/xray_fdr_log_records.h3
-rw-r--r--contrib/compiler-rt/lib/xray/xray_fdr_logging.cc206
-rw-r--r--contrib/compiler-rt/lib/xray/xray_fdr_logging.h1
-rw-r--r--contrib/compiler-rt/lib/xray/xray_fdr_logging_impl.h603
-rw-r--r--contrib/compiler-rt/lib/xray/xray_flags.h1
-rw-r--r--contrib/compiler-rt/lib/xray/xray_flags.inc28
-rw-r--r--contrib/compiler-rt/lib/xray/xray_init.cc35
-rw-r--r--contrib/compiler-rt/lib/xray/xray_inmemory_log.cc449
-rw-r--r--contrib/compiler-rt/lib/xray/xray_inmemory_log.h44
-rw-r--r--contrib/compiler-rt/lib/xray/xray_interface.cc312
-rw-r--r--contrib/compiler-rt/lib/xray/xray_interface_internal.h6
-rw-r--r--contrib/compiler-rt/lib/xray/xray_log_interface.cc63
-rw-r--r--contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S147
-rw-r--r--contrib/compiler-rt/lib/xray/xray_utils.cc3
-rw-r--r--contrib/compiler-rt/lib/xray/xray_x86_64.cc40
341 files changed, 14606 insertions, 4651 deletions
diff --git a/contrib/compiler-rt/include/sanitizer/allocator_interface.h b/contrib/compiler-rt/include/sanitizer/allocator_interface.h
index 5220631619fc..2d35804b3764 100644
--- a/contrib/compiler-rt/include/sanitizer/allocator_interface.h
+++ b/contrib/compiler-rt/include/sanitizer/allocator_interface.h
@@ -76,6 +76,13 @@ extern "C" {
void (*malloc_hook)(const volatile void *, size_t),
void (*free_hook)(const volatile void *));
+ /* Drains allocator quarantines (calling thread's and global ones), returns
+ freed memory back to OS and releases other non-essential internal allocator
+ resources in attempt to reduce process RSS.
+ Currently available with ASan only.
+ */
+ void __sanitizer_purge_allocator();
+
#ifdef __cplusplus
} // extern "C"
#endif
diff --git a/contrib/compiler-rt/include/sanitizer/asan_interface.h b/contrib/compiler-rt/include/sanitizer/asan_interface.h
index 97ba0ceb0b23..e689a730e2c3 100644
--- a/contrib/compiler-rt/include/sanitizer/asan_interface.h
+++ b/contrib/compiler-rt/include/sanitizer/asan_interface.h
@@ -144,6 +144,10 @@ extern "C" {
void *__asan_addr_is_in_fake_stack(void *fake_stack, void *addr, void **beg,
void **end);
+ // Performs cleanup before a [[noreturn]] function. Must be called
+ // before things like _exit and execl to avoid false positives on stack.
+ void __asan_handle_no_return(void);
+
#ifdef __cplusplus
} // extern "C"
#endif
diff --git a/contrib/compiler-rt/include/sanitizer/coverage_interface.h b/contrib/compiler-rt/include/sanitizer/coverage_interface.h
index 637379d47c41..081033cec8e5 100644
--- a/contrib/compiler-rt/include/sanitizer/coverage_interface.h
+++ b/contrib/compiler-rt/include/sanitizer/coverage_interface.h
@@ -22,8 +22,11 @@ extern "C" {
// Record and dump coverage info.
void __sanitizer_cov_dump();
- // Dump collected coverage info. Sorts pcs by module into individual
- // .sancov files.
+ // Clear collected coverage info.
+ void __sanitizer_cov_reset();
+
+ // Dump collected coverage info. Sorts pcs by module into individual .sancov
+ // files.
void __sanitizer_dump_coverage(const uintptr_t *pcs, uintptr_t len);
#ifdef __cplusplus
diff --git a/contrib/compiler-rt/include/sanitizer/hwasan_interface.h b/contrib/compiler-rt/include/sanitizer/hwasan_interface.h
new file mode 100644
index 000000000000..f70d87875968
--- /dev/null
+++ b/contrib/compiler-rt/include/sanitizer/hwasan_interface.h
@@ -0,0 +1,33 @@
+//===-- sanitizer/asan_interface.h ------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Public interface header.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_HWASAN_INTERFACE_H
+#define SANITIZER_HWASAN_INTERFACE_H
+
+#include <sanitizer/common_interface_defs.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ // This function may be optionally provided by user and should return
+ // a string containing HWASan runtime options. See asan_flags.h for details.
+ const char* __hwasan_default_options();
+
+ void __hwasan_enable_allocator_tagging();
+ void __hwasan_disable_allocator_tagging();
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // SANITIZER_HWASAN_INTERFACE_H
diff --git a/contrib/compiler-rt/include/sanitizer/lsan_interface.h b/contrib/compiler-rt/include/sanitizer/lsan_interface.h
index 8fb8e756da26..e3e509ff7d9f 100644
--- a/contrib/compiler-rt/include/sanitizer/lsan_interface.h
+++ b/contrib/compiler-rt/include/sanitizer/lsan_interface.h
@@ -64,8 +64,14 @@ extern "C" {
// for the program it is linked into (if the return value is non-zero). This
// function must be defined as returning a constant value; any behavior beyond
// that is unsupported.
+ // To avoid dead stripping, you may need to define this function with
+ // __attribute__((used))
int __lsan_is_turned_off();
+ // This function may be optionally provided by user and should return
+ // a string containing LSan runtime options. See lsan_flags.inc for details.
+ const char *__lsan_default_options();
+
// This function may be optionally provided by the user and should return
// a string containing LSan suppressions.
const char *__lsan_default_suppressions();
diff --git a/contrib/compiler-rt/include/sanitizer/scudo_interface.h b/contrib/compiler-rt/include/sanitizer/scudo_interface.h
new file mode 100644
index 000000000000..cab7e0f5f344
--- /dev/null
+++ b/contrib/compiler-rt/include/sanitizer/scudo_interface.h
@@ -0,0 +1,34 @@
+//===-- sanitizer/scudo_interface.h -----------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+/// Public Scudo interface header.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_SCUDO_INTERFACE_H_
+#define SANITIZER_SCUDO_INTERFACE_H_
+
+#include <sanitizer/common_interface_defs.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ // This function may be optionally provided by a user and should return
+ // a string containing Scudo runtime options. See scudo_flags.h for details.
+ const char* __scudo_default_options();
+
+ // This function allows to set the RSS limit at runtime. This can be either
+ // the hard limit (HardLimit=1) or the soft limit (HardLimit=0). The limit
+ // can be removed by setting LimitMb to 0. This function's parameters should
+ // be fully trusted to avoid security mishaps.
+ void __scudo_set_rss_limit(unsigned long LimitMb, int HardLimit);
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // SANITIZER_SCUDO_INTERFACE_H_
diff --git a/contrib/compiler-rt/include/sanitizer/tsan_interface.h b/contrib/compiler-rt/include/sanitizer/tsan_interface.h
index 5ea09ab5c1ff..530885014ac3 100644
--- a/contrib/compiler-rt/include/sanitizer/tsan_interface.h
+++ b/contrib/compiler-rt/include/sanitizer/tsan_interface.h
@@ -44,6 +44,11 @@ const unsigned __tsan_mutex_linker_init = 1 << 0;
const unsigned __tsan_mutex_write_reentrant = 1 << 1;
// Mutex is read reentrant.
const unsigned __tsan_mutex_read_reentrant = 1 << 2;
+// Mutex does not have static storage duration, and must not be used after
+// its destructor runs. The opposite of __tsan_mutex_linker_init.
+// If this flag is passed to __tsan_mutex_destroy, then the destruction
+// is ignored unless this flag was previously set on the mutex.
+const unsigned __tsan_mutex_not_static = 1 << 8;
// Mutex operation flags:
@@ -70,6 +75,7 @@ void __tsan_mutex_create(void *addr, unsigned flags);
// Annotate destruction of a mutex.
// Supported flags:
// - __tsan_mutex_linker_init
+// - __tsan_mutex_not_static
void __tsan_mutex_destroy(void *addr, unsigned flags);
// Annotate start of lock operation.
diff --git a/contrib/compiler-rt/include/xray/xray_interface.h b/contrib/compiler-rt/include/xray/xray_interface.h
index 564613417069..d08039a67fa2 100644
--- a/contrib/compiler-rt/include/xray/xray_interface.h
+++ b/contrib/compiler-rt/include/xray/xray_interface.h
@@ -106,6 +106,14 @@ extern uintptr_t __xray_function_address(int32_t FuncId);
/// encounter errors (when there are no instrumented functions, etc.).
extern size_t __xray_max_function_id();
+/// Initialize the required XRay data structures. This is useful in cases where
+/// users want to control precisely when the XRay instrumentation data
+/// structures are initialized, for example when the XRay library is built with
+/// the XRAY_NO_PREINIT preprocessor definition.
+///
+/// Calling __xray_init() more than once is safe across multiple threads.
+extern void __xray_init();
+
} // end extern "C"
#endif // XRAY_XRAY_INTERFACE_H
diff --git a/contrib/compiler-rt/include/xray/xray_log_interface.h b/contrib/compiler-rt/include/xray/xray_log_interface.h
index cdb20094dc26..6c53cdaebfe2 100644
--- a/contrib/compiler-rt/include/xray/xray_log_interface.h
+++ b/contrib/compiler-rt/include/xray/xray_log_interface.h
@@ -128,6 +128,16 @@ enum XRayLogFlushStatus {
XRAY_LOG_FLUSHED = 2,
};
+/// This enum indicates the installation state of a logging implementation, when
+/// associating a mode to a particular logging implementation through
+/// `__xray_log_register_impl(...)` or through `__xray_log_select_mode(...`.
+enum XRayLogRegisterStatus {
+ XRAY_REGISTRATION_OK = 0,
+ XRAY_DUPLICATE_MODE = 1,
+ XRAY_MODE_NOT_FOUND = 2,
+ XRAY_INCOMPLETE_IMPL = 3,
+};
+
/// A valid XRay logging implementation MUST provide all of the function
/// pointers in XRayLogImpl when being installed through `__xray_set_log_impl`.
/// To be precise, ALL the functions pointers MUST NOT be nullptr.
@@ -159,6 +169,9 @@ struct XRayLogImpl {
/// always have a handler for function entry and exit events. In case the
/// implementation wants to support arg1 (or other future extensions to XRay
/// logging) those MUST be installed by the installed 'log_init' handler.
+ ///
+ /// Because we didn't want to change the ABI of this struct, the arg1 handler
+ /// may be silently overwritten during initialization as well.
void (*handle_arg0)(int32_t, XRayEntryType);
/// The log implementation provided routine for when __xray_log_flushLog() is
@@ -186,6 +199,34 @@ struct XRayLogImpl {
/// called while in any other states.
void __xray_set_log_impl(XRayLogImpl Impl);
+/// This function registers a logging implementation against a "mode"
+/// identifier. This allows multiple modes to be registered, and chosen at
+/// runtime using the same mode identifier through
+/// `__xray_log_select_mode(...)`.
+///
+/// We treat the Mode identifier as a null-terminated byte string, as the
+/// identifier used when retrieving the log impl.
+///
+/// Returns:
+/// - XRAY_REGISTRATION_OK on success.
+/// - XRAY_DUPLICATE_MODE when an implementation is already associated with
+/// the provided Mode; does not update the already-registered
+/// implementation.
+XRayLogRegisterStatus __xray_log_register_mode(const char *Mode,
+ XRayLogImpl Impl);
+
+/// This function selects the implementation associated with Mode that has been
+/// registered through __xray_log_register_mode(...) and installs that
+/// implementation (as if through calling __xray_set_log_impl(...)). The same
+/// caveats apply to __xray_log_select_mode(...) as with
+/// __xray_log_set_log_impl(...).
+///
+/// Returns:
+/// - XRAY_REGISTRATION_OK on success.
+/// - XRAY_MODE_NOT_FOUND if there is no implementation associated with Mode;
+/// does not update the currently installed implementation.
+XRayLogRegisterStatus __xray_log_select_mode(const char *Mode);
+
/// This function removes the currently installed implementation. It will also
/// uninstall any handlers that have been previously installed. It does NOT
/// unpatch the instrumentation sleds.
@@ -220,12 +261,19 @@ XRayLogFlushStatus __xray_log_flushLog();
namespace __xray {
-// Options used by the LLVM XRay FDR implementation.
+/// Options used by the LLVM XRay FDR logging implementation.
struct FDRLoggingOptions {
bool ReportErrors = false;
int Fd = -1;
};
+/// Options used by the LLVM XRay Basic (Naive) logging implementation.
+struct BasicLoggingOptions {
+ int DurationFilterMicros = 0;
+ size_t MaxStackDepth = 0;
+ size_t ThreadBufferSize = 0;
+};
+
} // namespace __xray
#endif // XRAY_XRAY_LOG_INTERFACE_H
diff --git a/contrib/compiler-rt/include/xray/xray_records.h b/contrib/compiler-rt/include/xray/xray_records.h
index feb8d228b2fd..d4b7b4c31a9a 100644
--- a/contrib/compiler-rt/include/xray/xray_records.h
+++ b/contrib/compiler-rt/include/xray/xray_records.h
@@ -17,6 +17,8 @@
#ifndef XRAY_XRAY_RECORDS_H
#define XRAY_XRAY_RECORDS_H
+#include <cstdint>
+
namespace __xray {
enum FileTypes {
@@ -65,18 +67,23 @@ static_assert(sizeof(XRayFileHeader) == 32, "XRayFileHeader != 32 bytes");
enum RecordTypes {
NORMAL = 0,
+ ARG_PAYLOAD = 1,
};
struct alignas(32) XRayRecord {
// This is the type of the record being written. We use 16 bits to allow us to
// treat this as a discriminant, and so that the first 4 bytes get packed
// properly. See RecordTypes for more supported types.
- uint16_t RecordType = 0;
+ uint16_t RecordType = RecordTypes::NORMAL;
// The CPU where the thread is running. We assume number of CPUs <= 256.
uint8_t CPU = 0;
- // The type of the event. Usually either ENTER = 0 or EXIT = 1.
+ // The type of the event. One of the following:
+ // ENTER = 0
+ // EXIT = 1
+ // TAIL_EXIT = 2
+ // ENTER_ARG = 3
uint8_t Type = 0;
// The function ID for the record.
@@ -94,6 +101,32 @@ struct alignas(32) XRayRecord {
static_assert(sizeof(XRayRecord) == 32, "XRayRecord != 32 bytes");
+struct alignas(32) XRayArgPayload {
+ // We use the same 16 bits as a discriminant for the records in the log here
+ // too, and so that the first 4 bytes are packed properly.
+ uint16_t RecordType = RecordTypes::ARG_PAYLOAD;
+
+ // Add a few bytes to pad.
+ uint8_t Padding[2] = {};
+
+ // The function ID for the record.
+ int32_t FuncId = 0;
+
+ // The thread ID for the currently running thread.
+ uint32_t TId = 0;
+
+ // Add more padding.
+ uint8_t Padding2[4] = {};
+
+ // The argument payload.
+ uint64_t Arg = 0;
+
+ // The rest of this record ought to be left as padding.
+ uint8_t TailPadding[8] = {};
+} __attribute__((packed));
+
+static_assert(sizeof(XRayArgPayload) == 32, "XRayArgPayload != 32 bytes");
+
} // namespace __xray
#endif // XRAY_XRAY_RECORDS_H
diff --git a/contrib/compiler-rt/lib/BlocksRuntime/Block.h b/contrib/compiler-rt/lib/BlocksRuntime/Block.h
index 55cdd01a9123..e6cb142fd755 100644
--- a/contrib/compiler-rt/lib/BlocksRuntime/Block.h
+++ b/contrib/compiler-rt/lib/BlocksRuntime/Block.h
@@ -27,7 +27,7 @@
#if !defined(BLOCK_EXPORT)
# if defined(__cplusplus)
-# define BLOCK_EXPORT extern "C"
+# define BLOCK_EXPORT extern "C"
# else
# define BLOCK_EXPORT extern
# endif
diff --git a/contrib/compiler-rt/lib/BlocksRuntime/Block_private.h b/contrib/compiler-rt/lib/BlocksRuntime/Block_private.h
index 8ae821815ebe..91d8d8a9850b 100644
--- a/contrib/compiler-rt/lib/BlocksRuntime/Block_private.h
+++ b/contrib/compiler-rt/lib/BlocksRuntime/Block_private.h
@@ -27,7 +27,7 @@
#if !defined(BLOCK_EXPORT)
# if defined(__cplusplus)
-# define BLOCK_EXPORT extern "C"
+# define BLOCK_EXPORT extern "C"
# else
# define BLOCK_EXPORT extern
# endif
diff --git a/contrib/compiler-rt/lib/asan/asan_activation.cc b/contrib/compiler-rt/lib/asan/asan_activation.cc
index 66eba9ce2748..d642be93488d 100644
--- a/contrib/compiler-rt/lib/asan/asan_activation.cc
+++ b/contrib/compiler-rt/lib/asan/asan_activation.cc
@@ -16,8 +16,10 @@
#include "asan_allocator.h"
#include "asan_flags.h"
#include "asan_internal.h"
+#include "asan_mapping.h"
#include "asan_poisoning.h"
#include "asan_stack.h"
+#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_flags.h"
namespace __asan {
@@ -110,8 +112,9 @@ void AsanDeactivate() {
AllocatorOptions disabled = asan_deactivated_flags.allocator_options;
disabled.quarantine_size_mb = 0;
disabled.thread_local_quarantine_size_kb = 0;
- disabled.min_redzone = 16; // Redzone must be at least 16 bytes long.
- disabled.max_redzone = 16;
+ // Redzone must be at least Max(16, granularity) bytes long.
+ disabled.min_redzone = Max(16, (int)SHADOW_GRANULARITY);
+ disabled.max_redzone = disabled.min_redzone;
disabled.alloc_dealloc_mismatch = false;
disabled.may_return_null = true;
ReInitializeAllocator(disabled);
diff --git a/contrib/compiler-rt/lib/asan/asan_allocator.cc b/contrib/compiler-rt/lib/asan/asan_allocator.cc
index 92963ddfc4da..a437ae1cd3b1 100644
--- a/contrib/compiler-rt/lib/asan/asan_allocator.cc
+++ b/contrib/compiler-rt/lib/asan/asan_allocator.cc
@@ -84,7 +84,10 @@ struct ChunkHeader {
// This field is used for small sizes. For large sizes it is equal to
// SizeClassMap::kMaxSize and the actual size is stored in the
// SecondaryAllocator's metadata.
- u32 user_requested_size;
+ u32 user_requested_size : 29;
+ // align < 8 -> 0
+ // else -> log2(min(align, 512)) - 2
+ u32 user_requested_alignment_log : 3;
u32 alloc_context_id;
};
@@ -271,9 +274,9 @@ struct Allocator {
atomic_store(&max_redzone, options.max_redzone, memory_order_release);
}
- void Initialize(const AllocatorOptions &options) {
+ void InitLinkerInitialized(const AllocatorOptions &options) {
SetAllocatorMayReturnNull(options.may_return_null);
- allocator.Init(options.release_to_os_interval_ms);
+ allocator.InitLinkerInitialized(options.release_to_os_interval_ms);
SharedInitCode(options);
}
@@ -351,6 +354,20 @@ struct Allocator {
return Min(Max(rz_log, RZSize2Log(min_rz)), RZSize2Log(max_rz));
}
+ static uptr ComputeUserRequestedAlignmentLog(uptr user_requested_alignment) {
+ if (user_requested_alignment < 8)
+ return 0;
+ if (user_requested_alignment > 512)
+ user_requested_alignment = 512;
+ return Log2(user_requested_alignment) - 2;
+ }
+
+ static uptr ComputeUserAlignment(uptr user_requested_alignment_log) {
+ if (user_requested_alignment_log == 0)
+ return 0;
+ return 1LL << (user_requested_alignment_log + 2);
+ }
+
// We have an address between two chunks, and we want to report just one.
AsanChunk *ChooseChunk(uptr addr, AsanChunk *left_chunk,
AsanChunk *right_chunk) {
@@ -385,6 +402,8 @@ struct Allocator {
Flags &fl = *flags();
CHECK(stack);
const uptr min_alignment = SHADOW_GRANULARITY;
+ const uptr user_requested_alignment_log =
+ ComputeUserRequestedAlignmentLog(alignment);
if (alignment < min_alignment)
alignment = min_alignment;
if (size == 0) {
@@ -472,6 +491,7 @@ struct Allocator {
meta[0] = size;
meta[1] = chunk_beg;
}
+ m->user_requested_alignment_log = user_requested_alignment_log;
m->alloc_context_id = StackDepotPut(*stack);
@@ -573,8 +593,8 @@ struct Allocator {
}
}
- void Deallocate(void *ptr, uptr delete_size, BufferedStackTrace *stack,
- AllocType alloc_type) {
+ void Deallocate(void *ptr, uptr delete_size, uptr delete_alignment,
+ BufferedStackTrace *stack, AllocType alloc_type) {
uptr p = reinterpret_cast<uptr>(ptr);
if (p == 0) return;
@@ -601,11 +621,14 @@ struct Allocator {
ReportAllocTypeMismatch((uptr)ptr, stack, (AllocType)m->alloc_type,
(AllocType)alloc_type);
}
- }
-
- if (delete_size && flags()->new_delete_type_mismatch &&
- delete_size != m->UsedSize()) {
- ReportNewDeleteSizeMismatch(p, delete_size, stack);
+ } else {
+ if (flags()->new_delete_type_mismatch &&
+ (alloc_type == FROM_NEW || alloc_type == FROM_NEW_BR) &&
+ ((delete_size && delete_size != m->UsedSize()) ||
+ ComputeUserRequestedAlignmentLog(delete_alignment) !=
+ m->user_requested_alignment_log)) {
+ ReportNewDeleteTypeMismatch(p, delete_size, delete_alignment, stack);
+ }
}
QuarantineChunk(m, ptr, stack);
@@ -631,7 +654,7 @@ struct Allocator {
// If realloc() races with free(), we may start copying freed memory.
// However, we will report racy double-free later anyway.
REAL(memcpy)(new_ptr, old_ptr, memcpy_size);
- Deallocate(old_ptr, 0, stack, FROM_MALLOC);
+ Deallocate(old_ptr, 0, 0, stack, FROM_MALLOC);
}
return new_ptr;
}
@@ -716,6 +739,22 @@ struct Allocator {
return AsanChunkView(m1);
}
+ void Purge() {
+ AsanThread *t = GetCurrentThread();
+ if (t) {
+ AsanThreadLocalMallocStorage *ms = &t->malloc_storage();
+ quarantine.DrainAndRecycle(GetQuarantineCache(ms),
+ QuarantineCallback(GetAllocatorCache(ms)));
+ }
+ {
+ SpinMutexLock l(&fallback_mutex);
+ quarantine.DrainAndRecycle(&fallback_quarantine_cache,
+ QuarantineCallback(&fallback_allocator_cache));
+ }
+
+ allocator.ForceReleaseToOS();
+ }
+
void PrintStats() {
allocator.PrintStats();
quarantine.PrintStats();
@@ -750,6 +789,9 @@ bool AsanChunkView::IsQuarantined() const {
uptr AsanChunkView::Beg() const { return chunk_->Beg(); }
uptr AsanChunkView::End() const { return Beg() + UsedSize(); }
uptr AsanChunkView::UsedSize() const { return chunk_->UsedSize(); }
+u32 AsanChunkView::UserRequestedAlignment() const {
+ return Allocator::ComputeUserAlignment(chunk_->user_requested_alignment_log);
+}
uptr AsanChunkView::AllocTid() const { return chunk_->alloc_tid; }
uptr AsanChunkView::FreeTid() const { return chunk_->free_tid; }
AllocType AsanChunkView::GetAllocType() const {
@@ -775,7 +817,7 @@ StackTrace AsanChunkView::GetFreeStack() const {
}
void InitializeAllocator(const AllocatorOptions &options) {
- instance.Initialize(options);
+ instance.InitLinkerInitialized(options);
}
void ReInitializeAllocator(const AllocatorOptions &options) {
@@ -802,12 +844,12 @@ void PrintInternalAllocatorStats() {
}
void asan_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) {
- instance.Deallocate(ptr, 0, stack, alloc_type);
+ instance.Deallocate(ptr, 0, 0, stack, alloc_type);
}
-void asan_sized_free(void *ptr, uptr size, BufferedStackTrace *stack,
- AllocType alloc_type) {
- instance.Deallocate(ptr, size, stack, alloc_type);
+void asan_delete(void *ptr, uptr size, uptr alignment,
+ BufferedStackTrace *stack, AllocType alloc_type) {
+ instance.Deallocate(ptr, size, alignment, stack, alloc_type);
}
void *asan_malloc(uptr size, BufferedStackTrace *stack) {
@@ -823,7 +865,7 @@ void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) {
return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC, true));
if (size == 0) {
if (flags()->allocator_frees_and_returns_null_on_realloc_zero) {
- instance.Deallocate(p, 0, stack, FROM_MALLOC);
+ instance.Deallocate(p, 0, 0, stack, FROM_MALLOC);
return nullptr;
}
// Allocate a size of 1 if we shouldn't free() on Realloc to 0
@@ -839,6 +881,10 @@ void *asan_valloc(uptr size, BufferedStackTrace *stack) {
void *asan_pvalloc(uptr size, BufferedStackTrace *stack) {
uptr PageSize = GetPageSizeCached();
+ if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
+ errno = errno_ENOMEM;
+ return AsanAllocator::FailureHandler::OnBadRequest();
+ }
// pvalloc(0) should allocate one page.
size = size ? RoundUpTo(size, PageSize) : PageSize;
return SetErrnoOnNull(
@@ -1007,6 +1053,10 @@ uptr __sanitizer_get_allocated_size(const void *p) {
return allocated_size;
}
+void __sanitizer_purge_allocator() {
+ instance.Purge();
+}
+
#if !SANITIZER_SUPPORTS_WEAK_HOOKS
// Provide default (no-op) implementation of malloc hooks.
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook,
diff --git a/contrib/compiler-rt/lib/asan/asan_allocator.h b/contrib/compiler-rt/lib/asan/asan_allocator.h
index ad1aeb58a86b..26483db4c60e 100644
--- a/contrib/compiler-rt/lib/asan/asan_allocator.h
+++ b/contrib/compiler-rt/lib/asan/asan_allocator.h
@@ -58,6 +58,7 @@ class AsanChunkView {
uptr Beg() const; // First byte of user memory.
uptr End() const; // Last byte of user memory.
uptr UsedSize() const; // Size requested by the user.
+ u32 UserRequestedAlignment() const; // Originally requested alignment.
uptr AllocTid() const;
uptr FreeTid() const;
bool Eq(const AsanChunkView &c) const { return chunk_ == c.chunk_; }
@@ -119,7 +120,11 @@ struct AsanMapUnmapCallback {
};
#if SANITIZER_CAN_USE_ALLOCATOR64
-# if defined(__powerpc64__)
+# if SANITIZER_FUCHSIA
+const uptr kAllocatorSpace = ~(uptr)0;
+const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
+typedef DefaultSizeClassMap SizeClassMap;
+# elif defined(__powerpc64__)
const uptr kAllocatorSpace = 0xa0000000000ULL;
const uptr kAllocatorSize = 0x20000000000ULL; // 2T.
typedef DefaultSizeClassMap SizeClassMap;
@@ -193,8 +198,8 @@ struct AsanThreadLocalMallocStorage {
void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack,
AllocType alloc_type);
void asan_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type);
-void asan_sized_free(void *ptr, uptr size, BufferedStackTrace *stack,
- AllocType alloc_type);
+void asan_delete(void *ptr, uptr size, uptr alignment,
+ BufferedStackTrace *stack, AllocType alloc_type);
void *asan_malloc(uptr size, BufferedStackTrace *stack);
void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack);
diff --git a/contrib/compiler-rt/lib/asan/asan_descriptions.cc b/contrib/compiler-rt/lib/asan/asan_descriptions.cc
index 822a6a62d64e..86c6af7d9f75 100644
--- a/contrib/compiler-rt/lib/asan/asan_descriptions.cc
+++ b/contrib/compiler-rt/lib/asan/asan_descriptions.cc
@@ -122,6 +122,7 @@ static void GetAccessToHeapChunkInformation(ChunkAccess *descr,
}
descr->chunk_begin = chunk.Beg();
descr->chunk_size = chunk.UsedSize();
+ descr->user_requested_alignment = chunk.UserRequestedAlignment();
descr->alloc_type = chunk.GetAllocType();
}
@@ -150,7 +151,7 @@ static void PrintHeapChunkAccess(uptr addr, const ChunkAccess &descr) {
str.append(" %zu-byte region [%p,%p)\n", descr.chunk_size,
(void *)descr.chunk_begin,
(void *)(descr.chunk_begin + descr.chunk_size));
- str.append("%s", d.EndLocation());
+ str.append("%s", d.Default());
Printf("%s", str.data());
}
@@ -260,7 +261,7 @@ static void PrintAccessAndVarIntersection(const StackVarDescr &var, uptr addr,
// FIXME: we may want to also print the size of the access here,
// but in case of accesses generated by memset it may be confusing.
str.append("%s <== Memory access at offset %zd %s this variable%s\n",
- d.Location(), addr, pos_descr, d.EndLocation());
+ d.Location(), addr, pos_descr, d.Default());
} else {
str.append("\n");
}
@@ -295,7 +296,7 @@ static void DescribeAddressRelativeToGlobal(uptr addr, uptr access_size,
MaybeDemangleGlobalName(g.name));
PrintGlobalLocation(&str, g);
str.append("' (0x%zx) of size %zu\n", g.beg, g.size);
- str.append("%s", d.EndLocation());
+ str.append("%s", d.Default());
PrintGlobalNameIfASCII(&str, g);
Printf("%s", str.data());
}
@@ -335,6 +336,26 @@ void GlobalAddressDescription::Print(const char *bug_type) const {
}
}
+bool GlobalAddressDescription::PointsInsideTheSameVariable(
+ const GlobalAddressDescription &other) const {
+ if (size == 0 || other.size == 0) return false;
+
+ for (uptr i = 0; i < size; i++) {
+ const __asan_global &a = globals[i];
+ for (uptr j = 0; j < other.size; j++) {
+ const __asan_global &b = other.globals[j];
+ if (a.beg == b.beg &&
+ a.beg <= addr &&
+ b.beg <= other.addr &&
+ (addr + access_size) < (a.beg + a.size) &&
+ (other.addr + other.access_size) < (b.beg + b.size))
+ return true;
+ }
+ }
+
+ return false;
+}
+
void StackAddressDescription::Print() const {
Decorator d;
char tname[128];
@@ -343,10 +364,10 @@ void StackAddressDescription::Print() const {
ThreadNameWithParenthesis(tid, tname, sizeof(tname)));
if (!frame_descr) {
- Printf("%s\n", d.EndLocation());
+ Printf("%s\n", d.Default());
return;
}
- Printf(" at offset %zu in frame%s\n", offset, d.EndLocation());
+ Printf(" at offset %zu in frame%s\n", offset, d.Default());
// Now we print the frame where the alloca has happened.
// We print this frame as a stack trace with one element.
@@ -355,7 +376,7 @@ void StackAddressDescription::Print() const {
// previously. That's unfortunate, but I have no better solution,
// especially given that the alloca may be from entirely different place
// (e.g. use-after-scope, or different thread's stack).
- Printf("%s", d.EndLocation());
+ Printf("%s", d.Default());
StackTrace alloca_stack(&frame_pc, 1);
alloca_stack.Print();
@@ -405,18 +426,18 @@ void HeapAddressDescription::Print() const {
Printf("%sfreed by thread T%d%s here:%s\n", d.Allocation(),
free_thread->tid,
ThreadNameWithParenthesis(free_thread, tname, sizeof(tname)),
- d.EndAllocation());
+ d.Default());
StackTrace free_stack = GetStackTraceFromId(free_stack_id);
free_stack.Print();
Printf("%spreviously allocated by thread T%d%s here:%s\n", d.Allocation(),
alloc_thread->tid,
ThreadNameWithParenthesis(alloc_thread, tname, sizeof(tname)),
- d.EndAllocation());
+ d.Default());
} else {
Printf("%sallocated by thread T%d%s here:%s\n", d.Allocation(),
alloc_thread->tid,
ThreadNameWithParenthesis(alloc_thread, tname, sizeof(tname)),
- d.EndAllocation());
+ d.Default());
}
alloc_stack.Print();
DescribeThread(GetCurrentThread());
diff --git a/contrib/compiler-rt/lib/asan/asan_descriptions.h b/contrib/compiler-rt/lib/asan/asan_descriptions.h
index 0ee677eb7d0e..1a1b01cf20cd 100644
--- a/contrib/compiler-rt/lib/asan/asan_descriptions.h
+++ b/contrib/compiler-rt/lib/asan/asan_descriptions.h
@@ -34,11 +34,8 @@ class Decorator : public __sanitizer::SanitizerCommonDecorator {
public:
Decorator() : SanitizerCommonDecorator() {}
const char *Access() { return Blue(); }
- const char *EndAccess() { return Default(); }
const char *Location() { return Green(); }
- const char *EndLocation() { return Default(); }
const char *Allocation() { return Magenta(); }
- const char *EndAllocation() { return Default(); }
const char *ShadowByte(u8 byte) {
switch (byte) {
@@ -72,9 +69,6 @@ class Decorator : public __sanitizer::SanitizerCommonDecorator {
return Default();
}
}
- const char *EndShadowByte() { return Default(); }
- const char *MemoryByte() { return Magenta(); }
- const char *EndMemoryByte() { return Default(); }
};
enum ShadowKind : u8 {
@@ -108,6 +102,7 @@ struct ChunkAccess {
sptr offset;
uptr chunk_begin;
uptr chunk_size;
+ u32 user_requested_alignment : 12;
u32 access_type : 2;
u32 alloc_type : 2;
};
@@ -151,6 +146,10 @@ struct GlobalAddressDescription {
u8 size;
void Print(const char *bug_type = "") const;
+
+ // Returns true when this descriptions points inside the same global variable
+ // as other. Descriptions can have different address within the variable
+ bool PointsInsideTheSameVariable(const GlobalAddressDescription &other) const;
};
bool GetGlobalAddressInformation(uptr addr, uptr access_size,
diff --git a/contrib/compiler-rt/lib/asan/asan_errors.cc b/contrib/compiler-rt/lib/asan/asan_errors.cc
index b7a38eb7cece..0f4a3abff9a7 100644
--- a/contrib/compiler-rt/lib/asan/asan_errors.cc
+++ b/contrib/compiler-rt/lib/asan/asan_errors.cc
@@ -13,7 +13,6 @@
//===----------------------------------------------------------------------===//
#include "asan_errors.h"
-#include <signal.h>
#include "asan_descriptions.h"
#include "asan_mapping.h"
#include "asan_report.h"
@@ -22,82 +21,26 @@
namespace __asan {
-void ErrorStackOverflow::Print() {
- Decorator d;
- Printf("%s", d.Warning());
- Report(
- "ERROR: AddressSanitizer: %s on address %p"
- " (pc %p bp %p sp %p T%d)\n", scariness.GetDescription(),
- (void *)addr, (void *)pc, (void *)bp, (void *)sp, tid);
- Printf("%s", d.EndWarning());
- scariness.Print();
- BufferedStackTrace stack;
- GetStackTraceWithPcBpAndContext(&stack, kStackTraceMax, pc, bp, context,
- common_flags()->fast_unwind_on_fatal);
- stack.Print();
- ReportErrorSummary(scariness.GetDescription(), &stack);
-}
-
-static void MaybeDumpInstructionBytes(uptr pc) {
- if (!flags()->dump_instruction_bytes || (pc < GetPageSizeCached())) return;
- InternalScopedString str(1024);
- str.append("First 16 instruction bytes at pc: ");
- if (IsAccessibleMemoryRange(pc, 16)) {
- for (int i = 0; i < 16; ++i) {
- PrintMemoryByte(&str, "", ((u8 *)pc)[i], /*in_shadow*/ false, " ");
- }
- str.append("\n");
- } else {
- str.append("unaccessible\n");
- }
- Report("%s", str.data());
-}
-
-static void MaybeDumpRegisters(void *context) {
- if (!flags()->dump_registers) return;
- SignalContext::DumpAllRegisters(context);
-}
-
-static void MaybeReportNonExecRegion(uptr pc) {
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
- MemoryMappingLayout proc_maps(/*cache_enabled*/ true);
- MemoryMappedSegment segment;
- while (proc_maps.Next(&segment)) {
- if (pc >= segment.start && pc < segment.end && !segment.IsExecutable())
- Report("Hint: PC is at a non-executable region. Maybe a wild jump?\n");
- }
+static void OnStackUnwind(const SignalContext &sig,
+ const void *callback_context,
+ BufferedStackTrace *stack) {
+ bool fast = common_flags()->fast_unwind_on_fatal;
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
+ // On FreeBSD the slow unwinding that leverages _Unwind_Backtrace()
+ // yields the call stack of the signal's handler and not of the code
+ // that raised the signal (as it does on Linux).
+ fast = true;
#endif
+ // Tests and maybe some users expect that scariness is going to be printed
+ // just before the stack. As only asan has scariness score we have no
+ // corresponding code in the sanitizer_common and we use this callback to
+ // print it.
+ static_cast<const ScarinessScoreBase *>(callback_context)->Print();
+ GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context, fast);
}
void ErrorDeadlySignal::Print() {
- Decorator d;
- Printf("%s", d.Warning());
- const char *description = __sanitizer::DescribeSignalOrException(signo);
- Report(
- "ERROR: AddressSanitizer: %s on unknown address %p (pc %p bp %p sp %p "
- "T%d)\n",
- description, (void *)addr, (void *)pc, (void *)bp, (void *)sp, tid);
- Printf("%s", d.EndWarning());
- if (pc < GetPageSizeCached()) Report("Hint: pc points to the zero page.\n");
- if (is_memory_access) {
- const char *access_type =
- write_flag == SignalContext::WRITE
- ? "WRITE"
- : (write_flag == SignalContext::READ ? "READ" : "UNKNOWN");
- Report("The signal is caused by a %s memory access.\n", access_type);
- if (addr < GetPageSizeCached())
- Report("Hint: address points to the zero page.\n");
- }
- MaybeReportNonExecRegion(pc);
- scariness.Print();
- BufferedStackTrace stack;
- GetStackTraceWithPcBpAndContext(&stack, kStackTraceMax, pc, bp, context,
- common_flags()->fast_unwind_on_fatal);
- stack.Print();
- MaybeDumpInstructionBytes(pc);
- MaybeDumpRegisters(context);
- Printf("AddressSanitizer can not provide additional info.\n");
- ReportErrorSummary(description, &stack);
+ ReportDeadlySignal(signal, tid, &OnStackUnwind, &scariness);
}
void ErrorDoubleFree::Print() {
@@ -109,7 +52,7 @@ void ErrorDoubleFree::Print() {
"thread T%d%s:\n",
scariness.GetDescription(), addr_description.addr, tid,
ThreadNameWithParenthesis(tid, tname, sizeof(tname)));
- Printf("%s", d.EndWarning());
+ Printf("%s", d.Default());
scariness.Print();
GET_STACK_TRACE_FATAL(second_free_stack->trace[0],
second_free_stack->top_frame_bp);
@@ -118,7 +61,7 @@ void ErrorDoubleFree::Print() {
ReportErrorSummary(scariness.GetDescription(), &stack);
}
-void ErrorNewDeleteSizeMismatch::Print() {
+void ErrorNewDeleteTypeMismatch::Print() {
Decorator d;
Printf("%s", d.Warning());
char tname[128];
@@ -127,11 +70,29 @@ void ErrorNewDeleteSizeMismatch::Print() {
"T%d%s:\n",
scariness.GetDescription(), addr_description.addr, tid,
ThreadNameWithParenthesis(tid, tname, sizeof(tname)));
- Printf("%s object passed to delete has wrong type:\n", d.EndWarning());
- Printf(
- " size of the allocated type: %zd bytes;\n"
- " size of the deallocated type: %zd bytes.\n",
- addr_description.chunk_access.chunk_size, delete_size);
+ Printf("%s object passed to delete has wrong type:\n", d.Default());
+ if (delete_size != 0) {
+ Printf(
+ " size of the allocated type: %zd bytes;\n"
+ " size of the deallocated type: %zd bytes.\n",
+ addr_description.chunk_access.chunk_size, delete_size);
+ }
+ const uptr user_alignment =
+ addr_description.chunk_access.user_requested_alignment;
+ if (delete_alignment != user_alignment) {
+ char user_alignment_str[32];
+ char delete_alignment_str[32];
+ internal_snprintf(user_alignment_str, sizeof(user_alignment_str),
+ "%zd bytes", user_alignment);
+ internal_snprintf(delete_alignment_str, sizeof(delete_alignment_str),
+ "%zd bytes", delete_alignment);
+ static const char *kDefaultAlignment = "default-aligned";
+ Printf(
+ " alignment of the allocated type: %s;\n"
+ " alignment of the deallocated type: %s.\n",
+ user_alignment > 0 ? user_alignment_str : kDefaultAlignment,
+ delete_alignment > 0 ? delete_alignment_str : kDefaultAlignment);
+ }
CHECK_GT(free_stack->size, 0);
scariness.Print();
GET_STACK_TRACE_FATAL(free_stack->trace[0], free_stack->top_frame_bp);
@@ -152,7 +113,7 @@ void ErrorFreeNotMalloced::Print() {
"which was not malloc()-ed: %p in thread T%d%s\n",
addr_description.Address(), tid,
ThreadNameWithParenthesis(tid, tname, sizeof(tname)));
- Printf("%s", d.EndWarning());
+ Printf("%s", d.Default());
CHECK_GT(free_stack->size, 0);
scariness.Print();
GET_STACK_TRACE_FATAL(free_stack->trace[0], free_stack->top_frame_bp);
@@ -173,7 +134,7 @@ void ErrorAllocTypeMismatch::Print() {
scariness.GetDescription(),
alloc_names[alloc_type], dealloc_names[dealloc_type],
addr_description.addr);
- Printf("%s", d.EndWarning());
+ Printf("%s", d.Default());
CHECK_GT(dealloc_stack->size, 0);
scariness.Print();
GET_STACK_TRACE_FATAL(dealloc_stack->trace[0], dealloc_stack->top_frame_bp);
@@ -192,7 +153,7 @@ void ErrorMallocUsableSizeNotOwned::Print() {
"ERROR: AddressSanitizer: attempting to call malloc_usable_size() for "
"pointer which is not owned: %p\n",
addr_description.Address());
- Printf("%s", d.EndWarning());
+ Printf("%s", d.Default());
stack->Print();
addr_description.Print();
ReportErrorSummary(scariness.GetDescription(), stack);
@@ -205,7 +166,7 @@ void ErrorSanitizerGetAllocatedSizeNotOwned::Print() {
"ERROR: AddressSanitizer: attempting to call "
"__sanitizer_get_allocated_size() for pointer which is not owned: %p\n",
addr_description.Address());
- Printf("%s", d.EndWarning());
+ Printf("%s", d.Default());
stack->Print();
addr_description.Print();
ReportErrorSummary(scariness.GetDescription(), stack);
@@ -222,7 +183,7 @@ void ErrorStringFunctionMemoryRangesOverlap::Print() {
bug_type, addr1_description.Address(),
addr1_description.Address() + length1, addr2_description.Address(),
addr2_description.Address() + length2);
- Printf("%s", d.EndWarning());
+ Printf("%s", d.Default());
scariness.Print();
stack->Print();
addr1_description.Print();
@@ -235,7 +196,7 @@ void ErrorStringFunctionSizeOverflow::Print() {
Printf("%s", d.Warning());
Report("ERROR: AddressSanitizer: %s: (size=%zd)\n",
scariness.GetDescription(), size);
- Printf("%s", d.EndWarning());
+ Printf("%s", d.Default());
scariness.Print();
stack->Print();
addr_description.Print();
@@ -263,7 +224,7 @@ void ErrorODRViolation::Print() {
Printf("%s", d.Warning());
Report("ERROR: AddressSanitizer: %s (%p):\n", scariness.GetDescription(),
global1.beg);
- Printf("%s", d.EndWarning());
+ Printf("%s", d.Default());
InternalScopedString g1_loc(256), g2_loc(256);
PrintGlobalLocation(&g1_loc, global1);
PrintGlobalLocation(&g2_loc, global2);
@@ -292,7 +253,7 @@ void ErrorInvalidPointerPair::Print() {
Printf("%s", d.Warning());
Report("ERROR: AddressSanitizer: %s: %p %p\n", scariness.GetDescription(),
addr1_description.Address(), addr2_description.Address());
- Printf("%s", d.EndWarning());
+ Printf("%s", d.Default());
GET_STACK_TRACE_FATAL(pc, bp);
stack.Print();
addr1_description.Print();
@@ -477,9 +438,14 @@ static void PrintShadowMemoryForAddress(uptr addr) {
InternalScopedString str(4096 * 8);
str.append("Shadow bytes around the buggy address:\n");
for (int i = -5; i <= 5; i++) {
+ uptr row_shadow_addr = aligned_shadow + i * n_bytes_per_row;
+ // Skip rows that would be outside the shadow range. This can happen when
+ // the user address is near the bottom, top, or shadow gap of the address
+ // space.
+ if (!AddrIsInShadow(row_shadow_addr)) continue;
const char *prefix = (i == 0) ? "=>" : " ";
- PrintShadowBytes(&str, prefix, (u8 *)(aligned_shadow + i * n_bytes_per_row),
- (u8 *)shadow_addr, n_bytes_per_row);
+ PrintShadowBytes(&str, prefix, (u8 *)row_shadow_addr, (u8 *)shadow_addr,
+ n_bytes_per_row);
}
if (flags()->print_legend) PrintLegend(&str);
Printf("%s", str.data());
@@ -491,13 +457,13 @@ void ErrorGeneric::Print() {
uptr addr = addr_description.Address();
Report("ERROR: AddressSanitizer: %s on address %p at pc %p bp %p sp %p\n",
bug_descr, (void *)addr, pc, bp, sp);
- Printf("%s", d.EndWarning());
+ Printf("%s", d.Default());
char tname[128];
Printf("%s%s of size %zu at %p thread T%d%s%s\n", d.Access(),
access_size ? (is_write ? "WRITE" : "READ") : "ACCESS", access_size,
(void *)addr, tid,
- ThreadNameWithParenthesis(tid, tname, sizeof(tname)), d.EndAccess());
+ ThreadNameWithParenthesis(tid, tname, sizeof(tname)), d.Default());
scariness.Print();
GET_STACK_TRACE_FATAL(pc, bp);
diff --git a/contrib/compiler-rt/lib/asan/asan_errors.h b/contrib/compiler-rt/lib/asan/asan_errors.h
index 9a124924470e..518ba0c6945e 100644
--- a/contrib/compiler-rt/lib/asan/asan_errors.h
+++ b/contrib/compiler-rt/lib/asan/asan_errors.h
@@ -27,61 +27,28 @@ struct ErrorBase {
u32 tid;
};
-struct ErrorStackOverflow : ErrorBase {
- uptr addr, pc, bp, sp;
- // ErrorStackOverflow never owns the context.
- void *context;
- // VS2013 doesn't implement unrestricted unions, so we need a trivial default
- // constructor
- ErrorStackOverflow() = default;
- ErrorStackOverflow(u32 tid, const SignalContext &sig)
- : ErrorBase(tid),
- addr(sig.addr),
- pc(sig.pc),
- bp(sig.bp),
- sp(sig.sp),
- context(sig.context) {
- scariness.Clear();
- scariness.Scare(10, "stack-overflow");
- }
- void Print();
-};
-
struct ErrorDeadlySignal : ErrorBase {
- uptr addr, pc, bp, sp;
- // ErrorDeadlySignal never owns the context.
- void *context;
- int signo;
- SignalContext::WriteFlag write_flag;
- bool is_memory_access;
+ SignalContext signal;
// VS2013 doesn't implement unrestricted unions, so we need a trivial default
// constructor
ErrorDeadlySignal() = default;
- ErrorDeadlySignal(u32 tid, const SignalContext &sig, int signo_)
- : ErrorBase(tid),
- addr(sig.addr),
- pc(sig.pc),
- bp(sig.bp),
- sp(sig.sp),
- context(sig.context),
- signo(signo_),
- write_flag(sig.write_flag),
- is_memory_access(sig.is_memory_access) {
+ ErrorDeadlySignal(u32 tid, const SignalContext &sig)
+ : ErrorBase(tid), signal(sig) {
scariness.Clear();
- if (is_memory_access) {
- if (addr < GetPageSizeCached()) {
- scariness.Scare(10, "null-deref");
- } else if (addr == pc) {
- scariness.Scare(60, "wild-jump");
- } else if (write_flag == SignalContext::WRITE) {
- scariness.Scare(30, "wild-addr-write");
- } else if (write_flag == SignalContext::READ) {
- scariness.Scare(20, "wild-addr-read");
- } else {
- scariness.Scare(25, "wild-addr");
- }
- } else {
+ if (signal.IsStackOverflow()) {
+ scariness.Scare(10, "stack-overflow");
+ } else if (!signal.is_memory_access) {
scariness.Scare(10, "signal");
+ } else if (signal.addr < GetPageSizeCached()) {
+ scariness.Scare(10, "null-deref");
+ } else if (signal.addr == signal.pc) {
+ scariness.Scare(60, "wild-jump");
+ } else if (signal.write_flag == SignalContext::WRITE) {
+ scariness.Scare(30, "wild-addr-write");
+ } else if (signal.write_flag == SignalContext::READ) {
+ scariness.Scare(20, "wild-addr-read");
+ } else {
+ scariness.Scare(25, "wild-addr");
}
}
void Print();
@@ -104,17 +71,19 @@ struct ErrorDoubleFree : ErrorBase {
void Print();
};
-struct ErrorNewDeleteSizeMismatch : ErrorBase {
- // ErrorNewDeleteSizeMismatch doesn't own the stack trace.
+struct ErrorNewDeleteTypeMismatch : ErrorBase {
+ // ErrorNewDeleteTypeMismatch doesn't own the stack trace.
const BufferedStackTrace *free_stack;
HeapAddressDescription addr_description;
uptr delete_size;
+ uptr delete_alignment;
// VS2013 doesn't implement unrestricted unions, so we need a trivial default
// constructor
- ErrorNewDeleteSizeMismatch() = default;
- ErrorNewDeleteSizeMismatch(u32 tid, BufferedStackTrace *stack, uptr addr,
- uptr delete_size_)
- : ErrorBase(tid), free_stack(stack), delete_size(delete_size_) {
+ ErrorNewDeleteTypeMismatch() = default;
+ ErrorNewDeleteTypeMismatch(u32 tid, BufferedStackTrace *stack, uptr addr,
+ uptr delete_size_, uptr delete_alignment_)
+ : ErrorBase(tid), free_stack(stack), delete_size(delete_size_),
+ delete_alignment(delete_alignment_) {
GetHeapAddressInformation(addr, 1, &addr_description);
scariness.Clear();
scariness.Scare(10, "new-delete-type-mismatch");
@@ -324,10 +293,9 @@ struct ErrorGeneric : ErrorBase {
// clang-format off
#define ASAN_FOR_EACH_ERROR_KIND(macro) \
- macro(StackOverflow) \
macro(DeadlySignal) \
macro(DoubleFree) \
- macro(NewDeleteSizeMismatch) \
+ macro(NewDeleteTypeMismatch) \
macro(FreeNotMalloced) \
macro(AllocTypeMismatch) \
macro(MallocUsableSizeNotOwned) \
diff --git a/contrib/compiler-rt/lib/asan/asan_fake_stack.cc b/contrib/compiler-rt/lib/asan/asan_fake_stack.cc
index 017b7d2af129..1c6184e3c7fc 100644
--- a/contrib/compiler-rt/lib/asan/asan_fake_stack.cc
+++ b/contrib/compiler-rt/lib/asan/asan_fake_stack.cc
@@ -28,9 +28,9 @@ static const u64 kAllocaRedzoneMask = 31UL;
// For small size classes inline PoisonShadow for better performance.
ALWAYS_INLINE void SetShadow(uptr ptr, uptr size, uptr class_id, u64 magic) {
- CHECK_EQ(SHADOW_SCALE, 3); // This code expects SHADOW_SCALE=3.
u64 *shadow = reinterpret_cast<u64*>(MemToShadow(ptr));
- if (class_id <= 6) {
+ if (SHADOW_SCALE == 3 && class_id <= 6) {
+ // This code expects SHADOW_SCALE=3.
for (uptr i = 0; i < (((uptr)1) << class_id); i++) {
shadow[i] = magic;
// Make sure this does not become memset.
@@ -171,7 +171,7 @@ void FakeStack::ForEachFakeFrame(RangeIteratorCallback callback, void *arg) {
}
}
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_FUCHSIA
static THREADLOCAL FakeStack *fake_stack_tls;
FakeStack *GetTLSFakeStack() {
@@ -183,7 +183,7 @@ void SetTLSFakeStack(FakeStack *fs) {
#else
FakeStack *GetTLSFakeStack() { return 0; }
void SetTLSFakeStack(FakeStack *fs) { }
-#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
+#endif // (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_FUCHSIA
static FakeStack *GetFakeStack() {
AsanThread *t = GetCurrentThread();
diff --git a/contrib/compiler-rt/lib/asan/asan_flags.cc b/contrib/compiler-rt/lib/asan/asan_flags.cc
index 6be0d6e94b9a..562168e6d428 100644
--- a/contrib/compiler-rt/lib/asan/asan_flags.cc
+++ b/contrib/compiler-rt/lib/asan/asan_flags.cc
@@ -118,6 +118,10 @@ void InitializeFlags() {
const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions();
ubsan_parser.ParseString(ubsan_default_options);
#endif
+#if CAN_SANITIZE_LEAKS
+ const char *lsan_default_options = __lsan::MaybeCallLsanDefaultOptions();
+ lsan_parser.ParseString(lsan_default_options);
+#endif
// Override from command line.
asan_parser.ParseString(GetEnv("ASAN_OPTIONS"));
@@ -144,6 +148,9 @@ void InitializeFlags() {
SanitizerToolName);
Die();
}
+ // Ensure that redzone is at least SHADOW_GRANULARITY.
+ if (f->redzone < (int)SHADOW_GRANULARITY)
+ f->redzone = SHADOW_GRANULARITY;
// Make "strict_init_order" imply "check_initialization_order".
// TODO(samsonov): Use a single runtime flag for an init-order checker.
if (f->strict_init_order) {
diff --git a/contrib/compiler-rt/lib/asan/asan_flags.inc b/contrib/compiler-rt/lib/asan/asan_flags.inc
index f2216c2e9b3b..00071d39f041 100644
--- a/contrib/compiler-rt/lib/asan/asan_flags.inc
+++ b/contrib/compiler-rt/lib/asan/asan_flags.inc
@@ -79,6 +79,10 @@ ASAN_FLAG(
"Number of seconds to sleep between printing an error report and "
"terminating the program. Useful for debugging purposes (e.g. when one "
"needs to attach gdb).")
+ASAN_FLAG(
+ int, sleep_after_init, 0,
+ "Number of seconds to sleep after AddressSanitizer is initialized. "
+ "Useful for debugging purposes (e.g. when one needs to attach gdb).")
ASAN_FLAG(bool, check_malloc_usable_size, true,
"Allows the users to work around the bug in Nvidia drivers prior to "
"295.*.")
@@ -143,11 +147,6 @@ ASAN_FLAG(int, detect_odr_violation, 2,
"If >=2, detect violation of One-Definition-Rule (ODR); "
"If ==1, detect ODR-violation only if the two variables "
"have different sizes")
-ASAN_FLAG(bool, dump_instruction_bytes, false,
- "If true, dump 16 bytes starting at the instruction that caused SEGV")
-ASAN_FLAG(bool, dump_registers, true,
- "If true, dump values of CPU registers when SEGV happens. Only "
- "available on OS X for now.")
ASAN_FLAG(const char *, suppressions, "", "Suppressions file name.")
ASAN_FLAG(bool, halt_on_error, true,
"Crash the program after printing the first error report "
diff --git a/contrib/compiler-rt/lib/asan/asan_fuchsia.cc b/contrib/compiler-rt/lib/asan/asan_fuchsia.cc
new file mode 100644
index 000000000000..0b5bff4f565e
--- /dev/null
+++ b/contrib/compiler-rt/lib/asan/asan_fuchsia.cc
@@ -0,0 +1,218 @@
+//===-- asan_fuchsia.cc --------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Fuchsia-specific details.
+//===---------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_fuchsia.h"
+#if SANITIZER_FUCHSIA
+
+#include "asan_interceptors.h"
+#include "asan_internal.h"
+#include "asan_stack.h"
+#include "asan_thread.h"
+
+#include <limits.h>
+#include <zircon/sanitizer.h>
+#include <zircon/syscalls.h>
+#include <zircon/threads.h>
+
+namespace __asan {
+
+// The system already set up the shadow memory for us.
+// __sanitizer::GetMaxUserVirtualAddress has already been called by
+// AsanInitInternal->InitializeHighMemEnd (asan_rtl.cc).
+// Just do some additional sanity checks here.
+void InitializeShadowMemory() {
+ if (Verbosity()) PrintAddressSpaceLayout();
+
+ // Make sure SHADOW_OFFSET doesn't use __asan_shadow_memory_dynamic_address.
+ __asan_shadow_memory_dynamic_address = kDefaultShadowSentinel;
+ DCHECK(kLowShadowBeg != kDefaultShadowSentinel);
+ __asan_shadow_memory_dynamic_address = kLowShadowBeg;
+
+ CHECK_EQ(kShadowGapEnd, kHighShadowBeg - 1);
+ CHECK_EQ(kHighMemEnd, __sanitizer::ShadowBounds.memory_limit - 1);
+ CHECK_EQ(kHighMemBeg, __sanitizer::ShadowBounds.shadow_limit);
+ CHECK_EQ(kHighShadowBeg, __sanitizer::ShadowBounds.shadow_base);
+ CHECK_EQ(kShadowGapEnd, __sanitizer::ShadowBounds.shadow_base - 1);
+ CHECK_EQ(kLowShadowEnd, 0);
+ CHECK_EQ(kLowShadowBeg, 0);
+}
+
+void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
+ UNIMPLEMENTED();
+}
+
+void AsanCheckDynamicRTPrereqs() {}
+void AsanCheckIncompatibleRT() {}
+void InitializeAsanInterceptors() {}
+
+void *AsanDoesNotSupportStaticLinkage() { return nullptr; }
+
+void InitializePlatformExceptionHandlers() {}
+void AsanOnDeadlySignal(int signo, void *siginfo, void *context) {
+ UNIMPLEMENTED();
+}
+
+// We can use a plain thread_local variable for TSD.
+static thread_local void *per_thread;
+
+void *AsanTSDGet() { return per_thread; }
+
+void AsanTSDSet(void *tsd) { per_thread = tsd; }
+
+// There's no initialization needed, and the passed-in destructor
+// will never be called. Instead, our own thread destruction hook
+// (below) will call AsanThread::TSDDtor directly.
+void AsanTSDInit(void (*destructor)(void *tsd)) {
+ DCHECK(destructor == &PlatformTSDDtor);
+}
+
+void PlatformTSDDtor(void *tsd) { UNREACHABLE(__func__); }
+
+static inline size_t AsanThreadMmapSize() {
+ return RoundUpTo(sizeof(AsanThread), PAGE_SIZE);
+}
+
+struct AsanThread::InitOptions {
+ uptr stack_bottom, stack_size;
+};
+
+// Shared setup between thread creation and startup for the initial thread.
+static AsanThread *CreateAsanThread(StackTrace *stack, u32 parent_tid,
+ uptr user_id, bool detached,
+ const char *name, uptr stack_bottom,
+ uptr stack_size) {
+ // In lieu of AsanThread::Create.
+ AsanThread *thread = (AsanThread *)MmapOrDie(AsanThreadMmapSize(), __func__);
+
+ AsanThreadContext::CreateThreadContextArgs args = {thread, stack};
+ u32 tid =
+ asanThreadRegistry().CreateThread(user_id, detached, parent_tid, &args);
+ asanThreadRegistry().SetThreadName(tid, name);
+
+ // On other systems, AsanThread::Init() is called from the new
+ // thread itself. But on Fuchsia we already know the stack address
+ // range beforehand, so we can do most of the setup right now.
+ const AsanThread::InitOptions options = {stack_bottom, stack_size};
+ thread->Init(&options);
+
+ return thread;
+}
+
+// This gets the same arguments passed to Init by CreateAsanThread, above.
+// We're in the creator thread before the new thread is actually started,
+// but its stack address range is already known. We don't bother tracking
+// the static TLS address range because the system itself already uses an
+// ASan-aware allocator for that.
+void AsanThread::SetThreadStackAndTls(const AsanThread::InitOptions *options) {
+ DCHECK_NE(GetCurrentThread(), this);
+ DCHECK_NE(GetCurrentThread(), nullptr);
+ CHECK_NE(options->stack_bottom, 0);
+ CHECK_NE(options->stack_size, 0);
+ stack_bottom_ = options->stack_bottom;
+ stack_top_ = options->stack_bottom + options->stack_size;
+}
+
+// Called by __asan::AsanInitInternal (asan_rtl.c).
+AsanThread *CreateMainThread() {
+ thrd_t self = thrd_current();
+ char name[ZX_MAX_NAME_LEN];
+ CHECK_NE(__sanitizer::MainThreadStackBase, 0);
+ CHECK_GT(__sanitizer::MainThreadStackSize, 0);
+ AsanThread *t = CreateAsanThread(
+ nullptr, 0, reinterpret_cast<uptr>(self), true,
+ _zx_object_get_property(thrd_get_zx_handle(self), ZX_PROP_NAME, name,
+ sizeof(name)) == ZX_OK
+ ? name
+ : nullptr,
+ __sanitizer::MainThreadStackBase, __sanitizer::MainThreadStackSize);
+ SetCurrentThread(t);
+ return t;
+}
+
+// This is called before each thread creation is attempted. So, in
+// its first call, the calling thread is the initial and sole thread.
+static void *BeforeThreadCreateHook(uptr user_id, bool detached,
+ const char *name, uptr stack_bottom,
+ uptr stack_size) {
+ EnsureMainThreadIDIsCorrect();
+ // Strict init-order checking is thread-hostile.
+ if (flags()->strict_init_order) StopInitOrderChecking();
+
+ GET_STACK_TRACE_THREAD;
+ u32 parent_tid = GetCurrentTidOrInvalid();
+
+ return CreateAsanThread(&stack, parent_tid, user_id, detached, name,
+ stack_bottom, stack_size);
+}
+
+// This is called after creating a new thread (in the creating thread),
+// with the pointer returned by BeforeThreadCreateHook (above).
+static void ThreadCreateHook(void *hook, bool aborted) {
+ AsanThread *thread = static_cast<AsanThread *>(hook);
+ if (!aborted) {
+ // The thread was created successfully.
+ // ThreadStartHook is already running in the new thread.
+ } else {
+ // The thread wasn't created after all.
+ // Clean up everything we set up in BeforeThreadCreateHook.
+ asanThreadRegistry().FinishThread(thread->tid());
+ UnmapOrDie(thread, AsanThreadMmapSize());
+ }
+}
+
+// This is called in the newly-created thread before it runs anything else,
+// with the pointer returned by BeforeThreadCreateHook (above).
+// cf. asan_interceptors.cc:asan_thread_start
+static void ThreadStartHook(void *hook, uptr os_id) {
+ AsanThread *thread = static_cast<AsanThread *>(hook);
+ SetCurrentThread(thread);
+
+ // In lieu of AsanThread::ThreadStart.
+ asanThreadRegistry().StartThread(thread->tid(), os_id, /*workerthread*/ false,
+ nullptr);
+}
+
+// Each thread runs this just before it exits,
+// with the pointer returned by BeforeThreadCreateHook (above).
+// All per-thread destructors have already been called.
+static void ThreadExitHook(void *hook, uptr os_id) {
+ AsanThread::TSDDtor(per_thread);
+}
+
+} // namespace __asan
+
+// These are declared (in extern "C") by <zircon/sanitizer.h>.
+// The system runtime will call our definitions directly.
+
+void *__sanitizer_before_thread_create_hook(thrd_t thread, bool detached,
+ const char *name, void *stack_base,
+ size_t stack_size) {
+ return __asan::BeforeThreadCreateHook(
+ reinterpret_cast<uptr>(thread), detached, name,
+ reinterpret_cast<uptr>(stack_base), stack_size);
+}
+
+void __sanitizer_thread_create_hook(void *hook, thrd_t thread, int error) {
+ __asan::ThreadCreateHook(hook, error != thrd_success);
+}
+
+void __sanitizer_thread_start_hook(void *hook, thrd_t self) {
+ __asan::ThreadStartHook(hook, reinterpret_cast<uptr>(self));
+}
+
+void __sanitizer_thread_exit_hook(void *hook, thrd_t self) {
+ __asan::ThreadExitHook(hook, reinterpret_cast<uptr>(self));
+}
+
+#endif // SANITIZER_FUCHSIA
diff --git a/contrib/compiler-rt/lib/asan/asan_globals.cc b/contrib/compiler-rt/lib/asan/asan_globals.cc
index eebada804f07..0db65d010349 100644
--- a/contrib/compiler-rt/lib/asan/asan_globals.cc
+++ b/contrib/compiler-rt/lib/asan/asan_globals.cc
@@ -384,6 +384,10 @@ void __asan_register_globals(__asan_global *globals, uptr n) {
}
RegisterGlobal(&globals[i]);
}
+
+ // Poison the metadata. It should not be accessible to user code.
+ PoisonShadow(reinterpret_cast<uptr>(globals), n * sizeof(__asan_global),
+ kAsanGlobalRedzoneMagic);
}
// Unregister an array of globals.
@@ -399,6 +403,9 @@ void __asan_unregister_globals(__asan_global *globals, uptr n) {
}
UnregisterGlobal(&globals[i]);
}
+
+ // Unpoison the metadata.
+ PoisonShadow(reinterpret_cast<uptr>(globals), n * sizeof(__asan_global), 0);
}
// This method runs immediately prior to dynamic initialization in each TU,
diff --git a/contrib/compiler-rt/lib/asan/asan_init_version.h b/contrib/compiler-rt/lib/asan/asan_init_version.h
index f48cc19cc515..c49fcd740248 100644
--- a/contrib/compiler-rt/lib/asan/asan_init_version.h
+++ b/contrib/compiler-rt/lib/asan/asan_init_version.h
@@ -15,6 +15,8 @@
#ifndef ASAN_INIT_VERSION_H
#define ASAN_INIT_VERSION_H
+#include "sanitizer_common/sanitizer_platform.h"
+
extern "C" {
// Every time the ASan ABI changes we also change the version number in the
// __asan_init function name. Objects built with incompatible ASan ABI
@@ -32,7 +34,12 @@ extern "C" {
// v6=>v7: added 'odr_indicator' to __asan_global
// v7=>v8: added '__asan_(un)register_image_globals' functions for dead
// stripping support on Mach-O platforms
+#if SANITIZER_WORDSIZE == 32 && SANITIZER_ANDROID
+ // v8=>v9: 32-bit Android switched to dynamic shadow
+ #define __asan_version_mismatch_check __asan_version_mismatch_check_v9
+#else
#define __asan_version_mismatch_check __asan_version_mismatch_check_v8
+#endif
}
#endif // ASAN_INIT_VERSION_H
diff --git a/contrib/compiler-rt/lib/asan/asan_interceptors.cc b/contrib/compiler-rt/lib/asan/asan_interceptors.cc
index 34ca22b8616e..cb7dcb3b1ca8 100644
--- a/contrib/compiler-rt/lib/asan/asan_interceptors.cc
+++ b/contrib/compiler-rt/lib/asan/asan_interceptors.cc
@@ -24,6 +24,11 @@
#include "lsan/lsan_common.h"
#include "sanitizer_common/sanitizer_libc.h"
+// There is no general interception at all on Fuchsia.
+// Only the functions in asan_interceptors_memintrinsics.cc are
+// really defined to replace libc functions.
+#if !SANITIZER_FUCHSIA
+
#if SANITIZER_POSIX
#include "sanitizer_common/sanitizer_posix.h"
#endif
@@ -36,108 +41,6 @@
namespace __asan {
-// Return true if we can quickly decide that the region is unpoisoned.
-// We assume that a redzone is at least 16 bytes.
-static inline bool QuickCheckForUnpoisonedRegion(uptr beg, uptr size) {
- if (size == 0) return true;
- if (size <= 32)
- return !AddressIsPoisoned(beg) &&
- !AddressIsPoisoned(beg + size - 1) &&
- !AddressIsPoisoned(beg + size / 2);
- if (size <= 64)
- return !AddressIsPoisoned(beg) &&
- !AddressIsPoisoned(beg + size / 4) &&
- !AddressIsPoisoned(beg + size - 1) &&
- !AddressIsPoisoned(beg + 3 * size / 4) &&
- !AddressIsPoisoned(beg + size / 2);
- return false;
-}
-
-struct AsanInterceptorContext {
- const char *interceptor_name;
-};
-
-// We implement ACCESS_MEMORY_RANGE, ASAN_READ_RANGE,
-// and ASAN_WRITE_RANGE as macro instead of function so
-// that no extra frames are created, and stack trace contains
-// relevant information only.
-// We check all shadow bytes.
-#define ACCESS_MEMORY_RANGE(ctx, offset, size, isWrite) do { \
- uptr __offset = (uptr)(offset); \
- uptr __size = (uptr)(size); \
- uptr __bad = 0; \
- if (__offset > __offset + __size) { \
- GET_STACK_TRACE_FATAL_HERE; \
- ReportStringFunctionSizeOverflow(__offset, __size, &stack); \
- } \
- if (!QuickCheckForUnpoisonedRegion(__offset, __size) && \
- (__bad = __asan_region_is_poisoned(__offset, __size))) { \
- AsanInterceptorContext *_ctx = (AsanInterceptorContext *)ctx; \
- bool suppressed = false; \
- if (_ctx) { \
- suppressed = IsInterceptorSuppressed(_ctx->interceptor_name); \
- if (!suppressed && HaveStackTraceBasedSuppressions()) { \
- GET_STACK_TRACE_FATAL_HERE; \
- suppressed = IsStackTraceSuppressed(&stack); \
- } \
- } \
- if (!suppressed) { \
- GET_CURRENT_PC_BP_SP; \
- ReportGenericError(pc, bp, sp, __bad, isWrite, __size, 0, false);\
- } \
- } \
- } while (0)
-
-// memcpy is called during __asan_init() from the internals of printf(...).
-// We do not treat memcpy with to==from as a bug.
-// See http://llvm.org/bugs/show_bug.cgi?id=11763.
-#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \
- do { \
- if (UNLIKELY(!asan_inited)) return internal_memcpy(to, from, size); \
- if (asan_init_is_running) { \
- return REAL(memcpy)(to, from, size); \
- } \
- ENSURE_ASAN_INITED(); \
- if (flags()->replace_intrin) { \
- if (to != from) { \
- CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \
- } \
- ASAN_READ_RANGE(ctx, from, size); \
- ASAN_WRITE_RANGE(ctx, to, size); \
- } \
- return REAL(memcpy)(to, from, size); \
- } while (0)
-
-// memset is called inside Printf.
-#define ASAN_MEMSET_IMPL(ctx, block, c, size) \
- do { \
- if (UNLIKELY(!asan_inited)) return internal_memset(block, c, size); \
- if (asan_init_is_running) { \
- return REAL(memset)(block, c, size); \
- } \
- ENSURE_ASAN_INITED(); \
- if (flags()->replace_intrin) { \
- ASAN_WRITE_RANGE(ctx, block, size); \
- } \
- return REAL(memset)(block, c, size); \
- } while (0)
-
-#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \
- do { \
- if (UNLIKELY(!asan_inited)) return internal_memmove(to, from, size); \
- ENSURE_ASAN_INITED(); \
- if (flags()->replace_intrin) { \
- ASAN_READ_RANGE(ctx, from, size); \
- ASAN_WRITE_RANGE(ctx, to, size); \
- } \
- return internal_memmove(to, from, size); \
- } while (0)
-
-#define ASAN_READ_RANGE(ctx, offset, size) \
- ACCESS_MEMORY_RANGE(ctx, offset, size, false)
-#define ASAN_WRITE_RANGE(ctx, offset, size) \
- ACCESS_MEMORY_RANGE(ctx, offset, size, true)
-
#define ASAN_READ_STRING_OF_LEN(ctx, s, len, n) \
ASAN_READ_RANGE((ctx), (s), \
common_flags()->strict_string_checks ? (len) + 1 : (n))
@@ -145,23 +48,6 @@ struct AsanInterceptorContext {
#define ASAN_READ_STRING(ctx, s, n) \
ASAN_READ_STRING_OF_LEN((ctx), (s), REAL(strlen)(s), (n))
-// Behavior of functions like "memcpy" or "strcpy" is undefined
-// if memory intervals overlap. We report error in this case.
-// Macro is used to avoid creation of new frames.
-static inline bool RangesOverlap(const char *offset1, uptr length1,
- const char *offset2, uptr length2) {
- return !((offset1 + length1 <= offset2) || (offset2 + length2 <= offset1));
-}
-#define CHECK_RANGES_OVERLAP(name, _offset1, length1, _offset2, length2) do { \
- const char *offset1 = (const char*)_offset1; \
- const char *offset2 = (const char*)_offset2; \
- if (RangesOverlap(offset1, length1, offset2, length2)) { \
- GET_STACK_TRACE_FATAL_HERE; \
- ReportStringFunctionMemoryRangesOverlap(name, offset1, length1, \
- offset2, length2, &stack); \
- } \
-} while (0)
-
static inline uptr MaybeRealStrnlen(const char *s, uptr maxlen) {
#if SANITIZER_INTERCEPT_STRNLEN
if (REAL(strnlen)) {
@@ -275,6 +161,7 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
} while (false)
#include "sanitizer_common/sanitizer_common_interceptors.inc"
+#include "sanitizer_common/sanitizer_signal_interceptors.inc"
// Syscall interceptors don't have contexts, we don't support suppressions
// for them.
@@ -356,42 +243,6 @@ INTERCEPTOR(int, pthread_join, void *t, void **arg) {
DEFINE_REAL_PTHREAD_FUNCTIONS
#endif // ASAN_INTERCEPT_PTHREAD_CREATE
-#if ASAN_INTERCEPT_SIGNAL_AND_SIGACTION
-
-#if SANITIZER_ANDROID
-INTERCEPTOR(void*, bsd_signal, int signum, void *handler) {
- if (GetHandleSignalMode(signum) != kHandleSignalExclusive)
- return REAL(bsd_signal)(signum, handler);
- return 0;
-}
-#endif
-
-INTERCEPTOR(void*, signal, int signum, void *handler) {
- if (GetHandleSignalMode(signum) != kHandleSignalExclusive)
- return REAL(signal)(signum, handler);
- return nullptr;
-}
-
-INTERCEPTOR(int, sigaction, int signum, const struct sigaction *act,
- struct sigaction *oldact) {
- if (GetHandleSignalMode(signum) != kHandleSignalExclusive)
- return REAL(sigaction)(signum, act, oldact);
- return 0;
-}
-
-namespace __sanitizer {
-int real_sigaction(int signum, const void *act, void *oldact) {
- return REAL(sigaction)(signum, (const struct sigaction *)act,
- (struct sigaction *)oldact);
-}
-} // namespace __sanitizer
-
-#elif SANITIZER_POSIX
-// We need to have defined REAL(sigaction) on posix systems.
-DEFINE_REAL(int, sigaction, int signum, const struct sigaction *act,
- struct sigaction *oldact)
-#endif // ASAN_INTERCEPT_SIGNAL_AND_SIGACTION
-
#if ASAN_INTERCEPT_SWAPCONTEXT
static void ClearShadowMemoryForContextStack(uptr stack, uptr ssize) {
// Align to page size.
@@ -428,6 +279,11 @@ INTERCEPTOR(int, swapcontext, struct ucontext_t *oucp,
}
#endif // ASAN_INTERCEPT_SWAPCONTEXT
+#if SANITIZER_NETBSD
+#define longjmp __longjmp14
+#define siglongjmp __siglongjmp14
+#endif
+
INTERCEPTOR(void, longjmp, void *env, int val) {
__asan_handle_no_return();
REAL(longjmp)(env, val);
@@ -462,18 +318,6 @@ INTERCEPTOR(void, __cxa_throw, void *a, void *b, void *c) {
}
#endif
-void *__asan_memcpy(void *to, const void *from, uptr size) {
- ASAN_MEMCPY_IMPL(nullptr, to, from, size);
-}
-
-void *__asan_memset(void *block, int c, uptr size) {
- ASAN_MEMSET_IMPL(nullptr, block, c, size);
-}
-
-void *__asan_memmove(void *to, const void *from, uptr size) {
- ASAN_MEMMOVE_IMPL(nullptr, to, from, size);
-}
-
#if ASAN_INTERCEPT_INDEX
# if ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX
INTERCEPTOR(char*, index, const char *string, int c)
@@ -711,6 +555,7 @@ void InitializeAsanInterceptors() {
CHECK(!was_called_once);
was_called_once = true;
InitializeCommonInterceptors();
+ InitializeSignalInterceptors();
// Intercept str* functions.
ASAN_INTERCEPT_FUNC(strcat); // NOLINT
@@ -733,15 +578,9 @@ void InitializeAsanInterceptors() {
ASAN_INTERCEPT_FUNC(strtoll);
#endif
- // Intecept signal- and jump-related functions.
+ // Intecept jump-related functions.
ASAN_INTERCEPT_FUNC(longjmp);
-#if ASAN_INTERCEPT_SIGNAL_AND_SIGACTION
- ASAN_INTERCEPT_FUNC(sigaction);
-#if SANITIZER_ANDROID
- ASAN_INTERCEPT_FUNC(bsd_signal);
-#endif
- ASAN_INTERCEPT_FUNC(signal);
-#endif
+
#if ASAN_INTERCEPT_SWAPCONTEXT
ASAN_INTERCEPT_FUNC(swapcontext);
#endif
@@ -785,3 +624,5 @@ void InitializeAsanInterceptors() {
}
} // namespace __asan
+
+#endif // !SANITIZER_FUCHSIA
diff --git a/contrib/compiler-rt/lib/asan/asan_interceptors.h b/contrib/compiler-rt/lib/asan/asan_interceptors.h
index 93fca4f67366..e13bdecfdede 100644
--- a/contrib/compiler-rt/lib/asan/asan_interceptors.h
+++ b/contrib/compiler-rt/lib/asan/asan_interceptors.h
@@ -15,9 +15,30 @@
#define ASAN_INTERCEPTORS_H
#include "asan_internal.h"
+#include "asan_interceptors_memintrinsics.h"
#include "interception/interception.h"
#include "sanitizer_common/sanitizer_platform_interceptors.h"
+namespace __asan {
+
+void InitializeAsanInterceptors();
+void InitializePlatformInterceptors();
+
+#define ENSURE_ASAN_INITED() \
+ do { \
+ CHECK(!asan_init_is_running); \
+ if (UNLIKELY(!asan_inited)) { \
+ AsanInitFromRtl(); \
+ } \
+ } while (0)
+
+} // namespace __asan
+
+// There is no general interception at all on Fuchsia.
+// Only the functions in asan_interceptors_memintrinsics.h are
+// really defined to replace libc functions.
+#if !SANITIZER_FUCHSIA
+
// Use macro to describe if specific function should be
// intercepted on a given platform.
#if !SANITIZER_WINDOWS
@@ -34,25 +55,20 @@
# define ASAN_INTERCEPT_FORK 0
#endif
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS
# define ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX 1
#else
# define ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX 0
#endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_SOLARIS
# define ASAN_INTERCEPT_SWAPCONTEXT 1
#else
# define ASAN_INTERCEPT_SWAPCONTEXT 0
#endif
#if !SANITIZER_WINDOWS
-# define ASAN_INTERCEPT_SIGNAL_AND_SIGACTION 1
-#else
-# define ASAN_INTERCEPT_SIGNAL_AND_SIGACTION 0
-#endif
-
-#if !SANITIZER_WINDOWS
# define ASAN_INTERCEPT_SIGLONGJMP 1
#else
# define ASAN_INTERCEPT_SIGLONGJMP 0
@@ -66,7 +82,8 @@
// Android bug: https://code.google.com/p/android/issues/detail?id=61799
#if ASAN_HAS_EXCEPTIONS && !SANITIZER_WINDOWS && \
- !(SANITIZER_ANDROID && defined(__i386))
+ !(SANITIZER_ANDROID && defined(__i386)) && \
+ !SANITIZER_SOLARIS
# define ASAN_INTERCEPT___CXA_THROW 1
#else
# define ASAN_INTERCEPT___CXA_THROW 0
@@ -85,16 +102,11 @@
#endif
DECLARE_REAL(int, memcmp, const void *a1, const void *a2, uptr size)
-DECLARE_REAL(void*, memcpy, void *to, const void *from, uptr size)
-DECLARE_REAL(void*, memset, void *block, int c, uptr size)
DECLARE_REAL(char*, strchr, const char *str, int c)
DECLARE_REAL(SIZE_T, strlen, const char *s)
DECLARE_REAL(char*, strncpy, char *to, const char *from, uptr size)
DECLARE_REAL(uptr, strnlen, const char *s, uptr maxlen)
DECLARE_REAL(char*, strstr, const char *s1, const char *s2)
-struct sigaction;
-DECLARE_REAL(int, sigaction, int signum, const struct sigaction *act,
- struct sigaction *oldact)
#if !SANITIZER_MAC
#define ASAN_INTERCEPT_FUNC(name) \
@@ -113,18 +125,6 @@ DECLARE_REAL(int, sigaction, int signum, const struct sigaction *act,
#define ASAN_INTERCEPT_FUNC(name)
#endif // SANITIZER_MAC
-namespace __asan {
-
-void InitializeAsanInterceptors();
-void InitializePlatformInterceptors();
-
-#define ENSURE_ASAN_INITED() do { \
- CHECK(!asan_init_is_running); \
- if (UNLIKELY(!asan_inited)) { \
- AsanInitFromRtl(); \
- } \
-} while (0)
-
-} // namespace __asan
+#endif // !SANITIZER_FUCHSIA
#endif // ASAN_INTERCEPTORS_H
diff --git a/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cc b/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cc
new file mode 100644
index 000000000000..c89cb011492e
--- /dev/null
+++ b/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cc
@@ -0,0 +1,44 @@
+//===-- asan_interceptors_memintrinsics.cc --------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// ASan versions of memcpy, memmove, and memset.
+//===---------------------------------------------------------------------===//
+
+#include "asan_interceptors_memintrinsics.h"
+#include "asan_report.h"
+#include "asan_stack.h"
+#include "asan_suppressions.h"
+
+using namespace __asan; // NOLINT
+
+void *__asan_memcpy(void *to, const void *from, uptr size) {
+ ASAN_MEMCPY_IMPL(nullptr, to, from, size);
+}
+
+void *__asan_memset(void *block, int c, uptr size) {
+ ASAN_MEMSET_IMPL(nullptr, block, c, size);
+}
+
+void *__asan_memmove(void *to, const void *from, uptr size) {
+ ASAN_MEMMOVE_IMPL(nullptr, to, from, size);
+}
+
+#if SANITIZER_FUCHSIA
+
+// Fuchsia doesn't use sanitizer_common_interceptors.inc, but the only
+// things there it wants are these three. Just define them as aliases
+// here rather than repeating the contents.
+
+decltype(memcpy) memcpy[[gnu::alias("__asan_memcpy")]];
+decltype(memmove) memmove[[gnu::alias("__asan_memmove")]];
+decltype(memset) memset[[gnu::alias("__asan_memset")]];
+
+#endif // SANITIZER_FUCHSIA
diff --git a/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h b/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h
new file mode 100644
index 000000000000..5a8339a23e97
--- /dev/null
+++ b/contrib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h
@@ -0,0 +1,148 @@
+//===-- asan_interceptors_memintrinsics.h -----------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// ASan-private header for asan_memintrin.cc
+//===---------------------------------------------------------------------===//
+#ifndef ASAN_MEMINTRIN_H
+#define ASAN_MEMINTRIN_H
+
+#include "asan_interface_internal.h"
+#include "asan_internal.h"
+#include "asan_mapping.h"
+#include "interception/interception.h"
+
+DECLARE_REAL(void*, memcpy, void *to, const void *from, uptr size)
+DECLARE_REAL(void*, memset, void *block, int c, uptr size)
+
+namespace __asan {
+
+// Return true if we can quickly decide that the region is unpoisoned.
+// We assume that a redzone is at least 16 bytes.
+static inline bool QuickCheckForUnpoisonedRegion(uptr beg, uptr size) {
+ if (size == 0) return true;
+ if (size <= 32)
+ return !AddressIsPoisoned(beg) &&
+ !AddressIsPoisoned(beg + size - 1) &&
+ !AddressIsPoisoned(beg + size / 2);
+ if (size <= 64)
+ return !AddressIsPoisoned(beg) &&
+ !AddressIsPoisoned(beg + size / 4) &&
+ !AddressIsPoisoned(beg + size - 1) &&
+ !AddressIsPoisoned(beg + 3 * size / 4) &&
+ !AddressIsPoisoned(beg + size / 2);
+ return false;
+}
+
+struct AsanInterceptorContext {
+ const char *interceptor_name;
+};
+
+// We implement ACCESS_MEMORY_RANGE, ASAN_READ_RANGE,
+// and ASAN_WRITE_RANGE as macro instead of function so
+// that no extra frames are created, and stack trace contains
+// relevant information only.
+// We check all shadow bytes.
+#define ACCESS_MEMORY_RANGE(ctx, offset, size, isWrite) do { \
+ uptr __offset = (uptr)(offset); \
+ uptr __size = (uptr)(size); \
+ uptr __bad = 0; \
+ if (__offset > __offset + __size) { \
+ GET_STACK_TRACE_FATAL_HERE; \
+ ReportStringFunctionSizeOverflow(__offset, __size, &stack); \
+ } \
+ if (!QuickCheckForUnpoisonedRegion(__offset, __size) && \
+ (__bad = __asan_region_is_poisoned(__offset, __size))) { \
+ AsanInterceptorContext *_ctx = (AsanInterceptorContext *)ctx; \
+ bool suppressed = false; \
+ if (_ctx) { \
+ suppressed = IsInterceptorSuppressed(_ctx->interceptor_name); \
+ if (!suppressed && HaveStackTraceBasedSuppressions()) { \
+ GET_STACK_TRACE_FATAL_HERE; \
+ suppressed = IsStackTraceSuppressed(&stack); \
+ } \
+ } \
+ if (!suppressed) { \
+ GET_CURRENT_PC_BP_SP; \
+ ReportGenericError(pc, bp, sp, __bad, isWrite, __size, 0, false);\
+ } \
+ } \
+ } while (0)
+
+// memcpy is called during __asan_init() from the internals of printf(...).
+// We do not treat memcpy with to==from as a bug.
+// See http://llvm.org/bugs/show_bug.cgi?id=11763.
+#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \
+ do { \
+ if (UNLIKELY(!asan_inited)) return internal_memcpy(to, from, size); \
+ if (asan_init_is_running) { \
+ return REAL(memcpy)(to, from, size); \
+ } \
+ ENSURE_ASAN_INITED(); \
+ if (flags()->replace_intrin) { \
+ if (to != from) { \
+ CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \
+ } \
+ ASAN_READ_RANGE(ctx, from, size); \
+ ASAN_WRITE_RANGE(ctx, to, size); \
+ } \
+ return REAL(memcpy)(to, from, size); \
+ } while (0)
+
+// memset is called inside Printf.
+#define ASAN_MEMSET_IMPL(ctx, block, c, size) \
+ do { \
+ if (UNLIKELY(!asan_inited)) return internal_memset(block, c, size); \
+ if (asan_init_is_running) { \
+ return REAL(memset)(block, c, size); \
+ } \
+ ENSURE_ASAN_INITED(); \
+ if (flags()->replace_intrin) { \
+ ASAN_WRITE_RANGE(ctx, block, size); \
+ } \
+ return REAL(memset)(block, c, size); \
+ } while (0)
+
+#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \
+ do { \
+ if (UNLIKELY(!asan_inited)) return internal_memmove(to, from, size); \
+ ENSURE_ASAN_INITED(); \
+ if (flags()->replace_intrin) { \
+ ASAN_READ_RANGE(ctx, from, size); \
+ ASAN_WRITE_RANGE(ctx, to, size); \
+ } \
+ return internal_memmove(to, from, size); \
+ } while (0)
+
+#define ASAN_READ_RANGE(ctx, offset, size) \
+ ACCESS_MEMORY_RANGE(ctx, offset, size, false)
+#define ASAN_WRITE_RANGE(ctx, offset, size) \
+ ACCESS_MEMORY_RANGE(ctx, offset, size, true)
+
+// Behavior of functions like "memcpy" or "strcpy" is undefined
+// if memory intervals overlap. We report error in this case.
+// Macro is used to avoid creation of new frames.
+static inline bool RangesOverlap(const char *offset1, uptr length1,
+ const char *offset2, uptr length2) {
+ return !((offset1 + length1 <= offset2) || (offset2 + length2 <= offset1));
+}
+#define CHECK_RANGES_OVERLAP(name, _offset1, length1, _offset2, length2) do { \
+ const char *offset1 = (const char*)_offset1; \
+ const char *offset2 = (const char*)_offset2; \
+ if (RangesOverlap(offset1, length1, offset2, length2)) { \
+ GET_STACK_TRACE_FATAL_HERE; \
+ ReportStringFunctionMemoryRangesOverlap(name, offset1, length1, \
+ offset2, length2, &stack); \
+ } \
+} while (0)
+
+} // namespace __asan
+
+#endif // ASAN_MEMINTRIN_H
diff --git a/contrib/compiler-rt/lib/asan/asan_internal.h b/contrib/compiler-rt/lib/asan/asan_internal.h
index f09bbd83af25..19133e5291a9 100644
--- a/contrib/compiler-rt/lib/asan/asan_internal.h
+++ b/contrib/compiler-rt/lib/asan/asan_internal.h
@@ -69,8 +69,12 @@ void InitializePlatformExceptionHandlers();
bool IsSystemHeapAddress(uptr addr);
// asan_rtl.cc
+void PrintAddressSpaceLayout();
void NORETURN ShowStatsAndAbort();
+// asan_shadow_setup.cc
+void InitializeShadowMemory();
+
// asan_malloc_linux.cc / asan_malloc_mac.cc
void ReplaceSystemMalloc();
@@ -80,6 +84,9 @@ void *AsanDoesNotSupportStaticLinkage();
void AsanCheckDynamicRTPrereqs();
void AsanCheckIncompatibleRT();
+// asan_thread.cc
+AsanThread *CreateMainThread();
+
// Support function for __asan_(un)register_image_globals. Searches for the
// loaded image containing `needle' and then enumerates all global metadata
// structures declared in that image, applying `op' (e.g.,
diff --git a/contrib/compiler-rt/lib/asan/asan_linux.cc b/contrib/compiler-rt/lib/asan/asan_linux.cc
index 6d47ba432a61..047e1dbb72fa 100644
--- a/contrib/compiler-rt/lib/asan/asan_linux.cc
+++ b/contrib/compiler-rt/lib/asan/asan_linux.cc
@@ -13,10 +13,12 @@
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS
#include "asan_interceptors.h"
#include "asan_internal.h"
+#include "asan_premap_shadow.h"
#include "asan_thread.h"
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_freebsd.h"
@@ -39,9 +41,17 @@
#include <sys/link_elf.h>
#endif
-#if SANITIZER_ANDROID || SANITIZER_FREEBSD
+#if SANITIZER_SOLARIS
+#include <link.h>
+#endif
+
+#if SANITIZER_ANDROID || SANITIZER_FREEBSD || SANITIZER_SOLARIS
#include <ucontext.h>
extern "C" void* _DYNAMIC;
+#elif SANITIZER_NETBSD
+#include <link_elf.h>
+#include <ucontext.h>
+extern Elf_Dyn _DYNAMIC;
#else
#include <sys/ucontext.h>
#include <link.h>
@@ -77,9 +87,51 @@ void *AsanDoesNotSupportStaticLinkage() {
return &_DYNAMIC; // defined in link.h
}
+static void UnmapFromTo(uptr from, uptr to) {
+ CHECK(to >= from);
+ if (to == from) return;
+ uptr res = internal_munmap(reinterpret_cast<void *>(from), to - from);
+ if (UNLIKELY(internal_iserror(res))) {
+ Report(
+ "ERROR: AddresSanitizer failed to unmap 0x%zx (%zd) bytes at address "
+ "%p\n",
+ to - from, to - from, from);
+ CHECK("unable to unmap" && 0);
+ }
+}
+
+#if ASAN_PREMAP_SHADOW
+uptr FindPremappedShadowStart() {
+ uptr granularity = GetMmapGranularity();
+ uptr shadow_start = reinterpret_cast<uptr>(&__asan_shadow);
+ uptr premap_shadow_size = PremapShadowSize();
+ uptr shadow_size = RoundUpTo(kHighShadowEnd, granularity);
+ // We may have mapped too much. Release extra memory.
+ UnmapFromTo(shadow_start + shadow_size, shadow_start + premap_shadow_size);
+ return shadow_start;
+}
+#endif
+
uptr FindDynamicShadowStart() {
- UNREACHABLE("FindDynamicShadowStart is not available");
- return 0;
+#if ASAN_PREMAP_SHADOW
+ if (!PremapShadowFailed())
+ return FindPremappedShadowStart();
+#endif
+
+ uptr granularity = GetMmapGranularity();
+ uptr alignment = granularity * 8;
+ uptr left_padding = granularity;
+ uptr shadow_size = RoundUpTo(kHighShadowEnd, granularity);
+ uptr map_size = shadow_size + left_padding + alignment;
+
+ uptr map_start = (uptr)MmapNoAccess(map_size);
+ CHECK_NE(map_start, ~(uptr)0);
+
+ uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
+ UnmapFromTo(map_start, shadow_start - left_padding);
+ UnmapFromTo(shadow_start + shadow_size, map_start + map_size);
+
+ return shadow_start;
}
void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
@@ -93,6 +145,9 @@ void AsanCheckIncompatibleRT() {}
#else
static int FindFirstDSOCallback(struct dl_phdr_info *info, size_t size,
void *data) {
+ VReport(2, "info->dlpi_name = %s\tinfo->dlpi_addr = %p\n",
+ info->dlpi_name, info->dlpi_addr);
+
// Continue until the first dynamic library is found
if (!info->dlpi_name || info->dlpi_name[0] == 0)
return 0;
@@ -101,6 +156,21 @@ static int FindFirstDSOCallback(struct dl_phdr_info *info, size_t size,
if (internal_strncmp(info->dlpi_name, "linux-", sizeof("linux-") - 1) == 0)
return 0;
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
+ // Ignore first entry (the main program)
+ char **p = (char **)data;
+ if (!(*p)) {
+ *p = (char *)-1;
+ return 0;
+ }
+#endif
+
+#if SANITIZER_SOLARIS
+ // Ignore executable on Solaris
+ if (info->dlpi_addr == 0)
+ return 0;
+#endif
+
*(const char **)data = info->dlpi_name;
return 1;
}
@@ -179,4 +249,5 @@ void *AsanDlSymNext(const char *sym) {
} // namespace __asan
-#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD ||
+ // SANITIZER_SOLARIS
diff --git a/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc b/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc
index fd40f47db1c4..6697ff8764ed 100644
--- a/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc
+++ b/contrib/compiler-rt/lib/asan/asan_malloc_linux.cc
@@ -15,7 +15,8 @@
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_FUCHSIA || SANITIZER_LINUX || \
+ SANITIZER_NETBSD || SANITIZER_SOLARIS
#include "sanitizer_common/sanitizer_tls_get_addr.h"
#include "asan_allocator.h"
@@ -30,9 +31,9 @@ static uptr allocated_for_dlsym;
static const uptr kDlsymAllocPoolSize = 1024;
static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-static bool IsInDlsymAllocPool(const void *ptr) {
+static INLINE bool IsInDlsymAllocPool(const void *ptr) {
uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- return off < sizeof(alloc_memory_for_dlsym);
+ return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]);
}
static void *AllocateFromLocalPool(uptr size_in_bytes) {
@@ -43,6 +44,26 @@ static void *AllocateFromLocalPool(uptr size_in_bytes) {
return mem;
}
+static INLINE bool MaybeInDlsym() {
+ // Fuchsia doesn't use dlsym-based interceptors.
+ return !SANITIZER_FUCHSIA && asan_init_is_running;
+}
+
+static void *ReallocFromLocalPool(void *ptr, uptr size) {
+ const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+ const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
+ void *new_ptr;
+ if (UNLIKELY(MaybeInDlsym())) {
+ new_ptr = AllocateFromLocalPool(size);
+ } else {
+ ENSURE_ASAN_INITED();
+ GET_STACK_TRACE_MALLOC;
+ new_ptr = asan_malloc(size, &stack);
+ }
+ internal_memcpy(new_ptr, ptr, copy_size);
+ return new_ptr;
+}
+
INTERCEPTOR(void, free, void *ptr) {
GET_STACK_TRACE_FREE;
if (UNLIKELY(IsInDlsymAllocPool(ptr)))
@@ -60,7 +81,7 @@ INTERCEPTOR(void, cfree, void *ptr) {
#endif // SANITIZER_INTERCEPT_CFREE
INTERCEPTOR(void*, malloc, uptr size) {
- if (UNLIKELY(asan_init_is_running))
+ if (UNLIKELY(MaybeInDlsym()))
// Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
return AllocateFromLocalPool(size);
ENSURE_ASAN_INITED();
@@ -69,7 +90,7 @@ INTERCEPTOR(void*, malloc, uptr size) {
}
INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
- if (UNLIKELY(asan_init_is_running))
+ if (UNLIKELY(MaybeInDlsym()))
// Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
return AllocateFromLocalPool(nmemb * size);
ENSURE_ASAN_INITED();
@@ -78,21 +99,9 @@ INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
}
INTERCEPTOR(void*, realloc, void *ptr, uptr size) {
- if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
- const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
- void *new_ptr;
- if (UNLIKELY(asan_init_is_running)) {
- new_ptr = AllocateFromLocalPool(size);
- } else {
- ENSURE_ASAN_INITED();
- GET_STACK_TRACE_MALLOC;
- new_ptr = asan_malloc(size, &stack);
- }
- internal_memcpy(new_ptr, ptr, copy_size);
- return new_ptr;
- }
- if (UNLIKELY(asan_init_is_running))
+ if (UNLIKELY(IsInDlsymAllocPool(ptr)))
+ return ReallocFromLocalPool(ptr, size);
+ if (UNLIKELY(MaybeInDlsym()))
return AllocateFromLocalPool(size);
ENSURE_ASAN_INITED();
GET_STACK_TRACE_MALLOC;
@@ -226,4 +235,5 @@ void ReplaceSystemMalloc() {
} // namespace __asan
#endif // SANITIZER_ANDROID
-#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif // SANITIZER_FREEBSD || SANITIZER_FUCHSIA || SANITIZER_LINUX ||
+ // SANITIZER_NETBSD || SANITIZER_SOLARIS
diff --git a/contrib/compiler-rt/lib/asan/asan_mapping.h b/contrib/compiler-rt/lib/asan/asan_mapping.h
index 695740cd982f..d3f360ffc969 100644
--- a/contrib/compiler-rt/lib/asan/asan_mapping.h
+++ b/contrib/compiler-rt/lib/asan/asan_mapping.h
@@ -115,6 +115,13 @@
// || `[0x40000000, 0x47ffffff]` || LowShadow ||
// || `[0x00000000, 0x3fffffff]` || LowMem ||
//
+// Shadow mapping on NetBSD/x86-64 with SHADOW_OFFSET == 0x400000000000:
+// || `[0x4feffffffe01, 0x7f7ffffff000]` || HighMem ||
+// || `[0x49fdffffffc0, 0x4feffffffe00]` || HighShadow ||
+// || `[0x480000000000, 0x49fdffffffbf]` || ShadowGap ||
+// || `[0x400000000000, 0x47ffffffffff]` || LowShadow ||
+// || `[0x000000000000, 0x3fffffffffff]` || LowMem ||
+//
// Default Windows/i386 mapping:
// (the exact location of HighShadow/HighMem may vary depending
// on WoW64, /LARGEADDRESSAWARE, etc).
@@ -124,11 +131,16 @@
// || `[0x30000000, 0x35ffffff]` || LowShadow ||
// || `[0x00000000, 0x2fffffff]` || LowMem ||
+#if defined(ASAN_SHADOW_SCALE)
+static const u64 kDefaultShadowScale = ASAN_SHADOW_SCALE;
+#else
static const u64 kDefaultShadowScale = 3;
+#endif
static const u64 kDefaultShadowSentinel = ~(uptr)0;
static const u64 kDefaultShadowOffset32 = 1ULL << 29; // 0x20000000
static const u64 kDefaultShadowOffset64 = 1ULL << 44;
-static const u64 kDefaultShort64bitShadowOffset = 0x7FFF8000; // < 2G.
+static const u64 kDefaultShort64bitShadowOffset =
+ 0x7FFFFFFF & (~0xFFFULL << kDefaultShadowScale); // < 2G.
static const u64 kIosShadowOffset32 = 1ULL << 30; // 0x40000000
static const u64 kIosShadowOffset64 = 0x120200000;
static const u64 kIosSimShadowOffset32 = 1ULL << 30;
@@ -136,18 +148,20 @@ static const u64 kIosSimShadowOffset64 = kDefaultShadowOffset64;
static const u64 kAArch64_ShadowOffset64 = 1ULL << 36;
static const u64 kMIPS32_ShadowOffset32 = 0x0aaa0000;
static const u64 kMIPS64_ShadowOffset64 = 1ULL << 37;
-static const u64 kPPC64_ShadowOffset64 = 1ULL << 41;
+static const u64 kPPC64_ShadowOffset64 = 1ULL << 44;
static const u64 kSystemZ_ShadowOffset64 = 1ULL << 52;
static const u64 kFreeBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000
static const u64 kFreeBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000
+static const u64 kNetBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000
static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000
#define SHADOW_SCALE kDefaultShadowScale
-
-#if SANITIZER_WORDSIZE == 32
+#if SANITIZER_FUCHSIA
+# define SHADOW_OFFSET (0)
+#elif SANITIZER_WORDSIZE == 32
# if SANITIZER_ANDROID
-# define SHADOW_OFFSET (0)
+# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
# elif defined(__mips__)
# define SHADOW_OFFSET kMIPS32_ShadowOffset32
# elif SANITIZER_FREEBSD
@@ -178,6 +192,8 @@ static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000
# define SHADOW_OFFSET kSystemZ_ShadowOffset64
# elif SANITIZER_FREEBSD
# define SHADOW_OFFSET kFreeBSD_ShadowOffset64
+# elif SANITIZER_NETBSD
+# define SHADOW_OFFSET kNetBSD_ShadowOffset64
# elif SANITIZER_MAC
# define SHADOW_OFFSET kDefaultShadowOffset64
# elif defined(__mips64)
@@ -189,6 +205,12 @@ static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000
# endif
#endif
+#if SANITIZER_ANDROID && defined(__arm__)
+# define ASAN_PREMAP_SHADOW 1
+#else
+# define ASAN_PREMAP_SHADOW 0
+#endif
+
#define SHADOW_GRANULARITY (1ULL << SHADOW_SCALE)
#define MEM_TO_SHADOW(mem) (((mem) >> SHADOW_SCALE) + (SHADOW_OFFSET))
diff --git a/contrib/compiler-rt/lib/asan/asan_memory_profile.cc b/contrib/compiler-rt/lib/asan/asan_memory_profile.cc
index 05846c37cb6d..603284c8c688 100644
--- a/contrib/compiler-rt/lib/asan/asan_memory_profile.cc
+++ b/contrib/compiler-rt/lib/asan/asan_memory_profile.cc
@@ -107,6 +107,9 @@ static void MemoryProfileCB(const SuspendedThreadsList &suspended_threads_list,
__lsan::ForEachChunk(ChunkCallback, &hp);
uptr *Arg = reinterpret_cast<uptr*>(argument);
hp.Print(Arg[0], Arg[1]);
+
+ if (Verbosity())
+ __asan_print_accumulated_stats();
}
} // namespace __asan
diff --git a/contrib/compiler-rt/lib/asan/asan_new_delete.cc b/contrib/compiler-rt/lib/asan/asan_new_delete.cc
index e68c7f3e2400..072f027addd6 100644
--- a/contrib/compiler-rt/lib/asan/asan_new_delete.cc
+++ b/contrib/compiler-rt/lib/asan/asan_new_delete.cc
@@ -125,77 +125,69 @@ INTERCEPTOR(void *, _ZnwmRKSt9nothrow_t, size_t size, std::nothrow_t const&) {
INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&) {
OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/);
}
-#endif
+#endif // !SANITIZER_MAC
#define OPERATOR_DELETE_BODY(type) \
GET_STACK_TRACE_FREE;\
- asan_free(ptr, &stack, type);
+ asan_delete(ptr, 0, 0, &stack, type);
+
+#define OPERATOR_DELETE_BODY_SIZE(type) \
+ GET_STACK_TRACE_FREE;\
+ asan_delete(ptr, size, 0, &stack, type);
+
+#define OPERATOR_DELETE_BODY_ALIGN(type) \
+ GET_STACK_TRACE_FREE;\
+ asan_delete(ptr, 0, static_cast<uptr>(align), &stack, type);
+
+#define OPERATOR_DELETE_BODY_SIZE_ALIGN(type) \
+ GET_STACK_TRACE_FREE;\
+ asan_delete(ptr, size, static_cast<uptr>(align), &stack, type);
#if !SANITIZER_MAC
CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr) NOEXCEPT {
- OPERATOR_DELETE_BODY(FROM_NEW);
-}
+void operator delete(void *ptr) NOEXCEPT
+{ OPERATOR_DELETE_BODY(FROM_NEW); }
CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr) NOEXCEPT {
- OPERATOR_DELETE_BODY(FROM_NEW_BR);
-}
+void operator delete[](void *ptr) NOEXCEPT
+{ OPERATOR_DELETE_BODY(FROM_NEW_BR); }
CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, std::nothrow_t const&) {
- OPERATOR_DELETE_BODY(FROM_NEW);
-}
+void operator delete(void *ptr, std::nothrow_t const&)
+{ OPERATOR_DELETE_BODY(FROM_NEW); }
CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, std::nothrow_t const&) {
- OPERATOR_DELETE_BODY(FROM_NEW_BR);
-}
+void operator delete[](void *ptr, std::nothrow_t const&)
+{ OPERATOR_DELETE_BODY(FROM_NEW_BR); }
CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, size_t size) NOEXCEPT {
- GET_STACK_TRACE_FREE;
- asan_sized_free(ptr, size, &stack, FROM_NEW);
-}
+void operator delete(void *ptr, size_t size) NOEXCEPT
+{ OPERATOR_DELETE_BODY_SIZE(FROM_NEW); }
CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, size_t size) NOEXCEPT {
- GET_STACK_TRACE_FREE;
- asan_sized_free(ptr, size, &stack, FROM_NEW_BR);
-}
+void operator delete[](void *ptr, size_t size) NOEXCEPT
+{ OPERATOR_DELETE_BODY_SIZE(FROM_NEW_BR); }
CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, std::align_val_t) NOEXCEPT {
- OPERATOR_DELETE_BODY(FROM_NEW);
-}
+void operator delete(void *ptr, std::align_val_t align) NOEXCEPT
+{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW); }
CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, std::align_val_t) NOEXCEPT {
- OPERATOR_DELETE_BODY(FROM_NEW_BR);
-}
+void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT
+{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR); }
CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, std::align_val_t, std::nothrow_t const&) {
- OPERATOR_DELETE_BODY(FROM_NEW);
-}
+void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&)
+{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW); }
CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, std::align_val_t, std::nothrow_t const&) {
- OPERATOR_DELETE_BODY(FROM_NEW_BR);
-}
+void operator delete[](void *ptr, std::align_val_t align, std::nothrow_t const&)
+{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR); }
CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, size_t size, std::align_val_t) NOEXCEPT {
- GET_STACK_TRACE_FREE;
- asan_sized_free(ptr, size, &stack, FROM_NEW);
-}
+void operator delete(void *ptr, size_t size, std::align_val_t align) NOEXCEPT
+{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW); }
CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, size_t size, std::align_val_t) NOEXCEPT {
- GET_STACK_TRACE_FREE;
- asan_sized_free(ptr, size, &stack, FROM_NEW_BR);
-}
+void operator delete[](void *ptr, size_t size, std::align_val_t align) NOEXCEPT
+{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW_BR); }
#else // SANITIZER_MAC
-INTERCEPTOR(void, _ZdlPv, void *ptr) {
- OPERATOR_DELETE_BODY(FROM_NEW);
-}
-INTERCEPTOR(void, _ZdaPv, void *ptr) {
- OPERATOR_DELETE_BODY(FROM_NEW_BR);
-}
-INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) {
- OPERATOR_DELETE_BODY(FROM_NEW);
-}
-INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) {
- OPERATOR_DELETE_BODY(FROM_NEW_BR);
-}
-#endif
+INTERCEPTOR(void, _ZdlPv, void *ptr)
+{ OPERATOR_DELETE_BODY(FROM_NEW); }
+INTERCEPTOR(void, _ZdaPv, void *ptr)
+{ OPERATOR_DELETE_BODY(FROM_NEW_BR); }
+INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&)
+{ OPERATOR_DELETE_BODY(FROM_NEW); }
+INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&)
+{ OPERATOR_DELETE_BODY(FROM_NEW_BR); }
+#endif // !SANITIZER_MAC
diff --git a/contrib/compiler-rt/lib/asan/asan_poisoning.cc b/contrib/compiler-rt/lib/asan/asan_poisoning.cc
index abb75ab3bf92..c3a82aa0049e 100644
--- a/contrib/compiler-rt/lib/asan/asan_poisoning.cc
+++ b/contrib/compiler-rt/lib/asan/asan_poisoning.cc
@@ -217,7 +217,7 @@ uptr __asan_region_is_poisoned(uptr beg, uptr size) {
uptr __bad = __asan_region_is_poisoned(__p, __size); \
__asan_report_error(pc, bp, sp, __bad, isWrite, __size, 0);\
} \
- } while (false); \
+ } while (false)
extern "C" SANITIZER_INTERFACE_ATTRIBUTE
diff --git a/contrib/compiler-rt/lib/asan/asan_poisoning.h b/contrib/compiler-rt/lib/asan/asan_poisoning.h
index cc3281e08a71..1e00070bcf63 100644
--- a/contrib/compiler-rt/lib/asan/asan_poisoning.h
+++ b/contrib/compiler-rt/lib/asan/asan_poisoning.h
@@ -46,8 +46,11 @@ ALWAYS_INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size,
// for mapping shadow and zeroing out pages doesn't "just work", so we should
// probably provide higher-level interface for these operations.
// For now, just memset on Windows.
- if (value ||
- SANITIZER_WINDOWS == 1 ||
+ if (value || SANITIZER_WINDOWS == 1 ||
+ // TODO(mcgrathr): Fuchsia doesn't allow the shadow mapping to be
+ // changed at all. It doesn't currently have an efficient means
+ // to zero a bunch of pages, but maybe we should add one.
+ SANITIZER_FUCHSIA == 1 ||
shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) {
REAL(memset)((void*)shadow_beg, value, shadow_end - shadow_beg);
} else {
diff --git a/contrib/compiler-rt/lib/asan/asan_posix.cc b/contrib/compiler-rt/lib/asan/asan_posix.cc
index added746ace8..17c28b0aea2a 100644
--- a/contrib/compiler-rt/lib/asan/asan_posix.cc
+++ b/contrib/compiler-rt/lib/asan/asan_posix.cc
@@ -25,7 +25,6 @@
#include "sanitizer_common/sanitizer_procmaps.h"
#include <pthread.h>
-#include <signal.h>
#include <stdlib.h>
#include <sys/time.h>
#include <sys/resource.h>
@@ -34,58 +33,9 @@
namespace __asan {
void AsanOnDeadlySignal(int signo, void *siginfo, void *context) {
- ScopedDeadlySignal signal_scope(GetCurrentThread());
- int code = (int)((siginfo_t*)siginfo)->si_code;
- // Write the first message using fd=2, just in case.
- // It may actually fail to write in case stderr is closed.
- internal_write(2, "ASAN:DEADLYSIGNAL\n", 18);
- SignalContext sig = SignalContext::Create(siginfo, context);
-
- // Access at a reasonable offset above SP, or slightly below it (to account
- // for x86_64 or PowerPC redzone, ARM push of multiple registers, etc) is
- // probably a stack overflow.
-#ifdef __s390__
- // On s390, the fault address in siginfo points to start of the page, not
- // to the precise word that was accessed. Mask off the low bits of sp to
- // take it into account.
- bool IsStackAccess = sig.addr >= (sig.sp & ~0xFFF) &&
- sig.addr < sig.sp + 0xFFFF;
-#else
- bool IsStackAccess = sig.addr + 512 > sig.sp && sig.addr < sig.sp + 0xFFFF;
-#endif
-
-#if __powerpc__
- // Large stack frames can be allocated with e.g.
- // lis r0,-10000
- // stdux r1,r1,r0 # store sp to [sp-10000] and update sp by -10000
- // If the store faults then sp will not have been updated, so test above
- // will not work, because the fault address will be more than just "slightly"
- // below sp.
- if (!IsStackAccess && IsAccessibleMemoryRange(sig.pc, 4)) {
- u32 inst = *(unsigned *)sig.pc;
- u32 ra = (inst >> 16) & 0x1F;
- u32 opcd = inst >> 26;
- u32 xo = (inst >> 1) & 0x3FF;
- // Check for store-with-update to sp. The instructions we accept are:
- // stbu rs,d(ra) stbux rs,ra,rb
- // sthu rs,d(ra) sthux rs,ra,rb
- // stwu rs,d(ra) stwux rs,ra,rb
- // stdu rs,ds(ra) stdux rs,ra,rb
- // where ra is r1 (the stack pointer).
- if (ra == 1 &&
- (opcd == 39 || opcd == 45 || opcd == 37 || opcd == 62 ||
- (opcd == 31 && (xo == 247 || xo == 439 || xo == 183 || xo == 181))))
- IsStackAccess = true;
- }
-#endif // __powerpc__
-
- // We also check si_code to filter out SEGV caused by something else other
- // then hitting the guard page or unmapped memory, like, for example,
- // unaligned memory access.
- if (IsStackAccess && (code == si_SEGV_MAPERR || code == si_SEGV_ACCERR))
- ReportStackOverflow(sig);
- else
- ReportDeadlySignal(signo, sig);
+ StartReportDeadlySignal();
+ SignalContext sig(siginfo, context);
+ ReportDeadlySignal(sig);
}
// ---------------------- TSD ---------------- {{{1
diff --git a/contrib/compiler-rt/lib/asan/asan_premap_shadow.cc b/contrib/compiler-rt/lib/asan/asan_premap_shadow.cc
new file mode 100644
index 000000000000..229eba99fe0e
--- /dev/null
+++ b/contrib/compiler-rt/lib/asan/asan_premap_shadow.cc
@@ -0,0 +1,79 @@
+//===-- asan_premap_shadow.cc ---------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Reserve shadow memory with an ifunc resolver.
+//===----------------------------------------------------------------------===//
+
+#include "asan_mapping.h"
+
+#if ASAN_PREMAP_SHADOW
+
+#include "asan_premap_shadow.h"
+#include "sanitizer_common/sanitizer_posix.h"
+
+namespace __asan {
+
+// The code in this file needs to run in an unrelocated binary. It may not
+// access any external symbol, including its own non-hidden globals.
+
+// Conservative upper limit.
+uptr PremapShadowSize() {
+ uptr granularity = GetMmapGranularity();
+ return RoundUpTo(GetMaxVirtualAddress() >> SHADOW_SCALE, granularity);
+}
+
+// Returns an address aligned to 8 pages, such that one page on the left and
+// PremapShadowSize() bytes on the right of it are mapped r/o.
+uptr PremapShadow() {
+ uptr granularity = GetMmapGranularity();
+ uptr alignment = granularity * 8;
+ uptr left_padding = granularity;
+ uptr shadow_size = PremapShadowSize();
+ uptr map_size = shadow_size + left_padding + alignment;
+
+ uptr map_start = (uptr)MmapNoAccess(map_size);
+ CHECK_NE(map_start, ~(uptr)0);
+
+ uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
+ uptr shadow_end = shadow_start + shadow_size;
+ internal_munmap(reinterpret_cast<void *>(map_start),
+ shadow_start - left_padding - map_start);
+ internal_munmap(reinterpret_cast<void *>(shadow_end),
+ map_start + map_size - shadow_end);
+ return shadow_start;
+}
+
+bool PremapShadowFailed() {
+ uptr shadow = reinterpret_cast<uptr>(&__asan_shadow);
+ uptr resolver = reinterpret_cast<uptr>(&__asan_premap_shadow);
+ // shadow == resolver is how Android KitKat and older handles ifunc.
+ // shadow == 0 just in case.
+ if (shadow == 0 || shadow == resolver)
+ return true;
+ return false;
+}
+} // namespace __asan
+
+extern "C" {
+decltype(__asan_shadow)* __asan_premap_shadow() {
+ // The resolver may be called multiple times. Map the shadow just once.
+ static uptr premapped_shadow = 0;
+ if (!premapped_shadow) premapped_shadow = __asan::PremapShadow();
+ return reinterpret_cast<decltype(__asan_shadow)*>(premapped_shadow);
+}
+
+// __asan_shadow is a "function" that has the same address as the first byte of
+// the shadow mapping.
+INTERFACE_ATTRIBUTE __attribute__((ifunc("__asan_premap_shadow"))) void
+__asan_shadow();
+}
+
+#endif // ASAN_PREMAP_SHADOW
diff --git a/contrib/compiler-rt/lib/asan/asan_premap_shadow.h b/contrib/compiler-rt/lib/asan/asan_premap_shadow.h
new file mode 100644
index 000000000000..41acbdbbb69f
--- /dev/null
+++ b/contrib/compiler-rt/lib/asan/asan_premap_shadow.h
@@ -0,0 +1,30 @@
+//===-- asan_mapping.h ------------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Premap shadow range with an ifunc resolver.
+//===----------------------------------------------------------------------===//
+
+
+#ifndef ASAN_PREMAP_SHADOW_H
+#define ASAN_PREMAP_SHADOW_H
+
+#if ASAN_PREMAP_SHADOW
+namespace __asan {
+// Conservative upper limit.
+uptr PremapShadowSize();
+bool PremapShadowFailed();
+}
+#endif
+
+extern "C" INTERFACE_ATTRIBUTE void __asan_shadow();
+extern "C" decltype(__asan_shadow)* __asan_premap_shadow();
+
+#endif // ASAN_PREMAP_SHADOW_H
diff --git a/contrib/compiler-rt/lib/asan/asan_report.cc b/contrib/compiler-rt/lib/asan/asan_report.cc
index 2e477f258b8d..e3bc02994efd 100644
--- a/contrib/compiler-rt/lib/asan/asan_report.cc
+++ b/contrib/compiler-rt/lib/asan/asan_report.cc
@@ -60,9 +60,8 @@ void PrintMemoryByte(InternalScopedString *str, const char *before, u8 byte,
bool in_shadow, const char *after) {
Decorator d;
str->append("%s%s%x%x%s%s", before,
- in_shadow ? d.ShadowByte(byte) : d.MemoryByte(),
- byte >> 4, byte & 15,
- in_shadow ? d.EndShadowByte() : d.EndMemoryByte(), after);
+ in_shadow ? d.ShadowByte(byte) : d.MemoryByte(), byte >> 4,
+ byte & 15, d.Default(), after);
}
static void PrintZoneForPointer(uptr ptr, uptr zone_ptr,
@@ -123,53 +122,15 @@ bool ParseFrameDescription(const char *frame_descr,
// immediately after printing error report.
class ScopedInErrorReport {
public:
- explicit ScopedInErrorReport(bool fatal = false) {
- halt_on_error_ = fatal || flags()->halt_on_error;
-
- if (lock_.TryLock()) {
- StartReporting();
- return;
- }
-
- // ASan found two bugs in different threads simultaneously.
-
- u32 current_tid = GetCurrentTidOrInvalid();
- if (reporting_thread_tid_ == current_tid ||
- reporting_thread_tid_ == kInvalidTid) {
- // This is either asynch signal or nested error during error reporting.
- // Fail simple to avoid deadlocks in Report().
-
- // Can't use Report() here because of potential deadlocks
- // in nested signal handlers.
- const char msg[] = "AddressSanitizer: nested bug in the same thread, "
- "aborting.\n";
- WriteToFile(kStderrFd, msg, sizeof(msg));
-
- internal__exit(common_flags()->exitcode);
- }
-
- if (halt_on_error_) {
- // Do not print more than one report, otherwise they will mix up.
- // Error reporting functions shouldn't return at this situation, as
- // they are effectively no-returns.
-
- Report("AddressSanitizer: while reporting a bug found another one. "
- "Ignoring.\n");
-
- // Sleep long enough to make sure that the thread which started
- // to print an error report will finish doing it.
- SleepForSeconds(Max(100, flags()->sleep_before_dying + 1));
-
- // If we're still not dead for some reason, use raw _exit() instead of
- // Die() to bypass any additional checks.
- internal__exit(common_flags()->exitcode);
- } else {
- // The other thread will eventually finish reporting
- // so it's safe to wait
- lock_.Lock();
- }
-
- StartReporting();
+ explicit ScopedInErrorReport(bool fatal = false)
+ : halt_on_error_(fatal || flags()->halt_on_error) {
+ // Make sure the registry and sanitizer report mutexes are locked while
+ // we're printing an error report.
+ // We can lock them only here to avoid self-deadlock in case of
+ // recursive reports.
+ asanThreadRegistry().Lock();
+ Printf(
+ "=================================================================\n");
}
~ScopedInErrorReport() {
@@ -217,9 +178,6 @@ class ScopedInErrorReport {
if (!halt_on_error_)
internal_memset(&current_error_, 0, sizeof(current_error_));
- CommonSanitizerReportMutex.Unlock();
- reporting_thread_tid_ = kInvalidTid;
- lock_.Unlock();
if (halt_on_error_) {
Report("ABORTING\n");
Die();
@@ -237,39 +195,18 @@ class ScopedInErrorReport {
}
private:
- void StartReporting() {
- // Make sure the registry and sanitizer report mutexes are locked while
- // we're printing an error report.
- // We can lock them only here to avoid self-deadlock in case of
- // recursive reports.
- asanThreadRegistry().Lock();
- CommonSanitizerReportMutex.Lock();
- reporting_thread_tid_ = GetCurrentTidOrInvalid();
- Printf("===================================================="
- "=============\n");
- }
-
- static StaticSpinMutex lock_;
- static u32 reporting_thread_tid_;
+ ScopedErrorReportLock error_report_lock_;
// Error currently being reported. This enables the destructor to interact
// with the debugger and point it to an error description.
static ErrorDescription current_error_;
bool halt_on_error_;
};
-StaticSpinMutex ScopedInErrorReport::lock_;
-u32 ScopedInErrorReport::reporting_thread_tid_ = kInvalidTid;
ErrorDescription ScopedInErrorReport::current_error_;
-void ReportStackOverflow(const SignalContext &sig) {
+void ReportDeadlySignal(const SignalContext &sig) {
ScopedInErrorReport in_report(/*fatal*/ true);
- ErrorStackOverflow error(GetCurrentTidOrInvalid(), sig);
- in_report.ReportError(error);
-}
-
-void ReportDeadlySignal(int signo, const SignalContext &sig) {
- ScopedInErrorReport in_report(/*fatal*/ true);
- ErrorDeadlySignal error(GetCurrentTidOrInvalid(), sig, signo);
+ ErrorDeadlySignal error(GetCurrentTidOrInvalid(), sig);
in_report.ReportError(error);
}
@@ -279,11 +216,12 @@ void ReportDoubleFree(uptr addr, BufferedStackTrace *free_stack) {
in_report.ReportError(error);
}
-void ReportNewDeleteSizeMismatch(uptr addr, uptr delete_size,
+void ReportNewDeleteTypeMismatch(uptr addr, uptr delete_size,
+ uptr delete_alignment,
BufferedStackTrace *free_stack) {
ScopedInErrorReport in_report;
- ErrorNewDeleteSizeMismatch error(GetCurrentTidOrInvalid(), free_stack, addr,
- delete_size);
+ ErrorNewDeleteTypeMismatch error(GetCurrentTidOrInvalid(), free_stack, addr,
+ delete_size, delete_alignment);
in_report.ReportError(error);
}
@@ -360,17 +298,58 @@ static NOINLINE void ReportInvalidPointerPair(uptr pc, uptr bp, uptr sp,
in_report.ReportError(error);
}
+static bool IsInvalidPointerPair(uptr a1, uptr a2) {
+ if (a1 == a2)
+ return false;
+
+ // 256B in shadow memory can be iterated quite fast
+ static const uptr kMaxOffset = 2048;
+
+ uptr left = a1 < a2 ? a1 : a2;
+ uptr right = a1 < a2 ? a2 : a1;
+ uptr offset = right - left;
+ if (offset <= kMaxOffset)
+ return __asan_region_is_poisoned(left, offset);
+
+ AsanThread *t = GetCurrentThread();
+
+ // check whether left is a stack memory pointer
+ if (uptr shadow_offset1 = t->GetStackVariableShadowStart(left)) {
+ uptr shadow_offset2 = t->GetStackVariableShadowStart(right);
+ return shadow_offset2 == 0 || shadow_offset1 != shadow_offset2;
+ }
+
+ // check whether left is a heap memory address
+ HeapAddressDescription hdesc1, hdesc2;
+ if (GetHeapAddressInformation(left, 0, &hdesc1) &&
+ hdesc1.chunk_access.access_type == kAccessTypeInside)
+ return !GetHeapAddressInformation(right, 0, &hdesc2) ||
+ hdesc2.chunk_access.access_type != kAccessTypeInside ||
+ hdesc1.chunk_access.chunk_begin != hdesc2.chunk_access.chunk_begin;
+
+ // check whether left is an address of a global variable
+ GlobalAddressDescription gdesc1, gdesc2;
+ if (GetGlobalAddressInformation(left, 0, &gdesc1))
+ return !GetGlobalAddressInformation(right - 1, 0, &gdesc2) ||
+ !gdesc1.PointsInsideTheSameVariable(gdesc2);
+
+ if (t->GetStackVariableShadowStart(right) ||
+ GetHeapAddressInformation(right, 0, &hdesc2) ||
+ GetGlobalAddressInformation(right - 1, 0, &gdesc2))
+ return true;
+
+ // At this point we know nothing about both a1 and a2 addresses.
+ return false;
+}
+
static INLINE void CheckForInvalidPointerPair(void *p1, void *p2) {
if (!flags()->detect_invalid_pointer_pairs) return;
uptr a1 = reinterpret_cast<uptr>(p1);
uptr a2 = reinterpret_cast<uptr>(p2);
- AsanChunkView chunk1 = FindHeapChunkByAddress(a1);
- AsanChunkView chunk2 = FindHeapChunkByAddress(a2);
- bool valid1 = chunk1.IsAllocated();
- bool valid2 = chunk2.IsAllocated();
- if (!valid1 || !valid2 || !chunk1.Eq(chunk2)) {
+
+ if (IsInvalidPointerPair(a1, a2)) {
GET_CALLER_PC_BP_SP;
- return ReportInvalidPointerPair(pc, bp, sp, a1, a2);
+ ReportInvalidPointerPair(pc, bp, sp, a1, a2);
}
}
// ----------------------- Mac-specific reports ----------------- {{{1
diff --git a/contrib/compiler-rt/lib/asan/asan_report.h b/contrib/compiler-rt/lib/asan/asan_report.h
index 5a3533a319af..f2cdad47ee81 100644
--- a/contrib/compiler-rt/lib/asan/asan_report.h
+++ b/contrib/compiler-rt/lib/asan/asan_report.h
@@ -46,9 +46,9 @@ bool ParseFrameDescription(const char *frame_descr,
// Different kinds of error reports.
void ReportGenericError(uptr pc, uptr bp, uptr sp, uptr addr, bool is_write,
uptr access_size, u32 exp, bool fatal);
-void ReportStackOverflow(const SignalContext &sig);
-void ReportDeadlySignal(int signo, const SignalContext &sig);
-void ReportNewDeleteSizeMismatch(uptr addr, uptr delete_size,
+void ReportDeadlySignal(const SignalContext &sig);
+void ReportNewDeleteTypeMismatch(uptr addr, uptr delete_size,
+ uptr delete_alignment,
BufferedStackTrace *free_stack);
void ReportDoubleFree(uptr addr, BufferedStackTrace *free_stack);
void ReportFreeNotMalloced(uptr addr, BufferedStackTrace *free_stack);
diff --git a/contrib/compiler-rt/lib/asan/asan_rtl.cc b/contrib/compiler-rt/lib/asan/asan_rtl.cc
index 5ae3568ae04a..21fd0e240708 100644
--- a/contrib/compiler-rt/lib/asan/asan_rtl.cc
+++ b/contrib/compiler-rt/lib/asan/asan_rtl.cc
@@ -84,26 +84,6 @@ void ShowStatsAndAbort() {
Die();
}
-// ---------------------- mmap -------------------- {{{1
-// Reserve memory range [beg, end].
-// We need to use inclusive range because end+1 may not be representable.
-void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
- CHECK_EQ((beg % GetMmapGranularity()), 0);
- CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
- uptr size = end - beg + 1;
- DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb.
- void *res = MmapFixedNoReserve(beg, size, name);
- if (res != (void*)beg) {
- Report("ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
- "Perhaps you're using ulimit -v\n", size);
- Abort();
- }
- if (common_flags()->no_huge_pages_for_shadow)
- NoHugePagesInRegion(beg, size);
- if (common_flags()->use_madv_dontdump)
- DontDumpShadowMemory(beg, size);
-}
-
// --------------- LowLevelAllocateCallbac ---------- {{{1
static void OnLowLevelAllocate(uptr ptr, uptr size) {
PoisonShadow(ptr, size, kAsanInternalHeapMagic);
@@ -327,7 +307,7 @@ static void asan_atexit() {
static void InitializeHighMemEnd() {
#if !ASAN_FIXED_MAPPING
- kHighMemEnd = GetMaxVirtualAddress();
+ kHighMemEnd = GetMaxUserVirtualAddress();
// Increase kHighMemEnd to make sure it's properly
// aligned together with kHighMemBeg:
kHighMemEnd |= SHADOW_GRANULARITY * GetMmapGranularity() - 1;
@@ -335,46 +315,7 @@ static void InitializeHighMemEnd() {
CHECK_EQ((kHighMemBeg % GetMmapGranularity()), 0);
}
-static void ProtectGap(uptr addr, uptr size) {
- if (!flags()->protect_shadow_gap) {
- // The shadow gap is unprotected, so there is a chance that someone
- // is actually using this memory. Which means it needs a shadow...
- uptr GapShadowBeg = RoundDownTo(MEM_TO_SHADOW(addr), GetPageSizeCached());
- uptr GapShadowEnd =
- RoundUpTo(MEM_TO_SHADOW(addr + size), GetPageSizeCached()) - 1;
- if (Verbosity())
- Printf("protect_shadow_gap=0:"
- " not protecting shadow gap, allocating gap's shadow\n"
- "|| `[%p, %p]` || ShadowGap's shadow ||\n", GapShadowBeg,
- GapShadowEnd);
- ReserveShadowMemoryRange(GapShadowBeg, GapShadowEnd,
- "unprotected gap shadow");
- return;
- }
- void *res = MmapFixedNoAccess(addr, size, "shadow gap");
- if (addr == (uptr)res)
- return;
- // A few pages at the start of the address space can not be protected.
- // But we really want to protect as much as possible, to prevent this memory
- // being returned as a result of a non-FIXED mmap().
- if (addr == kZeroBaseShadowStart) {
- uptr step = GetMmapGranularity();
- while (size > step && addr < kZeroBaseMaxShadowStart) {
- addr += step;
- size -= step;
- void *res = MmapFixedNoAccess(addr, size, "shadow gap");
- if (addr == (uptr)res)
- return;
- }
- }
-
- Report("ERROR: Failed to protect the shadow gap. "
- "ASan cannot proceed correctly. ABORTING.\n");
- DumpProcessMap();
- Die();
-}
-
-static void PrintAddressSpaceLayout() {
+void PrintAddressSpaceLayout() {
Printf("|| `[%p, %p]` || HighMem ||\n",
(void*)kHighMemBeg, (void*)kHighMemEnd);
Printf("|| `[%p, %p]` || HighShadow ||\n",
@@ -426,71 +367,6 @@ static void PrintAddressSpaceLayout() {
kHighShadowBeg > kMidMemEnd);
}
-static void InitializeShadowMemory() {
- // Set the shadow memory address to uninitialized.
- __asan_shadow_memory_dynamic_address = kDefaultShadowSentinel;
-
- uptr shadow_start = kLowShadowBeg;
- // Detect if a dynamic shadow address must used and find a available location
- // when necessary. When dynamic address is used, the macro |kLowShadowBeg|
- // expands to |__asan_shadow_memory_dynamic_address| which is
- // |kDefaultShadowSentinel|.
- if (shadow_start == kDefaultShadowSentinel) {
- __asan_shadow_memory_dynamic_address = 0;
- CHECK_EQ(0, kLowShadowBeg);
- shadow_start = FindDynamicShadowStart();
- }
- // Update the shadow memory address (potentially) used by instrumentation.
- __asan_shadow_memory_dynamic_address = shadow_start;
-
- if (kLowShadowBeg)
- shadow_start -= GetMmapGranularity();
- bool full_shadow_is_available =
- MemoryRangeIsAvailable(shadow_start, kHighShadowEnd);
-
-#if SANITIZER_LINUX && defined(__x86_64__) && defined(_LP64) && \
- !ASAN_FIXED_MAPPING
- if (!full_shadow_is_available) {
- kMidMemBeg = kLowMemEnd < 0x3000000000ULL ? 0x3000000000ULL : 0;
- kMidMemEnd = kLowMemEnd < 0x3000000000ULL ? 0x4fffffffffULL : 0;
- }
-#endif
-
- if (Verbosity()) PrintAddressSpaceLayout();
-
- if (full_shadow_is_available) {
- // mmap the low shadow plus at least one page at the left.
- if (kLowShadowBeg)
- ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow");
- // mmap the high shadow.
- ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow");
- // protect the gap.
- ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1);
- CHECK_EQ(kShadowGapEnd, kHighShadowBeg - 1);
- } else if (kMidMemBeg &&
- MemoryRangeIsAvailable(shadow_start, kMidMemBeg - 1) &&
- MemoryRangeIsAvailable(kMidMemEnd + 1, kHighShadowEnd)) {
- CHECK(kLowShadowBeg != kLowShadowEnd);
- // mmap the low shadow plus at least one page at the left.
- ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow");
- // mmap the mid shadow.
- ReserveShadowMemoryRange(kMidShadowBeg, kMidShadowEnd, "mid shadow");
- // mmap the high shadow.
- ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow");
- // protect the gaps.
- ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1);
- ProtectGap(kShadowGap2Beg, kShadowGap2End - kShadowGap2Beg + 1);
- ProtectGap(kShadowGap3Beg, kShadowGap3End - kShadowGap3Beg + 1);
- } else {
- Report("Shadow memory range interleaves with an existing memory mapping. "
- "ASan cannot proceed correctly. ABORTING.\n");
- Report("ASan shadow was supposed to be located in the [%p-%p] range.\n",
- shadow_start, kHighShadowEnd);
- DumpProcessMap();
- Die();
- }
-}
-
static void AsanInitInternal() {
if (LIKELY(asan_inited)) return;
SanitizerToolName = "AddressSanitizer";
@@ -531,6 +407,7 @@ static void AsanInitInternal() {
MaybeReexec();
// Setup internal allocator callback.
+ SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY);
SetLowLevelAllocateCallback(OnLowLevelAllocate);
InitializeAsanInterceptors();
@@ -575,20 +452,18 @@ static void AsanInitInternal() {
InitTlsSize();
// Create main thread.
- AsanThread *main_thread = AsanThread::Create(
- /* start_routine */ nullptr, /* arg */ nullptr, /* parent_tid */ 0,
- /* stack */ nullptr, /* detached */ true);
+ AsanThread *main_thread = CreateMainThread();
CHECK_EQ(0, main_thread->tid());
- SetCurrentThread(main_thread);
- main_thread->ThreadStart(internal_getpid(),
- /* signal_thread_is_registered */ nullptr);
force_interface_symbols(); // no-op.
SanitizerInitializeUnwinder();
if (CAN_SANITIZE_LEAKS) {
__lsan::InitCommonLsan();
if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) {
- Atexit(__lsan::DoLeakCheck);
+ if (flags()->halt_on_error)
+ Atexit(__lsan::DoLeakCheck);
+ else
+ Atexit(__lsan::DoRecoverableLeakCheckVoid);
}
}
@@ -608,6 +483,11 @@ static void AsanInitInternal() {
}
VReport(1, "AddressSanitizer Init done\n");
+
+ if (flags()->sleep_after_init) {
+ Report("Sleeping for %d second(s)\n", flags()->sleep_after_init);
+ SleepForSeconds(flags()->sleep_after_init);
+ }
}
// Initialize as requested from some part of ASan runtime library (interceptors,
@@ -647,6 +527,7 @@ void NOINLINE __asan_handle_no_return() {
top = curr_thread->stack_top();
bottom = ((uptr)&local_stack - PageSize) & ~(PageSize - 1);
} else {
+ CHECK(!SANITIZER_FUCHSIA);
// If we haven't seen this thread, try asking the OS for stack bounds.
uptr tls_addr, tls_size, stack_size;
GetThreadStackAndTls(/*main=*/false, &bottom, &stack_size, &tls_addr,
diff --git a/contrib/compiler-rt/lib/asan/asan_scariness_score.h b/contrib/compiler-rt/lib/asan/asan_scariness_score.h
index 7f1571416fd5..7f095dd29f29 100644
--- a/contrib/compiler-rt/lib/asan/asan_scariness_score.h
+++ b/contrib/compiler-rt/lib/asan/asan_scariness_score.h
@@ -47,7 +47,7 @@ struct ScarinessScoreBase {
};
int GetScore() const { return score; }
const char *GetDescription() const { return descr; }
- void Print() {
+ void Print() const {
if (score && flags()->print_scariness)
Printf("SCARINESS: %d (%s)\n", score, descr);
}
diff --git a/contrib/compiler-rt/lib/asan/asan_shadow_setup.cc b/contrib/compiler-rt/lib/asan/asan_shadow_setup.cc
new file mode 100644
index 000000000000..b3cf0b5c1ce2
--- /dev/null
+++ b/contrib/compiler-rt/lib/asan/asan_shadow_setup.cc
@@ -0,0 +1,165 @@
+//===-- asan_shadow_setup.cc ----------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of AddressSanitizer, an address sanity checker.
+//
+// Set up the shadow memory.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+
+// asan_fuchsia.cc has its own InitializeShadowMemory implementation.
+#if !SANITIZER_FUCHSIA
+
+#include "asan_internal.h"
+#include "asan_mapping.h"
+
+namespace __asan {
+
+// ---------------------- mmap -------------------- {{{1
+// Reserve memory range [beg, end].
+// We need to use inclusive range because end+1 may not be representable.
+void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
+ CHECK_EQ((beg % GetMmapGranularity()), 0);
+ CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
+ uptr size = end - beg + 1;
+ DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb.
+ void *res = MmapFixedNoReserve(beg, size, name);
+ if (res != (void *)beg) {
+ Report(
+ "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
+ "Perhaps you're using ulimit -v\n",
+ size);
+ Abort();
+ }
+ if (common_flags()->no_huge_pages_for_shadow) NoHugePagesInRegion(beg, size);
+ if (common_flags()->use_madv_dontdump) DontDumpShadowMemory(beg, size);
+}
+
+static void ProtectGap(uptr addr, uptr size) {
+ if (!flags()->protect_shadow_gap) {
+ // The shadow gap is unprotected, so there is a chance that someone
+ // is actually using this memory. Which means it needs a shadow...
+ uptr GapShadowBeg = RoundDownTo(MEM_TO_SHADOW(addr), GetPageSizeCached());
+ uptr GapShadowEnd =
+ RoundUpTo(MEM_TO_SHADOW(addr + size), GetPageSizeCached()) - 1;
+ if (Verbosity())
+ Printf(
+ "protect_shadow_gap=0:"
+ " not protecting shadow gap, allocating gap's shadow\n"
+ "|| `[%p, %p]` || ShadowGap's shadow ||\n",
+ GapShadowBeg, GapShadowEnd);
+ ReserveShadowMemoryRange(GapShadowBeg, GapShadowEnd,
+ "unprotected gap shadow");
+ return;
+ }
+ void *res = MmapFixedNoAccess(addr, size, "shadow gap");
+ if (addr == (uptr)res) return;
+ // A few pages at the start of the address space can not be protected.
+ // But we really want to protect as much as possible, to prevent this memory
+ // being returned as a result of a non-FIXED mmap().
+ if (addr == kZeroBaseShadowStart) {
+ uptr step = GetMmapGranularity();
+ while (size > step && addr < kZeroBaseMaxShadowStart) {
+ addr += step;
+ size -= step;
+ void *res = MmapFixedNoAccess(addr, size, "shadow gap");
+ if (addr == (uptr)res) return;
+ }
+ }
+
+ Report(
+ "ERROR: Failed to protect the shadow gap. "
+ "ASan cannot proceed correctly. ABORTING.\n");
+ DumpProcessMap();
+ Die();
+}
+
+static void MaybeReportLinuxPIEBug() {
+#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__aarch64__))
+ Report("This might be related to ELF_ET_DYN_BASE change in Linux 4.12.\n");
+ Report(
+ "See https://github.com/google/sanitizers/issues/856 for possible "
+ "workarounds.\n");
+#endif
+}
+
+void InitializeShadowMemory() {
+ // Set the shadow memory address to uninitialized.
+ __asan_shadow_memory_dynamic_address = kDefaultShadowSentinel;
+
+ uptr shadow_start = kLowShadowBeg;
+ // Detect if a dynamic shadow address must used and find a available location
+ // when necessary. When dynamic address is used, the macro |kLowShadowBeg|
+ // expands to |__asan_shadow_memory_dynamic_address| which is
+ // |kDefaultShadowSentinel|.
+ bool full_shadow_is_available = false;
+ if (shadow_start == kDefaultShadowSentinel) {
+ __asan_shadow_memory_dynamic_address = 0;
+ CHECK_EQ(0, kLowShadowBeg);
+ shadow_start = FindDynamicShadowStart();
+ if (SANITIZER_LINUX) full_shadow_is_available = true;
+ }
+ // Update the shadow memory address (potentially) used by instrumentation.
+ __asan_shadow_memory_dynamic_address = shadow_start;
+
+ if (kLowShadowBeg) shadow_start -= GetMmapGranularity();
+
+ if (!full_shadow_is_available)
+ full_shadow_is_available =
+ MemoryRangeIsAvailable(shadow_start, kHighShadowEnd);
+
+#if SANITIZER_LINUX && defined(__x86_64__) && defined(_LP64) && \
+ !ASAN_FIXED_MAPPING
+ if (!full_shadow_is_available) {
+ kMidMemBeg = kLowMemEnd < 0x3000000000ULL ? 0x3000000000ULL : 0;
+ kMidMemEnd = kLowMemEnd < 0x3000000000ULL ? 0x4fffffffffULL : 0;
+ }
+#endif
+
+ if (Verbosity()) PrintAddressSpaceLayout();
+
+ if (full_shadow_is_available) {
+ // mmap the low shadow plus at least one page at the left.
+ if (kLowShadowBeg)
+ ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow");
+ // mmap the high shadow.
+ ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow");
+ // protect the gap.
+ ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1);
+ CHECK_EQ(kShadowGapEnd, kHighShadowBeg - 1);
+ } else if (kMidMemBeg &&
+ MemoryRangeIsAvailable(shadow_start, kMidMemBeg - 1) &&
+ MemoryRangeIsAvailable(kMidMemEnd + 1, kHighShadowEnd)) {
+ CHECK(kLowShadowBeg != kLowShadowEnd);
+ // mmap the low shadow plus at least one page at the left.
+ ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow");
+ // mmap the mid shadow.
+ ReserveShadowMemoryRange(kMidShadowBeg, kMidShadowEnd, "mid shadow");
+ // mmap the high shadow.
+ ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow");
+ // protect the gaps.
+ ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1);
+ ProtectGap(kShadowGap2Beg, kShadowGap2End - kShadowGap2Beg + 1);
+ ProtectGap(kShadowGap3Beg, kShadowGap3End - kShadowGap3Beg + 1);
+ } else {
+ Report(
+ "Shadow memory range interleaves with an existing memory mapping. "
+ "ASan cannot proceed correctly. ABORTING.\n");
+ Report("ASan shadow was supposed to be located in the [%p-%p] range.\n",
+ shadow_start, kHighShadowEnd);
+ MaybeReportLinuxPIEBug();
+ DumpProcessMap();
+ Die();
+ }
+}
+
+} // namespace __asan
+
+#endif // !SANITIZER_FUCHSIA
diff --git a/contrib/compiler-rt/lib/asan/asan_stack.h b/contrib/compiler-rt/lib/asan/asan_stack.h
index cc95e0f30a33..8e9df888d798 100644
--- a/contrib/compiler-rt/lib/asan/asan_stack.h
+++ b/contrib/compiler-rt/lib/asan/asan_stack.h
@@ -31,9 +31,8 @@ u32 GetMallocContextSize();
// The pc will be in the position 0 of the resulting stack trace.
// The bp may refer to the current frame or to the caller's frame.
ALWAYS_INLINE
-void GetStackTraceWithPcBpAndContext(BufferedStackTrace *stack, uptr max_depth,
- uptr pc, uptr bp, void *context,
- bool fast) {
+void GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, uptr bp,
+ void *context, bool fast) {
#if SANITIZER_WINDOWS
stack->Unwind(max_depth, pc, bp, context, 0, 0, fast);
#else
@@ -41,10 +40,6 @@ void GetStackTraceWithPcBpAndContext(BufferedStackTrace *stack, uptr max_depth,
stack->size = 0;
if (LIKELY(asan_inited)) {
if ((t = GetCurrentThread()) && !t->isUnwinding()) {
- // On FreeBSD the slow unwinding that leverages _Unwind_Backtrace()
- // yields the call stack of the signal's handler and not of the code
- // that raised the signal (as it does on Linux).
- if (SANITIZER_FREEBSD && t->isInDeadlySignal()) fast = true;
uptr stack_top = t->stack_top();
uptr stack_bottom = t->stack_bottom();
ScopedUnwinding unwind_scope(t);
@@ -66,32 +61,29 @@ void GetStackTraceWithPcBpAndContext(BufferedStackTrace *stack, uptr max_depth,
// as early as possible (in functions exposed to the user), as we generally
// don't want stack trace to contain functions from ASan internals.
-#define GET_STACK_TRACE(max_size, fast) \
- BufferedStackTrace stack; \
- if (max_size <= 2) { \
- stack.size = max_size; \
- if (max_size > 0) { \
- stack.top_frame_bp = GET_CURRENT_FRAME(); \
- stack.trace_buffer[0] = StackTrace::GetCurrentPc(); \
- if (max_size > 1) \
- stack.trace_buffer[1] = GET_CALLER_PC(); \
- } \
- } else { \
- GetStackTraceWithPcBpAndContext(&stack, max_size, \
- StackTrace::GetCurrentPc(), \
- GET_CURRENT_FRAME(), 0, fast); \
+#define GET_STACK_TRACE(max_size, fast) \
+ BufferedStackTrace stack; \
+ if (max_size <= 2) { \
+ stack.size = max_size; \
+ if (max_size > 0) { \
+ stack.top_frame_bp = GET_CURRENT_FRAME(); \
+ stack.trace_buffer[0] = StackTrace::GetCurrentPc(); \
+ if (max_size > 1) stack.trace_buffer[1] = GET_CALLER_PC(); \
+ } \
+ } else { \
+ GetStackTrace(&stack, max_size, StackTrace::GetCurrentPc(), \
+ GET_CURRENT_FRAME(), 0, fast); \
}
-#define GET_STACK_TRACE_FATAL(pc, bp) \
- BufferedStackTrace stack; \
- GetStackTraceWithPcBpAndContext(&stack, kStackTraceMax, pc, bp, 0, \
- common_flags()->fast_unwind_on_fatal)
+#define GET_STACK_TRACE_FATAL(pc, bp) \
+ BufferedStackTrace stack; \
+ GetStackTrace(&stack, kStackTraceMax, pc, bp, 0, \
+ common_flags()->fast_unwind_on_fatal)
-#define GET_STACK_TRACE_SIGNAL(sig) \
- BufferedStackTrace stack; \
- GetStackTraceWithPcBpAndContext(&stack, kStackTraceMax, \
- (sig).pc, (sig).bp, (sig).context, \
- common_flags()->fast_unwind_on_fatal)
+#define GET_STACK_TRACE_SIGNAL(sig) \
+ BufferedStackTrace stack; \
+ GetStackTrace(&stack, kStackTraceMax, (sig).pc, (sig).bp, (sig).context, \
+ common_flags()->fast_unwind_on_fatal)
#define GET_STACK_TRACE_FATAL_HERE \
GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal)
diff --git a/contrib/compiler-rt/lib/asan/asan_thread.cc b/contrib/compiler-rt/lib/asan/asan_thread.cc
index b1a0d9a3b37f..ad81512dff08 100644
--- a/contrib/compiler-rt/lib/asan/asan_thread.cc
+++ b/contrib/compiler-rt/lib/asan/asan_thread.cc
@@ -27,11 +27,6 @@ namespace __asan {
// AsanThreadContext implementation.
-struct CreateThreadContextArgs {
- AsanThread *thread;
- StackTrace *stack;
-};
-
void AsanThreadContext::OnCreated(void *arg) {
CreateThreadContextArgs *args = static_cast<CreateThreadContextArgs*>(arg);
if (args->stack)
@@ -88,7 +83,7 @@ AsanThread *AsanThread::Create(thread_callback_t start_routine, void *arg,
AsanThread *thread = (AsanThread*)MmapOrDie(size, __func__);
thread->start_routine_ = start_routine;
thread->arg_ = arg;
- CreateThreadContextArgs args = { thread, stack };
+ AsanThreadContext::CreateThreadContextArgs args = {thread, stack};
asanThreadRegistry().CreateThread(*reinterpret_cast<uptr *>(thread), detached,
parent_tid, &args);
@@ -223,12 +218,12 @@ FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() {
return nullptr;
}
-void AsanThread::Init() {
+void AsanThread::Init(const InitOptions *options) {
next_stack_top_ = next_stack_bottom_ = 0;
atomic_store(&stack_switching_, false, memory_order_release);
fake_stack_ = nullptr; // Will be initialized lazily if needed.
CHECK_EQ(this->stack_size(), 0U);
- SetThreadStackAndTls();
+ SetThreadStackAndTls(options);
CHECK_GT(this->stack_size(), 0U);
CHECK(AddrIsInMem(stack_bottom_));
CHECK(AddrIsInMem(stack_top_ - 1));
@@ -239,6 +234,10 @@ void AsanThread::Init() {
&local);
}
+// Fuchsia doesn't use ThreadStart.
+// asan_fuchsia.c defines CreateMainThread and SetThreadStackAndTls.
+#if !SANITIZER_FUCHSIA
+
thread_return_t AsanThread::ThreadStart(
tid_t os_id, atomic_uintptr_t *signal_thread_is_registered) {
Init();
@@ -270,7 +269,21 @@ thread_return_t AsanThread::ThreadStart(
return res;
}
-void AsanThread::SetThreadStackAndTls() {
+AsanThread *CreateMainThread() {
+ AsanThread *main_thread = AsanThread::Create(
+ /* start_routine */ nullptr, /* arg */ nullptr, /* parent_tid */ 0,
+ /* stack */ nullptr, /* detached */ true);
+ SetCurrentThread(main_thread);
+ main_thread->ThreadStart(internal_getpid(),
+ /* signal_thread_is_registered */ nullptr);
+ return main_thread;
+}
+
+// This implementation doesn't use the argument, which is just passed down
+// from the caller of Init (which see, above). It's only there to support
+// OS-specific implementations that need more information passed through.
+void AsanThread::SetThreadStackAndTls(const InitOptions *options) {
+ DCHECK_EQ(options, nullptr);
uptr tls_size = 0;
uptr stack_size = 0;
GetThreadStackAndTls(tid() == 0, const_cast<uptr *>(&stack_bottom_),
@@ -283,6 +296,8 @@ void AsanThread::SetThreadStackAndTls() {
CHECK(AddrIsInStack((uptr)&local));
}
+#endif // !SANITIZER_FUCHSIA
+
void AsanThread::ClearShadowForThreadStackAndTLS() {
PoisonShadow(stack_bottom_, stack_top_ - stack_bottom_, 0);
if (tls_begin_ != tls_end_)
@@ -302,7 +317,7 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr,
access->frame_descr = (const char *)((uptr*)bottom)[1];
return true;
}
- uptr aligned_addr = addr & ~(SANITIZER_WORDSIZE/8 - 1); // align addr.
+ uptr aligned_addr = RoundDownTo(addr, SANITIZER_WORDSIZE / 8); // align addr.
uptr mem_ptr = RoundDownTo(aligned_addr, SHADOW_GRANULARITY);
u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr);
u8 *shadow_bottom = (u8*)MemToShadow(bottom);
@@ -331,6 +346,29 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr,
return true;
}
+uptr AsanThread::GetStackVariableShadowStart(uptr addr) {
+ uptr bottom = 0;
+ if (AddrIsInStack(addr)) {
+ bottom = stack_bottom();
+ } else if (has_fake_stack()) {
+ bottom = fake_stack()->AddrIsInFakeStack(addr);
+ CHECK(bottom);
+ } else
+ return 0;
+
+ uptr aligned_addr = RoundDownTo(addr, SANITIZER_WORDSIZE / 8); // align addr.
+ u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr);
+ u8 *shadow_bottom = (u8*)MemToShadow(bottom);
+
+ while (shadow_ptr >= shadow_bottom &&
+ (*shadow_ptr != kAsanStackLeftRedzoneMagic &&
+ *shadow_ptr != kAsanStackMidRedzoneMagic &&
+ *shadow_ptr != kAsanStackRightRedzoneMagic))
+ shadow_ptr--;
+
+ return (uptr)shadow_ptr + 1;
+}
+
bool AsanThread::AddrIsInStack(uptr addr) {
const auto bounds = GetStackBounds();
return addr >= bounds.bottom && addr < bounds.top;
diff --git a/contrib/compiler-rt/lib/asan/asan_thread.h b/contrib/compiler-rt/lib/asan/asan_thread.h
index 424f9e68dfea..66091921101f 100644
--- a/contrib/compiler-rt/lib/asan/asan_thread.h
+++ b/contrib/compiler-rt/lib/asan/asan_thread.h
@@ -49,6 +49,11 @@ class AsanThreadContext : public ThreadContextBase {
void OnCreated(void *arg) override;
void OnFinished() override;
+
+ struct CreateThreadContextArgs {
+ AsanThread *thread;
+ StackTrace *stack;
+ };
};
// AsanThreadContext objects are never freed, so we need many of them.
@@ -62,7 +67,9 @@ class AsanThread {
static void TSDDtor(void *tsd);
void Destroy();
- void Init(); // Should be called from the thread itself.
+ struct InitOptions;
+ void Init(const InitOptions *options = nullptr);
+
thread_return_t ThreadStart(tid_t os_id,
atomic_uintptr_t *signal_thread_is_registered);
@@ -83,6 +90,9 @@ class AsanThread {
};
bool GetStackFrameAccessByAddr(uptr addr, StackFrameAccess *access);
+ // Returns a pointer to the start of the stack variable's shadow memory.
+ uptr GetStackVariableShadowStart(uptr addr);
+
bool AddrIsInStack(uptr addr);
void DeleteFakeStack(int tid) {
@@ -118,17 +128,15 @@ class AsanThread {
bool isUnwinding() const { return unwinding_; }
void setUnwinding(bool b) { unwinding_ = b; }
- // True if we are in a deadly signal handler.
- bool isInDeadlySignal() const { return in_deadly_signal_; }
- void setInDeadlySignal(bool b) { in_deadly_signal_ = b; }
-
AsanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; }
AsanStats &stats() { return stats_; }
private:
// NOTE: There is no AsanThread constructor. It is allocated
// via mmap() and *must* be valid in zero-initialized state.
- void SetThreadStackAndTls();
+
+ void SetThreadStackAndTls(const InitOptions *options);
+
void ClearShadowForThreadStackAndTLS();
FakeStack *AsyncSignalSafeLazyInitFakeStack();
@@ -158,7 +166,6 @@ class AsanThread {
AsanThreadLocalMallocStorage malloc_storage_;
AsanStats stats_;
bool unwinding_;
- bool in_deadly_signal_;
};
// ScopedUnwinding is a scope for stacktracing member of a context
@@ -173,20 +180,6 @@ class ScopedUnwinding {
AsanThread *thread;
};
-// ScopedDeadlySignal is a scope for handling deadly signals.
-class ScopedDeadlySignal {
- public:
- explicit ScopedDeadlySignal(AsanThread *t) : thread(t) {
- if (thread) thread->setInDeadlySignal(true);
- }
- ~ScopedDeadlySignal() {
- if (thread) thread->setInDeadlySignal(false);
- }
-
- private:
- AsanThread *thread;
-};
-
// Returns a single instance of registry.
ThreadRegistry &asanThreadRegistry();
diff --git a/contrib/compiler-rt/lib/asan/asan_win.cc b/contrib/compiler-rt/lib/asan/asan_win.cc
index 8a839d913f95..68eedd1863bc 100644
--- a/contrib/compiler-rt/lib/asan/asan_win.cc
+++ b/contrib/compiler-rt/lib/asan/asan_win.cc
@@ -57,8 +57,8 @@ long __asan_unhandled_exception_filter(EXCEPTION_POINTERS *info) {
// FIXME: Handle EXCEPTION_STACK_OVERFLOW here.
- SignalContext sig = SignalContext::Create(exception_record, context);
- ReportDeadlySignal(exception_record->ExceptionCode, sig);
+ SignalContext sig(exception_record, context);
+ ReportDeadlySignal(sig);
UNREACHABLE("returned from reporting deadly signal");
}
diff --git a/contrib/compiler-rt/lib/builtins/adddf3.c b/contrib/compiler-rt/lib/builtins/adddf3.c
index c528e9e21f51..9a3901312e51 100644
--- a/contrib/compiler-rt/lib/builtins/adddf3.c
+++ b/contrib/compiler-rt/lib/builtins/adddf3.c
@@ -20,8 +20,11 @@ COMPILER_RT_ABI double __adddf3(double a, double b){
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI double __aeabi_dadd(double a, double b) {
return __adddf3(a, b);
}
+#else
+AEABI_RTABI double __aeabi_dadd(double a, double b) COMPILER_RT_ALIAS(__adddf3);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/addsf3.c b/contrib/compiler-rt/lib/builtins/addsf3.c
index fe570687a25e..c5c1a41c3611 100644
--- a/contrib/compiler-rt/lib/builtins/addsf3.c
+++ b/contrib/compiler-rt/lib/builtins/addsf3.c
@@ -20,8 +20,11 @@ COMPILER_RT_ABI float __addsf3(float a, float b) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI float __aeabi_fadd(float a, float b) {
return __addsf3(a, b);
}
+#else
+AEABI_RTABI float __aeabi_fadd(float a, float b) COMPILER_RT_ALIAS(__addsf3);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S
index 3e7a8b86b739..87dd03dce94d 100644
--- a/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S
+++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S
@@ -30,7 +30,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmpeq)
push {r0-r3, lr}
bl __aeabi_cdcmpeq_check_nan
cmp r0, #1
-#if __ARM_ARCH_ISA_THUMB == 1
+#if defined(USE_THUMB_1)
beq 1f
// NaN has been ruled out, so __aeabi_cdcmple can't trap
mov r0, sp
@@ -46,9 +46,12 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmpeq)
pop {r0-r3, lr}
// NaN has been ruled out, so __aeabi_cdcmple can't trap
+ // Use "it ne" + unconditional branch to guarantee a supported relocation if
+ // __aeabi_cdcmple is in a different section for some builds.
+ IT(ne)
bne __aeabi_cdcmple
-#if defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7EM__)
+#if defined(USE_THUMB_2)
mov ip, #APSR_C
msr APSR_nzcvq, ip
#else
@@ -78,7 +81,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmple)
bl __aeabi_dcmplt
cmp r0, #1
-#if __ARM_ARCH_ISA_THUMB == 1
+#if defined(USE_THUMB_1)
bne 1f
// Z = 0, C = 0
movs r0, #1
diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S
index 1f304ffd964c..c5fee6b6a08e 100644
--- a/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S
+++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S
@@ -30,7 +30,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmpeq)
push {r0-r3, lr}
bl __aeabi_cfcmpeq_check_nan
cmp r0, #1
-#if __ARM_ARCH_ISA_THUMB == 1
+#if defined(USE_THUMB_1)
beq 1f
// NaN has been ruled out, so __aeabi_cfcmple can't trap
mov r0, sp
@@ -46,9 +46,12 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmpeq)
pop {r0-r3, lr}
// NaN has been ruled out, so __aeabi_cfcmple can't trap
+ // Use "it ne" + unconditional branch to guarantee a supported relocation if
+ // __aeabi_cfcmple is in a different section for some builds.
+ IT(ne)
bne __aeabi_cfcmple
-#if defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7EM__)
+#if defined(USE_THUMB_2)
mov ip, #APSR_C
msr APSR_nzcvq, ip
#else
@@ -78,7 +81,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmple)
bl __aeabi_fcmplt
cmp r0, #1
-#if __ARM_ARCH_ISA_THUMB == 1
+#if defined(USE_THUMB_1)
bne 1f
// Z = 0, C = 0
movs r0, #1
diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_idivmod.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_idivmod.S
index 4419929f62ac..9c9c80ab5a7b 100644
--- a/contrib/compiler-rt/lib/builtins/arm/aeabi_idivmod.S
+++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_idivmod.S
@@ -20,17 +20,10 @@
#endif
.syntax unified
- .syntax unified
.text
-#if defined(USE_THUMB_PROLOGUE)
- .thumb
-#endif
+ DEFINE_CODE_STATE
.p2align 2
-#if defined(USE_THUMB_PROLOGUE)
-DEFINE_COMPILERRT_THUMB_FUNCTION(__aeabi_idivmod)
-#else
DEFINE_COMPILERRT_FUNCTION(__aeabi_idivmod)
-#endif
#if defined(USE_THUMB_1)
push {r0, r1, lr}
bl SYMBOL_NAME(__divsi3)
diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_memcmp.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_memcmp.S
index 33ea54848b26..e86d6113760e 100644
--- a/contrib/compiler-rt/lib/builtins/arm/aeabi_memcmp.S
+++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_memcmp.S
@@ -14,7 +14,13 @@
.syntax unified
.p2align 2
DEFINE_COMPILERRT_FUNCTION(__aeabi_memcmp)
+#ifdef USE_THUMB_1
+ push {r7, lr}
+ bl memcmp
+ pop {r7, pc}
+#else
b memcmp
+#endif
END_COMPILERRT_FUNCTION(__aeabi_memcmp)
DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memcmp4, __aeabi_memcmp)
diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_memcpy.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_memcpy.S
index eabfa490494c..e83c5fd4dbb3 100644
--- a/contrib/compiler-rt/lib/builtins/arm/aeabi_memcpy.S
+++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_memcpy.S
@@ -14,7 +14,13 @@
.syntax unified
.p2align 2
DEFINE_COMPILERRT_FUNCTION(__aeabi_memcpy)
+#ifdef USE_THUMB_1
+ push {r7, lr}
+ bl memcpy
+ pop {r7, pc}
+#else
b memcpy
+#endif
END_COMPILERRT_FUNCTION(__aeabi_memcpy)
DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memcpy4, __aeabi_memcpy)
diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_memmove.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_memmove.S
index 1bf08c0d5b75..ee28300e46f2 100644
--- a/contrib/compiler-rt/lib/builtins/arm/aeabi_memmove.S
+++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_memmove.S
@@ -13,7 +13,13 @@
.p2align 2
DEFINE_COMPILERRT_FUNCTION(__aeabi_memmove)
+#ifdef USE_THUMB_1
+ push {r7, lr}
+ bl memmove
+ pop {r7, pc}
+#else
b memmove
+#endif
END_COMPILERRT_FUNCTION(__aeabi_memmove)
DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memmove4, __aeabi_memmove)
diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_memset.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_memset.S
index 633f592279b5..0a678d7627e7 100644
--- a/contrib/compiler-rt/lib/builtins/arm/aeabi_memset.S
+++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_memset.S
@@ -18,16 +18,29 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_memset)
mov r3, r1
mov r1, r2
mov r2, r3
+#ifdef USE_THUMB_1
+ push {r7, lr}
+ bl memset
+ pop {r7, pc}
+#else
b memset
+#endif
END_COMPILERRT_FUNCTION(__aeabi_memset)
DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memset4, __aeabi_memset)
DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memset8, __aeabi_memset)
+ .p2align 2
DEFINE_COMPILERRT_FUNCTION(__aeabi_memclr)
mov r2, r1
movs r1, #0
+#ifdef USE_THUMB_1
+ push {r7, lr}
+ bl memset
+ pop {r7, pc}
+#else
b memset
+#endif
END_COMPILERRT_FUNCTION(__aeabi_memclr)
DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_memclr4, __aeabi_memclr)
diff --git a/contrib/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S b/contrib/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S
index 37dae4a105cc..88a4a6d8bc12 100644
--- a/contrib/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S
+++ b/contrib/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S
@@ -22,16 +22,10 @@
.syntax unified
.text
-#if defined(USE_THUMB_PROLOGUE)
- .thumb
-#endif
+ DEFINE_CODE_STATE
.p2align 2
-#if defined(USE_THUMB_PROLOGUE)
-DEFINE_COMPILERRT_THUMB_FUNCTION(__aeabi_uidivmod)
-#else
DEFINE_COMPILERRT_FUNCTION(__aeabi_uidivmod)
-#endif
-#if __ARM_ARCH_ISA_THUMB == 1
+#if defined(USE_THUMB_1)
cmp r0, r1
bcc LOCAL_LABEL(case_denom_larger)
push {r0, r1, lr}
@@ -44,7 +38,7 @@ LOCAL_LABEL(case_denom_larger):
movs r1, r0
movs r0, #0
JMP (lr)
-#else
+#else // defined(USE_THUMB_1)
push { lr }
sub sp, sp, #4
mov r2, sp
diff --git a/contrib/compiler-rt/lib/builtins/arm/bswapdi2.S b/contrib/compiler-rt/lib/builtins/arm/bswapdi2.S
index 4e5a579cf16a..e9db8bac7994 100644
--- a/contrib/compiler-rt/lib/builtins/arm/bswapdi2.S
+++ b/contrib/compiler-rt/lib/builtins/arm/bswapdi2.S
@@ -11,9 +11,7 @@
.syntax unified
.text
-#if defined(USE_THUMB_PROLOGUE)
- .thumb
-#endif
+ DEFINE_CODE_STATE
//
// extern uint64_t __bswapdi2(uint64_t);
@@ -21,11 +19,7 @@
// Reverse all the bytes in a 64-bit integer.
//
.p2align 2
-#if defined(USE_THUMB_PROLOGUE)
-DEFINE_COMPILERRT_THUMB_FUNCTION(__bswapdi2)
-#else
DEFINE_COMPILERRT_FUNCTION(__bswapdi2)
-#endif
#if __ARM_ARCH < 6
// before armv6 does not have "rev" instruction
// r2 = rev(r0)
diff --git a/contrib/compiler-rt/lib/builtins/arm/bswapsi2.S b/contrib/compiler-rt/lib/builtins/arm/bswapsi2.S
index 60342ae0f2fb..1f6eed5c1bbf 100644
--- a/contrib/compiler-rt/lib/builtins/arm/bswapsi2.S
+++ b/contrib/compiler-rt/lib/builtins/arm/bswapsi2.S
@@ -11,9 +11,7 @@
.syntax unified
.text
-#if defined(USE_THUMB_PROLOGUE)
- .thumb
-#endif
+ DEFINE_CODE_STATE
//
// extern uint32_t __bswapsi2(uint32_t);
@@ -21,11 +19,7 @@
// Reverse all the bytes in a 32-bit integer.
//
.p2align 2
-#if defined(USE_THUMB_PROLOGUE)
-DEFINE_COMPILERRT_THUMB_FUNCTION(__bswapsi2)
-#else
DEFINE_COMPILERRT_FUNCTION(__bswapsi2)
-#endif
#if __ARM_ARCH < 6
// before armv6 does not have "rev" instruction
eor r1, r0, r0, ror #16
diff --git a/contrib/compiler-rt/lib/builtins/arm/clzdi2.S b/contrib/compiler-rt/lib/builtins/arm/clzdi2.S
index fe56a183f1b1..fc03b385cdfa 100644
--- a/contrib/compiler-rt/lib/builtins/arm/clzdi2.S
+++ b/contrib/compiler-rt/lib/builtins/arm/clzdi2.S
@@ -15,17 +15,10 @@
.syntax unified
.text
-#if defined(USE_THUMB_PROLOGUE)
- .thumb
-#endif
-
+ DEFINE_CODE_STATE
.p2align 2
-#if defined(USE_THUMB_PROLOGUE)
-DEFINE_COMPILERRT_THUMB_FUNCTION(__clzdi2)
-#else
DEFINE_COMPILERRT_FUNCTION(__clzdi2)
-#endif
#ifdef __ARM_FEATURE_CLZ
#ifdef __ARMEB__
cmp r0, 0
diff --git a/contrib/compiler-rt/lib/builtins/arm/clzsi2.S b/contrib/compiler-rt/lib/builtins/arm/clzsi2.S
index 28d43041eaf2..f2ce59c90119 100644
--- a/contrib/compiler-rt/lib/builtins/arm/clzsi2.S
+++ b/contrib/compiler-rt/lib/builtins/arm/clzsi2.S
@@ -15,16 +15,10 @@
.syntax unified
.text
-#if defined(USE_THUMB_PROLOGUE)
- .thumb
-#endif
+ DEFINE_CODE_STATE
.p2align 2
-#if defined(USE_THUMB_PROLOGUE)
-DEFINE_COMPILERRT_THUMB_FUNCTION(__clzsi2)
-#else
DEFINE_COMPILERRT_FUNCTION(__clzsi2)
-#endif
#ifdef __ARM_FEATURE_CLZ
clz r0, r0
JMP(lr)
diff --git a/contrib/compiler-rt/lib/builtins/arm/comparesf2.S b/contrib/compiler-rt/lib/builtins/arm/comparesf2.S
index 1f7031cbf665..c6c4cc067f07 100644
--- a/contrib/compiler-rt/lib/builtins/arm/comparesf2.S
+++ b/contrib/compiler-rt/lib/builtins/arm/comparesf2.S
@@ -40,25 +40,19 @@
#include "../assembly.h"
.syntax unified
.text
-#if defined(USE_THUMB_PROLOGUE)
- .thumb
-#endif
+ DEFINE_CODE_STATE
@ int __eqsf2(float a, float b)
.p2align 2
-#if defined(USE_THUMB_PROLOGUE)
-DEFINE_COMPILERRT_THUMB_FUNCTION(__eqsf2)
-#else
DEFINE_COMPILERRT_FUNCTION(__eqsf2)
-#endif
#if defined(COMPILER_RT_ARMHF_TARGET)
vmov r0, s0
vmov r1, s1
#endif
// Make copies of a and b with the sign bit shifted off the top. These will
// be used to detect zeros and NaNs.
-#if __ARM_ARCH_ISA_THUMB == 1
+#if defined(USE_THUMB_1)
push {r6, lr}
lsls r2, r0, #1
lsls r3, r1, #1
@@ -150,7 +144,7 @@ DEFINE_COMPILERRT_FUNCTION(__eqsf2)
// If a == b, then the Z flag is set, so we can get the correct final value
// into r0 by simply or'ing with 1 if Z is clear.
// For Thumb-1, r0 contains -1 if a < b, 0 if a > b and 0 if a == b.
-#if defined(USE_THUMB_1)
+#if !defined(USE_THUMB_1)
it ne
orrne r0, r0, #1
#endif
@@ -185,11 +179,7 @@ DEFINE_COMPILERRT_FUNCTION_ALIAS(__nesf2, __eqsf2)
@ int __gtsf2(float a, float b)
.p2align 2
-#if defined(USE_THUMB)
-DEFINE_COMPILERRT_THUMB_FUNCTION(__gtsf2)
-#else
DEFINE_COMPILERRT_FUNCTION(__gtsf2)
-#endif
// Identical to the preceding except in that we return -1 for NaN values.
// Given that the two paths share so much code, one might be tempted to
// unify them; however, the extra code needed to do so makes the code size
@@ -263,11 +253,7 @@ DEFINE_COMPILERRT_FUNCTION_ALIAS(__gesf2, __gtsf2)
@ int __unordsf2(float a, float b)
.p2align 2
-#if defined(USE_THUMB)
-DEFINE_COMPILERRT_THUMB_FUNCTION(__unordsf2)
-#else
DEFINE_COMPILERRT_FUNCTION(__unordsf2)
-#endif
#if defined(COMPILER_RT_ARMHF_TARGET)
vmov r0, s0
diff --git a/contrib/compiler-rt/lib/builtins/arm/divmodsi4.S b/contrib/compiler-rt/lib/builtins/arm/divmodsi4.S
index 3c8359852009..8a027b741efe 100644
--- a/contrib/compiler-rt/lib/builtins/arm/divmodsi4.S
+++ b/contrib/compiler-rt/lib/builtins/arm/divmodsi4.S
@@ -23,20 +23,14 @@
.syntax unified
.text
-#if defined(USE_THUMB_PROLOGUE)
- .thumb
-#endif
+ DEFINE_CODE_STATE
@ int __divmodsi4(int divident, int divisor, int *remainder)
@ Calculate the quotient and remainder of the (signed) division. The return
@ value is the quotient, the remainder is placed in the variable.
.p2align 3
-#if defined(USE_THUMB_PROLOGUE)
-DEFINE_COMPILERRT_THUMB_FUNCTION(__divmodsi4)
-#else
DEFINE_COMPILERRT_FUNCTION(__divmodsi4)
-#endif
#if __ARM_ARCH_EXT_IDIV__
tst r1, r1
beq LOCAL_LABEL(divzero)
diff --git a/contrib/compiler-rt/lib/builtins/arm/divsi3.S b/contrib/compiler-rt/lib/builtins/arm/divsi3.S
index ad96c9e5bfa9..19757af177eb 100644
--- a/contrib/compiler-rt/lib/builtins/arm/divsi3.S
+++ b/contrib/compiler-rt/lib/builtins/arm/divsi3.S
@@ -22,9 +22,7 @@
.syntax unified
.text
-#if defined(USE_THUMB_PROLOGUE)
- .thumb
-#endif
+ DEFINE_CODE_STATE
.p2align 3
// Ok, APCS and AAPCS agree on 32 bit args, so it's safe to use the same routine.
@@ -33,11 +31,7 @@ DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_idiv, __divsi3)
@ int __divsi3(int divident, int divisor)
@ Calculate and return the quotient of the (signed) division.
-#if defined(USE_THUMB_PROLOGUE)
-DEFINE_COMPILERRT_THUMB_FUNCTION(__divsi3)
-#else
DEFINE_COMPILERRT_FUNCTION(__divsi3)
-#endif
#if __ARM_ARCH_EXT_IDIV__
tst r1,r1
beq LOCAL_LABEL(divzero)
@@ -49,14 +43,14 @@ LOCAL_LABEL(divzero):
#else
ESTABLISH_FRAME
// Set aside the sign of the quotient.
-# if __ARM_ARCH_ISA_THUMB == 1
+# if defined(USE_THUMB_1)
movs r4, r0
eors r4, r1
# else
eor r4, r0, r1
# endif
// Take absolute value of a and b via abs(x) = (x^(x >> 31)) - (x >> 31).
-# if __ARM_ARCH_ISA_THUMB == 1
+# if defined(USE_THUMB_1)
asrs r2, r0, #31
asrs r3, r1, #31
eors r0, r2
diff --git a/contrib/compiler-rt/lib/builtins/arm/modsi3.S b/contrib/compiler-rt/lib/builtins/arm/modsi3.S
index 0986d84945a6..be263834d7f1 100644
--- a/contrib/compiler-rt/lib/builtins/arm/modsi3.S
+++ b/contrib/compiler-rt/lib/builtins/arm/modsi3.S
@@ -22,19 +22,13 @@
.syntax unified
.text
-#if defined(USE_THUMB_PROLOGUE)
- .thumb
-#endif
+ DEFINE_CODE_STATE
@ int __modsi3(int divident, int divisor)
@ Calculate and return the remainder of the (signed) division.
.p2align 3
-#if defined(USE_THUMB_PROLOGUE)
-DEFINE_COMPILERRT_THUMB_FUNCTION(__modsi3)
-#else
DEFINE_COMPILERRT_FUNCTION(__modsi3)
-#endif
#if __ARM_ARCH_EXT_IDIV__
tst r1, r1
beq LOCAL_LABEL(divzero)
diff --git a/contrib/compiler-rt/lib/builtins/arm/udivmodsi4.S b/contrib/compiler-rt/lib/builtins/arm/udivmodsi4.S
index 860fced63085..ee3950c9b0eb 100644
--- a/contrib/compiler-rt/lib/builtins/arm/udivmodsi4.S
+++ b/contrib/compiler-rt/lib/builtins/arm/udivmodsi4.S
@@ -16,9 +16,7 @@
.syntax unified
.text
-#if defined(USE_THUMB_PROLOGUE)
- .thumb
-#endif
+ DEFINE_CODE_STATE
@ unsigned int __udivmodsi4(unsigned int divident, unsigned int divisor,
@ unsigned int *remainder)
@@ -26,11 +24,7 @@
@ value is the quotient, the remainder is placed in the variable.
.p2align 2
-#if defined(USE_THUMB_PROLOGUE)
-DEFINE_COMPILERRT_THUMB_FUNCTION(__udivmodsi4)
-#else
DEFINE_COMPILERRT_FUNCTION(__udivmodsi4)
-#endif
#if __ARM_ARCH_EXT_IDIV__
tst r1, r1
beq LOCAL_LABEL(divby0)
diff --git a/contrib/compiler-rt/lib/builtins/arm/udivsi3.S b/contrib/compiler-rt/lib/builtins/arm/udivsi3.S
index 82db2cbb395f..6dea27d404ff 100644
--- a/contrib/compiler-rt/lib/builtins/arm/udivsi3.S
+++ b/contrib/compiler-rt/lib/builtins/arm/udivsi3.S
@@ -16,9 +16,8 @@
.syntax unified
.text
-#if defined(USE_THUMB_PROLOGUE)
- .thumb
-#endif
+
+DEFINE_CODE_STATE
.p2align 2
DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_uidiv, __udivsi3)
@@ -26,11 +25,7 @@ DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_uidiv, __udivsi3)
@ unsigned int __udivsi3(unsigned int divident, unsigned int divisor)
@ Calculate and return the quotient of the (unsigned) division.
-#if defined(USE_THUMB_PROLOGUE)
-DEFINE_COMPILERRT_THUMB_FUNCTION(__udivsi3)
-#else
DEFINE_COMPILERRT_FUNCTION(__udivsi3)
-#endif
#if __ARM_ARCH_EXT_IDIV__
tst r1, r1
beq LOCAL_LABEL(divby0)
diff --git a/contrib/compiler-rt/lib/builtins/arm/umodsi3.S b/contrib/compiler-rt/lib/builtins/arm/umodsi3.S
index 90f3e8575c77..069fad34cb9c 100644
--- a/contrib/compiler-rt/lib/builtins/arm/umodsi3.S
+++ b/contrib/compiler-rt/lib/builtins/arm/umodsi3.S
@@ -16,19 +16,13 @@
.syntax unified
.text
-#if defined(USE_THUMB_PROLOGUE)
- .thumb
-#endif
+ DEFINE_CODE_STATE
@ unsigned int __umodsi3(unsigned int divident, unsigned int divisor)
@ Calculate and return the remainder of the (unsigned) division.
.p2align 2
-#if defined(USE_THUMB_PROLOGUE)
-DEFINE_COMPILERRT_THUMB_FUNCTION(__umodsi3)
-#else
DEFINE_COMPILERRT_FUNCTION(__umodsi3)
-#endif
#if __ARM_ARCH_EXT_IDIV__
tst r1, r1
beq LOCAL_LABEL(divby0)
diff --git a/contrib/compiler-rt/lib/builtins/ashldi3.c b/contrib/compiler-rt/lib/builtins/ashldi3.c
index fcb0abdb1fce..a5c1836006b9 100644
--- a/contrib/compiler-rt/lib/builtins/ashldi3.c
+++ b/contrib/compiler-rt/lib/builtins/ashldi3.c
@@ -41,8 +41,5 @@ __ashldi3(di_int a, si_int b)
}
#if defined(__ARM_EABI__)
-AEABI_RTABI di_int __aeabi_llsl(di_int a, si_int b) {
- return __ashldi3(a, b);
-}
+AEABI_RTABI di_int __aeabi_llsl(di_int a, si_int b) COMPILER_RT_ALIAS(__ashldi3);
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/ashrdi3.c b/contrib/compiler-rt/lib/builtins/ashrdi3.c
index b4ab4c617ba0..84619965eca0 100644
--- a/contrib/compiler-rt/lib/builtins/ashrdi3.c
+++ b/contrib/compiler-rt/lib/builtins/ashrdi3.c
@@ -42,8 +42,5 @@ __ashrdi3(di_int a, si_int b)
}
#if defined(__ARM_EABI__)
-AEABI_RTABI di_int __aeabi_lasr(di_int a, si_int b) {
- return __ashrdi3(a, b);
-}
+AEABI_RTABI di_int __aeabi_lasr(di_int a, si_int b) COMPILER_RT_ALIAS(__ashrdi3);
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/assembly.h b/contrib/compiler-rt/lib/builtins/assembly.h
index 4893d483fd82..3f5e59b25442 100644
--- a/contrib/compiler-rt/lib/builtins/assembly.h
+++ b/contrib/compiler-rt/lib/builtins/assembly.h
@@ -68,10 +68,42 @@
#endif
#if defined(__arm__)
+
+/*
+ * Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros:
+ * - for '-mthumb -march=armv6' compiler defines '__thumb__'
+ * - for '-mthumb -march=armv7' compiler defines '__thumb__' and '__thumb2__'
+ */
+#if defined(__thumb2__) || defined(__thumb__)
+#define DEFINE_CODE_STATE .thumb SEPARATOR
+#define DECLARE_FUNC_ENCODING .thumb_func SEPARATOR
+#if defined(__thumb2__)
+#define USE_THUMB_2
+#define IT(cond) it cond
+#define ITT(cond) itt cond
+#define ITE(cond) ite cond
+#else
+#define USE_THUMB_1
+#define IT(cond)
+#define ITT(cond)
+#define ITE(cond)
+#endif // defined(__thumb__2)
+#else // !defined(__thumb2__) && !defined(__thumb__)
+#define DEFINE_CODE_STATE .arm SEPARATOR
+#define DECLARE_FUNC_ENCODING
+#define IT(cond)
+#define ITT(cond)
+#define ITE(cond)
+#endif
+
+#if defined(USE_THUMB_1) && defined(USE_THUMB_2)
+#error "USE_THUMB_1 and USE_THUMB_2 can't be defined together."
+#endif
+
#if defined(__ARM_ARCH_4T__) || __ARM_ARCH >= 5
#define ARM_HAS_BX
#endif
-#if !defined(__ARM_FEATURE_CLZ) && __ARM_ARCH_ISA_THUMB != 1 && \
+#if !defined(__ARM_FEATURE_CLZ) && !defined(USE_THUMB_1) && \
(__ARM_ARCH >= 6 || (__ARM_ARCH == 5 && !defined(__ARM_ARCH_5__)))
#define __ARM_FEATURE_CLZ
#endif
@@ -93,37 +125,15 @@
JMP(ip)
#endif
-/*
- * Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros:
- * - for '-mthumb -march=armv6' compiler defines '__thumb__'
- * - for '-mthumb -march=armv7' compiler defines '__thumb__' and '__thumb2__'
- */
-#if defined(__thumb2__)
-#define USE_THUMB_2 1
-#elif defined(__thumb__)
-#define USE_THUMB_1 1
-#endif
-
-#if defined(USE_THUMB_1) && defined(USE_THUMB_2)
-#error "USE_THUMB_1 and USE_THUMB_2 can't be defined together."
-#endif
-
-#if defined(USE_THUMB_1) || defined(USE_THUMB_1)
-#define USE_THUMB_PROLOGUE 1
-#endif
-
#if defined(USE_THUMB_2)
-#define IT(cond) it cond
-#define ITT(cond) itt cond
-#define ITE(cond) ite cond
#define WIDE(op) op.w
#else
-#define IT(cond)
-#define ITT(cond)
-#define ITE(cond)
#define WIDE(op) op
#endif
-#endif /* defined(__arm__) */
+#else // !defined(__arm)
+#define DECLARE_FUNC_ENCODING
+#define DEFINE_CODE_STATE
+#endif
#define GLUE2(a, b) a##b
#define GLUE(a, b) GLUE2(a, b)
@@ -137,13 +147,16 @@
#endif
#define DEFINE_COMPILERRT_FUNCTION(name) \
+ DEFINE_CODE_STATE \
FILE_LEVEL_DIRECTIVE SEPARATOR \
.globl SYMBOL_NAME(name) SEPARATOR \
SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \
DECLARE_SYMBOL_VISIBILITY(name) \
+ DECLARE_FUNC_ENCODING \
SYMBOL_NAME(name):
#define DEFINE_COMPILERRT_THUMB_FUNCTION(name) \
+ DEFINE_CODE_STATE \
FILE_LEVEL_DIRECTIVE SEPARATOR \
.globl SYMBOL_NAME(name) SEPARATOR \
SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \
@@ -152,16 +165,20 @@
SYMBOL_NAME(name):
#define DEFINE_COMPILERRT_PRIVATE_FUNCTION(name) \
+ DEFINE_CODE_STATE \
FILE_LEVEL_DIRECTIVE SEPARATOR \
.globl SYMBOL_NAME(name) SEPARATOR \
SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \
HIDDEN(SYMBOL_NAME(name)) SEPARATOR \
+ DECLARE_FUNC_ENCODING \
SYMBOL_NAME(name):
#define DEFINE_COMPILERRT_PRIVATE_FUNCTION_UNMANGLED(name) \
+ DEFINE_CODE_STATE \
.globl name SEPARATOR \
SYMBOL_IS_FUNC(name) SEPARATOR \
HIDDEN(name) SEPARATOR \
+ DECLARE_FUNC_ENCODING \
name:
#define DEFINE_COMPILERRT_FUNCTION_ALIAS(name, target) \
diff --git a/contrib/compiler-rt/lib/builtins/clear_cache.c b/contrib/compiler-rt/lib/builtins/clear_cache.c
index 7ccbe019dba9..4a01cb46d4ac 100644
--- a/contrib/compiler-rt/lib/builtins/clear_cache.c
+++ b/contrib/compiler-rt/lib/builtins/clear_cache.c
@@ -9,6 +9,7 @@
*/
#include "int_lib.h"
+#include <assert.h>
#include <stddef.h>
#if __APPLE__
@@ -23,7 +24,7 @@ uint32_t FlushInstructionCache(uintptr_t hProcess, void *lpBaseAddress,
uintptr_t GetCurrentProcess(void);
#endif
-#if (defined(__FreeBSD__) || defined(__Bitrig__)) && defined(__arm__)
+#if defined(__FreeBSD__) && defined(__arm__)
#include <sys/types.h>
#include <machine/sysarch.h>
#endif
@@ -32,7 +33,7 @@ uintptr_t GetCurrentProcess(void);
#include <machine/sysarch.h>
#endif
-#if defined(__mips__) && !defined(__FreeBSD__)
+#if defined(__linux__) && defined(__mips__)
#include <sys/cachectl.h>
#include <sys/syscall.h>
#include <unistd.h>
@@ -41,7 +42,7 @@ uintptr_t GetCurrentProcess(void);
* clear_mips_cache - Invalidates instruction cache for Mips.
*/
static void clear_mips_cache(const void* Addr, size_t Size) {
- asm volatile (
+ __asm__ volatile (
".set push\n"
".set noreorder\n"
".set noat\n"
@@ -96,7 +97,7 @@ void __clear_cache(void *start, void *end) {
* so there is nothing to do
*/
#elif defined(__arm__) && !defined(__APPLE__)
- #if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__Bitrig__)
+ #if defined(__FreeBSD__) || defined(__NetBSD__)
struct arm_sync_icache_args arg;
arg.addr = (uintptr_t)start;
@@ -121,15 +122,13 @@ void __clear_cache(void *start, void *end) {
: "=r"(start_reg)
: "r"(syscall_nr), "r"(start_reg), "r"(end_reg),
"r"(flags));
- if (start_reg != 0) {
- compilerrt_abort();
- }
+ assert(start_reg == 0 && "Cache flush syscall failed.");
#elif defined(_WIN32)
FlushInstructionCache(GetCurrentProcess(), start, end - start);
#else
compilerrt_abort();
#endif
-#elif defined(__mips__) && !defined(__FreeBSD__)
+#elif defined(__linux__) && defined(__mips__)
const uintptr_t start_int = (uintptr_t) start;
const uintptr_t end_int = (uintptr_t) end;
#if defined(__ANDROID__) && defined(__LP64__)
@@ -165,6 +164,21 @@ void __clear_cache(void *start, void *end) {
for (addr = xstart; addr < xend; addr += icache_line_size)
__asm __volatile("ic ivau, %0" :: "r"(addr));
__asm __volatile("isb sy");
+#elif defined (__powerpc64__)
+ const size_t line_size = 32;
+ const size_t len = (uintptr_t)end - (uintptr_t)start;
+
+ const uintptr_t mask = ~(line_size - 1);
+ const uintptr_t start_line = ((uintptr_t)start) & mask;
+ const uintptr_t end_line = ((uintptr_t)start + len + line_size - 1) & mask;
+
+ for (uintptr_t line = start_line; line < end_line; line += line_size)
+ __asm__ volatile("dcbf 0, %0" : : "r"(line));
+ __asm__ volatile("sync");
+
+ for (uintptr_t line = start_line; line < end_line; line += line_size)
+ __asm__ volatile("icbi 0, %0" : : "r"(line));
+ __asm__ volatile("isync");
#else
#if __APPLE__
/* On Darwin, sys_icache_invalidate() provides this functionality */
@@ -174,4 +188,3 @@ void __clear_cache(void *start, void *end) {
#endif
#endif
}
-
diff --git a/contrib/compiler-rt/lib/builtins/comparedf2.c b/contrib/compiler-rt/lib/builtins/comparedf2.c
index c5bb169d0021..44e5d2b288a6 100644
--- a/contrib/compiler-rt/lib/builtins/comparedf2.c
+++ b/contrib/compiler-rt/lib/builtins/comparedf2.c
@@ -143,8 +143,11 @@ __gtdf2(fp_t a, fp_t b) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI int __aeabi_dcmpun(fp_t a, fp_t b) {
return __unorddf2(a, b);
}
+#else
+AEABI_RTABI int __aeabi_dcmpun(fp_t a, fp_t b) COMPILER_RT_ALIAS(__unorddf2);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/comparesf2.c b/contrib/compiler-rt/lib/builtins/comparesf2.c
index 4badb5e1b9f7..43cd6a6a7003 100644
--- a/contrib/compiler-rt/lib/builtins/comparesf2.c
+++ b/contrib/compiler-rt/lib/builtins/comparesf2.c
@@ -143,8 +143,11 @@ __gtsf2(fp_t a, fp_t b) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI int __aeabi_fcmpun(fp_t a, fp_t b) {
return __unordsf2(a, b);
}
+#else
+AEABI_RTABI int __aeabi_fcmpun(fp_t a, fp_t b) COMPILER_RT_ALIAS(__unordsf2);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/cpu_model.c b/contrib/compiler-rt/lib/builtins/cpu_model.c
index 83ea7a49faf7..4c96e9cd85d5 100644
--- a/contrib/compiler-rt/lib/builtins/cpu_model.c
+++ b/contrib/compiler-rt/lib/builtins/cpu_model.c
@@ -54,6 +54,7 @@ enum ProcessorTypes {
AMD_BTVER1,
AMD_BTVER2,
AMDFAM17H,
+ INTEL_KNM,
CPU_TYPE_MAX
};
@@ -74,6 +75,7 @@ enum ProcessorSubtypes {
INTEL_COREI7_BROADWELL,
INTEL_COREI7_SKYLAKE,
INTEL_COREI7_SKYLAKE_AVX512,
+ INTEL_COREI7_CANNONLAKE,
CPU_SUBTYPE_MAX
};
@@ -339,6 +341,12 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
*Subtype = INTEL_COREI7_SKYLAKE_AVX512; // "skylake-avx512"
break;
+ // Cannonlake:
+ case 0x66:
+ *Type = INTEL_COREI7;
+ *Subtype = INTEL_COREI7_CANNONLAKE; // "cannonlake"
+ break;
+
case 0x1c: // Most 45 nm Intel Atom processors
case 0x26: // 45 nm Atom Lincroft
case 0x27: // 32 nm Atom Medfield
@@ -361,6 +369,10 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
*Type = INTEL_KNL; // knl
break;
+ case 0x85:
+ *Type = INTEL_KNM; // knm
+ break;
+
default: // Unknown family 6 CPU.
break;
break;
diff --git a/contrib/compiler-rt/lib/builtins/divdf3.c b/contrib/compiler-rt/lib/builtins/divdf3.c
index 492e32b851e9..04a4dc5571ca 100644
--- a/contrib/compiler-rt/lib/builtins/divdf3.c
+++ b/contrib/compiler-rt/lib/builtins/divdf3.c
@@ -183,8 +183,11 @@ __divdf3(fp_t a, fp_t b) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI fp_t __aeabi_ddiv(fp_t a, fp_t b) {
return __divdf3(a, b);
}
+#else
+AEABI_RTABI fp_t __aeabi_ddiv(fp_t a, fp_t b) COMPILER_RT_ALIAS(__divdf3);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/divsf3.c b/contrib/compiler-rt/lib/builtins/divsf3.c
index aa6289a6d70a..65294d70fc61 100644
--- a/contrib/compiler-rt/lib/builtins/divsf3.c
+++ b/contrib/compiler-rt/lib/builtins/divsf3.c
@@ -167,8 +167,11 @@ __divsf3(fp_t a, fp_t b) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI fp_t __aeabi_fdiv(fp_t a, fp_t b) {
return __divsf3(a, b);
}
+#else
+AEABI_RTABI fp_t __aeabi_fdiv(fp_t a, fp_t b) COMPILER_RT_ALIAS(__divsf3);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/divsi3.c b/contrib/compiler-rt/lib/builtins/divsi3.c
index 3852e3990b5b..75aea008ddc1 100644
--- a/contrib/compiler-rt/lib/builtins/divsi3.c
+++ b/contrib/compiler-rt/lib/builtins/divsi3.c
@@ -35,8 +35,5 @@ __divsi3(si_int a, si_int b)
}
#if defined(__ARM_EABI__)
-AEABI_RTABI si_int __aeabi_idiv(si_int a, si_int b) {
- return __divsi3(a, b);
-}
+AEABI_RTABI si_int __aeabi_idiv(si_int a, si_int b) COMPILER_RT_ALIAS(__divsi3);
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/emutls.c b/contrib/compiler-rt/lib/builtins/emutls.c
index 12aad3a42b76..5dd8dd154771 100644
--- a/contrib/compiler-rt/lib/builtins/emutls.c
+++ b/contrib/compiler-rt/lib/builtins/emutls.c
@@ -102,7 +102,6 @@ static __inline emutls_address_array* emutls_getspecific() {
#include <malloc.h>
#include <stdio.h>
#include <assert.h>
-#include <immintrin.h>
static LPCRITICAL_SECTION emutls_mutex;
static DWORD emutls_tls_index = TLS_OUT_OF_INDEXES;
@@ -203,25 +202,24 @@ static __inline emutls_address_array* emutls_getspecific() {
/* Provide atomic load/store functions for emutls_get_index if built with MSVC.
*/
#if !defined(__ATOMIC_RELEASE)
+#include <intrin.h>
enum { __ATOMIC_ACQUIRE = 2, __ATOMIC_RELEASE = 3 };
static __inline uintptr_t __atomic_load_n(void *ptr, unsigned type) {
assert(type == __ATOMIC_ACQUIRE);
+ // These return the previous value - but since we do an OR with 0,
+ // it's equivalent to a plain load.
#ifdef _WIN64
- return (uintptr_t) _load_be_u64(ptr);
+ return InterlockedOr64(ptr, 0);
#else
- return (uintptr_t) _load_be_u32(ptr);
+ return InterlockedOr(ptr, 0);
#endif
}
static __inline void __atomic_store_n(void *ptr, uintptr_t val, unsigned type) {
assert(type == __ATOMIC_RELEASE);
-#ifdef _WIN64
- _store_be_u64(ptr, val);
-#else
- _store_be_u32(ptr, val);
-#endif
+ InterlockedExchangePointer((void *volatile *)ptr, (void *)val);
}
#endif
diff --git a/contrib/compiler-rt/lib/builtins/enable_execute_stack.c b/contrib/compiler-rt/lib/builtins/enable_execute_stack.c
index 0dc3482c4467..327d460b4253 100644
--- a/contrib/compiler-rt/lib/builtins/enable_execute_stack.c
+++ b/contrib/compiler-rt/lib/builtins/enable_execute_stack.c
@@ -22,7 +22,7 @@
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
-#include <Windows.h>
+#include <windows.h>
#else
#ifndef __APPLE__
#include <unistd.h>
diff --git a/contrib/compiler-rt/lib/builtins/extendhfsf2.c b/contrib/compiler-rt/lib/builtins/extendhfsf2.c
index e7d9fde8abfc..d9c0db84b0ce 100644
--- a/contrib/compiler-rt/lib/builtins/extendhfsf2.c
+++ b/contrib/compiler-rt/lib/builtins/extendhfsf2.c
@@ -23,8 +23,11 @@ COMPILER_RT_ABI float __gnu_h2f_ieee(uint16_t a) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI float __aeabi_h2f(uint16_t a) {
return __extendhfsf2(a);
}
+#else
+AEABI_RTABI float __aeabi_h2f(uint16_t a) COMPILER_RT_ALIAS(__extendhfsf2);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/extendsfdf2.c b/contrib/compiler-rt/lib/builtins/extendsfdf2.c
index b9e7a7471a98..3d84529a6c53 100644
--- a/contrib/compiler-rt/lib/builtins/extendsfdf2.c
+++ b/contrib/compiler-rt/lib/builtins/extendsfdf2.c
@@ -17,8 +17,11 @@ COMPILER_RT_ABI double __extendsfdf2(float a) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI double __aeabi_f2d(float a) {
return __extendsfdf2(a);
}
+#else
+AEABI_RTABI double __aeabi_f2d(float a) COMPILER_RT_ALIAS(__extendsfdf2);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/fixdfdi.c b/contrib/compiler-rt/lib/builtins/fixdfdi.c
index 31d76df28255..54e312d3c8f7 100644
--- a/contrib/compiler-rt/lib/builtins/fixdfdi.c
+++ b/contrib/compiler-rt/lib/builtins/fixdfdi.c
@@ -45,13 +45,11 @@ __fixdfdi(fp_t a) {
#endif
#if defined(__ARM_EABI__)
-AEABI_RTABI di_int
-#if defined(__SOFT_FP__)
-__aeabi_d2lz(fp_t a) {
-#else
-__aeabi_d2lz(double a) {
-#endif
+#if defined(COMPILER_RT_ARMHF_TARGET)
+AEABI_RTABI di_int __aeabi_d2lz(fp_t a) {
return __fixdfdi(a);
}
+#else
+AEABI_RTABI di_int __aeabi_d2lz(fp_t a) COMPILER_RT_ALIAS(__fixdfdi);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/fixdfsi.c b/contrib/compiler-rt/lib/builtins/fixdfsi.c
index fc316dcd0545..5b9588175717 100644
--- a/contrib/compiler-rt/lib/builtins/fixdfsi.c
+++ b/contrib/compiler-rt/lib/builtins/fixdfsi.c
@@ -20,8 +20,11 @@ __fixdfsi(fp_t a) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI si_int __aeabi_d2iz(fp_t a) {
return __fixdfsi(a);
}
+#else
+AEABI_RTABI si_int __aeabi_d2iz(fp_t a) COMPILER_RT_ALIAS(__fixdfsi);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/fixsfdi.c b/contrib/compiler-rt/lib/builtins/fixsfdi.c
index c43473637d60..32e87c60889f 100644
--- a/contrib/compiler-rt/lib/builtins/fixsfdi.c
+++ b/contrib/compiler-rt/lib/builtins/fixsfdi.c
@@ -45,13 +45,11 @@ __fixsfdi(fp_t a) {
#endif
#if defined(__ARM_EABI__)
-AEABI_RTABI di_int
-#if defined(__SOFT_FP__)
-__aeabi_f2lz(fp_t a) {
-#else
-__aeabi_f2lz(float a) {
-#endif
+#if defined(COMPILER_RT_ARMHF_TARGET)
+AEABI_RTABI di_int __aeabi_f2lz(fp_t a) {
return __fixsfdi(a);
}
+#else
+AEABI_RTABI di_int __aeabi_f2lz(fp_t a) COMPILER_RT_ALIAS(__fixsfdi);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/fixsfsi.c b/contrib/compiler-rt/lib/builtins/fixsfsi.c
index 3276df966460..e94e5f3dcd68 100644
--- a/contrib/compiler-rt/lib/builtins/fixsfsi.c
+++ b/contrib/compiler-rt/lib/builtins/fixsfsi.c
@@ -20,8 +20,11 @@ __fixsfsi(fp_t a) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI si_int __aeabi_f2iz(fp_t a) {
return __fixsfsi(a);
}
+#else
+AEABI_RTABI si_int __aeabi_f2iz(fp_t a) COMPILER_RT_ALIAS(__fixsfsi);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/fixunsdfdi.c b/contrib/compiler-rt/lib/builtins/fixunsdfdi.c
index b734409709bf..bfe4dbb25656 100644
--- a/contrib/compiler-rt/lib/builtins/fixunsdfdi.c
+++ b/contrib/compiler-rt/lib/builtins/fixunsdfdi.c
@@ -42,13 +42,11 @@ __fixunsdfdi(fp_t a) {
#endif
#if defined(__ARM_EABI__)
-AEABI_RTABI du_int
-#if defined(__SOFT_FP__)
-__aeabi_d2ulz(fp_t a) {
-#else
-__aeabi_d2ulz(double a) {
-#endif
+#if defined(COMPILER_RT_ARMHF_TARGET)
+AEABI_RTABI du_int __aeabi_d2ulz(fp_t a) {
return __fixunsdfdi(a);
}
+#else
+AEABI_RTABI du_int __aeabi_d2ulz(fp_t a) COMPILER_RT_ALIAS(__fixunsdfdi);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/fixunsdfsi.c b/contrib/compiler-rt/lib/builtins/fixunsdfsi.c
index bb3d8e0f831b..3c5355beae1a 100644
--- a/contrib/compiler-rt/lib/builtins/fixunsdfsi.c
+++ b/contrib/compiler-rt/lib/builtins/fixunsdfsi.c
@@ -19,8 +19,11 @@ __fixunsdfsi(fp_t a) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI su_int __aeabi_d2uiz(fp_t a) {
return __fixunsdfsi(a);
}
+#else
+AEABI_RTABI su_int __aeabi_d2uiz(fp_t a) COMPILER_RT_ALIAS(__fixunsdfsi);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/fixunssfdi.c b/contrib/compiler-rt/lib/builtins/fixunssfdi.c
index 5d92245df0d9..080a25bb1e99 100644
--- a/contrib/compiler-rt/lib/builtins/fixunssfdi.c
+++ b/contrib/compiler-rt/lib/builtins/fixunssfdi.c
@@ -43,13 +43,11 @@ __fixunssfdi(fp_t a) {
#endif
#if defined(__ARM_EABI__)
-AEABI_RTABI du_int
-#if defined(__SOFT_FP__)
-__aeabi_f2ulz(fp_t a) {
-#else
-__aeabi_f2ulz(float a) {
-#endif
+#if defined(COMPILER_RT_ARMHF_TARGET)
+AEABI_RTABI du_int __aeabi_f2ulz(fp_t a) {
return __fixunssfdi(a);
}
+#else
+AEABI_RTABI du_int __aeabi_f2ulz(fp_t a) COMPILER_RT_ALIAS(__fixunssfdi);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/fixunssfsi.c b/contrib/compiler-rt/lib/builtins/fixunssfsi.c
index 91d5e8ae5d7f..eca2916a5c88 100644
--- a/contrib/compiler-rt/lib/builtins/fixunssfsi.c
+++ b/contrib/compiler-rt/lib/builtins/fixunssfsi.c
@@ -23,8 +23,11 @@ __fixunssfsi(fp_t a) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI su_int __aeabi_f2uiz(fp_t a) {
return __fixunssfsi(a);
}
+#else
+AEABI_RTABI su_int __aeabi_f2uiz(fp_t a) COMPILER_RT_ALIAS(__fixunssfsi);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/floatdidf.c b/contrib/compiler-rt/lib/builtins/floatdidf.c
index 681fecef9682..36b856e078d4 100644
--- a/contrib/compiler-rt/lib/builtins/floatdidf.c
+++ b/contrib/compiler-rt/lib/builtins/floatdidf.c
@@ -105,8 +105,11 @@ __floatdidf(di_int a)
#endif
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI double __aeabi_l2d(di_int a) {
return __floatdidf(a);
}
+#else
+AEABI_RTABI double __aeabi_l2d(di_int a) COMPILER_RT_ALIAS(__floatdidf);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/floatdisf.c b/contrib/compiler-rt/lib/builtins/floatdisf.c
index dd548165c373..a2f09eb2ed2c 100644
--- a/contrib/compiler-rt/lib/builtins/floatdisf.c
+++ b/contrib/compiler-rt/lib/builtins/floatdisf.c
@@ -78,8 +78,11 @@ __floatdisf(di_int a)
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI float __aeabi_l2f(di_int a) {
return __floatdisf(a);
}
+#else
+AEABI_RTABI float __aeabi_l2f(di_int a) COMPILER_RT_ALIAS(__floatdisf);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/floatsidf.c b/contrib/compiler-rt/lib/builtins/floatsidf.c
index 2ae395bdc1db..fe051123ce7c 100644
--- a/contrib/compiler-rt/lib/builtins/floatsidf.c
+++ b/contrib/compiler-rt/lib/builtins/floatsidf.c
@@ -51,8 +51,11 @@ __floatsidf(int a) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI fp_t __aeabi_i2d(int a) {
return __floatsidf(a);
}
+#else
+AEABI_RTABI fp_t __aeabi_i2d(int a) COMPILER_RT_ALIAS(__floatsidf);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/floatsisf.c b/contrib/compiler-rt/lib/builtins/floatsisf.c
index 08891fcdf201..bf087ee3c295 100644
--- a/contrib/compiler-rt/lib/builtins/floatsisf.c
+++ b/contrib/compiler-rt/lib/builtins/floatsisf.c
@@ -57,8 +57,11 @@ __floatsisf(int a) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI fp_t __aeabi_i2f(int a) {
return __floatsisf(a);
}
+#else
+AEABI_RTABI fp_t __aeabi_i2f(int a) COMPILER_RT_ALIAS(__floatsisf);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/floatundidf.c b/contrib/compiler-rt/lib/builtins/floatundidf.c
index 6c1a931ef2f3..8bc2a096324f 100644
--- a/contrib/compiler-rt/lib/builtins/floatundidf.c
+++ b/contrib/compiler-rt/lib/builtins/floatundidf.c
@@ -104,8 +104,11 @@ __floatundidf(du_int a)
#endif
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI double __aeabi_ul2d(du_int a) {
return __floatundidf(a);
}
+#else
+AEABI_RTABI double __aeabi_ul2d(du_int a) COMPILER_RT_ALIAS(__floatundidf);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/floatundisf.c b/contrib/compiler-rt/lib/builtins/floatundisf.c
index 86841a75dc66..844786ea7777 100644
--- a/contrib/compiler-rt/lib/builtins/floatundisf.c
+++ b/contrib/compiler-rt/lib/builtins/floatundisf.c
@@ -75,8 +75,11 @@ __floatundisf(du_int a)
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI float __aeabi_ul2f(du_int a) {
return __floatundisf(a);
}
+#else
+AEABI_RTABI float __aeabi_ul2f(du_int a) COMPILER_RT_ALIAS(__floatundisf);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/floatunsidf.c b/contrib/compiler-rt/lib/builtins/floatunsidf.c
index 8d4807194f0b..75cf6b9177df 100644
--- a/contrib/compiler-rt/lib/builtins/floatunsidf.c
+++ b/contrib/compiler-rt/lib/builtins/floatunsidf.c
@@ -40,8 +40,11 @@ __floatunsidf(unsigned int a) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI fp_t __aeabi_ui2d(unsigned int a) {
return __floatunsidf(a);
}
+#else
+AEABI_RTABI fp_t __aeabi_ui2d(unsigned int a) COMPILER_RT_ALIAS(__floatunsidf);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/floatunsisf.c b/contrib/compiler-rt/lib/builtins/floatunsisf.c
index f194c046d2fb..29525ccedbbe 100644
--- a/contrib/compiler-rt/lib/builtins/floatunsisf.c
+++ b/contrib/compiler-rt/lib/builtins/floatunsisf.c
@@ -48,8 +48,11 @@ __floatunsisf(unsigned int a) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI fp_t __aeabi_ui2f(unsigned int a) {
return __floatunsisf(a);
}
+#else
+AEABI_RTABI fp_t __aeabi_ui2f(unsigned int a) COMPILER_RT_ALIAS(__floatunsisf);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/int_endianness.h b/contrib/compiler-rt/lib/builtins/int_endianness.h
index 7995ddbb953c..e2586c56bac8 100644
--- a/contrib/compiler-rt/lib/builtins/int_endianness.h
+++ b/contrib/compiler-rt/lib/builtins/int_endianness.h
@@ -61,7 +61,7 @@
#endif /* *BSD */
-#if defined(__OpenBSD__) || defined(__Bitrig__)
+#if defined(__OpenBSD__)
#include <machine/endian.h>
#if _BYTE_ORDER == _BIG_ENDIAN
@@ -72,7 +72,7 @@
#define _YUGA_BIG_ENDIAN 0
#endif /* _BYTE_ORDER */
-#endif /* OpenBSD and Bitrig. */
+#endif /* OpenBSD */
/* .. */
diff --git a/contrib/compiler-rt/lib/builtins/int_lib.h b/contrib/compiler-rt/lib/builtins/int_lib.h
index 53caf3a53e63..93394cebf7ea 100644
--- a/contrib/compiler-rt/lib/builtins/int_lib.h
+++ b/contrib/compiler-rt/lib/builtins/int_lib.h
@@ -22,9 +22,11 @@
#if defined(__ELF__)
#define FNALIAS(alias_name, original_name) \
- void alias_name() __attribute__((alias(#original_name)))
+ void alias_name() __attribute__((__alias__(#original_name)))
+#define COMPILER_RT_ALIAS(aliasee) __attribute__((__alias__(#aliasee)))
#else
#define FNALIAS(alias, name) _Pragma("GCC error(\"alias unsupported on this file format\")")
+#define COMPILER_RT_ALIAS(aliasee) _Pragma("GCC error(\"alias unsupported on this file format\")")
#endif
/* ABI macro definitions */
diff --git a/contrib/compiler-rt/lib/builtins/lshrdi3.c b/contrib/compiler-rt/lib/builtins/lshrdi3.c
index becbbef4eb09..67b2a7668345 100644
--- a/contrib/compiler-rt/lib/builtins/lshrdi3.c
+++ b/contrib/compiler-rt/lib/builtins/lshrdi3.c
@@ -41,8 +41,5 @@ __lshrdi3(di_int a, si_int b)
}
#if defined(__ARM_EABI__)
-AEABI_RTABI di_int __aeabi_llsr(di_int a, si_int b) {
- return __lshrdi3(a, b);
-}
+AEABI_RTABI di_int __aeabi_llsr(di_int a, si_int b) COMPILER_RT_ALIAS(__lshrdi3);
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/muldf3.c b/contrib/compiler-rt/lib/builtins/muldf3.c
index 59a60190eba3..1bb103e38c13 100644
--- a/contrib/compiler-rt/lib/builtins/muldf3.c
+++ b/contrib/compiler-rt/lib/builtins/muldf3.c
@@ -20,8 +20,11 @@ COMPILER_RT_ABI fp_t __muldf3(fp_t a, fp_t b) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI fp_t __aeabi_dmul(fp_t a, fp_t b) {
return __muldf3(a, b);
}
+#else
+AEABI_RTABI fp_t __aeabi_dmul(fp_t a, fp_t b) COMPILER_RT_ALIAS(__muldf3);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/muldi3.c b/contrib/compiler-rt/lib/builtins/muldi3.c
index 6818a9e2f722..a187315e9165 100644
--- a/contrib/compiler-rt/lib/builtins/muldi3.c
+++ b/contrib/compiler-rt/lib/builtins/muldi3.c
@@ -54,8 +54,5 @@ __muldi3(di_int a, di_int b)
}
#if defined(__ARM_EABI__)
-AEABI_RTABI di_int __aeabi_lmul(di_int a, di_int b) {
- return __muldi3(a, b);
-}
+AEABI_RTABI di_int __aeabi_lmul(di_int a, di_int b) COMPILER_RT_ALIAS(__muldi3);
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/mulsf3.c b/contrib/compiler-rt/lib/builtins/mulsf3.c
index f141af1acc58..1e2cf3e717c9 100644
--- a/contrib/compiler-rt/lib/builtins/mulsf3.c
+++ b/contrib/compiler-rt/lib/builtins/mulsf3.c
@@ -20,8 +20,11 @@ COMPILER_RT_ABI fp_t __mulsf3(fp_t a, fp_t b) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI fp_t __aeabi_fmul(fp_t a, fp_t b) {
return __mulsf3(a, b);
}
+#else
+AEABI_RTABI fp_t __aeabi_fmul(fp_t a, fp_t b) COMPILER_RT_ALIAS(__mulsf3);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/negdf2.c b/contrib/compiler-rt/lib/builtins/negdf2.c
index 5e2544cdb4be..f0bfaad24743 100644
--- a/contrib/compiler-rt/lib/builtins/negdf2.c
+++ b/contrib/compiler-rt/lib/builtins/negdf2.c
@@ -20,8 +20,11 @@ __negdf2(fp_t a) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI fp_t __aeabi_dneg(fp_t a) {
return __negdf2(a);
}
+#else
+AEABI_RTABI fp_t __aeabi_dneg(fp_t a) COMPILER_RT_ALIAS(__negdf2);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/negsf2.c b/contrib/compiler-rt/lib/builtins/negsf2.c
index f90b34335680..05c97d4d5a11 100644
--- a/contrib/compiler-rt/lib/builtins/negsf2.c
+++ b/contrib/compiler-rt/lib/builtins/negsf2.c
@@ -20,8 +20,11 @@ __negsf2(fp_t a) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI fp_t __aeabi_fneg(fp_t a) {
return __negsf2(a);
}
+#else
+AEABI_RTABI fp_t __aeabi_fneg(fp_t a) COMPILER_RT_ALIAS(__negsf2);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/subdf3.c b/contrib/compiler-rt/lib/builtins/subdf3.c
index 38340dfab1a6..a892fa603cf2 100644
--- a/contrib/compiler-rt/lib/builtins/subdf3.c
+++ b/contrib/compiler-rt/lib/builtins/subdf3.c
@@ -22,8 +22,11 @@ __subdf3(fp_t a, fp_t b) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI fp_t __aeabi_dsub(fp_t a, fp_t b) {
return __subdf3(a, b);
}
+#else
+AEABI_RTABI fp_t __aeabi_dsub(fp_t a, fp_t b) COMPILER_RT_ALIAS(__subdf3);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/subsf3.c b/contrib/compiler-rt/lib/builtins/subsf3.c
index 34276b1447ba..4b2786177dcc 100644
--- a/contrib/compiler-rt/lib/builtins/subsf3.c
+++ b/contrib/compiler-rt/lib/builtins/subsf3.c
@@ -22,8 +22,11 @@ __subsf3(fp_t a, fp_t b) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI fp_t __aeabi_fsub(fp_t a, fp_t b) {
return __subsf3(a, b);
}
+#else
+AEABI_RTABI fp_t __aeabi_fsub(fp_t a, fp_t b) COMPILER_RT_ALIAS(__subsf3);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/truncdfhf2.c b/contrib/compiler-rt/lib/builtins/truncdfhf2.c
index 4bb71aa178a0..8354a41b8b6f 100644
--- a/contrib/compiler-rt/lib/builtins/truncdfhf2.c
+++ b/contrib/compiler-rt/lib/builtins/truncdfhf2.c
@@ -16,8 +16,11 @@ COMPILER_RT_ABI uint16_t __truncdfhf2(double a) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI uint16_t __aeabi_d2h(double a) {
return __truncdfhf2(a);
}
+#else
+AEABI_RTABI uint16_t __aeabi_d2h(double a) COMPILER_RT_ALIAS(__truncdfhf2);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/truncdfsf2.c b/contrib/compiler-rt/lib/builtins/truncdfsf2.c
index 8bf58bb23a3b..195d3e0656e7 100644
--- a/contrib/compiler-rt/lib/builtins/truncdfsf2.c
+++ b/contrib/compiler-rt/lib/builtins/truncdfsf2.c
@@ -16,8 +16,11 @@ COMPILER_RT_ABI float __truncdfsf2(double a) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI float __aeabi_d2f(double a) {
return __truncdfsf2(a);
}
+#else
+AEABI_RTABI float __aeabi_d2f(double a) COMPILER_RT_ALIAS(__truncdfsf2);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/truncsfhf2.c b/contrib/compiler-rt/lib/builtins/truncsfhf2.c
index f6ce1fa1de05..9c84ab4f938a 100644
--- a/contrib/compiler-rt/lib/builtins/truncsfhf2.c
+++ b/contrib/compiler-rt/lib/builtins/truncsfhf2.c
@@ -22,8 +22,11 @@ COMPILER_RT_ABI uint16_t __gnu_f2h_ieee(float a) {
}
#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
AEABI_RTABI uint16_t __aeabi_f2h(float a) {
return __truncsfhf2(a);
}
+#else
+AEABI_RTABI uint16_t __aeabi_f2h(float a) COMPILER_RT_ALIAS(__truncsfhf2);
+#endif
#endif
-
diff --git a/contrib/compiler-rt/lib/builtins/udivsi3.c b/contrib/compiler-rt/lib/builtins/udivsi3.c
index 8eccf102cc97..bb720f8c382b 100644
--- a/contrib/compiler-rt/lib/builtins/udivsi3.c
+++ b/contrib/compiler-rt/lib/builtins/udivsi3.c
@@ -64,8 +64,5 @@ __udivsi3(su_int n, su_int d)
}
#if defined(__ARM_EABI__)
-AEABI_RTABI su_int __aeabi_uidiv(su_int n, su_int d) {
- return __udivsi3(n, d);
-}
+AEABI_RTABI su_int __aeabi_uidiv(su_int n, su_int d) COMPILER_RT_ALIAS(__udivsi3);
#endif
-
diff --git a/contrib/compiler-rt/lib/cfi/cfi.cc b/contrib/compiler-rt/lib/cfi/cfi.cc
index f720230a70be..f7693b37d29c 100644
--- a/contrib/compiler-rt/lib/cfi/cfi.cc
+++ b/contrib/compiler-rt/lib/cfi/cfi.cc
@@ -280,7 +280,7 @@ void InitShadow() {
CHECK_EQ(0, GetShadow());
CHECK_EQ(0, GetShadowSize());
- uptr vma = GetMaxVirtualAddress();
+ uptr vma = GetMaxUserVirtualAddress();
// Shadow is 2 -> 2**kShadowGranularity.
SetShadowSize((vma >> (kShadowGranularity - 1)) + 1);
VReport(1, "CFI: VMA size %zx, shadow size %zx\n", vma, GetShadowSize());
diff --git a/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt b/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt
index cc111be8120e..d8c9d49e3cdd 100644
--- a/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt
+++ b/contrib/compiler-rt/lib/cfi/cfi_blacklist.txt
@@ -1,18 +1,4 @@
-# Standard library types.
-type:std::*
-
-# The stdext namespace contains Microsoft standard library extensions.
-type:stdext::*
-
-# Types with a uuid attribute, i.e. COM types.
-type:attr:uuid
-
-# STL allocators (T *allocator<T *>::allocate(size_type, const void*)).
-# The type signature mandates a cast from uninitialized void* to T*.
-# size_type can either be unsigned int (j) or unsigned long (m).
-fun:*8allocateEjPKv
-fun:*8allocateEmPKv
-
+[cfi-unrelated-cast]
# std::get_temporary_buffer, likewise (libstdc++, libc++).
fun:_ZSt20get_temporary_buffer*
fun:_ZNSt3__120get_temporary_buffer*
diff --git a/contrib/compiler-rt/lib/dfsan/dfsan.cc b/contrib/compiler-rt/lib/dfsan/dfsan.cc
index 3aa99b7f9c27..9e360c959fa7 100644
--- a/contrib/compiler-rt/lib/dfsan/dfsan.cc
+++ b/contrib/compiler-rt/lib/dfsan/dfsan.cc
@@ -21,6 +21,7 @@
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_file.h"
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_flag_parser.h"
#include "sanitizer_common/sanitizer_libc.h"
diff --git a/contrib/compiler-rt/lib/esan/esan_sideline.h b/contrib/compiler-rt/lib/esan/esan_sideline.h
index aa3fae1db318..04aff22f4cf0 100644
--- a/contrib/compiler-rt/lib/esan/esan_sideline.h
+++ b/contrib/compiler-rt/lib/esan/esan_sideline.h
@@ -17,6 +17,7 @@
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_platform_limits_posix.h"
namespace __esan {
@@ -46,7 +47,8 @@ public:
private:
static int runSideline(void *Arg);
static void registerSignal(int SigNum);
- static void handleSidelineSignal(int SigNum, void *SigInfo, void *Ctx);
+ static void handleSidelineSignal(int SigNum, __sanitizer_siginfo *SigInfo,
+ void *Ctx);
char *Stack;
SidelineFunc sampleFunc;
diff --git a/contrib/compiler-rt/lib/esan/esan_sideline_linux.cpp b/contrib/compiler-rt/lib/esan/esan_sideline_linux.cpp
index bc272dfe49f8..4a96910ece35 100644
--- a/contrib/compiler-rt/lib/esan/esan_sideline_linux.cpp
+++ b/contrib/compiler-rt/lib/esan/esan_sideline_linux.cpp
@@ -40,7 +40,8 @@ static const uptr SidelineIdUninitialized = 1;
static SidelineThread *TheThread;
// We aren't passing SA_NODEFER so the same signal is blocked while here.
-void SidelineThread::handleSidelineSignal(int SigNum, void *SigInfo,
+void SidelineThread::handleSidelineSignal(int SigNum,
+ __sanitizer_siginfo *SigInfo,
void *Ctx) {
VPrintf(3, "Sideline signal %d\n", SigNum);
CHECK_EQ(SigNum, SIGALRM);
diff --git a/contrib/compiler-rt/lib/esan/working_set_posix.cpp b/contrib/compiler-rt/lib/esan/working_set_posix.cpp
index fcfa87128857..5ec53b959261 100644
--- a/contrib/compiler-rt/lib/esan/working_set_posix.cpp
+++ b/contrib/compiler-rt/lib/esan/working_set_posix.cpp
@@ -34,7 +34,7 @@ bool processWorkingSetSignal(int SigNum, void (*Handler)(int),
VPrintf(2, "%s: %d\n", __FUNCTION__, SigNum);
if (SigNum == SIGSEGV) {
*Result = AppSigAct.handler;
- AppSigAct.sigaction = (void (*)(int, void*, void*))Handler;
+ AppSigAct.sigaction = (decltype(AppSigAct.sigaction))Handler;
return false; // Skip real call.
}
return true;
@@ -73,7 +73,7 @@ bool processWorkingSetSigprocmask(int How, void *Set, void *OldSet) {
static void reinstateDefaultHandler(int SigNum) {
__sanitizer_sigaction SigAct;
internal_memset(&SigAct, 0, sizeof(SigAct));
- SigAct.sigaction = (void (*)(int, void*, void*)) SIG_DFL;
+ SigAct.sigaction = (decltype(SigAct.sigaction))SIG_DFL;
int Res = internal_sigaction(SigNum, &SigAct, nullptr);
CHECK(Res == 0);
VPrintf(1, "Unregistered for %d handler\n", SigNum);
@@ -81,7 +81,8 @@ static void reinstateDefaultHandler(int SigNum) {
// If this is a shadow fault, we handle it here; otherwise, we pass it to the
// app to handle it just as the app would do without our tool in place.
-static void handleMemoryFault(int SigNum, void *Info, void *Ctx) {
+static void handleMemoryFault(int SigNum, __sanitizer_siginfo *Info,
+ void *Ctx) {
if (SigNum == SIGSEGV) {
// We rely on si_addr being filled in (thus we do not support old kernels).
siginfo_t *SigInfo = (siginfo_t *)Info;
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan.cc b/contrib/compiler-rt/lib/hwasan/hwasan.cc
new file mode 100644
index 000000000000..fcc40eb90182
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan.cc
@@ -0,0 +1,303 @@
+//===-- hwasan.cc -----------------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// HWAddressSanitizer runtime.
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "hwasan_thread.h"
+#include "hwasan_poisoning.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "ubsan/ubsan_flags.h"
+#include "ubsan/ubsan_init.h"
+
+// ACHTUNG! No system header includes in this file.
+
+using namespace __sanitizer;
+
+namespace __hwasan {
+
+void EnterSymbolizer() {
+ HwasanThread *t = GetCurrentThread();
+ CHECK(t);
+ t->EnterSymbolizer();
+}
+void ExitSymbolizer() {
+ HwasanThread *t = GetCurrentThread();
+ CHECK(t);
+ t->LeaveSymbolizer();
+}
+bool IsInSymbolizer() {
+ HwasanThread *t = GetCurrentThread();
+ return t && t->InSymbolizer();
+}
+
+static Flags hwasan_flags;
+
+Flags *flags() {
+ return &hwasan_flags;
+}
+
+int hwasan_inited = 0;
+bool hwasan_init_is_running;
+
+int hwasan_report_count = 0;
+
+void Flags::SetDefaults() {
+#define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
+#include "hwasan_flags.inc"
+#undef HWASAN_FLAG
+}
+
+static void RegisterHwasanFlags(FlagParser *parser, Flags *f) {
+#define HWASAN_FLAG(Type, Name, DefaultValue, Description) \
+ RegisterFlag(parser, #Name, Description, &f->Name);
+#include "hwasan_flags.inc"
+#undef HWASAN_FLAG
+}
+
+static void InitializeFlags() {
+ SetCommonFlagsDefaults();
+ {
+ CommonFlags cf;
+ cf.CopyFrom(*common_flags());
+ cf.external_symbolizer_path = GetEnv("HWASAN_SYMBOLIZER_PATH");
+ cf.malloc_context_size = 20;
+ cf.handle_ioctl = true;
+ // FIXME: test and enable.
+ cf.check_printf = false;
+ cf.intercept_tls_get_addr = true;
+ cf.exitcode = 99;
+ cf.handle_sigill = kHandleSignalExclusive;
+ OverrideCommonFlags(cf);
+ }
+
+ Flags *f = flags();
+ f->SetDefaults();
+
+ FlagParser parser;
+ RegisterHwasanFlags(&parser, f);
+ RegisterCommonFlags(&parser);
+
+#if HWASAN_CONTAINS_UBSAN
+ __ubsan::Flags *uf = __ubsan::flags();
+ uf->SetDefaults();
+
+ FlagParser ubsan_parser;
+ __ubsan::RegisterUbsanFlags(&ubsan_parser, uf);
+ RegisterCommonFlags(&ubsan_parser);
+#endif
+
+ // Override from user-specified string.
+ if (__hwasan_default_options)
+ parser.ParseString(__hwasan_default_options());
+#if HWASAN_CONTAINS_UBSAN
+ const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions();
+ ubsan_parser.ParseString(ubsan_default_options);
+#endif
+
+ const char *hwasan_options = GetEnv("HWASAN_OPTIONS");
+ parser.ParseString(hwasan_options);
+#if HWASAN_CONTAINS_UBSAN
+ ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS"));
+#endif
+ VPrintf(1, "HWASAN_OPTIONS: %s\n", hwasan_options ? hwasan_options : "<empty>");
+
+ InitializeCommonFlags();
+
+ if (Verbosity()) ReportUnrecognizedFlags();
+
+ if (common_flags()->help) parser.PrintFlagDescriptions();
+}
+
+void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp,
+ void *context, bool request_fast_unwind) {
+ HwasanThread *t = GetCurrentThread();
+ if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind)) {
+ // Block reports from our interceptors during _Unwind_Backtrace.
+ SymbolizerScope sym_scope;
+ return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind);
+ }
+ stack->Unwind(max_s, pc, bp, context, t->stack_top(), t->stack_bottom(),
+ request_fast_unwind);
+}
+
+void PrintWarning(uptr pc, uptr bp) {
+ GET_FATAL_STACK_TRACE_PC_BP(pc, bp);
+ ReportInvalidAccess(&stack, 0);
+}
+
+} // namespace __hwasan
+
+// Interface.
+
+using namespace __hwasan;
+
+void __hwasan_init() {
+ CHECK(!hwasan_init_is_running);
+ if (hwasan_inited) return;
+ hwasan_init_is_running = 1;
+ SanitizerToolName = "HWAddressSanitizer";
+
+ InitTlsSize();
+
+ CacheBinaryName();
+ InitializeFlags();
+
+ __sanitizer_set_report_path(common_flags()->log_path);
+
+ InitializeInterceptors();
+ InstallDeadlySignalHandlers(HwasanOnDeadlySignal);
+ InstallAtExitHandler(); // Needs __cxa_atexit interceptor.
+
+ DisableCoreDumperIfNecessary();
+ if (!InitShadow()) {
+ Printf("FATAL: HWAddressSanitizer can not mmap the shadow memory.\n");
+ Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
+ Printf("FATAL: Disabling ASLR is known to cause this error.\n");
+ Printf("FATAL: If running under GDB, try "
+ "'set disable-randomization off'.\n");
+ DumpProcessMap();
+ Die();
+ }
+
+ Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
+
+ InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
+
+ HwasanTSDInit(HwasanTSDDtor);
+
+ HwasanAllocatorInit();
+
+ HwasanThread *main_thread = HwasanThread::Create(nullptr, nullptr);
+ SetCurrentThread(main_thread);
+ main_thread->ThreadStart();
+
+#if HWASAN_CONTAINS_UBSAN
+ __ubsan::InitAsPlugin();
+#endif
+
+ VPrintf(1, "HWAddressSanitizer init done\n");
+
+ hwasan_init_is_running = 0;
+ hwasan_inited = 1;
+}
+
+void __hwasan_print_shadow(const void *x, uptr size) {
+ // FIXME:
+ Printf("FIXME: __hwasan_print_shadow unimplemented\n");
+}
+
+sptr __hwasan_test_shadow(const void *p, uptr sz) {
+ if (sz == 0)
+ return -1;
+ tag_t ptr_tag = GetTagFromPointer((uptr)p);
+ if (ptr_tag == 0)
+ return -1;
+ uptr ptr_raw = GetAddressFromPointer((uptr)p);
+ uptr shadow_first = MEM_TO_SHADOW(ptr_raw);
+ uptr shadow_last = MEM_TO_SHADOW(ptr_raw + sz - 1);
+ for (uptr s = shadow_first; s <= shadow_last; ++s)
+ if (*(tag_t*)s != ptr_tag)
+ return SHADOW_TO_MEM(s) - ptr_raw;
+ return -1;
+}
+
+u16 __sanitizer_unaligned_load16(const uu16 *p) {
+ return *p;
+}
+u32 __sanitizer_unaligned_load32(const uu32 *p) {
+ return *p;
+}
+u64 __sanitizer_unaligned_load64(const uu64 *p) {
+ return *p;
+}
+void __sanitizer_unaligned_store16(uu16 *p, u16 x) {
+ *p = x;
+}
+void __sanitizer_unaligned_store32(uu32 *p, u32 x) {
+ *p = x;
+}
+void __sanitizer_unaligned_store64(uu64 *p, u64 x) {
+ *p = x;
+}
+
+template<unsigned X>
+__attribute__((always_inline))
+static void SigIll() {
+#if defined(__aarch64__)
+ asm("hlt %0\n\t" ::"n"(X));
+#elif defined(__x86_64__) || defined(__i386__)
+ asm("ud2\n\t");
+#else
+ // FIXME: not always sigill.
+ __builtin_trap();
+#endif
+ // __builtin_unreachable();
+}
+
+template<bool IsStore, unsigned LogSize>
+__attribute__((always_inline, nodebug))
+static void CheckAddress(uptr p) {
+ tag_t ptr_tag = GetTagFromPointer(p);
+ uptr ptr_raw = p & ~kAddressTagMask;
+ tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(ptr_raw);
+ if (UNLIKELY(ptr_tag != mem_tag)) SigIll<0x100 + 0x10 * IsStore + LogSize>();
+}
+
+template<bool IsStore>
+__attribute__((always_inline, nodebug))
+static void CheckAddressSized(uptr p, uptr sz) {
+ CHECK_NE(0, sz);
+ tag_t ptr_tag = GetTagFromPointer(p);
+ uptr ptr_raw = p & ~kAddressTagMask;
+ tag_t *shadow_first = (tag_t *)MEM_TO_SHADOW(ptr_raw);
+ tag_t *shadow_last = (tag_t *)MEM_TO_SHADOW(ptr_raw + sz - 1);
+ for (tag_t *t = shadow_first; t <= shadow_last; ++t)
+ if (UNLIKELY(ptr_tag != *t)) SigIll<0x100 + 0x10 * IsStore + 0xf>();
+}
+
+void __hwasan_load(uptr p, uptr sz) { CheckAddressSized<false>(p, sz); }
+void __hwasan_load1(uptr p) { CheckAddress<false, 0>(p); }
+void __hwasan_load2(uptr p) { CheckAddress<false, 1>(p); }
+void __hwasan_load4(uptr p) { CheckAddress<false, 2>(p); }
+void __hwasan_load8(uptr p) { CheckAddress<false, 3>(p); }
+void __hwasan_load16(uptr p) { CheckAddress<false, 4>(p); }
+
+void __hwasan_store(uptr p, uptr sz) { CheckAddressSized<true>(p, sz); }
+void __hwasan_store1(uptr p) { CheckAddress<true, 0>(p); }
+void __hwasan_store2(uptr p) { CheckAddress<true, 1>(p); }
+void __hwasan_store4(uptr p) { CheckAddress<true, 2>(p); }
+void __hwasan_store8(uptr p) { CheckAddress<true, 3>(p); }
+void __hwasan_store16(uptr p) { CheckAddress<true, 4>(p); }
+
+#if !SANITIZER_SUPPORTS_WEAK_HOOKS
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+const char* __hwasan_default_options() { return ""; }
+} // extern "C"
+#endif
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_print_stack_trace() {
+ GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
+ stack.Print();
+}
+} // extern "C"
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan.h b/contrib/compiler-rt/lib/hwasan/hwasan.h
new file mode 100644
index 000000000000..bcf5282dce7c
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan.h
@@ -0,0 +1,172 @@
+//===-- hwasan.h --------------------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Private Hwasan header.
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_H
+#define HWASAN_H
+
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "hwasan_interface_internal.h"
+#include "hwasan_flags.h"
+#include "ubsan/ubsan_platform.h"
+
+#ifndef HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
+# define HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE 1
+#endif
+
+#ifndef HWASAN_CONTAINS_UBSAN
+# define HWASAN_CONTAINS_UBSAN CAN_SANITIZE_UB
+#endif
+
+typedef u8 tag_t;
+
+// Reasonable values are 4 (for 1/16th shadow) and 6 (for 1/64th).
+const uptr kShadowScale = 4;
+const uptr kShadowAlignment = 1UL << kShadowScale;
+
+#define MEM_TO_SHADOW_OFFSET(mem) ((uptr)(mem) >> kShadowScale)
+#define MEM_TO_SHADOW(mem) ((uptr)(mem) >> kShadowScale)
+#define SHADOW_TO_MEM(shadow) ((uptr)(shadow) << kShadowScale)
+
+#define MEM_IS_APP(mem) true
+
+// TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address
+// translation and can be used to store a tag.
+const unsigned kAddressTagShift = 56;
+const uptr kAddressTagMask = 0xFFUL << kAddressTagShift;
+
+static inline tag_t GetTagFromPointer(uptr p) {
+ return p >> kAddressTagShift;
+}
+
+static inline uptr GetAddressFromPointer(uptr p) {
+ return p & ~kAddressTagMask;
+}
+
+static inline void * GetAddressFromPointer(const void *p) {
+ return (void *)((uptr)p & ~kAddressTagMask);
+}
+
+static inline uptr AddTagToPointer(uptr p, tag_t tag) {
+ return (p & ~kAddressTagMask) | ((uptr)tag << kAddressTagShift);
+}
+
+namespace __hwasan {
+
+extern int hwasan_inited;
+extern bool hwasan_init_is_running;
+extern int hwasan_report_count;
+
+bool ProtectRange(uptr beg, uptr end);
+bool InitShadow();
+char *GetProcSelfMaps();
+void InitializeInterceptors();
+
+void HwasanAllocatorInit();
+void HwasanAllocatorThreadFinish();
+void HwasanDeallocate(StackTrace *stack, void *ptr);
+
+void *hwasan_malloc(uptr size, StackTrace *stack);
+void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack);
+void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack);
+void *hwasan_valloc(uptr size, StackTrace *stack);
+void *hwasan_pvalloc(uptr size, StackTrace *stack);
+void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack);
+void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack);
+int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
+ StackTrace *stack);
+
+void InstallTrapHandler();
+void InstallAtExitHandler();
+
+const char *GetStackOriginDescr(u32 id, uptr *pc);
+
+void EnterSymbolizer();
+void ExitSymbolizer();
+bool IsInSymbolizer();
+
+struct SymbolizerScope {
+ SymbolizerScope() { EnterSymbolizer(); }
+ ~SymbolizerScope() { ExitSymbolizer(); }
+};
+
+void PrintWarning(uptr pc, uptr bp);
+
+void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp,
+ void *context, bool request_fast_unwind);
+
+void ReportInvalidAccess(StackTrace *stack, u32 origin);
+void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size,
+ bool is_store);
+void ReportStats();
+void ReportAtExitStatistics();
+void DescribeMemoryRange(const void *x, uptr size);
+void ReportInvalidAccessInsideAddressRange(const char *what, const void *start, uptr size,
+ uptr offset);
+
+// Returns a "chained" origin id, pointing to the given stack trace followed by
+// the previous origin id.
+u32 ChainOrigin(u32 id, StackTrace *stack);
+
+const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1;
+
+#define GET_MALLOC_STACK_TRACE \
+ BufferedStackTrace stack; \
+ if (hwasan_inited) \
+ GetStackTrace(&stack, common_flags()->malloc_context_size, \
+ StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \
+ common_flags()->fast_unwind_on_malloc)
+
+#define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \
+ BufferedStackTrace stack; \
+ if (hwasan_inited) \
+ GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \
+ common_flags()->fast_unwind_on_fatal)
+
+class ScopedThreadLocalStateBackup {
+ public:
+ ScopedThreadLocalStateBackup() { Backup(); }
+ ~ScopedThreadLocalStateBackup() { Restore(); }
+ void Backup();
+ void Restore();
+ private:
+ u64 va_arg_overflow_size_tls;
+};
+
+void HwasanTSDInit(void (*destructor)(void *tsd));
+void *HwasanTSDGet();
+void HwasanTSDSet(void *tsd);
+void HwasanTSDDtor(void *tsd);
+
+void HwasanOnDeadlySignal(int signo, void *info, void *context);
+
+} // namespace __hwasan
+
+#define HWASAN_MALLOC_HOOK(ptr, size) \
+ do { \
+ if (&__sanitizer_malloc_hook) { \
+ __sanitizer_malloc_hook(ptr, size); \
+ } \
+ RunMallocHooks(ptr, size); \
+ } while (false)
+#define HWASAN_FREE_HOOK(ptr) \
+ do { \
+ if (&__sanitizer_free_hook) { \
+ __sanitizer_free_hook(ptr); \
+ } \
+ RunFreeHooks(ptr); \
+ } while (false)
+
+#endif // HWASAN_H
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan.syms.extra b/contrib/compiler-rt/lib/hwasan/hwasan.syms.extra
new file mode 100644
index 000000000000..8738627e06e8
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan.syms.extra
@@ -0,0 +1,2 @@
+__hwasan_*
+__ubsan_*
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc
new file mode 100644
index 000000000000..fbcaf78b88f0
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.cc
@@ -0,0 +1,330 @@
+//===-- hwasan_allocator.cc --------------------------- ---------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// HWAddressSanitizer allocator.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "hwasan.h"
+#include "hwasan_allocator.h"
+#include "hwasan_thread.h"
+#include "hwasan_poisoning.h"
+
+namespace __hwasan {
+
+enum {
+ CHUNK_INVALID = 0,
+ CHUNK_FREE = 1,
+ CHUNK_ALLOCATED = 2
+};
+
+struct Metadata {
+ u64 state : 2;
+ u64 requested_size : 62;
+ u32 alloc_context_id;
+ u32 free_context_id;
+};
+
+bool HwasanChunkView::IsValid() const {
+ return metadata_ && metadata_->state != CHUNK_INVALID;
+}
+bool HwasanChunkView::IsAllocated() const {
+ return metadata_ && metadata_->state == CHUNK_ALLOCATED;
+}
+uptr HwasanChunkView::Beg() const {
+ return block_;
+}
+uptr HwasanChunkView::End() const {
+ return Beg() + UsedSize();
+}
+uptr HwasanChunkView::UsedSize() const {
+ return metadata_->requested_size;
+}
+u32 HwasanChunkView::GetAllocStackId() const {
+ return metadata_->alloc_context_id;
+}
+u32 HwasanChunkView::GetFreeStackId() const {
+ return metadata_->free_context_id;
+}
+
+struct HwasanMapUnmapCallback {
+ void OnMap(uptr p, uptr size) const {}
+ void OnUnmap(uptr p, uptr size) const {
+ // We are about to unmap a chunk of user memory.
+ // It can return as user-requested mmap() or another thread stack.
+ // Make it accessible with zero-tagged pointer.
+ TagMemory(p, size, 0);
+ }
+};
+
+#if !defined(__aarch64__)
+#error unsupported platform
+#endif
+
+static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G
+static const uptr kRegionSizeLog = 20;
+static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog;
+typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap;
+
+struct AP32 {
+ static const uptr kSpaceBeg = 0;
+ static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
+ static const uptr kMetadataSize = sizeof(Metadata);
+ typedef __sanitizer::CompactSizeClassMap SizeClassMap;
+ static const uptr kRegionSizeLog = __hwasan::kRegionSizeLog;
+ typedef __hwasan::ByteMap ByteMap;
+ typedef HwasanMapUnmapCallback MapUnmapCallback;
+ static const uptr kFlags = 0;
+};
+typedef SizeClassAllocator32<AP32> PrimaryAllocator;
+typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache;
+typedef LargeMmapAllocator<HwasanMapUnmapCallback> SecondaryAllocator;
+typedef CombinedAllocator<PrimaryAllocator, AllocatorCache,
+ SecondaryAllocator> Allocator;
+
+static Allocator allocator;
+static AllocatorCache fallback_allocator_cache;
+static SpinMutex fallback_mutex;
+static atomic_uint8_t hwasan_allocator_tagging_enabled;
+
+void HwasanAllocatorInit() {
+ atomic_store_relaxed(&hwasan_allocator_tagging_enabled,
+ !flags()->disable_allocator_tagging);
+ SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+ allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
+}
+
+AllocatorCache *GetAllocatorCache(HwasanThreadLocalMallocStorage *ms) {
+ CHECK(ms);
+ CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
+ return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
+}
+
+void HwasanThreadLocalMallocStorage::CommitBack() {
+ allocator.SwallowCache(GetAllocatorCache(this));
+}
+
+static void *HwasanAllocate(StackTrace *stack, uptr size, uptr alignment,
+ bool zeroise) {
+ alignment = Max(alignment, kShadowAlignment);
+ size = RoundUpTo(size, kShadowAlignment);
+
+ if (size > kMaxAllowedMallocSize) {
+ Report("WARNING: HWAddressSanitizer failed to allocate %p bytes\n",
+ (void *)size);
+ return Allocator::FailureHandler::OnBadRequest();
+ }
+ HwasanThread *t = GetCurrentThread();
+ void *allocated;
+ if (t) {
+ AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+ allocated = allocator.Allocate(cache, size, alignment);
+ } else {
+ SpinMutexLock l(&fallback_mutex);
+ AllocatorCache *cache = &fallback_allocator_cache;
+ allocated = allocator.Allocate(cache, size, alignment);
+ }
+ Metadata *meta =
+ reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
+ meta->state = CHUNK_ALLOCATED;
+ meta->requested_size = size;
+ meta->alloc_context_id = StackDepotPut(*stack);
+ if (zeroise)
+ internal_memset(allocated, 0, size);
+
+ void *user_ptr = (flags()->tag_in_malloc &&
+ atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+ ? (void *)TagMemoryAligned((uptr)allocated, size, 0xBB)
+ : allocated;
+
+ HWASAN_MALLOC_HOOK(user_ptr, size);
+ return user_ptr;
+}
+
+void HwasanDeallocate(StackTrace *stack, void *user_ptr) {
+ CHECK(user_ptr);
+ HWASAN_FREE_HOOK(user_ptr);
+
+ void *p = GetAddressFromPointer(user_ptr);
+ Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
+ uptr size = meta->requested_size;
+ meta->state = CHUNK_FREE;
+ meta->requested_size = 0;
+ meta->free_context_id = StackDepotPut(*stack);
+ // This memory will not be reused by anyone else, so we are free to keep it
+ // poisoned.
+ if (flags()->tag_in_free &&
+ atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+ TagMemoryAligned((uptr)p, size, 0xBC);
+ HwasanThread *t = GetCurrentThread();
+ if (t) {
+ AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+ allocator.Deallocate(cache, p);
+ } else {
+ SpinMutexLock l(&fallback_mutex);
+ AllocatorCache *cache = &fallback_allocator_cache;
+ allocator.Deallocate(cache, p);
+ }
+}
+
+void *HwasanReallocate(StackTrace *stack, void *user_old_p, uptr new_size,
+ uptr alignment) {
+ alignment = Max(alignment, kShadowAlignment);
+ new_size = RoundUpTo(new_size, kShadowAlignment);
+
+ void *old_p = GetAddressFromPointer(user_old_p);
+ Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(old_p));
+ uptr old_size = meta->requested_size;
+ uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p);
+ if (new_size <= actually_allocated_size) {
+ // We are not reallocating here.
+ // FIXME: update stack trace for the allocation?
+ meta->requested_size = new_size;
+ if (!atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+ return user_old_p;
+ if (flags()->retag_in_realloc)
+ return (void *)TagMemoryAligned((uptr)old_p, new_size, 0xCC);
+ if (new_size > old_size) {
+ tag_t tag = GetTagFromPointer((uptr)user_old_p);
+ TagMemoryAligned((uptr)old_p + old_size, new_size - old_size, tag);
+ }
+ return user_old_p;
+ }
+ uptr memcpy_size = Min(new_size, old_size);
+ void *new_p = HwasanAllocate(stack, new_size, alignment, false /*zeroise*/);
+ if (new_p) {
+ internal_memcpy(new_p, old_p, memcpy_size);
+ HwasanDeallocate(stack, old_p);
+ }
+ return new_p;
+}
+
+HwasanChunkView FindHeapChunkByAddress(uptr address) {
+ void *block = allocator.GetBlockBegin(reinterpret_cast<void*>(address));
+ if (!block)
+ return HwasanChunkView();
+ Metadata *metadata =
+ reinterpret_cast<Metadata*>(allocator.GetMetaData(block));
+ return HwasanChunkView(reinterpret_cast<uptr>(block), metadata);
+}
+
+static uptr AllocationSize(const void *user_ptr) {
+ const void *p = GetAddressFromPointer(user_ptr);
+ if (!p) return 0;
+ const void *beg = allocator.GetBlockBegin(p);
+ if (beg != p) return 0;
+ Metadata *b = (Metadata *)allocator.GetMetaData(p);
+ return b->requested_size;
+}
+
+void *hwasan_malloc(uptr size, StackTrace *stack) {
+ return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false));
+}
+
+void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) {
+ if (UNLIKELY(CheckForCallocOverflow(size, nmemb)))
+ return SetErrnoOnNull(Allocator::FailureHandler::OnBadRequest());
+ return SetErrnoOnNull(HwasanAllocate(stack, nmemb * size, sizeof(u64), true));
+}
+
+void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) {
+ if (!ptr)
+ return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false));
+ if (size == 0) {
+ HwasanDeallocate(stack, ptr);
+ return nullptr;
+ }
+ return SetErrnoOnNull(HwasanReallocate(stack, ptr, size, sizeof(u64)));
+}
+
+void *hwasan_valloc(uptr size, StackTrace *stack) {
+ return SetErrnoOnNull(HwasanAllocate(stack, size, GetPageSizeCached(), false));
+}
+
+void *hwasan_pvalloc(uptr size, StackTrace *stack) {
+ uptr PageSize = GetPageSizeCached();
+ if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
+ errno = errno_ENOMEM;
+ return Allocator::FailureHandler::OnBadRequest();
+ }
+ // pvalloc(0) should allocate one page.
+ size = size ? RoundUpTo(size, PageSize) : PageSize;
+ return SetErrnoOnNull(HwasanAllocate(stack, size, PageSize, false));
+}
+
+void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) {
+ if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
+ errno = errno_EINVAL;
+ return Allocator::FailureHandler::OnBadRequest();
+ }
+ return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false));
+}
+
+void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack) {
+ if (UNLIKELY(!IsPowerOfTwo(alignment))) {
+ errno = errno_EINVAL;
+ return Allocator::FailureHandler::OnBadRequest();
+ }
+ return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false));
+}
+
+int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
+ StackTrace *stack) {
+ if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
+ Allocator::FailureHandler::OnBadRequest();
+ return errno_EINVAL;
+ }
+ void *ptr = HwasanAllocate(stack, size, alignment, false);
+ if (UNLIKELY(!ptr))
+ return errno_ENOMEM;
+ CHECK(IsAligned((uptr)ptr, alignment));
+ *memptr = ptr;
+ return 0;
+}
+
+} // namespace __hwasan
+
+using namespace __hwasan;
+
+void __hwasan_enable_allocator_tagging() {
+ atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 1);
+}
+
+void __hwasan_disable_allocator_tagging() {
+ atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 0);
+}
+
+uptr __sanitizer_get_current_allocated_bytes() {
+ uptr stats[AllocatorStatCount];
+ allocator.GetStats(stats);
+ return stats[AllocatorStatAllocated];
+}
+
+uptr __sanitizer_get_heap_size() {
+ uptr stats[AllocatorStatCount];
+ allocator.GetStats(stats);
+ return stats[AllocatorStatMapped];
+}
+
+uptr __sanitizer_get_free_bytes() { return 1; }
+
+uptr __sanitizer_get_unmapped_bytes() { return 1; }
+
+uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
+
+int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
+
+uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h
new file mode 100644
index 000000000000..d025112e9773
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan_allocator.h
@@ -0,0 +1,55 @@
+//===-- hwasan_allocator.h ----------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_ALLOCATOR_H
+#define HWASAN_ALLOCATOR_H
+
+#include "sanitizer_common/sanitizer_common.h"
+
+namespace __hwasan {
+
+struct HwasanThreadLocalMallocStorage {
+ uptr quarantine_cache[16];
+ // Allocator cache contains atomic_uint64_t which must be 8-byte aligned.
+ ALIGNED(8) uptr allocator_cache[96 * (512 * 8 + 16)]; // Opaque.
+ void CommitBack();
+
+ private:
+ // These objects are allocated via mmap() and are zero-initialized.
+ HwasanThreadLocalMallocStorage() {}
+};
+
+struct Metadata;
+
+class HwasanChunkView {
+ public:
+ HwasanChunkView() : block_(0), metadata_(nullptr) {}
+ HwasanChunkView(uptr block, Metadata *metadata)
+ : block_(block), metadata_(metadata) {}
+ bool IsValid() const; // Checks if it points to a valid allocated chunk
+ bool IsAllocated() const; // Checks if the memory is currently allocated
+ uptr Beg() const; // First byte of user memory
+ uptr End() const; // Last byte of user memory
+ uptr UsedSize() const; // Size requested by the user
+ u32 GetAllocStackId() const;
+ u32 GetFreeStackId() const;
+ private:
+ uptr block_;
+ Metadata *const metadata_;
+};
+
+HwasanChunkView FindHeapChunkByAddress(uptr address);
+
+} // namespace __hwasan
+
+#endif // HWASAN_ALLOCATOR_H
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_blacklist.txt b/contrib/compiler-rt/lib/hwasan/hwasan_blacklist.txt
new file mode 100644
index 000000000000..395ba28f0212
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan_blacklist.txt
@@ -0,0 +1,7 @@
+# Blacklist for HWAddressSanitizer. Turns off instrumentation of particular
+# functions or sources. Use with care. You may set location of blacklist
+# at compile-time using -fsanitize-blacklist=<path> flag.
+
+# Example usage:
+# fun:*bad_function_name*
+# src:file_with_tricky_code.cc
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_flags.h b/contrib/compiler-rt/lib/hwasan/hwasan_flags.h
new file mode 100644
index 000000000000..16d60c4d8ba5
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan_flags.h
@@ -0,0 +1,30 @@
+//===-- hwasan_flags.h --------------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+#ifndef HWASAN_FLAGS_H
+#define HWASAN_FLAGS_H
+
+namespace __hwasan {
+
+struct Flags {
+#define HWASAN_FLAG(Type, Name, DefaultValue, Description) Type Name;
+#include "hwasan_flags.inc"
+#undef HWASAN_FLAG
+
+ void SetDefaults();
+};
+
+Flags *flags();
+
+} // namespace __hwasan
+
+#endif // HWASAN_FLAGS_H
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc b/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc
new file mode 100644
index 000000000000..a2cb701ae93d
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan_flags.inc
@@ -0,0 +1,29 @@
+//===-- hwasan_flags.inc ------------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Hwasan runtime flags.
+//
+//===----------------------------------------------------------------------===//
+#ifndef HWASAN_FLAG
+# error "Define HWASAN_FLAG prior to including this file!"
+#endif
+
+// HWASAN_FLAG(Type, Name, DefaultValue, Description)
+// See COMMON_FLAG in sanitizer_flags.inc for more details.
+
+HWASAN_FLAG(bool, tag_in_malloc, true, "")
+HWASAN_FLAG(bool, tag_in_free, true, "")
+HWASAN_FLAG(bool, retag_in_realloc, true, "")
+HWASAN_FLAG(bool, print_stats, false, "")
+HWASAN_FLAG(bool, halt_on_error, true, "")
+HWASAN_FLAG(bool, atexit, false, "")
+
+// Test only flag to disable malloc/realloc/free memory tagging on startup.
+// Tagging can be reenabled with __hwasan_enable_allocator_tagging().
+HWASAN_FLAG(bool, disable_allocator_tagging, false, "")
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc b/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc
new file mode 100644
index 000000000000..fb39e9fdacf9
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan_interceptors.cc
@@ -0,0 +1,483 @@
+//===-- hwasan_interceptors.cc ----------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Interceptors for standard library functions.
+//
+// FIXME: move as many interceptors as possible into
+// sanitizer_common/sanitizer_common_interceptors.h
+//===----------------------------------------------------------------------===//
+
+#include "interception/interception.h"
+#include "hwasan.h"
+#include "hwasan_thread.h"
+#include "hwasan_poisoning.h"
+#include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+
+#include <stdarg.h>
+// ACHTUNG! No other system header includes in this file.
+// Ideally, we should get rid of stdarg.h as well.
+
+using namespace __hwasan;
+
+using __sanitizer::memory_order;
+using __sanitizer::atomic_load;
+using __sanitizer::atomic_store;
+using __sanitizer::atomic_uintptr_t;
+
+DECLARE_REAL(SIZE_T, strlen, const char *s)
+DECLARE_REAL(SIZE_T, strnlen, const char *s, SIZE_T maxlen)
+DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n)
+DECLARE_REAL(void *, memset, void *dest, int c, uptr n)
+
+bool IsInInterceptorScope() {
+ HwasanThread *t = GetCurrentThread();
+ return t && t->InInterceptorScope();
+}
+
+struct InterceptorScope {
+ InterceptorScope() {
+ HwasanThread *t = GetCurrentThread();
+ if (t)
+ t->EnterInterceptorScope();
+ }
+ ~InterceptorScope() {
+ HwasanThread *t = GetCurrentThread();
+ if (t)
+ t->LeaveInterceptorScope();
+ }
+};
+
+static uptr allocated_for_dlsym;
+static const uptr kDlsymAllocPoolSize = 1024;
+static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
+
+static bool IsInDlsymAllocPool(const void *ptr) {
+ uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+ return off < sizeof(alloc_memory_for_dlsym);
+}
+
+static void *AllocateFromLocalPool(uptr size_in_bytes) {
+ uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
+ void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
+ allocated_for_dlsym += size_in_words;
+ CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
+ return mem;
+}
+
+#define ENSURE_HWASAN_INITED() do { \
+ CHECK(!hwasan_init_is_running); \
+ if (!hwasan_inited) { \
+ __hwasan_init(); \
+ } \
+} while (0)
+
+
+
+#define HWASAN_READ_RANGE(ctx, offset, size) \
+ CHECK_UNPOISONED(offset, size)
+#define HWASAN_WRITE_RANGE(ctx, offset, size) \
+ CHECK_UNPOISONED(offset, size)
+
+
+
+// Check that [x, x+n) range is unpoisoned.
+#define CHECK_UNPOISONED_0(x, n) \
+ do { \
+ sptr __offset = __hwasan_test_shadow(x, n); \
+ if (__hwasan::IsInSymbolizer()) break; \
+ if (__offset >= 0) { \
+ GET_CALLER_PC_BP_SP; \
+ (void)sp; \
+ ReportInvalidAccessInsideAddressRange(__func__, x, n, __offset); \
+ __hwasan::PrintWarning(pc, bp); \
+ if (__hwasan::flags()->halt_on_error) { \
+ Printf("Exiting\n"); \
+ Die(); \
+ } \
+ } \
+ } while (0)
+
+// Check that [x, x+n) range is unpoisoned unless we are in a nested
+// interceptor.
+#define CHECK_UNPOISONED(x, n) \
+ do { \
+ if (!IsInInterceptorScope()) CHECK_UNPOISONED_0(x, n); \
+ } while (0)
+
+#define CHECK_UNPOISONED_STRING_OF_LEN(x, len, n) \
+ CHECK_UNPOISONED((x), \
+ common_flags()->strict_string_checks ? (len) + 1 : (n) )
+
+
+INTERCEPTOR(int, posix_memalign, void **memptr, SIZE_T alignment, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ CHECK_NE(memptr, 0);
+ int res = hwasan_posix_memalign(memptr, alignment, size, &stack);
+ return res;
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(void *, memalign, SIZE_T alignment, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_memalign(alignment, size, &stack);
+}
+#define HWASAN_MAYBE_INTERCEPT_MEMALIGN INTERCEPT_FUNCTION(memalign)
+#else
+#define HWASAN_MAYBE_INTERCEPT_MEMALIGN
+#endif
+
+INTERCEPTOR(void *, aligned_alloc, SIZE_T alignment, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_aligned_alloc(alignment, size, &stack);
+}
+
+INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ void *ptr = hwasan_memalign(alignment, size, &stack);
+ if (ptr)
+ DTLS_on_libc_memalign(ptr, size);
+ return ptr;
+}
+
+INTERCEPTOR(void *, valloc, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_valloc(size, &stack);
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(void *, pvalloc, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_pvalloc(size, &stack);
+}
+#define HWASAN_MAYBE_INTERCEPT_PVALLOC INTERCEPT_FUNCTION(pvalloc)
+#else
+#define HWASAN_MAYBE_INTERCEPT_PVALLOC
+#endif
+
+INTERCEPTOR(void, free, void *ptr) {
+ GET_MALLOC_STACK_TRACE;
+ if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
+ HwasanDeallocate(&stack, ptr);
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(void, cfree, void *ptr) {
+ GET_MALLOC_STACK_TRACE;
+ if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
+ HwasanDeallocate(&stack, ptr);
+}
+#define HWASAN_MAYBE_INTERCEPT_CFREE INTERCEPT_FUNCTION(cfree)
+#else
+#define HWASAN_MAYBE_INTERCEPT_CFREE
+#endif
+
+INTERCEPTOR(uptr, malloc_usable_size, void *ptr) {
+ return __sanitizer_get_allocated_size(ptr);
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+// This function actually returns a struct by value, but we can't unpoison a
+// temporary! The following is equivalent on all supported platforms but
+// aarch64 (which uses a different register for sret value). We have a test
+// to confirm that.
+INTERCEPTOR(void, mallinfo, __sanitizer_mallinfo *sret) {
+#ifdef __aarch64__
+ uptr r8;
+ asm volatile("mov %0,x8" : "=r" (r8));
+ sret = reinterpret_cast<__sanitizer_mallinfo*>(r8);
+#endif
+ REAL(memset)(sret, 0, sizeof(*sret));
+}
+#define HWASAN_MAYBE_INTERCEPT_MALLINFO INTERCEPT_FUNCTION(mallinfo)
+#else
+#define HWASAN_MAYBE_INTERCEPT_MALLINFO
+#endif
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(int, mallopt, int cmd, int value) {
+ return -1;
+}
+#define HWASAN_MAYBE_INTERCEPT_MALLOPT INTERCEPT_FUNCTION(mallopt)
+#else
+#define HWASAN_MAYBE_INTERCEPT_MALLOPT
+#endif
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(void, malloc_stats, void) {
+ // FIXME: implement, but don't call REAL(malloc_stats)!
+}
+#define HWASAN_MAYBE_INTERCEPT_MALLOC_STATS INTERCEPT_FUNCTION(malloc_stats)
+#else
+#define HWASAN_MAYBE_INTERCEPT_MALLOC_STATS
+#endif
+
+
+INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(!hwasan_inited))
+ // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
+ return AllocateFromLocalPool(nmemb * size);
+ return hwasan_calloc(nmemb, size, &stack);
+}
+
+INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
+ uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+ uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
+ void *new_ptr;
+ if (UNLIKELY(!hwasan_inited)) {
+ new_ptr = AllocateFromLocalPool(copy_size);
+ } else {
+ copy_size = size;
+ new_ptr = hwasan_malloc(copy_size, &stack);
+ }
+ internal_memcpy(new_ptr, ptr, copy_size);
+ return new_ptr;
+ }
+ return hwasan_realloc(ptr, size, &stack);
+}
+
+INTERCEPTOR(void *, malloc, SIZE_T size) {
+ GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(!hwasan_inited))
+ // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
+ return AllocateFromLocalPool(size);
+ return hwasan_malloc(size, &stack);
+}
+
+
+INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags,
+ int fd, OFF_T offset) {
+ if (hwasan_init_is_running)
+ return REAL(mmap)(addr, length, prot, flags, fd, offset);
+ ENSURE_HWASAN_INITED();
+ if (addr && !MEM_IS_APP(addr)) {
+ if (flags & map_fixed) {
+ errno = errno_EINVAL;
+ return (void *)-1;
+ } else {
+ addr = nullptr;
+ }
+ }
+ void *res = REAL(mmap)(addr, length, prot, flags, fd, offset);
+ return res;
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags,
+ int fd, OFF64_T offset) {
+ ENSURE_HWASAN_INITED();
+ if (addr && !MEM_IS_APP(addr)) {
+ if (flags & map_fixed) {
+ errno = errno_EINVAL;
+ return (void *)-1;
+ } else {
+ addr = nullptr;
+ }
+ }
+ void *res = REAL(mmap64)(addr, length, prot, flags, fd, offset);
+ return res;
+}
+#define HWASAN_MAYBE_INTERCEPT_MMAP64 INTERCEPT_FUNCTION(mmap64)
+#else
+#define HWASAN_MAYBE_INTERCEPT_MMAP64
+#endif
+
+extern "C" int pthread_attr_init(void *attr);
+extern "C" int pthread_attr_destroy(void *attr);
+
+static void *HwasanThreadStartFunc(void *arg) {
+ HwasanThread *t = (HwasanThread *)arg;
+ SetCurrentThread(t);
+ return t->ThreadStart();
+}
+
+INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*),
+ void * param) {
+ ENSURE_HWASAN_INITED(); // for GetTlsSize()
+ __sanitizer_pthread_attr_t myattr;
+ if (!attr) {
+ pthread_attr_init(&myattr);
+ attr = &myattr;
+ }
+
+ AdjustStackSize(attr);
+
+ HwasanThread *t = HwasanThread::Create(callback, param);
+
+ int res = REAL(pthread_create)(th, attr, HwasanThreadStartFunc, t);
+
+ if (attr == &myattr)
+ pthread_attr_destroy(&myattr);
+ return res;
+}
+
+static void BeforeFork() {
+ StackDepotLockAll();
+}
+
+static void AfterFork() {
+ StackDepotUnlockAll();
+}
+
+INTERCEPTOR(int, fork, void) {
+ ENSURE_HWASAN_INITED();
+ BeforeFork();
+ int pid = REAL(fork)();
+ AfterFork();
+ return pid;
+}
+
+
+struct HwasanInterceptorContext {
+ bool in_interceptor_scope;
+};
+
+namespace __hwasan {
+
+int OnExit() {
+ // FIXME: ask frontend whether we need to return failure.
+ return 0;
+}
+
+} // namespace __hwasan
+
+// A version of CHECK_UNPOISONED using a saved scope value. Used in common
+// interceptors.
+#define CHECK_UNPOISONED_CTX(ctx, x, n) \
+ do { \
+ if (!((HwasanInterceptorContext *)ctx)->in_interceptor_scope) \
+ CHECK_UNPOISONED_0(x, n); \
+ } while (0)
+
+#define HWASAN_INTERCEPT_FUNC(name) \
+ do { \
+ if ((!INTERCEPT_FUNCTION(name) || !REAL(name))) \
+ VReport(1, "HWAddressSanitizer: failed to intercept '" #name "'\n"); \
+ } while (0)
+
+#define HWASAN_INTERCEPT_FUNC_VER(name, ver) \
+ do { \
+ if ((!INTERCEPT_FUNCTION_VER(name, ver) || !REAL(name))) \
+ VReport( \
+ 1, "HWAddressSanitizer: failed to intercept '" #name "@@" #ver "'\n"); \
+ } while (0)
+
+#define COMMON_INTERCEPT_FUNCTION(name) HWASAN_INTERCEPT_FUNC(name)
+#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
+ HWASAN_INTERCEPT_FUNC_VER(name, ver)
+#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
+ CHECK_UNPOISONED_CTX(ctx, ptr, size)
+#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
+ CHECK_UNPOISONED_CTX(ctx, ptr, size)
+#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(ptr, size) \
+ HWASAN_WRITE_RANGE(ctx, ptr, size)
+#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
+ if (hwasan_init_is_running) return REAL(func)(__VA_ARGS__); \
+ ENSURE_HWASAN_INITED(); \
+ HwasanInterceptorContext hwasan_ctx = {IsInInterceptorScope()}; \
+ ctx = (void *)&hwasan_ctx; \
+ (void)ctx; \
+ InterceptorScope interceptor_scope;
+#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \
+ do { \
+ } while (false)
+#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
+ do { \
+ } while (false)
+#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
+ do { \
+ } while (false)
+#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \
+ do { \
+ } while (false)
+#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
+ do { \
+ } while (false) // FIXME
+#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
+ do { \
+ } while (false) // FIXME
+#define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name)
+#define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit()
+
+#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \
+ if (HwasanThread *t = GetCurrentThread()) { \
+ *begin = t->tls_begin(); \
+ *end = t->tls_end(); \
+ } else { \
+ *begin = *end = 0; \
+ }
+
+#include "sanitizer_common/sanitizer_platform_interceptors.h"
+#include "sanitizer_common/sanitizer_common_interceptors.inc"
+#include "sanitizer_common/sanitizer_signal_interceptors.inc"
+
+#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) CHECK_UNPOISONED(p, s)
+#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
+ do { \
+ (void)(p); \
+ (void)(s); \
+ } while (false)
+#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
+ do { \
+ (void)(p); \
+ (void)(s); \
+ } while (false)
+#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
+ do { \
+ (void)(p); \
+ (void)(s); \
+ } while (false)
+#include "sanitizer_common/sanitizer_common_syscalls.inc"
+
+
+
+namespace __hwasan {
+
+void InitializeInterceptors() {
+ static int inited = 0;
+ CHECK_EQ(inited, 0);
+ InitializeCommonInterceptors();
+ InitializeSignalInterceptors();
+
+ INTERCEPT_FUNCTION(mmap);
+ HWASAN_MAYBE_INTERCEPT_MMAP64;
+ INTERCEPT_FUNCTION(posix_memalign);
+ HWASAN_MAYBE_INTERCEPT_MEMALIGN;
+ INTERCEPT_FUNCTION(__libc_memalign);
+ INTERCEPT_FUNCTION(valloc);
+ HWASAN_MAYBE_INTERCEPT_PVALLOC;
+ INTERCEPT_FUNCTION(malloc);
+ INTERCEPT_FUNCTION(calloc);
+ INTERCEPT_FUNCTION(realloc);
+ INTERCEPT_FUNCTION(free);
+ HWASAN_MAYBE_INTERCEPT_CFREE;
+ INTERCEPT_FUNCTION(malloc_usable_size);
+ HWASAN_MAYBE_INTERCEPT_MALLINFO;
+ HWASAN_MAYBE_INTERCEPT_MALLOPT;
+ HWASAN_MAYBE_INTERCEPT_MALLOC_STATS;
+ INTERCEPT_FUNCTION(pthread_create);
+ INTERCEPT_FUNCTION(fork);
+
+ inited = 1;
+}
+} // namespace __hwasan
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h b/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h
new file mode 100644
index 000000000000..08b77534e0ae
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan_interface_internal.h
@@ -0,0 +1,97 @@
+//===-- hwasan_interface_internal.h -------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Private Hwasan interface header.
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_INTERFACE_INTERNAL_H
+#define HWASAN_INTERFACE_INTERNAL_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_init();
+
+using __sanitizer::uptr;
+using __sanitizer::sptr;
+using __sanitizer::uu64;
+using __sanitizer::uu32;
+using __sanitizer::uu16;
+using __sanitizer::u64;
+using __sanitizer::u32;
+using __sanitizer::u16;
+using __sanitizer::u8;
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load(uptr, uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load1(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load2(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load4(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load8(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load16(uptr);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store(uptr, uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store1(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store2(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store4(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store8(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store16(uptr);
+
+// Returns the offset of the first tag mismatch or -1 if the whole range is
+// good.
+SANITIZER_INTERFACE_ATTRIBUTE
+sptr __hwasan_test_shadow(const void *x, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+/* OPTIONAL */ const char* __hwasan_default_options();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_print_shadow(const void *x, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u16 __sanitizer_unaligned_load16(const uu16 *p);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u32 __sanitizer_unaligned_load32(const uu32 *p);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u64 __sanitizer_unaligned_load64(const uu64 *p);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store16(uu16 *p, u16 x);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store32(uu32 *p, u32 x);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store64(uu64 *p, u64 x);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_enable_allocator_tagging();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_disable_allocator_tagging();
+
+} // extern "C"
+
+#endif // HWASAN_INTERFACE_INTERNAL_H
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc b/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc
new file mode 100644
index 000000000000..9b8613171cbd
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan_linux.cc
@@ -0,0 +1,251 @@
+//===-- hwasan_linux.cc -----------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Linux-, NetBSD- and FreeBSD-specific code.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
+
+#include "hwasan.h"
+#include "hwasan_thread.h"
+
+#include <elf.h>
+#include <link.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <unistd.h>
+#include <unwind.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+
+namespace __hwasan {
+
+void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
+ CHECK_EQ((beg % GetMmapGranularity()), 0);
+ CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
+ uptr size = end - beg + 1;
+ DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb.
+ void *res = MmapFixedNoReserve(beg, size, name);
+ if (res != (void *)beg) {
+ Report(
+ "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
+ "Perhaps you're using ulimit -v\n",
+ size);
+ Abort();
+ }
+ if (common_flags()->no_huge_pages_for_shadow) NoHugePagesInRegion(beg, size);
+ if (common_flags()->use_madv_dontdump) DontDumpShadowMemory(beg, size);
+}
+
+static void ProtectGap(uptr addr, uptr size) {
+ void *res = MmapFixedNoAccess(addr, size, "shadow gap");
+ if (addr == (uptr)res) return;
+ // A few pages at the start of the address space can not be protected.
+ // But we really want to protect as much as possible, to prevent this memory
+ // being returned as a result of a non-FIXED mmap().
+ if (addr == 0) {
+ uptr step = GetMmapGranularity();
+ while (size > step) {
+ addr += step;
+ size -= step;
+ void *res = MmapFixedNoAccess(addr, size, "shadow gap");
+ if (addr == (uptr)res) return;
+ }
+ }
+
+ Report(
+ "ERROR: Failed to protect the shadow gap. "
+ "ASan cannot proceed correctly. ABORTING.\n");
+ DumpProcessMap();
+ Die();
+}
+
+bool InitShadow() {
+ const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
+
+ // LowMem covers as much of the first 4GB as possible.
+ const uptr kLowMemEnd = 1UL<<32;
+ const uptr kLowShadowEnd = kLowMemEnd >> kShadowScale;
+ const uptr kLowShadowStart = kLowShadowEnd >> kShadowScale;
+
+ // HighMem covers the upper part of the address space.
+ const uptr kHighShadowEnd = (maxVirtualAddress >> kShadowScale) + 1;
+ const uptr kHighShadowStart = Max(kLowMemEnd, kHighShadowEnd >> kShadowScale);
+ CHECK(kHighShadowStart < kHighShadowEnd);
+
+ const uptr kHighMemStart = kHighShadowStart << kShadowScale;
+ CHECK(kHighShadowEnd <= kHighMemStart);
+
+ if (Verbosity()) {
+ Printf("|| `[%p, %p]` || HighMem ||\n", (void *)kHighMemStart,
+ (void *)maxVirtualAddress);
+ if (kHighMemStart > kHighShadowEnd)
+ Printf("|| `[%p, %p]` || ShadowGap2 ||\n", (void *)kHighShadowEnd,
+ (void *)kHighMemStart);
+ Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowStart,
+ (void *)kHighShadowEnd);
+ if (kHighShadowStart > kLowMemEnd)
+ Printf("|| `[%p, %p]` || ShadowGap2 ||\n", (void *)kHighShadowEnd,
+ (void *)kHighMemStart);
+ Printf("|| `[%p, %p]` || LowMem ||\n", (void *)kLowShadowEnd,
+ (void *)kLowMemEnd);
+ Printf("|| `[%p, %p]` || LowShadow ||\n", (void *)kLowShadowStart,
+ (void *)kLowShadowEnd);
+ Printf("|| `[%p, %p]` || ShadowGap1 ||\n", (void *)0,
+ (void *)kLowShadowStart);
+ }
+
+ ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd - 1, "low shadow");
+ ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd - 1, "high shadow");
+ ProtectGap(0, kLowShadowStart);
+ if (kHighShadowStart > kLowMemEnd)
+ ProtectGap(kLowMemEnd, kHighShadowStart - kLowMemEnd);
+ if (kHighMemStart > kHighShadowEnd)
+ ProtectGap(kHighShadowEnd, kHighMemStart - kHighShadowEnd);
+
+ return true;
+}
+
+static void HwasanAtExit(void) {
+ if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
+ ReportStats();
+ if (hwasan_report_count > 0) {
+ // ReportAtExitStatistics();
+ if (common_flags()->exitcode)
+ internal__exit(common_flags()->exitcode);
+ }
+}
+
+void InstallAtExitHandler() {
+ atexit(HwasanAtExit);
+}
+
+// ---------------------- TSD ---------------- {{{1
+
+static pthread_key_t tsd_key;
+static bool tsd_key_inited = false;
+
+void HwasanTSDInit(void (*destructor)(void *tsd)) {
+ CHECK(!tsd_key_inited);
+ tsd_key_inited = true;
+ CHECK_EQ(0, pthread_key_create(&tsd_key, destructor));
+}
+
+HwasanThread *GetCurrentThread() {
+ return (HwasanThread*)pthread_getspecific(tsd_key);
+}
+
+void SetCurrentThread(HwasanThread *t) {
+ // Make sure that HwasanTSDDtor gets called at the end.
+ CHECK(tsd_key_inited);
+ // Make sure we do not reset the current HwasanThread.
+ CHECK_EQ(0, pthread_getspecific(tsd_key));
+ pthread_setspecific(tsd_key, (void *)t);
+}
+
+void HwasanTSDDtor(void *tsd) {
+ HwasanThread *t = (HwasanThread*)tsd;
+ if (t->destructor_iterations_ > 1) {
+ t->destructor_iterations_--;
+ CHECK_EQ(0, pthread_setspecific(tsd_key, tsd));
+ return;
+ }
+ // Make sure that signal handler can not see a stale current thread pointer.
+ atomic_signal_fence(memory_order_seq_cst);
+ HwasanThread::TSDDtor(tsd);
+}
+
+struct AccessInfo {
+ uptr addr;
+ uptr size;
+ bool is_store;
+ bool is_load;
+};
+
+#if defined(__aarch64__)
+static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
+ // Access type is encoded in HLT immediate as 0x1XY,
+ // where X is 1 for store, 0 for load.
+ // Valid values of Y are 0 to 4, which are interpreted as log2(access_size),
+ // and 0xF, which means that access size is stored in X1 register.
+ // Access address is always in X0 register.
+ AccessInfo ai;
+ uptr pc = (uptr)info->si_addr;
+ unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
+ if ((code & 0xff00) != 0x100)
+ return AccessInfo{0, 0, false, false}; // Not ours.
+ bool is_store = code & 0x10;
+ unsigned size_log = code & 0xff;
+ if (size_log > 4 && size_log != 0xf)
+ return AccessInfo{0, 0, false, false}; // Not ours.
+
+ ai.is_store = is_store;
+ ai.is_load = !is_store;
+ ai.addr = uc->uc_mcontext.regs[0];
+ if (size_log == 0xf)
+ ai.size = uc->uc_mcontext.regs[1];
+ else
+ ai.size = 1U << size_log;
+ return ai;
+}
+#else
+static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
+ return AccessInfo{0, 0, false, false};
+}
+#endif
+
+static bool HwasanOnSIGILL(int signo, siginfo_t *info, ucontext_t *uc) {
+ SignalContext sig{info, uc};
+ AccessInfo ai = GetAccessInfo(info, uc);
+ if (!ai.is_store && !ai.is_load)
+ return false;
+
+ InternalScopedBuffer<BufferedStackTrace> stack_buffer(1);
+ BufferedStackTrace *stack = stack_buffer.data();
+ stack->Reset();
+ GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, uc,
+ common_flags()->fast_unwind_on_fatal);
+
+ ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store);
+
+ ++hwasan_report_count;
+ if (flags()->halt_on_error)
+ Die();
+
+ uc->uc_mcontext.pc += 4;
+ return true;
+}
+
+static void OnStackUnwind(const SignalContext &sig, const void *,
+ BufferedStackTrace *stack) {
+ GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context,
+ common_flags()->fast_unwind_on_fatal);
+}
+
+void HwasanOnDeadlySignal(int signo, void *info, void *context) {
+ // Probably a tag mismatch.
+ if (signo == SIGILL)
+ if (HwasanOnSIGILL(signo, (siginfo_t *)info, (ucontext_t*)context))
+ return;
+
+ HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
+}
+
+
+} // namespace __hwasan
+
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc b/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc
new file mode 100644
index 000000000000..3ccc26734bf6
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan_new_delete.cc
@@ -0,0 +1,66 @@
+//===-- hwasan_new_delete.cc ------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Interceptors for operators new and delete.
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator.h"
+
+#if HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
+
+#include <stddef.h>
+
+using namespace __hwasan; // NOLINT
+
+// Fake std::nothrow_t to avoid including <new>.
+namespace std {
+ struct nothrow_t {};
+} // namespace std
+
+
+// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
+#define OPERATOR_NEW_BODY(nothrow) \
+ GET_MALLOC_STACK_TRACE; \
+ void *res = hwasan_malloc(size, &stack);\
+ if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\
+ return res
+
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size, std::nothrow_t const&) {
+ OPERATOR_NEW_BODY(true /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size, std::nothrow_t const&) {
+ OPERATOR_NEW_BODY(true /*nothrow*/);
+}
+
+#define OPERATOR_DELETE_BODY \
+ GET_MALLOC_STACK_TRACE; \
+ if (ptr) HwasanDeallocate(&stack, ptr)
+
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, std::nothrow_t const&) {
+ OPERATOR_DELETE_BODY;
+}
+
+#endif // HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc
new file mode 100644
index 000000000000..411fd05b10d5
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.cc
@@ -0,0 +1,36 @@
+//===-- hwasan_poisoning.cc ---------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "hwasan_poisoning.h"
+
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_common.h"
+
+namespace __hwasan {
+
+uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
+ CHECK(IsAligned(p, kShadowAlignment));
+ CHECK(IsAligned(size, kShadowAlignment));
+ uptr shadow_start = MEM_TO_SHADOW(p);
+ uptr shadow_size = MEM_TO_SHADOW_OFFSET(size);
+ internal_memset((void *)shadow_start, tag, shadow_size);
+ return AddTagToPointer(p, tag);
+}
+
+uptr TagMemory(uptr p, uptr size, tag_t tag) {
+ uptr start = RoundDownTo(p, kShadowAlignment);
+ uptr end = RoundUpTo(p + size, kShadowAlignment);
+ return TagMemoryAligned(start, end - start, tag);
+}
+
+} // namespace __hwasan
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h
new file mode 100644
index 000000000000..b44a91f975f5
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan_poisoning.h
@@ -0,0 +1,25 @@
+//===-- hwasan_poisoning.h ----------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_POISONING_H
+#define HWASAN_POISONING_H
+
+#include "hwasan.h"
+
+namespace __hwasan {
+uptr TagMemory(uptr p, uptr size, tag_t tag);
+uptr TagMemoryAligned(uptr p, uptr size, tag_t tag);
+
+} // namespace __hwasan
+
+#endif // HWASAN_POISONING_H
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_report.cc b/contrib/compiler-rt/lib/hwasan/hwasan_report.cc
new file mode 100644
index 000000000000..a3c6709491d3
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan_report.cc
@@ -0,0 +1,133 @@
+//===-- hwasan_report.cc ----------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Error reporting.
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "hwasan_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_mutex.h"
+#include "sanitizer_common/sanitizer_report_decorator.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+
+using namespace __sanitizer;
+
+namespace __hwasan {
+
+static StackTrace GetStackTraceFromId(u32 id) {
+ CHECK(id);
+ StackTrace res = StackDepotGet(id);
+ CHECK(res.trace);
+ return res;
+}
+
+class Decorator: public __sanitizer::SanitizerCommonDecorator {
+ public:
+ Decorator() : SanitizerCommonDecorator() { }
+ const char *Allocation() { return Magenta(); }
+ const char *Origin() { return Magenta(); }
+ const char *Name() { return Green(); }
+};
+
+struct HeapAddressDescription {
+ uptr addr;
+ u32 alloc_stack_id;
+ u32 free_stack_id;
+
+ void Print() const {
+ Decorator d;
+ if (free_stack_id) {
+ Printf("%sfreed here:%s\n", d.Allocation(), d.Default());
+ GetStackTraceFromId(free_stack_id).Print();
+ Printf("%spreviously allocated here:%s\n", d.Allocation(), d.Default());
+ } else {
+ Printf("%sallocated here:%s\n", d.Allocation(), d.Default());
+ }
+ GetStackTraceFromId(alloc_stack_id).Print();
+ }
+};
+
+bool GetHeapAddressInformation(uptr addr, uptr access_size,
+ HeapAddressDescription *description) {
+ HwasanChunkView chunk = FindHeapChunkByAddress(addr);
+ if (!chunk.IsValid())
+ return false;
+ description->addr = addr;
+ description->alloc_stack_id = chunk.GetAllocStackId();
+ description->free_stack_id = chunk.GetFreeStackId();
+ return true;
+}
+
+void PrintAddressDescription(uptr addr, uptr access_size) {
+ HeapAddressDescription heap_description;
+ if (GetHeapAddressInformation(addr, access_size, &heap_description)) {
+ heap_description.Print();
+ return;
+ }
+ // We exhausted our possibilities. Bail out.
+ Printf("HWAddressSanitizer can not describe address in more detail.\n");
+}
+
+void ReportInvalidAccess(StackTrace *stack, u32 origin) {
+ ScopedErrorReportLock l;
+
+ Decorator d;
+ Printf("%s", d.Warning());
+ Report("WARNING: HWAddressSanitizer: invalid access\n");
+ Printf("%s", d.Default());
+ stack->Print();
+ ReportErrorSummary("invalid-access", stack);
+}
+
+void ReportStats() {}
+
+void ReportInvalidAccessInsideAddressRange(const char *what, const void *start,
+ uptr size, uptr offset) {
+ ScopedErrorReportLock l;
+
+ Decorator d;
+ Printf("%s", d.Warning());
+ Printf("%sTag mismatch in %s%s%s at offset %zu inside [%p, %zu)%s\n",
+ d.Warning(), d.Name(), what, d.Warning(), offset, start, size,
+ d.Default());
+ PrintAddressDescription((uptr)start + offset, 1);
+ // if (__sanitizer::Verbosity())
+ // DescribeMemoryRange(start, size);
+}
+
+void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size,
+ bool is_store) {
+ ScopedErrorReportLock l;
+
+ Decorator d;
+ Printf("%s", d.Warning());
+ uptr address = GetAddressFromPointer(addr);
+ Printf("%s of size %zu at %p\n", is_store ? "WRITE" : "READ", access_size,
+ address);
+
+ tag_t ptr_tag = GetTagFromPointer(addr);
+ tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(address);
+ Printf("pointer tag 0x%x\nmemory tag 0x%x\n", ptr_tag, mem_tag);
+ Printf("%s", d.Default());
+
+ stack->Print();
+
+ PrintAddressDescription(address, access_size);
+
+ ReportErrorSummary("tag-mismatch", stack);
+}
+
+
+} // namespace __hwasan
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc b/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc
new file mode 100644
index 000000000000..d6551ffc6fdc
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan_thread.cc
@@ -0,0 +1,75 @@
+
+#include "hwasan.h"
+#include "hwasan_thread.h"
+#include "hwasan_poisoning.h"
+#include "hwasan_interface_internal.h"
+
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+
+namespace __hwasan {
+
+HwasanThread *HwasanThread::Create(thread_callback_t start_routine,
+ void *arg) {
+ uptr PageSize = GetPageSizeCached();
+ uptr size = RoundUpTo(sizeof(HwasanThread), PageSize);
+ HwasanThread *thread = (HwasanThread*)MmapOrDie(size, __func__);
+ thread->start_routine_ = start_routine;
+ thread->arg_ = arg;
+ thread->destructor_iterations_ = GetPthreadDestructorIterations();
+
+ return thread;
+}
+
+void HwasanThread::SetThreadStackAndTls() {
+ uptr tls_size = 0;
+ uptr stack_size = 0;
+ GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size,
+ &tls_begin_, &tls_size);
+ stack_top_ = stack_bottom_ + stack_size;
+ tls_end_ = tls_begin_ + tls_size;
+
+ int local;
+ CHECK(AddrIsInStack((uptr)&local));
+}
+
+void HwasanThread::Init() {
+ SetThreadStackAndTls();
+ CHECK(MEM_IS_APP(stack_bottom_));
+ CHECK(MEM_IS_APP(stack_top_ - 1));
+}
+
+void HwasanThread::TSDDtor(void *tsd) {
+ HwasanThread *t = (HwasanThread*)tsd;
+ t->Destroy();
+}
+
+void HwasanThread::ClearShadowForThreadStackAndTLS() {
+ TagMemory(stack_bottom_, stack_top_ - stack_bottom_, 0);
+ if (tls_begin_ != tls_end_)
+ TagMemory(tls_begin_, tls_end_ - tls_begin_, 0);
+}
+
+void HwasanThread::Destroy() {
+ malloc_storage().CommitBack();
+ ClearShadowForThreadStackAndTLS();
+ uptr size = RoundUpTo(sizeof(HwasanThread), GetPageSizeCached());
+ UnmapOrDie(this, size);
+ DTLS_Destroy();
+}
+
+thread_return_t HwasanThread::ThreadStart() {
+ Init();
+
+ if (!start_routine_) {
+ // start_routine_ == 0 if we're on the main thread or on one of the
+ // OS X libdispatch worker threads. But nobody is supposed to call
+ // ThreadStart() for the worker threads.
+ return 0;
+ }
+
+ thread_return_t res = start_routine_(arg_);
+
+ return res;
+}
+
+} // namespace __hwasan
diff --git a/contrib/compiler-rt/lib/hwasan/hwasan_thread.h b/contrib/compiler-rt/lib/hwasan/hwasan_thread.h
new file mode 100644
index 000000000000..96f1bb813adf
--- /dev/null
+++ b/contrib/compiler-rt/lib/hwasan/hwasan_thread.h
@@ -0,0 +1,81 @@
+//===-- hwasan_thread.h -------------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_THREAD_H
+#define HWASAN_THREAD_H
+
+#include "hwasan_allocator.h"
+#include "sanitizer_common/sanitizer_common.h"
+
+namespace __hwasan {
+
+class HwasanThread {
+ public:
+ static HwasanThread *Create(thread_callback_t start_routine, void *arg);
+ static void TSDDtor(void *tsd);
+ void Destroy();
+
+ void Init(); // Should be called from the thread itself.
+ thread_return_t ThreadStart();
+
+ uptr stack_top() { return stack_top_; }
+ uptr stack_bottom() { return stack_bottom_; }
+ uptr tls_begin() { return tls_begin_; }
+ uptr tls_end() { return tls_end_; }
+ bool IsMainThread() { return start_routine_ == nullptr; }
+
+ bool AddrIsInStack(uptr addr) {
+ return addr >= stack_bottom_ && addr < stack_top_;
+ }
+
+ bool InSignalHandler() { return in_signal_handler_; }
+ void EnterSignalHandler() { in_signal_handler_++; }
+ void LeaveSignalHandler() { in_signal_handler_--; }
+
+ bool InSymbolizer() { return in_symbolizer_; }
+ void EnterSymbolizer() { in_symbolizer_++; }
+ void LeaveSymbolizer() { in_symbolizer_--; }
+
+ bool InInterceptorScope() { return in_interceptor_scope_; }
+ void EnterInterceptorScope() { in_interceptor_scope_++; }
+ void LeaveInterceptorScope() { in_interceptor_scope_--; }
+
+ HwasanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; }
+
+ int destructor_iterations_;
+
+ private:
+ // NOTE: There is no HwasanThread constructor. It is allocated
+ // via mmap() and *must* be valid in zero-initialized state.
+ void SetThreadStackAndTls();
+ void ClearShadowForThreadStackAndTLS();
+ thread_callback_t start_routine_;
+ void *arg_;
+ uptr stack_top_;
+ uptr stack_bottom_;
+ uptr tls_begin_;
+ uptr tls_end_;
+
+ unsigned in_signal_handler_;
+ unsigned in_symbolizer_;
+ unsigned in_interceptor_scope_;
+
+ HwasanThreadLocalMallocStorage malloc_storage_;
+};
+
+HwasanThread *GetCurrentThread();
+void SetCurrentThread(HwasanThread *t);
+
+} // namespace __hwasan
+
+#endif // HWASAN_THREAD_H
diff --git a/contrib/compiler-rt/lib/interception/interception.h b/contrib/compiler-rt/lib/interception/interception.h
index d79fa67babfa..cba484936eac 100644
--- a/contrib/compiler-rt/lib/interception/interception.h
+++ b/contrib/compiler-rt/lib/interception/interception.h
@@ -15,13 +15,14 @@
#ifndef INTERCEPTION_H
#define INTERCEPTION_H
-#if !defined(__linux__) && !defined(__FreeBSD__) && \
- !defined(__APPLE__) && !defined(_WIN32)
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_MAC && \
+ !SANITIZER_NETBSD && !SANITIZER_WINDOWS && !SANITIZER_FUCHSIA && \
+ !SANITIZER_SOLARIS
# error "Interception doesn't work on this operating system."
#endif
-#include "sanitizer_common/sanitizer_internal_defs.h"
-
// These typedefs should be used only in the interceptor definitions to replace
// the standard system types (e.g. SSIZE_T instead of ssize_t)
typedef __sanitizer::uptr SIZE_T;
@@ -87,7 +88,7 @@ typedef __sanitizer::OFF64_T OFF64_T;
// As it's decided at compile time which functions are to be intercepted on Mac,
// INTERCEPT_FUNCTION() is effectively a no-op on this system.
-#if defined(__APPLE__)
+#if SANITIZER_MAC
#include <sys/cdefs.h> // For __DARWIN_ALIAS_C().
// Just a pair of pointers.
@@ -121,7 +122,7 @@ const interpose_substitution substitution_##func_name[] \
# define INTERCEPTOR_ATTRIBUTE
# define DECLARE_WRAPPER(ret_type, func, ...)
-#elif defined(_WIN32)
+#elif SANITIZER_WINDOWS
# define WRAP(x) __asan_wrap_##x
# define WRAPPER_NAME(x) "__asan_wrap_"#x
# define INTERCEPTOR_ATTRIBUTE __declspec(dllexport)
@@ -129,7 +130,7 @@ const interpose_substitution substitution_##func_name[] \
extern "C" ret_type func(__VA_ARGS__);
# define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \
extern "C" __declspec(dllimport) ret_type __stdcall func(__VA_ARGS__);
-#elif defined(__FreeBSD__)
+#elif SANITIZER_FREEBSD || SANITIZER_NETBSD
# define WRAP(x) __interceptor_ ## x
# define WRAPPER_NAME(x) "__interceptor_" #x
# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
@@ -139,7 +140,7 @@ const interpose_substitution substitution_##func_name[] \
# define DECLARE_WRAPPER(ret_type, func, ...) \
extern "C" ret_type func(__VA_ARGS__) \
__attribute__((alias("__interceptor_" #func), visibility("default")));
-#else
+#elif !SANITIZER_FUCHSIA
# define WRAP(x) __interceptor_ ## x
# define WRAPPER_NAME(x) "__interceptor_" #x
# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
@@ -148,7 +149,15 @@ const interpose_substitution substitution_##func_name[] \
__attribute__((weak, alias("__interceptor_" #func), visibility("default")));
#endif
-#if !defined(__APPLE__)
+#if SANITIZER_FUCHSIA
+// There is no general interception at all on Fuchsia.
+// Sanitizer runtimes just define functions directly to preempt them,
+// and have bespoke ways to access the underlying libc functions.
+# include <zircon/sanitizer.h>
+# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
+# define REAL(x) __unsanitized_##x
+# define DECLARE_REAL(ret_type, func, ...)
+#elif !SANITIZER_MAC
# define PTR_TO_REAL(x) real_##x
# define REAL(x) __interception::PTR_TO_REAL(x)
# define FUNC_TYPE(x) x##_f
@@ -159,22 +168,26 @@ const interpose_substitution substitution_##func_name[] \
extern FUNC_TYPE(func) PTR_TO_REAL(func); \
}
# define ASSIGN_REAL(dst, src) REAL(dst) = REAL(src)
-#else // __APPLE__
+#else // SANITIZER_MAC
# define REAL(x) x
# define DECLARE_REAL(ret_type, func, ...) \
extern "C" ret_type func(__VA_ARGS__);
# define ASSIGN_REAL(x, y)
-#endif // __APPLE__
+#endif // SANITIZER_MAC
+#if !SANITIZER_FUCHSIA
#define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \
DECLARE_REAL(ret_type, func, __VA_ARGS__) \
extern "C" ret_type WRAP(func)(__VA_ARGS__);
+#else
+#define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...)
+#endif
// Generally, you don't need to use DEFINE_REAL by itself, as INTERCEPTOR
// macros does its job. In exceptional cases you may need to call REAL(foo)
// without defining INTERCEPTOR(..., foo, ...). For example, if you override
// foo with an interceptor for other function.
-#if !defined(__APPLE__)
+#if !SANITIZER_MAC && !SANITIZER_FUCHSIA
# define DEFINE_REAL(ret_type, func, ...) \
typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \
namespace __interception { \
@@ -184,7 +197,18 @@ const interpose_substitution substitution_##func_name[] \
# define DEFINE_REAL(ret_type, func, ...)
#endif
-#if !defined(__APPLE__)
+#if SANITIZER_FUCHSIA
+
+// We need to define the __interceptor_func name just to get
+// sanitizer_common/scripts/gen_dynamic_list.py to export func.
+// But we don't need to export __interceptor_func to get that.
+#define INTERCEPTOR(ret_type, func, ...) \
+ extern "C"[[ gnu::alias(#func), gnu::visibility("hidden") ]] ret_type \
+ __interceptor_##func(__VA_ARGS__); \
+ extern "C" INTERCEPTOR_ATTRIBUTE ret_type func(__VA_ARGS__)
+
+#elif !SANITIZER_MAC
+
#define INTERCEPTOR(ret_type, func, ...) \
DEFINE_REAL(ret_type, func, __VA_ARGS__) \
DECLARE_WRAPPER(ret_type, func, __VA_ARGS__) \
@@ -196,7 +220,7 @@ const interpose_substitution substitution_##func_name[] \
#define INTERCEPTOR_WITH_SUFFIX(ret_type, func, ...) \
INTERCEPTOR(ret_type, func, __VA_ARGS__)
-#else // __APPLE__
+#else // SANITIZER_MAC
#define INTERCEPTOR_ZZZ(suffix, ret_type, func, ...) \
extern "C" ret_type func(__VA_ARGS__) suffix; \
@@ -215,7 +239,7 @@ const interpose_substitution substitution_##func_name[] \
INTERPOSER_2(overridee, WRAP(overrider))
#endif
-#if defined(_WIN32)
+#if SANITIZER_WINDOWS
# define INTERCEPTOR_WINAPI(ret_type, func, ...) \
typedef ret_type (__stdcall *FUNC_TYPE(func))(__VA_ARGS__); \
namespace __interception { \
@@ -241,17 +265,19 @@ typedef unsigned long uptr; // NOLINT
#define INCLUDED_FROM_INTERCEPTION_LIB
-#if defined(__linux__) || defined(__FreeBSD__)
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS
+
# include "interception_linux.h"
# define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)
# define INTERCEPT_FUNCTION_VER(func, symver) \
INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver)
-#elif defined(__APPLE__)
+#elif SANITIZER_MAC
# include "interception_mac.h"
# define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_MAC(func)
# define INTERCEPT_FUNCTION_VER(func, symver) \
INTERCEPT_FUNCTION_VER_MAC(func, symver)
-#else // defined(_WIN32)
+#elif SANITIZER_WINDOWS
# include "interception_win.h"
# define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_WIN(func)
# define INTERCEPT_FUNCTION_VER(func, symver) \
diff --git a/contrib/compiler-rt/lib/interception/interception_linux.cc b/contrib/compiler-rt/lib/interception/interception_linux.cc
index 6e908ac017b2..c991550a4f72 100644
--- a/contrib/compiler-rt/lib/interception/interception_linux.cc
+++ b/contrib/compiler-rt/lib/interception/interception_linux.cc
@@ -12,25 +12,44 @@
// Linux-specific interception methods.
//===----------------------------------------------------------------------===//
-#if defined(__linux__) || defined(__FreeBSD__)
#include "interception.h"
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS
+
#include <dlfcn.h> // for dlsym() and dlvsym()
+#if SANITIZER_NETBSD
+#include "sanitizer_common/sanitizer_libc.h"
+#endif
+
namespace __interception {
bool GetRealFunctionAddress(const char *func_name, uptr *func_addr,
uptr real, uptr wrapper) {
+#if SANITIZER_NETBSD
+ // XXX: Find a better way to handle renames
+ if (internal_strcmp(func_name, "sigaction") == 0) func_name = "__sigaction14";
+#endif
*func_addr = (uptr)dlsym(RTLD_NEXT, func_name);
+ if (!*func_addr) {
+ // If the lookup using RTLD_NEXT failed, the sanitizer runtime library is
+ // later in the library search order than the DSO that we are trying to
+ // intercept, which means that we cannot intercept this function. We still
+ // want the address of the real definition, though, so look it up using
+ // RTLD_DEFAULT.
+ *func_addr = (uptr)dlsym(RTLD_DEFAULT, func_name);
+ }
return real == wrapper;
}
-#if !defined(__ANDROID__) // android does not have dlvsym
+// Android and Solaris do not have dlvsym
+#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS
void *GetFuncAddrVer(const char *func_name, const char *ver) {
return dlvsym(RTLD_NEXT, func_name, ver);
}
-#endif // !defined(__ANDROID__)
+#endif // !SANITIZER_ANDROID
} // namespace __interception
-
-#endif // __linux__ || __FreeBSD__
+#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD ||
+ // SANITIZER_SOLARIS
diff --git a/contrib/compiler-rt/lib/interception/interception_linux.h b/contrib/compiler-rt/lib/interception/interception_linux.h
index 27a66c882041..98fe51b858fc 100644
--- a/contrib/compiler-rt/lib/interception/interception_linux.h
+++ b/contrib/compiler-rt/lib/interception/interception_linux.h
@@ -12,7 +12,8 @@
// Linux-specific interception methods.
//===----------------------------------------------------------------------===//
-#if defined(__linux__) || defined(__FreeBSD__)
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS
#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)
# error "interception_linux.h should be included from interception library only"
@@ -34,14 +35,16 @@ void *GetFuncAddrVer(const char *func_name, const char *ver);
(::__interception::uptr) & (func), \
(::__interception::uptr) & WRAP(func))
-#if !defined(__ANDROID__) // android does not have dlvsym
+// Android and Solaris do not have dlvsym
+#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS
#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \
(::__interception::real_##func = (func##_f)( \
unsigned long)::__interception::GetFuncAddrVer(#func, symver))
#else
#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \
INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)
-#endif // !defined(__ANDROID__)
+#endif // !SANITIZER_ANDROID && !SANITIZER_SOLARIS
#endif // INTERCEPTION_LINUX_H
-#endif // __linux__ || __FreeBSD__
+#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD ||
+ // SANITIZER_SOLARIS
diff --git a/contrib/compiler-rt/lib/interception/interception_mac.cc b/contrib/compiler-rt/lib/interception/interception_mac.cc
index b035cf998140..ea8072f8dd73 100644
--- a/contrib/compiler-rt/lib/interception/interception_mac.cc
+++ b/contrib/compiler-rt/lib/interception/interception_mac.cc
@@ -12,9 +12,8 @@
// Mac-specific interception methods.
//===----------------------------------------------------------------------===//
-#ifdef __APPLE__
-
#include "interception.h"
+#if SANITIZER_MAC
-#endif // __APPLE__
+#endif // SANITIZER_MAC
diff --git a/contrib/compiler-rt/lib/interception/interception_mac.h b/contrib/compiler-rt/lib/interception/interception_mac.h
index e5a35c6971cd..6f31fed47b84 100644
--- a/contrib/compiler-rt/lib/interception/interception_mac.h
+++ b/contrib/compiler-rt/lib/interception/interception_mac.h
@@ -12,7 +12,7 @@
// Mac-specific interception methods.
//===----------------------------------------------------------------------===//
-#ifdef __APPLE__
+#if SANITIZER_MAC
#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)
# error "interception_mac.h should be included from interception.h only"
@@ -25,4 +25,4 @@
#define INTERCEPT_FUNCTION_VER_MAC(func, symver)
#endif // INTERCEPTION_MAC_H
-#endif // __APPLE__
+#endif // SANITIZER_MAC
diff --git a/contrib/compiler-rt/lib/interception/interception_type_test.cc b/contrib/compiler-rt/lib/interception/interception_type_test.cc
index 98ce2e6afc1b..2b3a6d509bfb 100644
--- a/contrib/compiler-rt/lib/interception/interception_type_test.cc
+++ b/contrib/compiler-rt/lib/interception/interception_type_test.cc
@@ -12,9 +12,10 @@
// Compile-time tests of the internal type definitions.
//===----------------------------------------------------------------------===//
-#if defined(__linux__) || defined(__APPLE__)
-
#include "interception.h"
+
+#if SANITIZER_LINUX || SANITIZER_MAC
+
#include <sys/types.h>
#include <stddef.h>
#include <stdint.h>
@@ -24,14 +25,14 @@ COMPILER_CHECK(sizeof(::SSIZE_T) == sizeof(ssize_t));
COMPILER_CHECK(sizeof(::PTRDIFF_T) == sizeof(ptrdiff_t));
COMPILER_CHECK(sizeof(::INTMAX_T) == sizeof(intmax_t));
-#ifndef __APPLE__
+#if !SANITIZER_MAC
COMPILER_CHECK(sizeof(::OFF64_T) == sizeof(off64_t));
#endif
// The following are the cases when pread (and friends) is used instead of
// pread64. In those cases we need OFF_T to match off_t. We don't care about the
// rest (they depend on _FILE_OFFSET_BITS setting when building an application).
-# if defined(__ANDROID__) || !defined _FILE_OFFSET_BITS || \
+# if SANITIZER_ANDROID || !defined _FILE_OFFSET_BITS || \
_FILE_OFFSET_BITS != 64
COMPILER_CHECK(sizeof(::OFF_T) == sizeof(off_t));
# endif
diff --git a/contrib/compiler-rt/lib/interception/interception_win.cc b/contrib/compiler-rt/lib/interception/interception_win.cc
index b2902d57f542..dc3fe35242ef 100644
--- a/contrib/compiler-rt/lib/interception/interception_win.cc
+++ b/contrib/compiler-rt/lib/interception/interception_win.cc
@@ -125,9 +125,9 @@
// addr2: .bytes <body>
//===----------------------------------------------------------------------===//
-#ifdef _WIN32
-
#include "interception.h"
+
+#if SANITIZER_WINDOWS
#include "sanitizer_common/sanitizer_platform.h"
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
@@ -223,9 +223,8 @@ static bool IsMemoryPadding(uptr address, uptr size) {
return true;
}
-static const u8 kHintNop10Bytes[] = {
- 0x66, 0x66, 0x0F, 0x1F, 0x84,
- 0x00, 0x00, 0x00, 0x00, 0x00
+static const u8 kHintNop9Bytes[] = {
+ 0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00
};
template<class T>
@@ -240,8 +239,8 @@ static bool FunctionHasPrefix(uptr address, const T &pattern) {
static bool FunctionHasPadding(uptr address, uptr size) {
if (IsMemoryPadding(address - size, size))
return true;
- if (size <= sizeof(kHintNop10Bytes) &&
- FunctionHasPrefix(address, kHintNop10Bytes))
+ if (size <= sizeof(kHintNop9Bytes) &&
+ FunctionHasPrefix(address, kHintNop9Bytes))
return true;
return false;
}
@@ -554,7 +553,10 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
case 0x246c8948: // 48 89 6C 24 XX : mov QWORD ptr [rsp + XX], rbp
case 0x245c8948: // 48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx
case 0x24748948: // 48 89 74 24 XX : mov QWORD PTR [rsp + XX], rsi
+ case 0x244C8948: // 48 89 4C 24 XX : mov QWORD PTR [rsp + XX], rcx
return 5;
+ case 0x24648348: // 48 83 64 24 XX : and QWORD PTR [rsp + XX], YY
+ return 6;
}
#else
@@ -833,6 +835,7 @@ bool OverrideFunction(
static void **InterestingDLLsAvailable() {
static const char *InterestingDLLs[] = {
"kernel32.dll",
+ "msvcr100.dll", // VS2010
"msvcr110.dll", // VS2012
"msvcr120.dll", // VS2013
"vcruntime140.dll", // VS2015
@@ -1010,4 +1013,4 @@ bool OverrideImportedFunction(const char *module_to_patch,
} // namespace __interception
-#endif // _WIN32
+#endif // SANITIZER_MAC
diff --git a/contrib/compiler-rt/lib/interception/interception_win.h b/contrib/compiler-rt/lib/interception/interception_win.h
index 9061f9ed4c21..71a44428f5c7 100644
--- a/contrib/compiler-rt/lib/interception/interception_win.h
+++ b/contrib/compiler-rt/lib/interception/interception_win.h
@@ -12,7 +12,7 @@
// Windows-specific interception methods.
//===----------------------------------------------------------------------===//
-#ifdef _WIN32
+#if SANITIZER_WINDOWS
#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)
# error "interception_win.h should be included from interception library only"
@@ -81,4 +81,4 @@ void TestOnlyReleaseTrampolineRegions();
(::__interception::uptr *)&REAL(func))
#endif // INTERCEPTION_WIN_H
-#endif // _WIN32
+#endif // SANITIZER_WINDOWS
diff --git a/contrib/compiler-rt/lib/lsan/lsan.cc b/contrib/compiler-rt/lib/lsan/lsan.cc
index 6c4767d61252..a9f7e399e14c 100644
--- a/contrib/compiler-rt/lib/lsan/lsan.cc
+++ b/contrib/compiler-rt/lib/lsan/lsan.cc
@@ -56,6 +56,9 @@ static void InitializeFlags() {
RegisterLsanFlags(&parser, f);
RegisterCommonFlags(&parser);
+ // Override from user-specified string.
+ const char *lsan_default_options = MaybeCallLsanDefaultOptions();
+ parser.ParseString(lsan_default_options);
parser.ParseString(GetEnv("LSAN_OPTIONS"));
SetVerbosity(common_flags()->verbosity);
@@ -65,6 +68,17 @@ static void InitializeFlags() {
if (common_flags()->help) parser.PrintFlagDescriptions();
}
+static void OnStackUnwind(const SignalContext &sig, const void *,
+ BufferedStackTrace *stack) {
+ GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context,
+ common_flags()->fast_unwind_on_fatal);
+}
+
+static void LsanOnDeadlySignal(int signo, void *siginfo, void *context) {
+ HandleDeadlySignal(siginfo, context, GetCurrentThread(), &OnStackUnwind,
+ nullptr);
+}
+
extern "C" void __lsan_init() {
CHECK(!lsan_init_is_running);
if (lsan_inited)
@@ -80,6 +94,7 @@ extern "C" void __lsan_init() {
InitTlsSize();
InitializeInterceptors();
InitializeThreadRegistry();
+ InstallDeadlySignalHandlers(LsanOnDeadlySignal);
u32 tid = ThreadCreate(0, 0, true);
CHECK_EQ(tid, 0);
ThreadStart(tid, GetTid());
diff --git a/contrib/compiler-rt/lib/lsan/lsan.h b/contrib/compiler-rt/lib/lsan/lsan.h
index 7446e9507e0d..6baee81d00cd 100644
--- a/contrib/compiler-rt/lib/lsan/lsan.h
+++ b/contrib/compiler-rt/lib/lsan/lsan.h
@@ -12,24 +12,14 @@
//
//===----------------------------------------------------------------------===//
+#include "lsan_thread.h"
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_stacktrace.h"
-#define GET_STACK_TRACE(max_size, fast) \
- BufferedStackTrace stack; \
- { \
- uptr stack_top = 0, stack_bottom = 0; \
- ThreadContext *t; \
- if (fast && (t = CurrentThreadContext())) { \
- stack_top = t->stack_end(); \
- stack_bottom = t->stack_begin(); \
- } \
- if (!SANITIZER_MIPS || \
- IsValidFrame(GET_CURRENT_FRAME(), stack_top, stack_bottom)) { \
- stack.Unwind(max_size, StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), \
- /* context */ 0, stack_top, stack_bottom, fast); \
- } \
- }
+#define GET_STACK_TRACE(max_size, fast) \
+ __sanitizer::BufferedStackTrace stack; \
+ GetStackTrace(&stack, max_size, StackTrace::GetCurrentPc(), \
+ GET_CURRENT_FRAME(), nullptr, fast);
#define GET_STACK_TRACE_FATAL \
GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal)
@@ -51,9 +41,27 @@ void ReplaceSystemMalloc();
__lsan_init(); \
} while (0)
+// Get the stack trace with the given pc and bp.
+// The pc will be in the position 0 of the resulting stack trace.
+// The bp may refer to the current frame or to the caller's frame.
+ALWAYS_INLINE
+void GetStackTrace(__sanitizer::BufferedStackTrace *stack,
+ __sanitizer::uptr max_depth, __sanitizer::uptr pc,
+ __sanitizer::uptr bp, void *context, bool fast) {
+ uptr stack_top = 0, stack_bottom = 0;
+ ThreadContext *t;
+ if (fast && (t = CurrentThreadContext())) {
+ stack_top = t->stack_end();
+ stack_bottom = t->stack_begin();
+ }
+ if (!SANITIZER_MIPS || IsValidFrame(bp, stack_top, stack_bottom)) {
+ stack->Unwind(max_depth, pc, bp, context, stack_top, stack_bottom, fast);
+ }
+}
+
} // namespace __lsan
extern bool lsan_inited;
extern bool lsan_init_is_running;
-extern "C" void __lsan_init();
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __lsan_init();
diff --git a/contrib/compiler-rt/lib/lsan/lsan_allocator.h b/contrib/compiler-rt/lib/lsan/lsan_allocator.h
index 5a0d94c71415..4006f7929269 100644
--- a/contrib/compiler-rt/lib/lsan/lsan_allocator.h
+++ b/contrib/compiler-rt/lib/lsan/lsan_allocator.h
@@ -68,9 +68,16 @@ struct AP32 {
};
typedef SizeClassAllocator32<AP32> PrimaryAllocator;
#elif defined(__x86_64__) || defined(__powerpc64__)
+# if defined(__powerpc64__)
+const uptr kAllocatorSpace = 0xa0000000000ULL;
+const uptr kAllocatorSize = 0x20000000000ULL; // 2T.
+# else
+const uptr kAllocatorSpace = 0x600000000000ULL;
+const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
+# endif
struct AP64 { // Allocator64 parameters. Deliberately using a short name.
- static const uptr kSpaceBeg = 0x600000000000ULL;
- static const uptr kSpaceSize = 0x40000000000ULL; // 4T.
+ static const uptr kSpaceBeg = kAllocatorSpace;
+ static const uptr kSpaceSize = kAllocatorSize;
static const uptr kMetadataSize = sizeof(ChunkMetadata);
typedef DefaultSizeClassMap SizeClassMap;
typedef NoOpMapUnmapCallback MapUnmapCallback;
diff --git a/contrib/compiler-rt/lib/lsan/lsan_common.cc b/contrib/compiler-rt/lib/lsan/lsan_common.cc
index c121e6a8fb24..69ffda539a26 100644
--- a/contrib/compiler-rt/lib/lsan/lsan_common.cc
+++ b/contrib/compiler-rt/lib/lsan/lsan_common.cc
@@ -57,12 +57,12 @@ void RegisterLsanFlags(FlagParser *parser, Flags *f) {
#define LOG_POINTERS(...) \
do { \
if (flags()->log_pointers) Report(__VA_ARGS__); \
- } while (0);
+ } while (0)
#define LOG_THREADS(...) \
do { \
if (flags()->log_threads) Report(__VA_ARGS__); \
- } while (0);
+ } while (0)
ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)];
static SuppressionContext *suppression_ctx = nullptr;
@@ -107,6 +107,10 @@ void InitializeRootRegions() {
root_regions = new(placeholder) InternalMmapVector<RootRegion>(1);
}
+const char *MaybeCallLsanDefaultOptions() {
+ return (&__lsan_default_options) ? __lsan_default_options() : "";
+}
+
void InitCommonLsan() {
InitializeRootRegions();
if (common_flags()->detect_leaks) {
@@ -122,7 +126,6 @@ class Decorator: public __sanitizer::SanitizerCommonDecorator {
Decorator() : SanitizerCommonDecorator() { }
const char *Error() { return Red(); }
const char *Leak() { return Blue(); }
- const char *End() { return Default(); }
};
static inline bool CanBeAHeapPointer(uptr p) {
@@ -408,8 +411,9 @@ static void MarkInvalidPCCb(uptr chunk, void *arg) {
}
}
-// On Linux, handles dynamically allocated TLS blocks by treating all chunks
-// allocated from ld-linux.so as reachable.
+// On Linux, treats all chunks allocated from ld-linux.so as reachable, which
+// covers dynamically allocated TLS blocks, internal dynamic loader's loaded
+// modules accounting etc.
// Dynamic TLS blocks contain the TLS variables of dynamically loaded modules.
// They are allocated with a __libc_memalign() call in allocate_and_init()
// (elf/dl-tls.c). Glibc won't tell us the address ranges occupied by those
@@ -564,7 +568,7 @@ static bool CheckForLeaks() {
"\n");
Printf("%s", d.Error());
Report("ERROR: LeakSanitizer: detected memory leaks\n");
- Printf("%s", d.End());
+ Printf("%s", d.Default());
param.leak_report.ReportTopLeaks(flags()->max_leaks);
}
if (common_flags()->print_suppressions)
@@ -594,6 +598,8 @@ static int DoRecoverableLeakCheck() {
return have_leaks ? 1 : 0;
}
+void DoRecoverableLeakCheckVoid() { DoRecoverableLeakCheck(); }
+
static Suppression *GetSuppressionForAddr(uptr addr) {
Suppression *s = nullptr;
@@ -698,7 +704,7 @@ void LeakReport::PrintReportForLeak(uptr index) {
Printf("%s leak of %zu byte(s) in %zu object(s) allocated from:\n",
leaks_[index].is_directly_leaked ? "Direct" : "Indirect",
leaks_[index].total_size, leaks_[index].hit_count);
- Printf("%s", d.End());
+ Printf("%s", d.Default());
PrintStackTraceById(leaks_[index].stack_trace_id);
@@ -756,6 +762,7 @@ uptr LeakReport::UnsuppressedLeakCount() {
namespace __lsan {
void InitCommonLsan() { }
void DoLeakCheck() { }
+void DoRecoverableLeakCheckVoid() { }
void DisableInThisThread() { }
void EnableInThisThread() { }
}
@@ -854,6 +861,11 @@ int __lsan_do_recoverable_leak_check() {
#if !SANITIZER_SUPPORTS_WEAK_HOOKS
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+const char * __lsan_default_options() {
+ return "";
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
int __lsan_is_turned_off() {
return 0;
}
diff --git a/contrib/compiler-rt/lib/lsan/lsan_common.h b/contrib/compiler-rt/lib/lsan/lsan_common.h
index 31bf3eb1df42..f3863309d893 100644
--- a/contrib/compiler-rt/lib/lsan/lsan_common.h
+++ b/contrib/compiler-rt/lib/lsan/lsan_common.h
@@ -27,9 +27,9 @@
// because of "small" (4 bytes) pointer size that leads to high false negative
// ratio on large leaks. But we still want to have it for some 32 bit arches
// (e.g. x86), see https://github.com/google/sanitizers/issues/403.
-// To enable LeakSanitizer on new architecture, one need to implement
-// internal_clone function as well as (probably) adjust TLS machinery for
-// new architecture inside sanitizer library.
+// To enable LeakSanitizer on a new architecture, one needs to implement the
+// internal_clone function as well as (probably) adjust the TLS machinery for
+// the new architecture inside the sanitizer library.
#if (SANITIZER_LINUX && !SANITIZER_ANDROID || SANITIZER_MAC) && \
(SANITIZER_WORDSIZE == 64) && \
(defined(__x86_64__) || defined(__mips64) || defined(__aarch64__) || \
@@ -143,8 +143,10 @@ enum IgnoreObjectResult {
};
// Functions called from the parent tool.
+const char *MaybeCallLsanDefaultOptions();
void InitCommonLsan();
void DoLeakCheck();
+void DoRecoverableLeakCheckVoid();
void DisableCounterUnderflow();
bool DisabledInThisThread();
@@ -250,6 +252,9 @@ class LsanMetadata {
extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+const char *__lsan_default_options();
+
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
int __lsan_is_turned_off();
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
diff --git a/contrib/compiler-rt/lib/lsan/lsan_common_linux.cc b/contrib/compiler-rt/lib/lsan/lsan_common_linux.cc
index 5042c7b3ada5..cdd7f032a192 100644
--- a/contrib/compiler-rt/lib/lsan/lsan_common_linux.cc
+++ b/contrib/compiler-rt/lib/lsan/lsan_common_linux.cc
@@ -20,6 +20,7 @@
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_getauxval.h"
#include "sanitizer_common/sanitizer_linux.h"
#include "sanitizer_common/sanitizer_stackdepot.h"
@@ -30,8 +31,12 @@ static const char kLinkerName[] = "ld";
static char linker_placeholder[sizeof(LoadedModule)] ALIGNED(64);
static LoadedModule *linker = nullptr;
-static bool IsLinker(const char* full_name) {
- return LibraryNameIs(full_name, kLinkerName);
+static bool IsLinker(const LoadedModule& module) {
+#if SANITIZER_USE_GETAUXVAL
+ return module.base_address() == getauxval(AT_BASE);
+#else
+ return LibraryNameIs(module.full_name(), kLinkerName);
+#endif // SANITIZER_USE_GETAUXVAL
}
__attribute__((tls_model("initial-exec")))
@@ -49,22 +54,25 @@ void InitializePlatformSpecificModules() {
ListOfModules modules;
modules.init();
for (LoadedModule &module : modules) {
- if (!IsLinker(module.full_name())) continue;
+ if (!IsLinker(module))
+ continue;
if (linker == nullptr) {
linker = reinterpret_cast<LoadedModule *>(linker_placeholder);
*linker = module;
module = LoadedModule();
} else {
VReport(1, "LeakSanitizer: Multiple modules match \"%s\". "
- "TLS will not be handled correctly.\n", kLinkerName);
+ "TLS and other allocations originating from linker might be "
+ "falsely reported as leaks.\n", kLinkerName);
linker->clear();
linker = nullptr;
return;
}
}
if (linker == nullptr) {
- VReport(1, "LeakSanitizer: Dynamic linker not found. "
- "TLS will not be handled correctly.\n");
+ VReport(1, "LeakSanitizer: Dynamic linker not found. TLS and other "
+ "allocations originating from linker might be falsely reported "
+ "as leaks.\n");
}
}
diff --git a/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc b/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc
index ade94340ae81..ac27c7af6e35 100644
--- a/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc
+++ b/contrib/compiler-rt/lib/lsan/lsan_common_mac.cc
@@ -92,8 +92,25 @@ LoadedModule *GetLinker() { return nullptr; }
// required on Darwin.
void InitializePlatformSpecificModules() {}
+// Sections which can't contain contain global pointers. This list errs on the
+// side of caution to avoid false positives, at the expense of performance.
+//
+// Other potentially safe sections include:
+// __all_image_info, __crash_info, __const, __got, __interpose, __objc_msg_break
+//
+// Sections which definitely cannot be included here are:
+// __objc_data, __objc_const, __data, __bss, __common, __thread_data,
+// __thread_bss, __thread_vars, __objc_opt_rw, __objc_opt_ptrs
+static const char *kSkippedSecNames[] = {
+ "__cfstring", "__la_symbol_ptr", "__mod_init_func",
+ "__mod_term_func", "__nl_symbol_ptr", "__objc_classlist",
+ "__objc_classrefs", "__objc_imageinfo", "__objc_nlclslist",
+ "__objc_protolist", "__objc_selrefs", "__objc_superrefs"};
+
// Scans global variables for heap pointers.
void ProcessGlobalRegions(Frontier *frontier) {
+ for (auto name : kSkippedSecNames) CHECK(ARRAY_SIZE(name) < kMaxSegName);
+
MemoryMappingLayout memory_mapping(false);
InternalMmapVector<LoadedModule> modules(/*initial_capacity*/ 128);
memory_mapping.DumpListOfModules(&modules);
@@ -107,6 +124,10 @@ void ProcessGlobalRegions(Frontier *frontier) {
// Sections storing global variables are writable and non-executable
if (range.executable || !range.writable) continue;
+ for (auto name : kSkippedSecNames) {
+ if (!internal_strcmp(range.name, name)) continue;
+ }
+
ScanGlobalRange(range.beg, range.end, frontier);
}
}
diff --git a/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc b/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc
index 168868b012bc..b3e73e3896d5 100644
--- a/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc
+++ b/contrib/compiler-rt/lib/lsan/lsan_interceptors.cc
@@ -20,6 +20,7 @@
#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_linux.h"
#include "sanitizer_common/sanitizer_platform_interceptors.h"
+#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
#include "sanitizer_common/sanitizer_platform_limits_posix.h"
#include "sanitizer_common/sanitizer_posix.h"
#include "sanitizer_common/sanitizer_tls_get_addr.h"
@@ -44,6 +45,7 @@ int pthread_setspecific(unsigned key, const void *v);
namespace std {
struct nothrow_t;
+ enum class align_val_t: size_t;
}
#if !SANITIZER_MAC
@@ -203,13 +205,19 @@ INTERCEPTOR(int, mprobe, void *ptr) {
#define OPERATOR_NEW_BODY(nothrow) \
ENSURE_LSAN_INITED; \
GET_STACK_TRACE_MALLOC; \
- void *res = Allocate(stack, size, 1, kAlwaysClearMemory);\
- if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\
+ void *res = lsan_malloc(size, stack); \
+ if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM(); \
+ return res;
+#define OPERATOR_NEW_BODY_ALIGN(nothrow) \
+ ENSURE_LSAN_INITED; \
+ GET_STACK_TRACE_MALLOC; \
+ void *res = lsan_memalign((uptr)align, size, stack); \
+ if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM(); \
return res;
#define OPERATOR_DELETE_BODY \
ENSURE_LSAN_INITED; \
- Deallocate(ptr);
+ lsan_free(ptr);
// On OS X it's not enough to just provide our own 'operator new' and
// 'operator delete' implementations, because they're going to be in the runtime
@@ -229,6 +237,18 @@ void *operator new(size_t size, std::nothrow_t const&)
INTERCEPTOR_ATTRIBUTE
void *operator new[](size_t size, std::nothrow_t const&)
{ OPERATOR_NEW_BODY(true /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size, std::align_val_t align)
+{ OPERATOR_NEW_BODY_ALIGN(false /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size, std::align_val_t align)
+{ OPERATOR_NEW_BODY_ALIGN(false /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size, std::align_val_t align, std::nothrow_t const&)
+{ OPERATOR_NEW_BODY_ALIGN(true /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size, std::align_val_t align, std::nothrow_t const&)
+{ OPERATOR_NEW_BODY_ALIGN(true /*nothrow*/); }
INTERCEPTOR_ATTRIBUTE
void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
@@ -239,6 +259,30 @@ void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; }
INTERCEPTOR_ATTRIBUTE
void operator delete[](void *ptr, std::nothrow_t const &)
{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, size_t size) NOEXCEPT
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, size_t size) NOEXCEPT
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, std::align_val_t) NOEXCEPT
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, std::align_val_t) NOEXCEPT
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, std::align_val_t, std::nothrow_t const&)
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, std::align_val_t, std::nothrow_t const&)
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, size_t size, std::align_val_t) NOEXCEPT
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, size_t size, std::align_val_t) NOEXCEPT
+{ OPERATOR_DELETE_BODY; }
#else // SANITIZER_MAC
@@ -265,6 +309,7 @@ INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&)
///// Thread initialization and finalization. /////
+#if !SANITIZER_NETBSD
static unsigned g_thread_finalize_key;
static void thread_finalize(void *v) {
@@ -278,6 +323,18 @@ static void thread_finalize(void *v) {
}
ThreadFinish();
}
+#endif
+
+#if SANITIZER_NETBSD
+INTERCEPTOR(void, _lwp_exit) {
+ ENSURE_LSAN_INITED;
+ ThreadFinish();
+ REAL(_lwp_exit)();
+}
+#define LSAN_MAYBE_INTERCEPT__LWP_EXIT INTERCEPT_FUNCTION(_lwp_exit)
+#else
+#define LSAN_MAYBE_INTERCEPT__LWP_EXIT
+#endif
struct ThreadParam {
void *(*callback)(void *arg);
@@ -291,11 +348,13 @@ extern "C" void *__lsan_thread_start_func(void *arg) {
void *param = p->param;
// Wait until the last iteration to maximize the chance that we are the last
// destructor to run.
+#if !SANITIZER_NETBSD
if (pthread_setspecific(g_thread_finalize_key,
(void*)GetPthreadDestructorIterations())) {
Report("LeakSanitizer: failed to set thread key.\n");
Die();
}
+#endif
int tid = 0;
while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0)
internal_sched_yield();
@@ -357,9 +416,14 @@ INTERCEPTOR(void, _exit, int status) {
REAL(_exit)(status);
}
+#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
+#include "sanitizer_common/sanitizer_signal_interceptors.inc"
+
namespace __lsan {
void InitializeInterceptors() {
+ InitializeSignalInterceptors();
+
INTERCEPT_FUNCTION(malloc);
INTERCEPT_FUNCTION(free);
LSAN_MAYBE_INTERCEPT_CFREE;
@@ -378,10 +442,14 @@ void InitializeInterceptors() {
INTERCEPT_FUNCTION(pthread_join);
INTERCEPT_FUNCTION(_exit);
+ LSAN_MAYBE_INTERCEPT__LWP_EXIT;
+
+#if !SANITIZER_NETBSD
if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) {
Report("LeakSanitizer: failed to create thread key.\n");
Die();
}
+#endif
}
} // namespace __lsan
diff --git a/contrib/compiler-rt/lib/lsan/weak_symbols.txt b/contrib/compiler-rt/lib/lsan/weak_symbols.txt
index da4f994da865..692255679bee 100644
--- a/contrib/compiler-rt/lib/lsan/weak_symbols.txt
+++ b/contrib/compiler-rt/lib/lsan/weak_symbols.txt
@@ -1,2 +1,3 @@
+___lsan_default_options
___lsan_default_suppressions
___lsan_is_turned_off
diff --git a/contrib/compiler-rt/lib/msan/msan.cc b/contrib/compiler-rt/lib/msan/msan.cc
index d2981f0b0edb..e6226ba7670f 100644
--- a/contrib/compiler-rt/lib/msan/msan.cc
+++ b/contrib/compiler-rt/lib/msan/msan.cc
@@ -218,14 +218,14 @@ static void InitializeFlags() {
}
void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp,
- bool request_fast_unwind) {
+ void *context, bool request_fast_unwind) {
MsanThread *t = GetCurrentThread();
if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind)) {
// Block reports from our interceptors during _Unwind_Backtrace.
SymbolizerScope sym_scope;
- return stack->Unwind(max_s, pc, bp, nullptr, 0, 0, request_fast_unwind);
+ return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind);
}
- stack->Unwind(max_s, pc, bp, nullptr, t->stack_top(), t->stack_bottom(),
+ stack->Unwind(max_s, pc, bp, context, t->stack_top(), t->stack_bottom(),
request_fast_unwind);
}
@@ -369,6 +369,16 @@ void __msan_warning_noreturn() {
Die();
}
+static void OnStackUnwind(const SignalContext &sig, const void *,
+ BufferedStackTrace *stack) {
+ GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context,
+ common_flags()->fast_unwind_on_fatal);
+}
+
+static void MsanOnDeadlySignal(int signo, void *siginfo, void *context) {
+ HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr);
+}
+
void __msan_init() {
CHECK(!msan_init_is_running);
if (msan_inited) return;
@@ -384,6 +394,7 @@ void __msan_init() {
__sanitizer_set_report_path(common_flags()->log_path);
InitializeInterceptors();
+ InstallDeadlySignalHandlers(MsanOnDeadlySignal);
InstallAtExitHandler(); // Needs __cxa_atexit interceptor.
DisableCoreDumperIfNecessary();
diff --git a/contrib/compiler-rt/lib/msan/msan.h b/contrib/compiler-rt/lib/msan/msan.h
index fa9c15b88bef..cbae444127ee 100644
--- a/contrib/compiler-rt/lib/msan/msan.h
+++ b/contrib/compiler-rt/lib/msan/msan.h
@@ -160,21 +160,25 @@ const MappingDesc kMemoryLayout[] = {
# define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x1000000000ULL)
#elif SANITIZER_LINUX && SANITIZER_PPC64
-
const MappingDesc kMemoryLayout[] = {
- {0x000000000000ULL, 0x000100000000ULL, MappingDesc::APP, "low memory"},
- {0x000100000000ULL, 0x080000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x080000000000ULL, 0x180100000000ULL, MappingDesc::SHADOW, "shadow"},
- {0x180100000000ULL, 0x1C0000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x1C0000000000ULL, 0x2C0100000000ULL, MappingDesc::ORIGIN, "origin"},
- {0x2C0100000000ULL, 0x300000000000ULL, MappingDesc::INVALID, "invalid"},
- {0x300000000000ULL, 0x400000000000ULL, MappingDesc::APP, "high memory"}};
+ {0x000000000000ULL, 0x000200000000ULL, MappingDesc::APP, "low memory"},
+ {0x000200000000ULL, 0x080000000000ULL, MappingDesc::INVALID, "invalid"},
+ {0x080000000000ULL, 0x180200000000ULL, MappingDesc::SHADOW, "shadow"},
+ {0x180200000000ULL, 0x1C0000000000ULL, MappingDesc::INVALID, "invalid"},
+ {0x1C0000000000ULL, 0x2C0200000000ULL, MappingDesc::ORIGIN, "origin"},
+ {0x2C0200000000ULL, 0x300000000000ULL, MappingDesc::INVALID, "invalid"},
+ {0x300000000000ULL, 0x800000000000ULL, MappingDesc::APP, "high memory"}};
+// Various kernels use different low end ranges but we can combine them into one
+// big range. They also use different high end ranges but we can map them all to
+// one range.
// Maps low and high app ranges to contiguous space with zero base:
-// Low: 0000 0000 0000 - 0000 ffff ffff -> 1000 0000 0000 - 1000 ffff ffff
+// Low: 0000 0000 0000 - 0001 ffff ffff -> 1000 0000 0000 - 1001 ffff ffff
// High: 3000 0000 0000 - 3fff ffff ffff -> 0000 0000 0000 - 0fff ffff ffff
+// High: 4000 0000 0000 - 4fff ffff ffff -> 0000 0000 0000 - 0fff ffff ffff
+// High: 7000 0000 0000 - 7fff ffff ffff -> 0000 0000 0000 - 0fff ffff ffff
#define LINEARIZE_MEM(mem) \
- (((uptr)(mem) & ~0x200000000000ULL) ^ 0x100000000000ULL)
+ (((uptr)(mem) & ~0xE00000000000ULL) ^ 0x100000000000ULL)
#define MEM_TO_SHADOW(mem) (LINEARIZE_MEM((mem)) + 0x080000000000ULL)
#define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x140000000000ULL)
@@ -199,7 +203,7 @@ const MappingDesc kMemoryLayout[] = {
#define MEM_TO_SHADOW(mem) (LINEARIZE_MEM((mem)) + 0x100000000000ULL)
#define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x280000000000)
-#elif SANITIZER_LINUX && SANITIZER_WORDSIZE == 64
+#elif SANITIZER_NETBSD || (SANITIZER_LINUX && SANITIZER_WORDSIZE == 64)
#ifdef MSAN_LINUX_X86_64_OLD_MAPPING
// Requries PIE binary and ASLR enabled.
@@ -310,7 +314,7 @@ void PrintWarning(uptr pc, uptr bp);
void PrintWarningWithOrigin(uptr pc, uptr bp, u32 origin);
void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp,
- bool request_fast_unwind);
+ void *context, bool request_fast_unwind);
void ReportUMR(StackTrace *stack, u32 origin);
void ReportExpectedUMRNotFound(StackTrace *stack);
@@ -330,32 +334,32 @@ u32 ChainOrigin(u32 id, StackTrace *stack);
const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1;
-#define GET_MALLOC_STACK_TRACE \
- BufferedStackTrace stack; \
- if (__msan_get_track_origins() && msan_inited) \
- GetStackTrace(&stack, common_flags()->malloc_context_size, \
- StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), \
+#define GET_MALLOC_STACK_TRACE \
+ BufferedStackTrace stack; \
+ if (__msan_get_track_origins() && msan_inited) \
+ GetStackTrace(&stack, common_flags()->malloc_context_size, \
+ StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \
common_flags()->fast_unwind_on_malloc)
// For platforms which support slow unwinder only, we restrict the store context
// size to 1, basically only storing the current pc. We do this because the slow
// unwinder which is based on libunwind is not async signal safe and causes
// random freezes in forking applications as well as in signal handlers.
-#define GET_STORE_STACK_TRACE_PC_BP(pc, bp) \
- BufferedStackTrace stack; \
- if (__msan_get_track_origins() > 1 && msan_inited) { \
- if (!SANITIZER_CAN_FAST_UNWIND) \
- GetStackTrace(&stack, Min(1, flags()->store_context_size), pc, bp, \
- false); \
- else \
- GetStackTrace(&stack, flags()->store_context_size, pc, bp, \
- common_flags()->fast_unwind_on_malloc); \
+#define GET_STORE_STACK_TRACE_PC_BP(pc, bp) \
+ BufferedStackTrace stack; \
+ if (__msan_get_track_origins() > 1 && msan_inited) { \
+ if (!SANITIZER_CAN_FAST_UNWIND) \
+ GetStackTrace(&stack, Min(1, flags()->store_context_size), pc, bp, \
+ nullptr, false); \
+ else \
+ GetStackTrace(&stack, flags()->store_context_size, pc, bp, nullptr, \
+ common_flags()->fast_unwind_on_malloc); \
}
-#define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \
- BufferedStackTrace stack; \
- if (msan_inited) \
- GetStackTrace(&stack, kStackTraceMax, pc, bp, \
+#define GET_FATAL_STACK_TRACE_PC_BP(pc, bp) \
+ BufferedStackTrace stack; \
+ if (msan_inited) \
+ GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \
common_flags()->fast_unwind_on_fatal)
#define GET_STORE_STACK_TRACE \
diff --git a/contrib/compiler-rt/lib/msan/msan_allocator.cc b/contrib/compiler-rt/lib/msan/msan_allocator.cc
index 1034dbdf9b55..0f9942324931 100644
--- a/contrib/compiler-rt/lib/msan/msan_allocator.cc
+++ b/contrib/compiler-rt/lib/msan/msan_allocator.cc
@@ -62,7 +62,8 @@ struct MsanMapUnmapCallback {
};
typedef SizeClassAllocator32<AP32> PrimaryAllocator;
#elif defined(__x86_64__)
-#if SANITIZER_LINUX && !defined(MSAN_LINUX_X86_64_OLD_MAPPING)
+#if SANITIZER_NETBSD || \
+ (SANITIZER_LINUX && !defined(MSAN_LINUX_X86_64_OLD_MAPPING))
static const uptr kAllocatorSpace = 0x700000000000ULL;
#else
static const uptr kAllocatorSpace = 0x600000000000ULL;
@@ -255,8 +256,12 @@ void *msan_valloc(uptr size, StackTrace *stack) {
void *msan_pvalloc(uptr size, StackTrace *stack) {
uptr PageSize = GetPageSizeCached();
+ if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
+ errno = errno_ENOMEM;
+ return Allocator::FailureHandler::OnBadRequest();
+ }
// pvalloc(0) should allocate one page.
- size = size == 0 ? PageSize : RoundUpTo(size, PageSize);
+ size = size ? RoundUpTo(size, PageSize) : PageSize;
return SetErrnoOnNull(MsanAllocate(stack, size, PageSize, false));
}
diff --git a/contrib/compiler-rt/lib/msan/msan_interceptors.cc b/contrib/compiler-rt/lib/msan/msan_interceptors.cc
index b5d22baca08d..a7fe09b25ffb 100644
--- a/contrib/compiler-rt/lib/msan/msan_interceptors.cc
+++ b/contrib/compiler-rt/lib/msan/msan_interceptors.cc
@@ -22,6 +22,7 @@
#include "msan_thread.h"
#include "msan_poisoning.h"
#include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
#include "sanitizer_common/sanitizer_allocator.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
#include "sanitizer_common/sanitizer_allocator_internal.h"
@@ -33,6 +34,11 @@
#include "sanitizer_common/sanitizer_linux.h"
#include "sanitizer_common/sanitizer_tls_get_addr.h"
+#if SANITIZER_NETBSD
+#define gettimeofday __gettimeofday50
+#define getrusage __getrusage50
+#endif
+
#include <stdarg.h>
// ACHTUNG! No other system header includes in this file.
// Ideally, we should get rid of stdarg.h as well.
@@ -86,22 +92,21 @@ static void *AllocateFromLocalPool(uptr size_in_bytes) {
} while (0)
// Check that [x, x+n) range is unpoisoned.
-#define CHECK_UNPOISONED_0(x, n) \
- do { \
- sptr offset = __msan_test_shadow(x, n); \
- if (__msan::IsInSymbolizer()) \
- break; \
- if (offset >= 0 && __msan::flags()->report_umrs) { \
- GET_CALLER_PC_BP_SP; \
- (void) sp; \
- ReportUMRInsideAddressRange(__func__, x, n, offset); \
- __msan::PrintWarningWithOrigin( \
- pc, bp, __msan_get_origin((const char *)x + offset)); \
- if (__msan::flags()->halt_on_error) { \
- Printf("Exiting\n"); \
- Die(); \
- } \
- } \
+#define CHECK_UNPOISONED_0(x, n) \
+ do { \
+ sptr __offset = __msan_test_shadow(x, n); \
+ if (__msan::IsInSymbolizer()) break; \
+ if (__offset >= 0 && __msan::flags()->report_umrs) { \
+ GET_CALLER_PC_BP_SP; \
+ (void)sp; \
+ ReportUMRInsideAddressRange(__func__, x, n, __offset); \
+ __msan::PrintWarningWithOrigin( \
+ pc, bp, __msan_get_origin((const char *)x + __offset)); \
+ if (__msan::flags()->halt_on_error) { \
+ Printf("Exiting\n"); \
+ Die(); \
+ } \
+ } \
} while (0)
// Check that [x, x+n) range is unpoisoned unless we are in a nested
@@ -109,7 +114,7 @@ static void *AllocateFromLocalPool(uptr size_in_bytes) {
#define CHECK_UNPOISONED(x, n) \
do { \
if (!IsInInterceptorScope()) CHECK_UNPOISONED_0(x, n); \
- } while (0);
+ } while (0)
#define CHECK_UNPOISONED_STRING_OF_LEN(x, len, n) \
CHECK_UNPOISONED((x), \
@@ -118,7 +123,7 @@ static void *AllocateFromLocalPool(uptr size_in_bytes) {
#define CHECK_UNPOISONED_STRING(x, n) \
CHECK_UNPOISONED_STRING_OF_LEN((x), internal_strlen(x), (n))
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(SIZE_T, fread_unlocked, void *ptr, SIZE_T size, SIZE_T nmemb,
void *file) {
ENSURE_MSAN_INITED();
@@ -134,16 +139,21 @@ INTERCEPTOR(SIZE_T, fread_unlocked, void *ptr, SIZE_T size, SIZE_T nmemb,
INTERCEPTOR(SSIZE_T, readlink, const char *path, char *buf, SIZE_T bufsiz) {
ENSURE_MSAN_INITED();
- CHECK_UNPOISONED_STRING(path, 0)
+ CHECK_UNPOISONED_STRING(path, 0);
SSIZE_T res = REAL(readlink)(path, buf, bufsiz);
if (res > 0)
__msan_unpoison(buf, res);
return res;
}
+#if !SANITIZER_NETBSD
INTERCEPTOR(void *, mempcpy, void *dest, const void *src, SIZE_T n) {
return (char *)__msan_memcpy(dest, src, n) + n;
}
+#define MSAN_MAYBE_INTERCEPT_MEMPCPY INTERCEPT_FUNCTION(mempcpy)
+#else
+#define MSAN_MAYBE_INTERCEPT_MEMPCPY
+#endif
INTERCEPTOR(void *, memccpy, void *dest, const void *src, int c, SIZE_T n) {
ENSURE_MSAN_INITED();
@@ -168,7 +178,7 @@ INTERCEPTOR(int, posix_memalign, void **memptr, SIZE_T alignment, SIZE_T size) {
return res;
}
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(void *, memalign, SIZE_T alignment, SIZE_T size) {
GET_MALLOC_STACK_TRACE;
return msan_memalign(alignment, size, &stack);
@@ -183,6 +193,7 @@ INTERCEPTOR(void *, aligned_alloc, SIZE_T alignment, SIZE_T size) {
return msan_aligned_alloc(alignment, size, &stack);
}
+#if !SANITIZER_NETBSD
INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) {
GET_MALLOC_STACK_TRACE;
void *ptr = msan_memalign(alignment, size, &stack);
@@ -190,13 +201,17 @@ INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) {
DTLS_on_libc_memalign(ptr, size);
return ptr;
}
+#define MSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN INTERCEPT_FUNCTION(__libc_memalign)
+#else
+#define MSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN
+#endif
INTERCEPTOR(void *, valloc, SIZE_T size) {
GET_MALLOC_STACK_TRACE;
return msan_valloc(size, &stack);
}
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(void *, pvalloc, SIZE_T size) {
GET_MALLOC_STACK_TRACE;
return msan_pvalloc(size, &stack);
@@ -212,7 +227,7 @@ INTERCEPTOR(void, free, void *ptr) {
MsanDeallocate(&stack, ptr);
}
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(void, cfree, void *ptr) {
GET_MALLOC_STACK_TRACE;
if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
@@ -223,11 +238,17 @@ INTERCEPTOR(void, cfree, void *ptr) {
#define MSAN_MAYBE_INTERCEPT_CFREE
#endif
+#if !SANITIZER_NETBSD
INTERCEPTOR(uptr, malloc_usable_size, void *ptr) {
return __sanitizer_get_allocated_size(ptr);
}
+#define MSAN_MAYBE_INTERCEPT_MALLOC_USABLE_SIZE \
+ INTERCEPT_FUNCTION(malloc_usable_size)
+#else
+#define MSAN_MAYBE_INTERCEPT_MALLOC_USABLE_SIZE
+#endif
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
// This function actually returns a struct by value, but we can't unpoison a
// temporary! The following is equivalent on all supported platforms but
// aarch64 (which uses a different register for sret value). We have a test
@@ -246,7 +267,7 @@ INTERCEPTOR(void, mallinfo, __sanitizer_mallinfo *sret) {
#define MSAN_MAYBE_INTERCEPT_MALLINFO
#endif
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(int, mallopt, int cmd, int value) {
return -1;
}
@@ -255,7 +276,7 @@ INTERCEPTOR(int, mallopt, int cmd, int value) {
#define MSAN_MAYBE_INTERCEPT_MALLOPT
#endif
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(void, malloc_stats, void) {
// FIXME: implement, but don't call REAL(malloc_stats)!
}
@@ -286,6 +307,7 @@ INTERCEPTOR(char *, strncpy, char *dest, const char *src, SIZE_T n) { // NOLINT
return res;
}
+#if !SANITIZER_NETBSD
INTERCEPTOR(char *, stpcpy, char *dest, const char *src) { // NOLINT
ENSURE_MSAN_INITED();
GET_STORE_STACK_TRACE;
@@ -295,6 +317,10 @@ INTERCEPTOR(char *, stpcpy, char *dest, const char *src) { // NOLINT
CopyShadowAndOrigin(dest, src, n + 1, &stack);
return res;
}
+#define MSAN_MAYBE_INTERCEPT_STPCPY INTERCEPT_FUNCTION(stpcpy)
+#else
+#define MSAN_MAYBE_INTERCEPT_STPCPY
+#endif
INTERCEPTOR(char *, strdup, char *src) {
ENSURE_MSAN_INITED();
@@ -308,7 +334,7 @@ INTERCEPTOR(char *, strdup, char *src) {
return res;
}
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(char *, __strdup, char *src) {
ENSURE_MSAN_INITED();
GET_STORE_STACK_TRACE;
@@ -323,6 +349,7 @@ INTERCEPTOR(char *, __strdup, char *src) {
#define MSAN_MAYBE_INTERCEPT___STRDUP
#endif
+#if !SANITIZER_NETBSD
INTERCEPTOR(char *, gcvt, double number, SIZE_T ndigit, char *buf) {
ENSURE_MSAN_INITED();
char *res = REAL(gcvt)(number, ndigit, buf);
@@ -330,6 +357,10 @@ INTERCEPTOR(char *, gcvt, double number, SIZE_T ndigit, char *buf) {
__msan_unpoison(buf, n + 1);
return res;
}
+#define MSAN_MAYBE_INTERCEPT_GCVT INTERCEPT_FUNCTION(gcvt)
+#else
+#define MSAN_MAYBE_INTERCEPT_GCVT
+#endif
INTERCEPTOR(char *, strcat, char *dest, const char *src) { // NOLINT
ENSURE_MSAN_INITED();
@@ -386,6 +417,16 @@ INTERCEPTOR(char *, strncat, char *dest, const char *src, SIZE_T n) { // NOLINT
INTERCEPTOR_STRTO_BODY(ret_type, func, nptr, endptr, base, loc); \
}
+#if SANITIZER_NETBSD
+#define INTERCEPTORS_STRTO(ret_type, func, char_type) \
+ INTERCEPTOR_STRTO(ret_type, func, char_type) \
+ INTERCEPTOR_STRTO_LOC(ret_type, func##_l, char_type)
+
+#define INTERCEPTORS_STRTO_BASE(ret_type, func, char_type) \
+ INTERCEPTOR_STRTO_BASE(ret_type, func, char_type) \
+ INTERCEPTOR_STRTO_BASE_LOC(ret_type, func##_l, char_type)
+
+#else
#define INTERCEPTORS_STRTO(ret_type, func, char_type) \
INTERCEPTOR_STRTO(ret_type, func, char_type) \
INTERCEPTOR_STRTO_LOC(ret_type, func##_l, char_type) \
@@ -397,6 +438,7 @@ INTERCEPTOR(char *, strncat, char *dest, const char *src, SIZE_T n) { // NOLINT
INTERCEPTOR_STRTO_BASE_LOC(ret_type, func##_l, char_type) \
INTERCEPTOR_STRTO_BASE_LOC(ret_type, __##func##_l, char_type) \
INTERCEPTOR_STRTO_BASE_LOC(ret_type, __##func##_internal, char_type)
+#endif
INTERCEPTORS_STRTO(double, strtod, char) // NOLINT
INTERCEPTORS_STRTO(float, strtof, char) // NOLINT
@@ -405,6 +447,7 @@ INTERCEPTORS_STRTO_BASE(long, strtol, char) // NOLINT
INTERCEPTORS_STRTO_BASE(long long, strtoll, char) // NOLINT
INTERCEPTORS_STRTO_BASE(unsigned long, strtoul, char) // NOLINT
INTERCEPTORS_STRTO_BASE(unsigned long long, strtoull, char) // NOLINT
+INTERCEPTORS_STRTO_BASE(u64, strtouq, char) // NOLINT
INTERCEPTORS_STRTO(double, wcstod, wchar_t) // NOLINT
INTERCEPTORS_STRTO(float, wcstof, wchar_t) // NOLINT
@@ -414,11 +457,17 @@ INTERCEPTORS_STRTO_BASE(long long, wcstoll, wchar_t) // NOLINT
INTERCEPTORS_STRTO_BASE(unsigned long, wcstoul, wchar_t) // NOLINT
INTERCEPTORS_STRTO_BASE(unsigned long long, wcstoull, wchar_t) // NOLINT
+#if SANITIZER_NETBSD
+#define INTERCEPT_STRTO(func) \
+ INTERCEPT_FUNCTION(func); \
+ INTERCEPT_FUNCTION(func##_l);
+#else
#define INTERCEPT_STRTO(func) \
INTERCEPT_FUNCTION(func); \
INTERCEPT_FUNCTION(func##_l); \
INTERCEPT_FUNCTION(__##func##_l); \
INTERCEPT_FUNCTION(__##func##_internal);
+#endif
// FIXME: support *wprintf in common format interceptors.
@@ -457,6 +506,20 @@ INTERCEPTOR(SIZE_T, strxfrm_l, char *dest, const char *src, SIZE_T n,
return res;
}
+#if SANITIZER_LINUX
+INTERCEPTOR(SIZE_T, __strxfrm_l, char *dest, const char *src, SIZE_T n,
+ void *loc) {
+ ENSURE_MSAN_INITED();
+ CHECK_UNPOISONED(src, REAL(strlen)(src) + 1);
+ SIZE_T res = REAL(__strxfrm_l)(dest, src, n, loc);
+ if (res < n) __msan_unpoison(dest, res + 1);
+ return res;
+}
+#define MSAN_MAYBE_INTERCEPT___STRXFRM_L INTERCEPT_FUNCTION(__strxfrm_l)
+#else
+#define MSAN_MAYBE_INTERCEPT___STRXFRM_L
+#endif
+
#define INTERCEPTOR_STRFTIME_BODY(char_type, ret_type, func, s, ...) \
ENSURE_MSAN_INITED(); \
ret_type res = REAL(func)(s, __VA_ARGS__); \
@@ -473,7 +536,7 @@ INTERCEPTOR(SIZE_T, strftime_l, char *s, SIZE_T max, const char *format,
INTERCEPTOR_STRFTIME_BODY(char, SIZE_T, strftime_l, s, max, format, tm, loc);
}
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(SIZE_T, __strftime_l, char *s, SIZE_T max, const char *format,
__sanitizer_tm *tm, void *loc) {
INTERCEPTOR_STRFTIME_BODY(char, SIZE_T, __strftime_l, s, max, format, tm,
@@ -495,7 +558,7 @@ INTERCEPTOR(SIZE_T, wcsftime_l, wchar_t *s, SIZE_T max, const wchar_t *format,
loc);
}
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(SIZE_T, __wcsftime_l, wchar_t *s, SIZE_T max, const wchar_t *format,
__sanitizer_tm *tm, void *loc) {
INTERCEPTOR_STRFTIME_BODY(wchar_t, SIZE_T, __wcsftime_l, s, max, format, tm,
@@ -513,7 +576,8 @@ INTERCEPTOR(int, mbtowc, wchar_t *dest, const char *src, SIZE_T n) {
return res;
}
-INTERCEPTOR(int, mbrtowc, wchar_t *dest, const char *src, SIZE_T n, void *ps) {
+INTERCEPTOR(SIZE_T, mbrtowc, wchar_t *dest, const char *src, SIZE_T n,
+ void *ps) {
ENSURE_MSAN_INITED();
SIZE_T res = REAL(mbrtowc)(dest, src, n, ps);
if (res != (SIZE_T)-1 && dest) __msan_unpoison(dest, sizeof(wchar_t));
@@ -529,6 +593,7 @@ INTERCEPTOR(wchar_t *, wmemcpy, wchar_t *dest, const wchar_t *src, SIZE_T n) {
return res;
}
+#if !SANITIZER_NETBSD
INTERCEPTOR(wchar_t *, wmempcpy, wchar_t *dest, const wchar_t *src, SIZE_T n) {
ENSURE_MSAN_INITED();
GET_STORE_STACK_TRACE;
@@ -536,6 +601,10 @@ INTERCEPTOR(wchar_t *, wmempcpy, wchar_t *dest, const wchar_t *src, SIZE_T n) {
CopyShadowAndOrigin(dest, src, n * sizeof(wchar_t), &stack);
return res;
}
+#define MSAN_MAYBE_INTERCEPT_WMEMPCPY INTERCEPT_FUNCTION(wmempcpy)
+#else
+#define MSAN_MAYBE_INTERCEPT_WMEMPCPY
+#endif
INTERCEPTOR(wchar_t *, wmemset, wchar_t *s, wchar_t c, SIZE_T n) {
CHECK(MEM_IS_APP(s));
@@ -569,6 +638,7 @@ INTERCEPTOR(int, gettimeofday, void *tv, void *tz) {
return res;
}
+#if !SANITIZER_NETBSD
INTERCEPTOR(char *, fcvt, double x, int a, int *b, int *c) {
ENSURE_MSAN_INITED();
char *res = REAL(fcvt)(x, a, b, c);
@@ -577,6 +647,10 @@ INTERCEPTOR(char *, fcvt, double x, int a, int *b, int *c) {
if (res) __msan_unpoison(res, REAL(strlen)(res) + 1);
return res;
}
+#define MSAN_MAYBE_INTERCEPT_FCVT INTERCEPT_FUNCTION(fcvt)
+#else
+#define MSAN_MAYBE_INTERCEPT_FCVT
+#endif
INTERCEPTOR(char *, getenv, char *name) {
if (msan_init_is_running)
@@ -601,7 +675,7 @@ static void UnpoisonEnviron() {
INTERCEPTOR(int, setenv, const char *name, const char *value, int overwrite) {
ENSURE_MSAN_INITED();
- CHECK_UNPOISONED_STRING(name, 0)
+ CHECK_UNPOISONED_STRING(name, 0);
int res = REAL(setenv)(name, value, overwrite);
if (!res) UnpoisonEnviron();
return res;
@@ -614,7 +688,7 @@ INTERCEPTOR(int, putenv, char *string) {
return res;
}
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(int, __fxstat, int magic, int fd, void *buf) {
ENSURE_MSAN_INITED();
int res = REAL(__fxstat)(magic, fd, buf);
@@ -627,7 +701,7 @@ INTERCEPTOR(int, __fxstat, int magic, int fd, void *buf) {
#define MSAN_MAYBE_INTERCEPT___FXSTAT
#endif
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(int, __fxstat64, int magic, int fd, void *buf) {
ENSURE_MSAN_INITED();
int res = REAL(__fxstat64)(magic, fd, buf);
@@ -640,7 +714,7 @@ INTERCEPTOR(int, __fxstat64, int magic, int fd, void *buf) {
#define MSAN_MAYBE_INTERCEPT___FXSTAT64
#endif
-#if SANITIZER_FREEBSD
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
INTERCEPTOR(int, fstatat, int fd, char *pathname, void *buf, int flags) {
ENSURE_MSAN_INITED();
int res = REAL(fstatat)(fd, pathname, buf, flags);
@@ -659,7 +733,7 @@ INTERCEPTOR(int, __fxstatat, int magic, int fd, char *pathname, void *buf,
# define MSAN_INTERCEPT_FSTATAT INTERCEPT_FUNCTION(__fxstatat)
#endif
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(int, __fxstatat64, int magic, int fd, char *pathname, void *buf,
int flags) {
ENSURE_MSAN_INITED();
@@ -706,7 +780,7 @@ INTERCEPTOR(char *, fgets, char *s, int size, void *stream) {
return res;
}
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(char *, fgets_unlocked, char *s, int size, void *stream) {
ENSURE_MSAN_INITED();
char *res = REAL(fgets_unlocked)(s, size, stream);
@@ -729,7 +803,7 @@ INTERCEPTOR(int, getrlimit, int resource, void *rlim) {
return res;
}
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(int, getrlimit64, int resource, void *rlim) {
if (msan_init_is_running) return REAL(getrlimit64)(resource, rlim);
ENSURE_MSAN_INITED();
@@ -805,7 +879,7 @@ INTERCEPTOR(int, gethostname, char *name, SIZE_T len) {
return res;
}
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(int, epoll_wait, int epfd, void *events, int maxevents,
int timeout) {
ENSURE_MSAN_INITED();
@@ -820,7 +894,7 @@ INTERCEPTOR(int, epoll_wait, int epfd, void *events, int maxevents,
#define MSAN_MAYBE_INTERCEPT_EPOLL_WAIT
#endif
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(int, epoll_pwait, int epfd, void *events, int maxevents,
int timeout, void *sigmask) {
ENSURE_MSAN_INITED();
@@ -909,7 +983,7 @@ INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags,
return res;
}
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags,
int fd, OFF64_T offset) {
ENSURE_MSAN_INITED();
@@ -983,59 +1057,13 @@ static void SignalAction(int signo, void *si, void *uc) {
cb(signo, si, uc);
}
-INTERCEPTOR(int, sigaction, int signo, const __sanitizer_sigaction *act,
- __sanitizer_sigaction *oldact) {
- ENSURE_MSAN_INITED();
- // FIXME: check that *act is unpoisoned.
- // That requires intercepting all of sigemptyset, sigfillset, etc.
- int res;
- if (flags()->wrap_signals) {
- SpinMutexLock lock(&sigactions_mu);
- CHECK_LT(signo, kMaxSignals);
- uptr old_cb = atomic_load(&sigactions[signo], memory_order_relaxed);
- __sanitizer_sigaction new_act;
- __sanitizer_sigaction *pnew_act = act ? &new_act : nullptr;
- if (act) {
- REAL(memcpy)(pnew_act, act, sizeof(__sanitizer_sigaction));
- uptr cb = (uptr)pnew_act->sigaction;
- uptr new_cb = (pnew_act->sa_flags & __sanitizer::sa_siginfo)
- ? (uptr)SignalAction
- : (uptr)SignalHandler;
- if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) {
- atomic_store(&sigactions[signo], cb, memory_order_relaxed);
- pnew_act->sigaction = (void (*)(int, void *, void *))new_cb;
- }
- }
- res = REAL(sigaction)(signo, pnew_act, oldact);
- if (res == 0 && oldact) {
- uptr cb = (uptr)oldact->sigaction;
- if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) {
- oldact->sigaction = (void (*)(int, void *, void *))old_cb;
- }
- }
- } else {
- res = REAL(sigaction)(signo, act, oldact);
- }
-
- if (res == 0 && oldact) {
- __msan_unpoison(oldact, sizeof(__sanitizer_sigaction));
- }
- return res;
-}
-
-INTERCEPTOR(int, signal, int signo, uptr cb) {
- ENSURE_MSAN_INITED();
- if (flags()->wrap_signals) {
- CHECK_LT(signo, kMaxSignals);
- SpinMutexLock lock(&sigactions_mu);
- if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) {
- atomic_store(&sigactions[signo], cb, memory_order_relaxed);
- cb = (uptr) SignalHandler;
- }
- return REAL(signal)(signo, cb);
- } else {
- return REAL(signal)(signo, cb);
- }
+static void read_sigaction(const __sanitizer_sigaction *act) {
+ CHECK_UNPOISONED(&act->sa_flags, sizeof(act->sa_flags));
+ if (act->sa_flags & __sanitizer::sa_siginfo)
+ CHECK_UNPOISONED(&act->sigaction, sizeof(act->sigaction));
+ else
+ CHECK_UNPOISONED(&act->handler, sizeof(act->handler));
+ CHECK_UNPOISONED(&act->sa_mask, sizeof(act->sa_mask));
}
extern "C" int pthread_attr_init(void *attr);
@@ -1080,6 +1108,11 @@ INTERCEPTOR(int, pthread_key_create, __sanitizer_pthread_key_t *key,
return res;
}
+#if SANITIZER_NETBSD
+INTERCEPTOR(void, __libc_thr_keycreate, void *m, void (*dtor)(void *value)) \
+ ALIAS(WRAPPER_NAME(pthread_key_create));
+#endif
+
INTERCEPTOR(int, pthread_join, void *th, void **retval) {
ENSURE_MSAN_INITED();
int res = REAL(pthread_join)(th, retval);
@@ -1124,21 +1157,6 @@ INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg,
return REAL(__cxa_atexit)(MSanAtExitWrapper, r, dso_handle);
}
-DECLARE_REAL(int, shmctl, int shmid, int cmd, void *buf)
-
-INTERCEPTOR(void *, shmat, int shmid, const void *shmaddr, int shmflg) {
- ENSURE_MSAN_INITED();
- void *p = REAL(shmat)(shmid, shmaddr, shmflg);
- if (p != (void *)-1) {
- __sanitizer_shmid_ds ds;
- int res = REAL(shmctl)(shmid, shmctl_ipc_stat, &ds);
- if (!res) {
- __msan_unpoison(p, ds.shm_segsz);
- }
- }
- return p;
-}
-
static void BeforeFork() {
StackDepotLockAll();
ChainedOriginDepotLockAll();
@@ -1293,6 +1311,73 @@ int OnExit() {
#include "sanitizer_common/sanitizer_platform_interceptors.h"
#include "sanitizer_common/sanitizer_common_interceptors.inc"
+static uptr signal_impl(int signo, uptr cb);
+static int sigaction_impl(int signo, const __sanitizer_sigaction *act,
+ __sanitizer_sigaction *oldact);
+
+#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \
+ { return sigaction_impl(signo, act, oldact); }
+
+#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \
+ { \
+ handler = signal_impl(signo, handler); \
+ return REAL(func)(signo, handler); \
+ }
+
+#include "sanitizer_common/sanitizer_signal_interceptors.inc"
+
+static int sigaction_impl(int signo, const __sanitizer_sigaction *act,
+ __sanitizer_sigaction *oldact) {
+ ENSURE_MSAN_INITED();
+ if (act) read_sigaction(act);
+ int res;
+ if (flags()->wrap_signals) {
+ SpinMutexLock lock(&sigactions_mu);
+ CHECK_LT(signo, kMaxSignals);
+ uptr old_cb = atomic_load(&sigactions[signo], memory_order_relaxed);
+ __sanitizer_sigaction new_act;
+ __sanitizer_sigaction *pnew_act = act ? &new_act : nullptr;
+ if (act) {
+ REAL(memcpy)(pnew_act, act, sizeof(__sanitizer_sigaction));
+ uptr cb = (uptr)pnew_act->sigaction;
+ uptr new_cb = (pnew_act->sa_flags & __sanitizer::sa_siginfo)
+ ? (uptr)SignalAction
+ : (uptr)SignalHandler;
+ if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) {
+ atomic_store(&sigactions[signo], cb, memory_order_relaxed);
+ pnew_act->sigaction = (decltype(pnew_act->sigaction))new_cb;
+ }
+ }
+ res = REAL(SIGACTION_SYMNAME)(signo, pnew_act, oldact);
+ if (res == 0 && oldact) {
+ uptr cb = (uptr)oldact->sigaction;
+ if (cb == (uptr)SignalAction || cb == (uptr)SignalHandler) {
+ oldact->sigaction = (decltype(oldact->sigaction))old_cb;
+ }
+ }
+ } else {
+ res = REAL(SIGACTION_SYMNAME)(signo, act, oldact);
+ }
+
+ if (res == 0 && oldact) {
+ __msan_unpoison(oldact, sizeof(__sanitizer_sigaction));
+ }
+ return res;
+}
+
+static uptr signal_impl(int signo, uptr cb) {
+ ENSURE_MSAN_INITED();
+ if (flags()->wrap_signals) {
+ CHECK_LT(signo, kMaxSignals);
+ SpinMutexLock lock(&sigactions_mu);
+ if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) {
+ atomic_store(&sigactions[signo], cb, memory_order_relaxed);
+ cb = (uptr)&SignalHandler;
+ }
+ }
+ return cb;
+}
+
#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) CHECK_UNPOISONED(p, s)
#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
do { \
@@ -1353,6 +1438,19 @@ static int msan_dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size,
return cbdata->callback(info, size, cbdata->data);
}
+INTERCEPTOR(void *, shmat, int shmid, const void *shmaddr, int shmflg) {
+ ENSURE_MSAN_INITED();
+ void *p = REAL(shmat)(shmid, shmaddr, shmflg);
+ if (p != (void *)-1) {
+ __sanitizer_shmid_ds ds;
+ int res = REAL(shmctl)(shmid, shmctl_ipc_stat, &ds);
+ if (!res) {
+ __msan_unpoison(p, ds.shm_segsz);
+ }
+ }
+ return p;
+}
+
INTERCEPTOR(int, dl_iterate_phdr, dl_iterate_phdr_cb callback, void *data) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, dl_iterate_phdr, callback, data);
@@ -1455,12 +1553,13 @@ void InitializeInterceptors() {
static int inited = 0;
CHECK_EQ(inited, 0);
InitializeCommonInterceptors();
+ InitializeSignalInterceptors();
INTERCEPT_FUNCTION(mmap);
MSAN_MAYBE_INTERCEPT_MMAP64;
INTERCEPT_FUNCTION(posix_memalign);
MSAN_MAYBE_INTERCEPT_MEMALIGN;
- INTERCEPT_FUNCTION(__libc_memalign);
+ MSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN;
INTERCEPT_FUNCTION(valloc);
MSAN_MAYBE_INTERCEPT_PVALLOC;
INTERCEPT_FUNCTION(malloc);
@@ -1468,7 +1567,7 @@ void InitializeInterceptors() {
INTERCEPT_FUNCTION(realloc);
INTERCEPT_FUNCTION(free);
MSAN_MAYBE_INTERCEPT_CFREE;
- INTERCEPT_FUNCTION(malloc_usable_size);
+ MSAN_MAYBE_INTERCEPT_MALLOC_USABLE_SIZE;
MSAN_MAYBE_INTERCEPT_MALLINFO;
MSAN_MAYBE_INTERCEPT_MALLOPT;
MSAN_MAYBE_INTERCEPT_MALLOC_STATS;
@@ -1476,18 +1575,18 @@ void InitializeInterceptors() {
MSAN_MAYBE_INTERCEPT_FREAD_UNLOCKED;
INTERCEPT_FUNCTION(readlink);
INTERCEPT_FUNCTION(memccpy);
- INTERCEPT_FUNCTION(mempcpy);
+ MSAN_MAYBE_INTERCEPT_MEMPCPY;
INTERCEPT_FUNCTION(bcopy);
INTERCEPT_FUNCTION(wmemset);
INTERCEPT_FUNCTION(wmemcpy);
- INTERCEPT_FUNCTION(wmempcpy);
+ MSAN_MAYBE_INTERCEPT_WMEMPCPY;
INTERCEPT_FUNCTION(wmemmove);
INTERCEPT_FUNCTION(strcpy); // NOLINT
- INTERCEPT_FUNCTION(stpcpy); // NOLINT
+ MSAN_MAYBE_INTERCEPT_STPCPY; // NOLINT
INTERCEPT_FUNCTION(strdup);
MSAN_MAYBE_INTERCEPT___STRDUP;
INTERCEPT_FUNCTION(strncpy); // NOLINT
- INTERCEPT_FUNCTION(gcvt);
+ MSAN_MAYBE_INTERCEPT_GCVT;
INTERCEPT_FUNCTION(strcat); // NOLINT
INTERCEPT_FUNCTION(strncat); // NOLINT
INTERCEPT_STRTO(strtod);
@@ -1497,6 +1596,7 @@ void InitializeInterceptors() {
INTERCEPT_STRTO(strtoul);
INTERCEPT_STRTO(strtoll);
INTERCEPT_STRTO(strtoull);
+ INTERCEPT_STRTO(strtouq);
INTERCEPT_STRTO(wcstod);
INTERCEPT_STRTO(wcstof);
INTERCEPT_STRTO(wcstold);
@@ -1513,6 +1613,7 @@ void InitializeInterceptors() {
#endif
INTERCEPT_FUNCTION(strxfrm);
INTERCEPT_FUNCTION(strxfrm_l);
+ MSAN_MAYBE_INTERCEPT___STRXFRM_L;
INTERCEPT_FUNCTION(strftime);
INTERCEPT_FUNCTION(strftime_l);
MSAN_MAYBE_INTERCEPT___STRFTIME_L;
@@ -1531,7 +1632,7 @@ void InitializeInterceptors() {
INTERCEPT_FUNCTION(setenv);
INTERCEPT_FUNCTION(putenv);
INTERCEPT_FUNCTION(gettimeofday);
- INTERCEPT_FUNCTION(fcvt);
+ MSAN_MAYBE_INTERCEPT_FCVT;
MSAN_MAYBE_INTERCEPT___FXSTAT;
MSAN_INTERCEPT_FSTATAT;
MSAN_MAYBE_INTERCEPT___FXSTAT64;
@@ -1553,14 +1654,17 @@ void InitializeInterceptors() {
INTERCEPT_FUNCTION(dlerror);
INTERCEPT_FUNCTION(dl_iterate_phdr);
INTERCEPT_FUNCTION(getrusage);
- INTERCEPT_FUNCTION(sigaction);
- INTERCEPT_FUNCTION(signal);
#if defined(__mips__)
INTERCEPT_FUNCTION_VER(pthread_create, "GLIBC_2.2");
#else
INTERCEPT_FUNCTION(pthread_create);
#endif
INTERCEPT_FUNCTION(pthread_key_create);
+
+#if SANITIZER_NETBSD
+ INTERCEPT_FUNCTION(__libc_thr_keycreate);
+#endif
+
INTERCEPT_FUNCTION(pthread_join);
INTERCEPT_FUNCTION(tzset);
INTERCEPT_FUNCTION(__cxa_atexit);
diff --git a/contrib/compiler-rt/lib/msan/msan_linux.cc b/contrib/compiler-rt/lib/msan/msan_linux.cc
index 0a687f620c94..4e6321fcb918 100644
--- a/contrib/compiler-rt/lib/msan/msan_linux.cc
+++ b/contrib/compiler-rt/lib/msan/msan_linux.cc
@@ -9,11 +9,11 @@
//
// This file is a part of MemorySanitizer.
//
-// Linux- and FreeBSD-specific code.
+// Linux-, NetBSD- and FreeBSD-specific code.
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
#include "msan.h"
#include "msan_thread.h"
@@ -120,7 +120,7 @@ bool InitShadow(bool init_origins) {
return false;
}
- const uptr maxVirtualAddress = GetMaxVirtualAddress();
+ const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
for (unsigned i = 0; i < kMemoryLayoutSize; ++i) {
uptr start = kMemoryLayout[i].start;
@@ -213,4 +213,4 @@ void MsanTSDDtor(void *tsd) {
} // namespace __msan
-#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
diff --git a/contrib/compiler-rt/lib/msan/msan_report.cc b/contrib/compiler-rt/lib/msan/msan_report.cc
index 9a35c9c13de5..28c9bbabb3e9 100644
--- a/contrib/compiler-rt/lib/msan/msan_report.cc
+++ b/contrib/compiler-rt/lib/msan/msan_report.cc
@@ -30,10 +30,8 @@ namespace __msan {
class Decorator: public __sanitizer::SanitizerCommonDecorator {
public:
Decorator() : SanitizerCommonDecorator() { }
- const char *Warning() { return Red(); }
const char *Origin() { return Magenta(); }
const char *Name() { return Green(); }
- const char *End() { return Default(); }
};
static void DescribeStackOrigin(const char *so, uptr pc) {
@@ -47,7 +45,7 @@ static void DescribeStackOrigin(const char *so, uptr pc) {
" %sUninitialized value was created by an allocation of '%s%s%s'"
" in the stack frame of function '%s%s%s'%s\n",
d.Origin(), d.Name(), s, d.Origin(), d.Name(), sep + 1, d.Origin(),
- d.End());
+ d.Default());
InternalFree(s);
if (pc) {
@@ -66,7 +64,7 @@ static void DescribeOrigin(u32 id) {
StackTrace stack;
o = o.getNextChainedOrigin(&stack);
Printf(" %sUninitialized value was stored to memory at%s\n", d.Origin(),
- d.End());
+ d.Default());
stack.Print();
}
if (o.isStackOrigin()) {
@@ -78,18 +76,19 @@ static void DescribeOrigin(u32 id) {
switch (stack.tag) {
case StackTrace::TAG_ALLOC:
Printf(" %sUninitialized value was created by a heap allocation%s\n",
- d.Origin(), d.End());
+ d.Origin(), d.Default());
break;
case StackTrace::TAG_DEALLOC:
Printf(" %sUninitialized value was created by a heap deallocation%s\n",
- d.Origin(), d.End());
+ d.Origin(), d.Default());
break;
case STACK_TRACE_TAG_POISON:
Printf(" %sMemory was marked as uninitialized%s\n", d.Origin(),
- d.End());
+ d.Default());
break;
default:
- Printf(" %sUninitialized value was created%s\n", d.Origin(), d.End());
+ Printf(" %sUninitialized value was created%s\n", d.Origin(),
+ d.Default());
break;
}
stack.Print();
@@ -99,12 +98,12 @@ static void DescribeOrigin(u32 id) {
void ReportUMR(StackTrace *stack, u32 origin) {
if (!__msan::flags()->report_umrs) return;
- SpinMutexLock l(&CommonSanitizerReportMutex);
+ ScopedErrorReportLock l;
Decorator d;
Printf("%s", d.Warning());
Report("WARNING: MemorySanitizer: use-of-uninitialized-value\n");
- Printf("%s", d.End());
+ Printf("%s", d.Default());
stack->Print();
if (origin) {
DescribeOrigin(origin);
@@ -113,14 +112,14 @@ void ReportUMR(StackTrace *stack, u32 origin) {
}
void ReportExpectedUMRNotFound(StackTrace *stack) {
- SpinMutexLock l(&CommonSanitizerReportMutex);
+ ScopedErrorReportLock l;
Printf("WARNING: Expected use of uninitialized value not found\n");
stack->Print();
}
void ReportStats() {
- SpinMutexLock l(&CommonSanitizerReportMutex);
+ ScopedErrorReportLock l;
if (__msan_get_track_origins() > 0) {
StackDepotStats *stack_depot_stats = StackDepotGetStats();
@@ -138,13 +137,13 @@ void ReportStats() {
}
void ReportAtExitStatistics() {
- SpinMutexLock l(&CommonSanitizerReportMutex);
+ ScopedErrorReportLock l;
if (msan_report_count > 0) {
Decorator d;
Printf("%s", d.Warning());
Printf("MemorySanitizer: %d warnings reported.\n", msan_report_count);
- Printf("%s", d.End());
+ Printf("%s", d.Default());
}
}
@@ -203,7 +202,7 @@ void DescribeMemoryRange(const void *x, uptr size) {
Decorator d;
Printf("%s", d.Warning());
Printf("Shadow map of [%p, %p), %zu bytes:\n", start, end, end - start);
- Printf("%s", d.End());
+ Printf("%s", d.Default());
while (s < e) {
// Line start.
if (pos % 16 == 0) {
@@ -265,7 +264,7 @@ void ReportUMRInsideAddressRange(const char *what, const void *start, uptr size,
Printf("%s", d.Warning());
Printf("%sUninitialized bytes in %s%s%s at offset %zu inside [%p, %zu)%s\n",
d.Warning(), d.Name(), what, d.Warning(), offset, start, size,
- d.End());
+ d.Default());
if (__sanitizer::Verbosity())
DescribeMemoryRange(start, size);
}
diff --git a/contrib/compiler-rt/lib/profile/GCDAProfiling.c b/contrib/compiler-rt/lib/profile/GCDAProfiling.c
index 138af6ec4033..ef299f002e20 100644
--- a/contrib/compiler-rt/lib/profile/GCDAProfiling.c
+++ b/contrib/compiler-rt/lib/profile/GCDAProfiling.c
@@ -20,9 +20,6 @@
|*
\*===----------------------------------------------------------------------===*/
-#include "InstrProfilingPort.h"
-#include "InstrProfilingUtil.h"
-
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
@@ -34,9 +31,6 @@
#else
#include <sys/mman.h>
#include <sys/file.h>
-#ifndef MAP_FILE
-#define MAP_FILE 0
-#endif
#endif
#if defined(__FreeBSD__) && defined(__i386__)
@@ -62,6 +56,9 @@ typedef unsigned int uint32_t;
typedef unsigned long long uint64_t;
#endif
+#include "InstrProfiling.h"
+#include "InstrProfilingUtil.h"
+
/* #define DEBUG_GCDAPROFILING */
/*
@@ -238,17 +235,17 @@ void llvm_gcda_start_file(const char *orig_filename, const char version[4],
/* Try just opening the file. */
new_file = 0;
- fd = open(filename, O_RDWR);
+ fd = open(filename, O_RDWR | O_BINARY);
if (fd == -1) {
/* Try opening the file, creating it if necessary. */
new_file = 1;
mode = "w+b";
- fd = open(filename, O_RDWR | O_CREAT, 0644);
+ fd = open(filename, O_RDWR | O_CREAT | O_BINARY, 0644);
if (fd == -1) {
/* Try creating the directories first then opening the file. */
__llvm_profile_recursive_mkdir(filename);
- fd = open(filename, O_RDWR | O_CREAT, 0644);
+ fd = open(filename, O_RDWR | O_CREAT | O_BINARY, 0644);
if (fd == -1) {
/* Bah! It's hopeless. */
int errnum = errno;
@@ -263,7 +260,7 @@ void llvm_gcda_start_file(const char *orig_filename, const char version[4],
* same GCDA. This can fail if the filesystem doesn't support it, but in that
* case we'll just carry on with the old racy behaviour and hope for the best.
*/
- flock(fd, LOCK_EX);
+ lprofLockFd(fd);
output_file = fdopen(fd, mode);
/* Initialize the write buffer. */
@@ -462,7 +459,7 @@ void llvm_gcda_end_file() {
unmap_file();
}
- flock(fd, LOCK_UN);
+ lprofUnlockFd(fd);
fclose(output_file);
output_file = NULL;
write_buffer = NULL;
diff --git a/contrib/compiler-rt/lib/profile/InstrProfData.inc b/contrib/compiler-rt/lib/profile/InstrProfData.inc
index be0dd4ad04bf..6a98dc7b9b85 100644
--- a/contrib/compiler-rt/lib/profile/InstrProfData.inc
+++ b/contrib/compiler-rt/lib/profile/InstrProfData.inc
@@ -628,9 +628,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
/* Raw profile format version (start from 1). */
#define INSTR_PROF_RAW_VERSION 4
/* Indexed profile format version (start from 1). */
-#define INSTR_PROF_INDEX_VERSION 4
+#define INSTR_PROF_INDEX_VERSION 5
/* Coverage mapping format vresion (start from 0). */
-#define INSTR_PROF_COVMAP_VERSION 1
+#define INSTR_PROF_COVMAP_VERSION 2
/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
* version for other variants of profile. We set the lowest bit of the upper 8
diff --git a/contrib/compiler-rt/lib/profile/InstrProfiling.c b/contrib/compiler-rt/lib/profile/InstrProfiling.c
index fe66fec50658..00b31e1ee7d4 100644
--- a/contrib/compiler-rt/lib/profile/InstrProfiling.c
+++ b/contrib/compiler-rt/lib/profile/InstrProfiling.c
@@ -7,12 +7,14 @@
|*
\*===----------------------------------------------------------------------===*/
-#include "InstrProfiling.h"
-#include "InstrProfilingInternal.h"
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+
#define INSTR_PROF_VALUE_PROF_DATA
#include "InstrProfData.inc"
diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingFile.c b/contrib/compiler-rt/lib/profile/InstrProfilingFile.c
index d038bb9cb5b0..d7c0abbc16e3 100644
--- a/contrib/compiler-rt/lib/profile/InstrProfilingFile.c
+++ b/contrib/compiler-rt/lib/profile/InstrProfilingFile.c
@@ -7,9 +7,6 @@
|*
\*===----------------------------------------------------------------------===*/
-#include "InstrProfiling.h"
-#include "InstrProfilingInternal.h"
-#include "InstrProfilingUtil.h"
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
@@ -31,6 +28,10 @@
#endif
#endif
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+#include "InstrProfilingUtil.h"
+
/* From where is profile name specified.
* The order the enumerators define their
* precedence. Re-order them may lead to
@@ -85,7 +86,6 @@ typedef struct lprofFilename {
COMPILER_RT_WEAK lprofFilename lprofCurFilename = {0, 0, 0, {0}, {0},
0, 0, 0, PNS_unknown};
-int getpid(void);
static int getCurFilenameLength();
static const char *getCurFilename(char *FilenameBuf);
static unsigned doMerging() { return lprofCurFilename.MergePoolSize; }
@@ -325,7 +325,7 @@ static int parseFilenamePattern(const char *FilenamePat,
if (FilenamePat[I] == '%') {
if (FilenamePat[++I] == 'p') {
if (!NumPids++) {
- if (snprintf(PidChars, MAX_PID_SIZE, "%d", getpid()) <= 0) {
+ if (snprintf(PidChars, MAX_PID_SIZE, "%ld", (long)getpid()) <= 0) {
PROF_WARN("Unable to get pid for filename pattern %s. Using the "
"default name.",
FilenamePat);
@@ -519,8 +519,10 @@ void __llvm_profile_initialize_file(void) {
EnvFilenamePat = getFilenamePatFromEnv();
if (EnvFilenamePat) {
- SelectedPat = EnvFilenamePat;
- PNS = PNS_environment;
+ /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid
+ at the moment when __llvm_profile_write_file() gets executed. */
+ parseAndSetFilename(EnvFilenamePat, PNS_environment, 1);
+ return;
} else if (hasCommandLineOverrider) {
SelectedPat = INSTR_PROF_PROFILE_NAME_VAR;
PNS = PNS_command_line;
diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingInternal.h b/contrib/compiler-rt/lib/profile/InstrProfilingInternal.h
index 36490ef7d433..40540ab5f205 100644
--- a/contrib/compiler-rt/lib/profile/InstrProfilingInternal.h
+++ b/contrib/compiler-rt/lib/profile/InstrProfilingInternal.h
@@ -10,8 +10,9 @@
#ifndef PROFILE_INSTRPROFILING_INTERNALH_
#define PROFILE_INSTRPROFILING_INTERNALH_
+#include <stddef.h>
+
#include "InstrProfiling.h"
-#include "stddef.h"
/*!
* \brief Write instrumentation data to the given buffer, given explicit
diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
index b6c780ff514f..89f1ab4cfeda 100644
--- a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
+++ b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c
@@ -7,11 +7,13 @@
|*
\*===----------------------------------------------------------------------===*/
-#include "InstrProfiling.h"
+#if defined(__linux__) || defined(__FreeBSD__) || \
+ (defined(__sun__) && defined(__svr4__))
-#if defined(__linux__) || defined(__FreeBSD__)
#include <stdlib.h>
+#include "InstrProfiling.h"
+
#define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME)
#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_SECT_NAME)
#define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_SECT_NAME)
diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c
index b25966487e91..a339abc7f883 100644
--- a/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c
+++ b/contrib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c
@@ -7,12 +7,13 @@
|*
\*===----------------------------------------------------------------------===*/
-#include "InstrProfiling.h"
-
-#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__)
+#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \
+ !(defined(__sun__) && defined(__svr4__))
#include <stdlib.h>
+#include "InstrProfiling.h"
+
static const __llvm_profile_data *DataFirst = NULL;
static const __llvm_profile_data *DataLast = NULL;
static const char *NamesFirst = NULL;
diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingPort.h b/contrib/compiler-rt/lib/profile/InstrProfilingPort.h
index 5789351956b9..cd1264b0e575 100644
--- a/contrib/compiler-rt/lib/profile/InstrProfilingPort.h
+++ b/contrib/compiler-rt/lib/profile/InstrProfilingPort.h
@@ -7,6 +7,9 @@
|*
\*===----------------------------------------------------------------------===*/
+/* This header must be included after all others so it can provide fallback
+ definitions for stuff missing in system headers. */
+
#ifndef PROFILE_INSTRPROFILING_PORT_H_
#define PROFILE_INSTRPROFILING_PORT_H_
@@ -44,9 +47,6 @@
#define COMPILER_RT_GETHOSTNAME(Name, Len) ((void)(Name), (void)(Len), (-1))
#else
#define COMPILER_RT_GETHOSTNAME(Name, Len) lprofGetHostName(Name, Len)
-#ifndef _MSC_VER
-#define COMPILER_RT_HAS_UNAME 1
-#endif
#endif
#if COMPILER_RT_HAS_ATOMICS == 1
@@ -107,6 +107,14 @@
#define PROF_NOTE(Format, ...) \
fprintf(stderr, "LLVM Profile Note: " Format, __VA_ARGS__);
+#ifndef MAP_FILE
+#define MAP_FILE 0
+#endif
+
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+
#if defined(__FreeBSD__)
#include <inttypes.h>
diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingUtil.c b/contrib/compiler-rt/lib/profile/InstrProfilingUtil.c
index fb68f30a5e1f..110562383412 100644
--- a/contrib/compiler-rt/lib/profile/InstrProfilingUtil.c
+++ b/contrib/compiler-rt/lib/profile/InstrProfilingUtil.c
@@ -7,13 +7,10 @@
|*
\*===----------------------------------------------------------------------===*/
-#include "InstrProfilingUtil.h"
-#include "InstrProfiling.h"
-
#ifdef _WIN32
#include <direct.h>
-#include <io.h>
#include <windows.h>
+#include "WindowsMMap.h"
#else
#include <sys/stat.h>
#include <sys/types.h>
@@ -34,6 +31,9 @@
#include <sys/prctl.h>
#endif
+#include "InstrProfiling.h"
+#include "InstrProfilingUtil.h"
+
COMPILER_RT_VISIBILITY
void __llvm_profile_recursive_mkdir(char *path) {
int i;
@@ -86,16 +86,16 @@ COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) {
#elif defined(COMPILER_RT_HAS_UNAME)
COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) {
struct utsname N;
- int R;
- if (!(R = uname(&N)))
+ int R = uname(&N);
+ if (R >= 0) {
strncpy(Name, N.nodename, Len);
+ return 0;
+ }
return R;
}
#endif
-COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) {
- FILE *f;
- int fd;
+COMPILER_RT_VISIBILITY int lprofLockFd(int fd) {
#ifdef COMPILER_RT_HAS_FCNTL_LCK
struct flock s_flock;
@@ -103,21 +103,59 @@ COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) {
s_flock.l_start = 0;
s_flock.l_len = 0; /* Until EOF. */
s_flock.l_pid = getpid();
-
s_flock.l_type = F_WRLCK;
- fd = open(ProfileName, O_RDWR | O_CREAT, 0666);
- if (fd < 0)
- return NULL;
while (fcntl(fd, F_SETLKW, &s_flock) == -1) {
if (errno != EINTR) {
if (errno == ENOLCK) {
- PROF_WARN("Data may be corrupted during profile merging : %s\n",
- "Fail to obtain file lock due to system limit.");
+ return -1;
}
break;
}
}
+ return 0;
+#else
+ flock(fd, LOCK_EX);
+ return 0;
+#endif
+}
+
+COMPILER_RT_VISIBILITY int lprofUnlockFd(int fd) {
+#ifdef COMPILER_RT_HAS_FCNTL_LCK
+ struct flock s_flock;
+
+ s_flock.l_whence = SEEK_SET;
+ s_flock.l_start = 0;
+ s_flock.l_len = 0; /* Until EOF. */
+ s_flock.l_pid = getpid();
+ s_flock.l_type = F_UNLCK;
+
+ while (fcntl(fd, F_SETLKW, &s_flock) == -1) {
+ if (errno != EINTR) {
+ if (errno == ENOLCK) {
+ return -1;
+ }
+ break;
+ }
+ }
+ return 0;
+#else
+ flock(fd, LOCK_UN);
+ return 0;
+#endif
+}
+
+COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) {
+ FILE *f;
+ int fd;
+#ifdef COMPILER_RT_HAS_FCNTL_LCK
+ fd = open(ProfileName, O_RDWR | O_CREAT, 0666);
+ if (fd < 0)
+ return NULL;
+
+ if (lprofLockFd(fd) != 0)
+ PROF_WARN("Data may be corrupted during profile merging : %s\n",
+ "Fail to obtain file lock due to system limit.");
f = fdopen(fd, "r+b");
#elif defined(_WIN32)
diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingUtil.h b/contrib/compiler-rt/lib/profile/InstrProfilingUtil.h
index 9698599606e0..2f2ea1b876a8 100644
--- a/contrib/compiler-rt/lib/profile/InstrProfilingUtil.h
+++ b/contrib/compiler-rt/lib/profile/InstrProfilingUtil.h
@@ -16,6 +16,9 @@
/*! \brief Create a directory tree. */
void __llvm_profile_recursive_mkdir(char *Pathname);
+int lprofLockFd(int fd);
+int lprofUnlockFd(int fd);
+
/*! Open file \c Filename for read+write with write
* lock for exclusive access. The caller will block
* if the lock is already held by another process. */
diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingValue.c b/contrib/compiler-rt/lib/profile/InstrProfilingValue.c
index 44263da80097..3db0de8a6723 100644
--- a/contrib/compiler-rt/lib/profile/InstrProfilingValue.c
+++ b/contrib/compiler-rt/lib/profile/InstrProfilingValue.c
@@ -7,13 +7,15 @@
|*
\*===----------------------------------------------------------------------===*/
-#include "InstrProfiling.h"
-#include "InstrProfilingInternal.h"
-#include "InstrProfilingUtil.h" /* For PS4 getenv shim. */
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+#include "InstrProfilingUtil.h"
+
#define INSTR_PROF_VALUE_PROF_DATA
#define INSTR_PROF_COMMON_API_IMPL
#include "InstrProfData.inc"
@@ -22,7 +24,7 @@ static int hasStaticCounters = 1;
static int OutOfNodesWarnings = 0;
static int hasNonDefaultValsPerSite = 0;
#define INSTR_PROF_MAX_VP_WARNS 10
-#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 8
+#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 16
#define INSTR_PROF_VNODE_POOL_SIZE 1024
#ifndef _MSC_VER
diff --git a/contrib/compiler-rt/lib/profile/InstrProfilingWriter.c b/contrib/compiler-rt/lib/profile/InstrProfilingWriter.c
index d4c9b9bd663c..7c6061d2c65a 100644
--- a/contrib/compiler-rt/lib/profile/InstrProfilingWriter.c
+++ b/contrib/compiler-rt/lib/profile/InstrProfilingWriter.c
@@ -7,14 +7,15 @@
|*
\*===----------------------------------------------------------------------===*/
-#include "InstrProfiling.h"
-#include "InstrProfilingInternal.h"
#ifdef _MSC_VER
/* For _alloca */
#include <malloc.h>
#endif
#include <string.h>
+#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+
#define INSTR_PROF_VALUE_PROF_DATA
#include "InstrProfData.inc"
diff --git a/contrib/compiler-rt/lib/profile/WindowsMMap.c b/contrib/compiler-rt/lib/profile/WindowsMMap.c
index f81d7da5389e..dc87a888ae7b 100644
--- a/contrib/compiler-rt/lib/profile/WindowsMMap.c
+++ b/contrib/compiler-rt/lib/profile/WindowsMMap.c
@@ -18,11 +18,12 @@
#if defined(_WIN32)
#include "WindowsMMap.h"
-#include "InstrProfiling.h"
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
+#include "InstrProfiling.h"
+
#ifdef __USE_FILE_OFFSET64
# define DWORD_HI(x) (x >> 32)
# define DWORD_LO(x) ((x) & 0xffffffff)
@@ -120,9 +121,61 @@ int msync(void *addr, size_t length, int flags)
}
COMPILER_RT_VISIBILITY
-int flock(int fd, int operation)
-{
- return -1; /* Not supported. */
+int lock(HANDLE handle, DWORD lockType, BOOL blocking) {
+ DWORD flags = lockType;
+ if (!blocking)
+ flags |= LOCKFILE_FAIL_IMMEDIATELY;
+
+ OVERLAPPED overlapped;
+ ZeroMemory(&overlapped, sizeof(OVERLAPPED));
+ overlapped.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
+ BOOL result = LockFileEx(handle, flags, 0, MAXDWORD, MAXDWORD, &overlapped);
+ if (!result) {
+ DWORD dw = GetLastError();
+
+ // In non-blocking mode, return an error if the file is locked.
+ if (!blocking && dw == ERROR_LOCK_VIOLATION)
+ return -1; // EWOULDBLOCK
+
+ // If the error is ERROR_IO_PENDING, we need to wait until the operation
+ // finishes. Otherwise, we return an error.
+ if (dw != ERROR_IO_PENDING)
+ return -1;
+
+ DWORD dwNumBytes;
+ if (!GetOverlappedResult(handle, &overlapped, &dwNumBytes, TRUE))
+ return -1;
+ }
+
+ return 0;
+}
+
+COMPILER_RT_VISIBILITY
+int flock(int fd, int operation) {
+ HANDLE handle = (HANDLE)_get_osfhandle(fd);
+ if (handle == INVALID_HANDLE_VALUE)
+ return -1;
+
+ BOOL blocking = (operation & LOCK_NB) == 0;
+ int op = operation & ~LOCK_NB;
+
+ switch (op) {
+ case LOCK_EX:
+ return lock(handle, LOCKFILE_EXCLUSIVE_LOCK, blocking);
+
+ case LOCK_SH:
+ return lock(handle, 0, blocking);
+
+ case LOCK_UN:
+ if (!UnlockFile(handle, 0, 0, MAXDWORD, MAXDWORD))
+ return -1;
+ break;
+
+ default:
+ return -1;
+ }
+
+ return 0;
}
#undef DWORD_HI
diff --git a/contrib/compiler-rt/lib/safestack/safestack.cc b/contrib/compiler-rt/lib/safestack/safestack.cc
index b194b6cfa8f9..d783cd5a9b29 100644
--- a/contrib/compiler-rt/lib/safestack/safestack.cc
+++ b/contrib/compiler-rt/lib/safestack/safestack.cc
@@ -21,7 +21,9 @@
#include <unistd.h>
#include <sys/resource.h>
#include <sys/types.h>
+#if !defined(__NetBSD__)
#include <sys/user.h>
+#endif
#include "interception/interception.h"
#include "sanitizer_common/sanitizer_common.h"
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc
index 84f523c5e431..fc4f7a75ae34 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc
@@ -178,11 +178,13 @@ void InternalFree(void *addr, InternalAllocatorCache *cache) {
}
// LowLevelAllocator
+constexpr uptr kLowLevelAllocatorDefaultAlignment = 8;
+static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment;
static LowLevelAllocateCallback low_level_alloc_callback;
void *LowLevelAllocator::Allocate(uptr size) {
// Align allocation size.
- size = RoundUpTo(size, 8);
+ size = RoundUpTo(size, low_level_alloc_min_alignment);
if (allocated_end_ - allocated_current_ < (sptr)size) {
uptr size_to_allocate = Max(size, GetPageSizeCached());
allocated_current_ =
@@ -199,6 +201,11 @@ void *LowLevelAllocator::Allocate(uptr size) {
return res;
}
+void SetLowLevelAllocateMinAlignment(uptr alignment) {
+ CHECK(IsPowerOfTwo(alignment));
+ low_level_alloc_min_alignment = Max(alignment, low_level_alloc_min_alignment);
+}
+
void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback) {
low_level_alloc_callback = callback;
}
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h
index 8c5696ea789c..38368361de6f 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h
@@ -56,6 +56,19 @@ struct NoOpMapUnmapCallback {
// Callback type for iterating over chunks.
typedef void (*ForEachChunkCallback)(uptr chunk, void *arg);
+INLINE u32 Rand(u32 *state) { // ANSI C linear congruential PRNG.
+ return (*state = *state * 1103515245 + 12345) >> 16;
+}
+
+INLINE u32 RandN(u32 *state, u32 n) { return Rand(state) % n; } // [0, n)
+
+template<typename T>
+INLINE void RandomShuffle(T *a, u32 n, u32 *rand_state) {
+ if (n <= 1) return;
+ for (u32 i = n - 1; i > 0; i--)
+ Swap(a[i], a[RandN(rand_state, i + 1)]);
+}
+
#include "sanitizer_allocator_size_class_map.h"
#include "sanitizer_allocator_stats.h"
#include "sanitizer_allocator_primary64.h"
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.cc
new file mode 100644
index 000000000000..dc263dbef5fc
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.cc
@@ -0,0 +1,23 @@
+//===-- sanitizer_allocator_checks.cc ---------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Various checks shared between ThreadSanitizer, MemorySanitizer, etc. memory
+// allocators.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_errno.h"
+
+namespace __sanitizer {
+
+void SetErrnoToENOMEM() {
+ errno = errno_ENOMEM;
+}
+
+} // namespace __sanitizer
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h
index 202916eae348..b61a8b2eb3b6 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_checks.h
@@ -15,17 +15,22 @@
#ifndef SANITIZER_ALLOCATOR_CHECKS_H
#define SANITIZER_ALLOCATOR_CHECKS_H
-#include "sanitizer_errno.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_common.h"
#include "sanitizer_platform.h"
namespace __sanitizer {
+// The following is defined in a separate compilation unit to avoid pulling in
+// sanitizer_errno.h in this header, which leads to conflicts when other system
+// headers include errno.h. This is usually the result of an unlikely event,
+// and as such we do not care as much about having it inlined.
+void SetErrnoToENOMEM();
+
// A common errno setting logic shared by almost all sanitizer allocator APIs.
INLINE void *SetErrnoOnNull(void *ptr) {
if (UNLIKELY(!ptr))
- errno = errno_ENOMEM;
+ SetErrnoToENOMEM();
return ptr;
}
@@ -59,6 +64,12 @@ INLINE bool CheckForCallocOverflow(uptr size, uptr n) {
return (max / size) < n;
}
+// Returns true if the size passed to pvalloc overflows when rounded to the next
+// multiple of page_size.
+INLINE bool CheckForPvallocOverflow(uptr size, uptr page_size) {
+ return RoundUpTo(size, page_size) < size;
+}
+
} // namespace __sanitizer
#endif // SANITIZER_ALLOCATOR_CHECKS_H
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h
index efd25cadfe74..0d8a2a174bb5 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h
@@ -77,6 +77,10 @@ class CombinedAllocator {
primary_.SetReleaseToOSIntervalMs(release_to_os_interval_ms);
}
+ void ForceReleaseToOS() {
+ primary_.ForceReleaseToOS();
+ }
+
void Deallocate(AllocatorCache *cache, void *p) {
if (!p) return;
if (primary_.PointerIsMine(p))
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h
index 13910e719e78..2f5ce3151ee4 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h
@@ -39,6 +39,9 @@ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
void __sanitizer_free_hook(void *ptr);
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
+__sanitizer_purge_allocator();
+
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
__sanitizer_print_memory_profile(uptr top_percent, uptr max_number_of_contexts);
} // extern "C"
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_local_cache.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_local_cache.h
index ec0742c20be2..1b3c2c0c19d1 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_local_cache.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_local_cache.h
@@ -26,9 +26,6 @@ struct SizeClassAllocatorLocalCache
template <class SizeClassAllocator>
struct SizeClassAllocator64LocalCache {
typedef SizeClassAllocator Allocator;
- static const uptr kNumClasses = SizeClassAllocator::kNumClasses;
- typedef typename Allocator::SizeClassMapT SizeClassMap;
- typedef typename Allocator::CompactPtrT CompactPtrT;
void Init(AllocatorGlobalStats *s) {
stats_.Init();
@@ -76,14 +73,18 @@ struct SizeClassAllocator64LocalCache {
}
void Drain(SizeClassAllocator *allocator) {
- for (uptr class_id = 0; class_id < kNumClasses; class_id++) {
- PerClass *c = &per_class_[class_id];
+ for (uptr i = 0; i < kNumClasses; i++) {
+ PerClass *c = &per_class_[i];
while (c->count > 0)
- Drain(c, allocator, class_id, c->count);
+ Drain(c, allocator, i, c->count);
}
}
- // private:
+ private:
+ typedef typename Allocator::SizeClassMapT SizeClassMap;
+ static const uptr kNumClasses = SizeClassMap::kNumClasses;
+ typedef typename Allocator::CompactPtrT CompactPtrT;
+
struct PerClass {
u32 count;
u32 max_count;
@@ -94,7 +95,7 @@ struct SizeClassAllocator64LocalCache {
AllocatorStats stats_;
void InitCache() {
- if (per_class_[1].max_count)
+ if (LIKELY(per_class_[1].max_count))
return;
for (uptr i = 0; i < kNumClasses; i++) {
PerClass *c = &per_class_[i];
@@ -130,7 +131,6 @@ template <class SizeClassAllocator>
struct SizeClassAllocator32LocalCache {
typedef SizeClassAllocator Allocator;
typedef typename Allocator::TransferBatch TransferBatch;
- static const uptr kNumClasses = SizeClassAllocator::kNumClasses;
void Init(AllocatorGlobalStats *s) {
stats_.Init();
@@ -138,6 +138,21 @@ struct SizeClassAllocator32LocalCache {
s->Register(&stats_);
}
+ // Returns a TransferBatch suitable for class_id.
+ TransferBatch *CreateBatch(uptr class_id, SizeClassAllocator *allocator,
+ TransferBatch *b) {
+ if (uptr batch_class_id = per_class_[class_id].batch_class_id)
+ return (TransferBatch*)Allocate(allocator, batch_class_id);
+ return b;
+ }
+
+ // Destroys TransferBatch b.
+ void DestroyBatch(uptr class_id, SizeClassAllocator *allocator,
+ TransferBatch *b) {
+ if (uptr batch_class_id = per_class_[class_id].batch_class_id)
+ Deallocate(allocator, batch_class_id, b);
+ }
+
void Destroy(SizeClassAllocator *allocator, AllocatorGlobalStats *s) {
Drain(allocator);
if (s)
@@ -173,66 +188,57 @@ struct SizeClassAllocator32LocalCache {
}
void Drain(SizeClassAllocator *allocator) {
- for (uptr class_id = 0; class_id < kNumClasses; class_id++) {
- PerClass *c = &per_class_[class_id];
+ for (uptr i = 0; i < kNumClasses; i++) {
+ PerClass *c = &per_class_[i];
while (c->count > 0)
- Drain(allocator, class_id);
+ Drain(allocator, i);
}
}
- // private:
- typedef typename SizeClassAllocator::SizeClassMapT SizeClassMap;
+ private:
+ typedef typename Allocator::SizeClassMapT SizeClassMap;
+ static const uptr kBatchClassID = SizeClassMap::kBatchClassID;
+ static const uptr kNumClasses = SizeClassMap::kNumClasses;
+ // If kUseSeparateSizeClassForBatch is true, all TransferBatch objects are
+ // allocated from kBatchClassID size class (except for those that are needed
+ // for kBatchClassID itself). The goal is to have TransferBatches in a totally
+ // different region of RAM to improve security.
+ static const bool kUseSeparateSizeClassForBatch =
+ Allocator::kUseSeparateSizeClassForBatch;
+
struct PerClass {
uptr count;
uptr max_count;
uptr class_size;
- uptr class_id_for_transfer_batch;
+ uptr batch_class_id;
void *batch[2 * TransferBatch::kMaxNumCached];
};
PerClass per_class_[kNumClasses];
AllocatorStats stats_;
void InitCache() {
- if (per_class_[1].max_count)
+ if (LIKELY(per_class_[1].max_count))
return;
- // TransferBatch class is declared in SizeClassAllocator.
- uptr class_id_for_transfer_batch =
- SizeClassMap::ClassID(sizeof(TransferBatch));
+ const uptr batch_class_id = SizeClassMap::ClassID(sizeof(TransferBatch));
for (uptr i = 0; i < kNumClasses; i++) {
PerClass *c = &per_class_[i];
uptr max_cached = TransferBatch::MaxCached(i);
c->max_count = 2 * max_cached;
c->class_size = Allocator::ClassIdToSize(i);
- // We transfer chunks between central and thread-local free lists in
- // batches. For small size classes we allocate batches separately. For
- // large size classes we may use one of the chunks to store the batch.
- // sizeof(TransferBatch) must be a power of 2 for more efficient
- // allocation.
- c->class_id_for_transfer_batch = (c->class_size <
+ // Precompute the class id to use to store batches for the current class
+ // id. 0 means the class size is large enough to store a batch within one
+ // of the chunks. If using a separate size class, it will always be
+ // kBatchClassID, except for kBatchClassID itself.
+ if (kUseSeparateSizeClassForBatch) {
+ c->batch_class_id = (i == kBatchClassID) ? 0 : kBatchClassID;
+ } else {
+ c->batch_class_id = (c->class_size <
TransferBatch::AllocationSizeRequiredForNElements(max_cached)) ?
- class_id_for_transfer_batch : 0;
+ batch_class_id : 0;
+ }
}
}
- // Returns a TransferBatch suitable for class_id.
- // For small size classes allocates the batch from the allocator.
- // For large size classes simply returns b.
- TransferBatch *CreateBatch(uptr class_id, SizeClassAllocator *allocator,
- TransferBatch *b) {
- if (uptr batch_class_id = per_class_[class_id].class_id_for_transfer_batch)
- return (TransferBatch*)Allocate(allocator, batch_class_id);
- return b;
- }
-
- // Destroys TransferBatch b.
- // For small size classes deallocates b to the allocator.
- // Does notthing for large size classes.
- void DestroyBatch(uptr class_id, SizeClassAllocator *allocator,
- TransferBatch *b) {
- if (uptr batch_class_id = per_class_[class_id].class_id_for_transfer_batch)
- Deallocate(allocator, batch_class_id, b);
- }
-
NOINLINE bool Refill(SizeClassAllocator *allocator, uptr class_id) {
InitCache();
PerClass *c = &per_class_[class_id];
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h
index e85821543d27..6c682aff64a2 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h
@@ -41,6 +41,7 @@ template<class SizeClassAllocator> struct SizeClassAllocator32LocalCache;
struct SizeClassAllocator32FlagMasks { // Bit masks.
enum {
kRandomShuffleChunks = 1,
+ kUseSeparateSizeClassForBatch = 2,
};
};
@@ -55,8 +56,10 @@ class SizeClassAllocator32 {
typedef typename Params::ByteMap ByteMap;
typedef typename Params::MapUnmapCallback MapUnmapCallback;
- static const bool kRandomShuffleChunks =
- Params::kFlags & SizeClassAllocator32FlagMasks::kRandomShuffleChunks;
+ static const bool kRandomShuffleChunks = Params::kFlags &
+ SizeClassAllocator32FlagMasks::kRandomShuffleChunks;
+ static const bool kUseSeparateSizeClassForBatch = Params::kFlags &
+ SizeClassAllocator32FlagMasks::kUseSeparateSizeClassForBatch;
struct TransferBatch {
static const uptr kMaxNumCached = SizeClassMap::kMaxNumCachedHint - 2;
@@ -94,11 +97,11 @@ class SizeClassAllocator32 {
static const uptr kBatchSize = sizeof(TransferBatch);
COMPILER_CHECK((kBatchSize & (kBatchSize - 1)) == 0);
- COMPILER_CHECK(sizeof(TransferBatch) ==
- SizeClassMap::kMaxNumCachedHint * sizeof(uptr));
+ COMPILER_CHECK(kBatchSize == SizeClassMap::kMaxNumCachedHint * sizeof(uptr));
static uptr ClassIdToSize(uptr class_id) {
- return SizeClassMap::Size(class_id);
+ return (class_id == SizeClassMap::kBatchClassID) ?
+ kBatchSize : SizeClassMap::Size(class_id);
}
typedef SizeClassAllocator32<Params> ThisT;
@@ -117,6 +120,10 @@ class SizeClassAllocator32 {
// This is empty here. Currently only implemented in 64-bit allocator.
}
+ void ForceReleaseToOS() {
+ // Currently implemented in 64-bit allocator only.
+ }
+
void *MapWithCallback(uptr size) {
void *res = MmapOrDie(size, "SizeClassAllocator32");
MapUnmapCallback().OnMap((uptr)res, size);
@@ -161,9 +168,9 @@ class SizeClassAllocator32 {
NOINLINE void DeallocateBatch(AllocatorStats *stat, uptr class_id,
TransferBatch *b) {
CHECK_LT(class_id, kNumClasses);
+ CHECK_GT(b->Count(), 0);
SizeClassInfo *sci = GetSizeClassInfo(class_id);
SpinMutexLock l(&sci->mutex);
- CHECK_GT(b->Count(), 0);
sci->free_list.push_front(b);
}
@@ -261,7 +268,8 @@ class SizeClassAllocator32 {
struct SizeClassInfo {
SpinMutex mutex;
IntrusiveList<TransferBatch> free_list;
- char padding[kCacheLineSize - sizeof(uptr) -
+ u32 rand_state;
+ char padding[kCacheLineSize - 2 * sizeof(uptr) -
sizeof(IntrusiveList<TransferBatch>)];
};
COMPILER_CHECK(sizeof(SizeClassInfo) == kCacheLineSize);
@@ -294,29 +302,62 @@ class SizeClassAllocator32 {
return &size_class_info_array[class_id];
}
+ bool PopulateBatches(AllocatorCache *c, SizeClassInfo *sci, uptr class_id,
+ TransferBatch **current_batch, uptr max_count,
+ uptr *pointers_array, uptr count) {
+ // If using a separate class for batches, we do not need to shuffle it.
+ if (kRandomShuffleChunks && (!kUseSeparateSizeClassForBatch ||
+ class_id != SizeClassMap::kBatchClassID))
+ RandomShuffle(pointers_array, count, &sci->rand_state);
+ TransferBatch *b = *current_batch;
+ for (uptr i = 0; i < count; i++) {
+ if (!b) {
+ b = c->CreateBatch(class_id, this, (TransferBatch*)pointers_array[i]);
+ if (UNLIKELY(!b))
+ return false;
+ b->Clear();
+ }
+ b->Add((void*)pointers_array[i]);
+ if (b->Count() == max_count) {
+ sci->free_list.push_back(b);
+ b = nullptr;
+ }
+ }
+ *current_batch = b;
+ return true;
+ }
+
bool PopulateFreeList(AllocatorStats *stat, AllocatorCache *c,
SizeClassInfo *sci, uptr class_id) {
uptr size = ClassIdToSize(class_id);
uptr reg = AllocateRegion(stat, class_id);
if (UNLIKELY(!reg))
return false;
+ if (kRandomShuffleChunks)
+ if (UNLIKELY(sci->rand_state == 0))
+ // The random state is initialized from ASLR (PIE) and time.
+ sci->rand_state = reinterpret_cast<uptr>(sci) ^ NanoTime();
uptr n_chunks = kRegionSize / (size + kMetadataSize);
uptr max_count = TransferBatch::MaxCached(class_id);
CHECK_GT(max_count, 0);
TransferBatch *b = nullptr;
+ const uptr kShuffleArraySize = 48;
+ uptr shuffle_array[kShuffleArraySize];
+ uptr count = 0;
for (uptr i = reg; i < reg + n_chunks * size; i += size) {
- if (!b) {
- b = c->CreateBatch(class_id, this, (TransferBatch*)i);
- if (UNLIKELY(!b))
+ shuffle_array[count++] = i;
+ if (count == kShuffleArraySize) {
+ if (UNLIKELY(!PopulateBatches(c, sci, class_id, &b, max_count,
+ shuffle_array, count)))
return false;
- b->Clear();
- }
- b->Add((void*)i);
- if (b->Count() == max_count) {
- sci->free_list.push_back(b);
- b = nullptr;
+ count = 0;
}
}
+ if (count) {
+ if (UNLIKELY(!PopulateBatches(c, sci, class_id, &b, max_count,
+ shuffle_array, count)))
+ return false;
+ }
if (b) {
CHECK_GT(b->Count(), 0);
sci->free_list.push_back(b);
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
index 0c2e72ce7eb1..651a64b04f2e 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
@@ -62,21 +62,20 @@ class SizeClassAllocator64 {
// as a 4-byte integer (offset from the region start shifted right by 4).
typedef u32 CompactPtrT;
static const uptr kCompactPtrScale = 4;
- CompactPtrT PointerToCompactPtr(uptr base, uptr ptr) {
+ CompactPtrT PointerToCompactPtr(uptr base, uptr ptr) const {
return static_cast<CompactPtrT>((ptr - base) >> kCompactPtrScale);
}
- uptr CompactPtrToPointer(uptr base, CompactPtrT ptr32) {
+ uptr CompactPtrToPointer(uptr base, CompactPtrT ptr32) const {
return base + (static_cast<uptr>(ptr32) << kCompactPtrScale);
}
void Init(s32 release_to_os_interval_ms) {
uptr TotalSpaceSize = kSpaceSize + AdditionalSize();
if (kUsingConstantSpaceBeg) {
- CHECK_EQ(kSpaceBeg, reinterpret_cast<uptr>(
- MmapFixedNoAccess(kSpaceBeg, TotalSpaceSize)));
+ CHECK_EQ(kSpaceBeg, address_range.Init(TotalSpaceSize, AllocatorName(),
+ kSpaceBeg));
} else {
- NonConstSpaceBeg =
- reinterpret_cast<uptr>(MmapNoAccess(TotalSpaceSize));
+ NonConstSpaceBeg = address_range.Init(TotalSpaceSize, AllocatorName());
CHECK_NE(NonConstSpaceBeg, ~(uptr)0);
}
SetReleaseToOSIntervalMs(release_to_os_interval_ms);
@@ -92,6 +91,13 @@ class SizeClassAllocator64 {
memory_order_relaxed);
}
+ void ForceReleaseToOS() {
+ for (uptr class_id = 1; class_id < kNumClasses; class_id++) {
+ BlockingMutexLock l(&GetRegionInfo(class_id)->mutex);
+ MaybeReleaseToOS(class_id, true /*force*/);
+ }
+ }
+
static bool CanAllocate(uptr size, uptr alignment) {
return size <= SizeClassMap::kMaxSize &&
alignment <= SizeClassMap::kMaxSize;
@@ -116,7 +122,7 @@ class SizeClassAllocator64 {
region->num_freed_chunks = new_num_freed_chunks;
region->stats.n_freed += n_chunks;
- MaybeReleaseToOS(class_id);
+ MaybeReleaseToOS(class_id, false /*force*/);
}
NOINLINE bool GetFromAllocator(AllocatorStats *stat, uptr class_id,
@@ -155,7 +161,7 @@ class SizeClassAllocator64 {
space_beg;
}
- uptr GetRegionBeginBySizeClass(uptr class_id) {
+ uptr GetRegionBeginBySizeClass(uptr class_id) const {
return SpaceBeg() + kRegionSize * class_id;
}
@@ -223,30 +229,39 @@ class SizeClassAllocator64 {
uptr avail_chunks = region->allocated_user / ClassIdToSize(class_id);
Printf(
"%s %02zd (%6zd): mapped: %6zdK allocs: %7zd frees: %7zd inuse: %6zd "
- "num_freed_chunks %7zd avail: %6zd rss: %6zdK releases: %6zd\n",
+ "num_freed_chunks %7zd avail: %6zd rss: %6zdK releases: %6zd "
+ "last released: %6zdK region: 0x%zx\n",
region->exhausted ? "F" : " ", class_id, ClassIdToSize(class_id),
region->mapped_user >> 10, region->stats.n_allocated,
region->stats.n_freed, in_use, region->num_freed_chunks, avail_chunks,
- rss >> 10, region->rtoi.num_releases);
+ rss >> 10, region->rtoi.num_releases,
+ region->rtoi.last_released_bytes >> 10,
+ SpaceBeg() + kRegionSize * class_id);
}
void PrintStats() {
+ uptr rss_stats[kNumClasses];
+ for (uptr class_id = 0; class_id < kNumClasses; class_id++)
+ rss_stats[class_id] = SpaceBeg() + kRegionSize * class_id;
+ GetMemoryProfile(FillMemoryProfile, rss_stats, kNumClasses);
+
uptr total_mapped = 0;
+ uptr total_rss = 0;
uptr n_allocated = 0;
uptr n_freed = 0;
for (uptr class_id = 1; class_id < kNumClasses; class_id++) {
RegionInfo *region = GetRegionInfo(class_id);
- total_mapped += region->mapped_user;
+ if (region->mapped_user != 0) {
+ total_mapped += region->mapped_user;
+ total_rss += rss_stats[class_id];
+ }
n_allocated += region->stats.n_allocated;
n_freed += region->stats.n_freed;
}
- Printf("Stats: SizeClassAllocator64: %zdM mapped in %zd allocations; "
- "remains %zd\n",
- total_mapped >> 20, n_allocated, n_allocated - n_freed);
- uptr rss_stats[kNumClasses];
- for (uptr class_id = 0; class_id < kNumClasses; class_id++)
- rss_stats[class_id] = SpaceBeg() + kRegionSize * class_id;
- GetMemoryProfile(FillMemoryProfile, rss_stats, kNumClasses);
+
+ Printf("Stats: SizeClassAllocator64: %zdM mapped (%zdM rss) in "
+ "%zd allocations; remains %zd\n", total_mapped >> 20,
+ total_rss >> 20, n_allocated, n_allocated - n_freed);
for (uptr class_id = 1; class_id < kNumClasses; class_id++)
PrintStats(class_id, rss_stats[class_id]);
}
@@ -294,7 +309,243 @@ class SizeClassAllocator64 {
static const uptr kNumClasses = SizeClassMap::kNumClasses;
static const uptr kNumClassesRounded = SizeClassMap::kNumClassesRounded;
+ // A packed array of counters. Each counter occupies 2^n bits, enough to store
+ // counter's max_value. Ctor will try to allocate the required buffer via
+ // mapper->MapPackedCounterArrayBuffer and the caller is expected to check
+ // whether the initialization was successful by checking IsAllocated() result.
+ // For the performance sake, none of the accessors check the validity of the
+ // arguments, it is assumed that index is always in [0, n) range and the value
+ // is not incremented past max_value.
+ template<class MemoryMapperT>
+ class PackedCounterArray {
+ public:
+ PackedCounterArray(u64 num_counters, u64 max_value, MemoryMapperT *mapper)
+ : n(num_counters), memory_mapper(mapper) {
+ CHECK_GT(num_counters, 0);
+ CHECK_GT(max_value, 0);
+ constexpr u64 kMaxCounterBits = sizeof(*buffer) * 8ULL;
+ // Rounding counter storage size up to the power of two allows for using
+ // bit shifts calculating particular counter's index and offset.
+ uptr counter_size_bits =
+ RoundUpToPowerOfTwo(MostSignificantSetBitIndex(max_value) + 1);
+ CHECK_LE(counter_size_bits, kMaxCounterBits);
+ counter_size_bits_log = Log2(counter_size_bits);
+ counter_mask = ~0ULL >> (kMaxCounterBits - counter_size_bits);
+
+ uptr packing_ratio = kMaxCounterBits >> counter_size_bits_log;
+ CHECK_GT(packing_ratio, 0);
+ packing_ratio_log = Log2(packing_ratio);
+ bit_offset_mask = packing_ratio - 1;
+
+ buffer_size =
+ (RoundUpTo(n, 1ULL << packing_ratio_log) >> packing_ratio_log) *
+ sizeof(*buffer);
+ buffer = reinterpret_cast<u64*>(
+ memory_mapper->MapPackedCounterArrayBuffer(buffer_size));
+ }
+ ~PackedCounterArray() {
+ if (buffer) {
+ memory_mapper->UnmapPackedCounterArrayBuffer(
+ reinterpret_cast<uptr>(buffer), buffer_size);
+ }
+ }
+
+ bool IsAllocated() const {
+ return !!buffer;
+ }
+
+ u64 GetCount() const {
+ return n;
+ }
+
+ uptr Get(uptr i) const {
+ DCHECK_LT(i, n);
+ uptr index = i >> packing_ratio_log;
+ uptr bit_offset = (i & bit_offset_mask) << counter_size_bits_log;
+ return (buffer[index] >> bit_offset) & counter_mask;
+ }
+
+ void Inc(uptr i) const {
+ DCHECK_LT(Get(i), counter_mask);
+ uptr index = i >> packing_ratio_log;
+ uptr bit_offset = (i & bit_offset_mask) << counter_size_bits_log;
+ buffer[index] += 1ULL << bit_offset;
+ }
+
+ void IncRange(uptr from, uptr to) const {
+ DCHECK_LE(from, to);
+ for (uptr i = from; i <= to; i++)
+ Inc(i);
+ }
+
+ private:
+ const u64 n;
+ u64 counter_size_bits_log;
+ u64 counter_mask;
+ u64 packing_ratio_log;
+ u64 bit_offset_mask;
+
+ MemoryMapperT* const memory_mapper;
+ u64 buffer_size;
+ u64* buffer;
+ };
+
+ template<class MemoryMapperT>
+ class FreePagesRangeTracker {
+ public:
+ explicit FreePagesRangeTracker(MemoryMapperT* mapper)
+ : memory_mapper(mapper),
+ page_size_scaled_log(Log2(GetPageSizeCached() >> kCompactPtrScale)),
+ in_the_range(false), current_page(0), current_range_start_page(0) {}
+
+ void NextPage(bool freed) {
+ if (freed) {
+ if (!in_the_range) {
+ current_range_start_page = current_page;
+ in_the_range = true;
+ }
+ } else {
+ CloseOpenedRange();
+ }
+ current_page++;
+ }
+
+ void Done() {
+ CloseOpenedRange();
+ }
+
+ private:
+ void CloseOpenedRange() {
+ if (in_the_range) {
+ memory_mapper->ReleasePageRangeToOS(
+ current_range_start_page << page_size_scaled_log,
+ current_page << page_size_scaled_log);
+ in_the_range = false;
+ }
+ }
+
+ MemoryMapperT* const memory_mapper;
+ const uptr page_size_scaled_log;
+ bool in_the_range;
+ uptr current_page;
+ uptr current_range_start_page;
+ };
+
+ // Iterates over the free_array to identify memory pages containing freed
+ // chunks only and returns these pages back to OS.
+ // allocated_pages_count is the total number of pages allocated for the
+ // current bucket.
+ template<class MemoryMapperT>
+ static void ReleaseFreeMemoryToOS(CompactPtrT *free_array,
+ uptr free_array_count, uptr chunk_size,
+ uptr allocated_pages_count,
+ MemoryMapperT *memory_mapper) {
+ const uptr page_size = GetPageSizeCached();
+
+ // Figure out the number of chunks per page and whether we can take a fast
+ // path (the number of chunks per page is the same for all pages).
+ uptr full_pages_chunk_count_max;
+ bool same_chunk_count_per_page;
+ if (chunk_size <= page_size && page_size % chunk_size == 0) {
+ // Same number of chunks per page, no cross overs.
+ full_pages_chunk_count_max = page_size / chunk_size;
+ same_chunk_count_per_page = true;
+ } else if (chunk_size <= page_size && page_size % chunk_size != 0 &&
+ chunk_size % (page_size % chunk_size) == 0) {
+ // Some chunks are crossing page boundaries, which means that the page
+ // contains one or two partial chunks, but all pages contain the same
+ // number of chunks.
+ full_pages_chunk_count_max = page_size / chunk_size + 1;
+ same_chunk_count_per_page = true;
+ } else if (chunk_size <= page_size) {
+ // Some chunks are crossing page boundaries, which means that the page
+ // contains one or two partial chunks.
+ full_pages_chunk_count_max = page_size / chunk_size + 2;
+ same_chunk_count_per_page = false;
+ } else if (chunk_size > page_size && chunk_size % page_size == 0) {
+ // One chunk covers multiple pages, no cross overs.
+ full_pages_chunk_count_max = 1;
+ same_chunk_count_per_page = true;
+ } else if (chunk_size > page_size) {
+ // One chunk covers multiple pages, Some chunks are crossing page
+ // boundaries. Some pages contain one chunk, some contain two.
+ full_pages_chunk_count_max = 2;
+ same_chunk_count_per_page = false;
+ } else {
+ UNREACHABLE("All chunk_size/page_size ratios must be handled.");
+ }
+
+ PackedCounterArray<MemoryMapperT> counters(allocated_pages_count,
+ full_pages_chunk_count_max,
+ memory_mapper);
+ if (!counters.IsAllocated())
+ return;
+
+ const uptr chunk_size_scaled = chunk_size >> kCompactPtrScale;
+ const uptr page_size_scaled = page_size >> kCompactPtrScale;
+ const uptr page_size_scaled_log = Log2(page_size_scaled);
+
+ // Iterate over free chunks and count how many free chunks affect each
+ // allocated page.
+ if (chunk_size <= page_size && page_size % chunk_size == 0) {
+ // Each chunk affects one page only.
+ for (uptr i = 0; i < free_array_count; i++)
+ counters.Inc(free_array[i] >> page_size_scaled_log);
+ } else {
+ // In all other cases chunks might affect more than one page.
+ for (uptr i = 0; i < free_array_count; i++) {
+ counters.IncRange(
+ free_array[i] >> page_size_scaled_log,
+ (free_array[i] + chunk_size_scaled - 1) >> page_size_scaled_log);
+ }
+ }
+
+ // Iterate over pages detecting ranges of pages with chunk counters equal
+ // to the expected number of chunks for the particular page.
+ FreePagesRangeTracker<MemoryMapperT> range_tracker(memory_mapper);
+ if (same_chunk_count_per_page) {
+ // Fast path, every page has the same number of chunks affecting it.
+ for (uptr i = 0; i < counters.GetCount(); i++)
+ range_tracker.NextPage(counters.Get(i) == full_pages_chunk_count_max);
+ } else {
+ // Show path, go through the pages keeping count how many chunks affect
+ // each page.
+ const uptr pn =
+ chunk_size < page_size ? page_size_scaled / chunk_size_scaled : 1;
+ const uptr pnc = pn * chunk_size_scaled;
+ // The idea is to increment the current page pointer by the first chunk
+ // size, middle portion size (the portion of the page covered by chunks
+ // except the first and the last one) and then the last chunk size, adding
+ // up the number of chunks on the current page and checking on every step
+ // whether the page boundary was crossed.
+ uptr prev_page_boundary = 0;
+ uptr current_boundary = 0;
+ for (uptr i = 0; i < counters.GetCount(); i++) {
+ uptr page_boundary = prev_page_boundary + page_size_scaled;
+ uptr chunks_per_page = pn;
+ if (current_boundary < page_boundary) {
+ if (current_boundary > prev_page_boundary)
+ chunks_per_page++;
+ current_boundary += pnc;
+ if (current_boundary < page_boundary) {
+ chunks_per_page++;
+ current_boundary += chunk_size_scaled;
+ }
+ }
+ prev_page_boundary = page_boundary;
+
+ range_tracker.NextPage(counters.Get(i) == chunks_per_page);
+ }
+ }
+ range_tracker.Done();
+ }
+
private:
+ friend class MemoryMapper;
+
+ ReservedAddressRange address_range;
+ static const char *AllocatorName() { return "sanitizer_allocator"; }
+
static const uptr kRegionSize = kSpaceSize / kNumClassesRounded;
// FreeArray is the array of free-d chunks (stored as 4-byte offsets).
// In the worst case it may reguire kRegionSize/SizeClassMap::kMinSize
@@ -330,6 +581,7 @@ class SizeClassAllocator64 {
uptr n_freed_at_last_release;
uptr num_releases;
u64 last_release_at_ns;
+ u64 last_released_bytes;
};
struct RegionInfo {
@@ -347,30 +599,18 @@ class SizeClassAllocator64 {
};
COMPILER_CHECK(sizeof(RegionInfo) >= kCacheLineSize);
- u32 Rand(u32 *state) { // ANSI C linear congruential PRNG.
- return (*state = *state * 1103515245 + 12345) >> 16;
- }
-
- u32 RandN(u32 *state, u32 n) { return Rand(state) % n; } // [0, n)
-
- void RandomShuffle(u32 *a, u32 n, u32 *rand_state) {
- if (n <= 1) return;
- for (u32 i = n - 1; i > 0; i--)
- Swap(a[i], a[RandN(rand_state, i + 1)]);
- }
-
- RegionInfo *GetRegionInfo(uptr class_id) {
+ RegionInfo *GetRegionInfo(uptr class_id) const {
CHECK_LT(class_id, kNumClasses);
RegionInfo *regions =
reinterpret_cast<RegionInfo *>(SpaceBeg() + kSpaceSize);
return &regions[class_id];
}
- uptr GetMetadataEnd(uptr region_beg) {
+ uptr GetMetadataEnd(uptr region_beg) const {
return region_beg + kRegionSize - kFreeArraySize;
}
- uptr GetChunkIdx(uptr chunk, uptr size) {
+ uptr GetChunkIdx(uptr chunk, uptr size) const {
if (!kUsingConstantSpaceBeg)
chunk -= SpaceBeg();
@@ -382,13 +622,12 @@ class SizeClassAllocator64 {
return (u32)offset / (u32)size;
}
- CompactPtrT *GetFreeArray(uptr region_beg) {
- return reinterpret_cast<CompactPtrT *>(region_beg + kRegionSize -
- kFreeArraySize);
+ CompactPtrT *GetFreeArray(uptr region_beg) const {
+ return reinterpret_cast<CompactPtrT *>(GetMetadataEnd(region_beg));
}
bool MapWithCallback(uptr beg, uptr size) {
- uptr mapped = reinterpret_cast<uptr>(MmapFixedOrDieOnFatalError(beg, size));
+ uptr mapped = address_range.Map(beg, size);
if (UNLIKELY(!mapped))
return false;
CHECK_EQ(beg, mapped);
@@ -397,21 +636,21 @@ class SizeClassAllocator64 {
}
void MapWithCallbackOrDie(uptr beg, uptr size) {
- CHECK_EQ(beg, reinterpret_cast<uptr>(MmapFixedOrDie(beg, size)));
+ CHECK_EQ(beg, address_range.MapOrDie(beg, size));
MapUnmapCallback().OnMap(beg, size);
}
void UnmapWithCallbackOrDie(uptr beg, uptr size) {
MapUnmapCallback().OnUnmap(beg, size);
- UnmapOrDie(reinterpret_cast<void *>(beg), size);
+ address_range.Unmap(beg, size);
}
bool EnsureFreeArraySpace(RegionInfo *region, uptr region_beg,
uptr num_freed_chunks) {
uptr needed_space = num_freed_chunks * sizeof(CompactPtrT);
if (region->mapped_free_array < needed_space) {
- CHECK_LE(needed_space, kFreeArraySize);
uptr new_mapped_free_array = RoundUpTo(needed_space, kFreeArrayMapSize);
+ CHECK_LE(new_mapped_free_array, kFreeArraySize);
uptr current_map_end = reinterpret_cast<uptr>(GetFreeArray(region_beg)) +
region->mapped_free_array;
uptr new_map_size = new_mapped_free_array - region->mapped_free_array;
@@ -422,55 +661,74 @@ class SizeClassAllocator64 {
return true;
}
+ // Check whether this size class is exhausted.
+ bool IsRegionExhausted(RegionInfo *region, uptr class_id,
+ uptr additional_map_size) {
+ if (LIKELY(region->mapped_user + region->mapped_meta +
+ additional_map_size <= kRegionSize - kFreeArraySize))
+ return false;
+ if (!region->exhausted) {
+ region->exhausted = true;
+ Printf("%s: Out of memory. ", SanitizerToolName);
+ Printf("The process has exhausted %zuMB for size class %zu.\n",
+ kRegionSize >> 20, ClassIdToSize(class_id));
+ }
+ return true;
+ }
+
NOINLINE bool PopulateFreeArray(AllocatorStats *stat, uptr class_id,
RegionInfo *region, uptr requested_count) {
// region->mutex is held.
- const uptr size = ClassIdToSize(class_id);
- const uptr new_space_beg = region->allocated_user;
- const uptr new_space_end = new_space_beg + requested_count * size;
const uptr region_beg = GetRegionBeginBySizeClass(class_id);
+ const uptr size = ClassIdToSize(class_id);
+ const uptr total_user_bytes =
+ region->allocated_user + requested_count * size;
// Map more space for chunks, if necessary.
- if (new_space_end > region->mapped_user) {
- if (!kUsingConstantSpaceBeg && region->mapped_user == 0)
- region->rand_state = static_cast<u32>(region_beg >> 12); // From ASLR.
+ if (LIKELY(total_user_bytes > region->mapped_user)) {
+ if (UNLIKELY(region->mapped_user == 0)) {
+ if (!kUsingConstantSpaceBeg && kRandomShuffleChunks)
+ // The random state is initialized from ASLR.
+ region->rand_state = static_cast<u32>(region_beg >> 12);
+ // Postpone the first release to OS attempt for ReleaseToOSIntervalMs,
+ // preventing just allocated memory from being released sooner than
+ // necessary and also preventing extraneous ReleaseMemoryPagesToOS calls
+ // for short lived processes.
+ // Do it only when the feature is turned on, to avoid a potentially
+ // extraneous syscall.
+ if (ReleaseToOSIntervalMs() >= 0)
+ region->rtoi.last_release_at_ns = MonotonicNanoTime();
+ }
// Do the mmap for the user memory.
- uptr map_size = kUserMapSize;
- while (new_space_end > region->mapped_user + map_size)
- map_size += kUserMapSize;
- CHECK_GE(region->mapped_user + map_size, new_space_end);
+ const uptr user_map_size =
+ RoundUpTo(total_user_bytes - region->mapped_user, kUserMapSize);
+ if (UNLIKELY(IsRegionExhausted(region, class_id, user_map_size)))
+ return false;
if (UNLIKELY(!MapWithCallback(region_beg + region->mapped_user,
- map_size)))
+ user_map_size)))
return false;
- stat->Add(AllocatorStatMapped, map_size);
- region->mapped_user += map_size;
+ stat->Add(AllocatorStatMapped, user_map_size);
+ region->mapped_user += user_map_size;
}
- const uptr new_chunks_count = (region->mapped_user - new_space_beg) / size;
+ const uptr new_chunks_count =
+ (region->mapped_user - region->allocated_user) / size;
- // Calculate the required space for metadata.
- const uptr requested_allocated_meta =
- region->allocated_meta + new_chunks_count * kMetadataSize;
- uptr requested_mapped_meta = region->mapped_meta;
- while (requested_allocated_meta > requested_mapped_meta)
- requested_mapped_meta += kMetaMapSize;
- // Check whether this size class is exhausted.
- if (region->mapped_user + requested_mapped_meta >
- kRegionSize - kFreeArraySize) {
- if (!region->exhausted) {
- region->exhausted = true;
- Printf("%s: Out of memory. ", SanitizerToolName);
- Printf("The process has exhausted %zuMB for size class %zu.\n",
- kRegionSize >> 20, size);
+ if (kMetadataSize) {
+ // Calculate the required space for metadata.
+ const uptr total_meta_bytes =
+ region->allocated_meta + new_chunks_count * kMetadataSize;
+ const uptr meta_map_size = (total_meta_bytes > region->mapped_meta) ?
+ RoundUpTo(total_meta_bytes - region->mapped_meta, kMetaMapSize) : 0;
+ // Map more space for metadata, if necessary.
+ if (meta_map_size) {
+ if (UNLIKELY(IsRegionExhausted(region, class_id, meta_map_size)))
+ return false;
+ if (UNLIKELY(!MapWithCallback(
+ GetMetadataEnd(region_beg) - region->mapped_meta - meta_map_size,
+ meta_map_size)))
+ return false;
+ region->mapped_meta += meta_map_size;
}
- return false;
- }
- // Map more space for metadata, if necessary.
- if (requested_mapped_meta > region->mapped_meta) {
- if (UNLIKELY(!MapWithCallback(
- GetMetadataEnd(region_beg) - requested_mapped_meta,
- requested_mapped_meta - region->mapped_meta)))
- return false;
- region->mapped_meta = requested_mapped_meta;
}
// If necessary, allocate more space for the free array and populate it with
@@ -479,7 +737,7 @@ class SizeClassAllocator64 {
if (UNLIKELY(!EnsureFreeArraySpace(region, region_beg, total_freed_chunks)))
return false;
CompactPtrT *free_array = GetFreeArray(region_beg);
- for (uptr i = 0, chunk = new_space_beg; i < new_chunks_count;
+ for (uptr i = 0, chunk = region->allocated_user; i < new_chunks_count;
i++, chunk += size)
free_array[total_freed_chunks - 1 - i] = PointerToCompactPtr(0, chunk);
if (kRandomShuffleChunks)
@@ -491,27 +749,66 @@ class SizeClassAllocator64 {
region->num_freed_chunks += new_chunks_count;
region->allocated_user += new_chunks_count * size;
CHECK_LE(region->allocated_user, region->mapped_user);
- region->allocated_meta = requested_allocated_meta;
+ region->allocated_meta += new_chunks_count * kMetadataSize;
CHECK_LE(region->allocated_meta, region->mapped_meta);
region->exhausted = false;
+ // TODO(alekseyshl): Consider bumping last_release_at_ns here to prevent
+ // MaybeReleaseToOS from releasing just allocated pages or protect these
+ // not yet used chunks some other way.
+
return true;
}
- void MaybeReleaseChunkRange(uptr region_beg, uptr chunk_size,
- CompactPtrT first, CompactPtrT last) {
- uptr beg_ptr = CompactPtrToPointer(region_beg, first);
- uptr end_ptr = CompactPtrToPointer(region_beg, last) + chunk_size;
- ReleaseMemoryPagesToOS(beg_ptr, end_ptr);
- }
+ class MemoryMapper {
+ public:
+ MemoryMapper(const ThisT& base_allocator, uptr class_id)
+ : allocator(base_allocator),
+ region_base(base_allocator.GetRegionBeginBySizeClass(class_id)),
+ released_ranges_count(0),
+ released_bytes(0) {
+ }
+
+ uptr GetReleasedRangesCount() const {
+ return released_ranges_count;
+ }
+
+ uptr GetReleasedBytes() const {
+ return released_bytes;
+ }
+
+ uptr MapPackedCounterArrayBuffer(uptr buffer_size) {
+ // TODO(alekseyshl): The idea to explore is to check if we have enough
+ // space between num_freed_chunks*sizeof(CompactPtrT) and
+ // mapped_free_array to fit buffer_size bytes and use that space instead
+ // of mapping a temporary one.
+ return reinterpret_cast<uptr>(
+ MmapOrDieOnFatalError(buffer_size, "ReleaseToOSPageCounters"));
+ }
- // Attempts to release some RAM back to OS. The region is expected to be
- // locked.
- // Algorithm:
- // * Sort the chunks.
- // * Find ranges fully covered by free-d chunks
- // * Release them to OS with madvise.
- void MaybeReleaseToOS(uptr class_id) {
+ void UnmapPackedCounterArrayBuffer(uptr buffer, uptr buffer_size) {
+ UnmapOrDie(reinterpret_cast<void *>(buffer), buffer_size);
+ }
+
+ // Releases [from, to) range of pages back to OS.
+ void ReleasePageRangeToOS(CompactPtrT from, CompactPtrT to) {
+ const uptr from_page = allocator.CompactPtrToPointer(region_base, from);
+ const uptr to_page = allocator.CompactPtrToPointer(region_base, to);
+ ReleaseMemoryPagesToOS(from_page, to_page);
+ released_ranges_count++;
+ released_bytes += to_page - from_page;
+ }
+
+ private:
+ const ThisT& allocator;
+ const uptr region_base;
+ uptr released_ranges_count;
+ uptr released_bytes;
+ };
+
+ // Attempts to release RAM occupied by freed chunks back to OS. The region is
+ // expected to be locked.
+ void MaybeReleaseToOS(uptr class_id, bool force) {
RegionInfo *region = GetRegionInfo(class_id);
const uptr chunk_size = ClassIdToSize(class_id);
const uptr page_size = GetPageSizeCached();
@@ -524,37 +821,29 @@ class SizeClassAllocator64 {
return; // Nothing new to release.
}
- s32 interval_ms = ReleaseToOSIntervalMs();
- if (interval_ms < 0)
- return;
+ if (!force) {
+ s32 interval_ms = ReleaseToOSIntervalMs();
+ if (interval_ms < 0)
+ return;
- u64 now_ns = NanoTime();
- if (region->rtoi.last_release_at_ns + interval_ms * 1000000ULL > now_ns)
- return; // Memory was returned recently.
- region->rtoi.last_release_at_ns = now_ns;
+ if (region->rtoi.last_release_at_ns + interval_ms * 1000000ULL >
+ MonotonicNanoTime()) {
+ return; // Memory was returned recently.
+ }
+ }
- uptr region_beg = GetRegionBeginBySizeClass(class_id);
- CompactPtrT *free_array = GetFreeArray(region_beg);
- SortArray(free_array, n);
+ MemoryMapper memory_mapper(*this, class_id);
- const uptr scaled_chunk_size = chunk_size >> kCompactPtrScale;
- const uptr kScaledGranularity = page_size >> kCompactPtrScale;
+ ReleaseFreeMemoryToOS<MemoryMapper>(
+ GetFreeArray(GetRegionBeginBySizeClass(class_id)), n, chunk_size,
+ RoundUpTo(region->allocated_user, page_size) / page_size,
+ &memory_mapper);
- uptr range_beg = free_array[0];
- uptr prev = free_array[0];
- for (uptr i = 1; i < n; i++) {
- uptr chunk = free_array[i];
- CHECK_GT(chunk, prev);
- if (chunk - prev != scaled_chunk_size) {
- CHECK_GT(chunk - prev, scaled_chunk_size);
- if (prev + scaled_chunk_size - range_beg >= kScaledGranularity) {
- MaybeReleaseChunkRange(region_beg, chunk_size, range_beg, prev);
- region->rtoi.n_freed_at_last_release = region->stats.n_freed;
- region->rtoi.num_releases++;
- }
- range_beg = chunk;
- }
- prev = chunk;
+ if (memory_mapper.GetReleasedRangesCount() > 0) {
+ region->rtoi.n_freed_at_last_release = region->stats.n_freed;
+ region->rtoi.num_releases += memory_mapper.GetReleasedRangesCount();
+ region->rtoi.last_released_bytes = memory_mapper.GetReleasedBytes();
}
+ region->rtoi.last_release_at_ns = MonotonicNanoTime();
}
};
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h
index 7151a4636056..2bd83b2eb5a4 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_size_class_map.h
@@ -134,8 +134,9 @@ class SizeClassMap {
static const uptr kMaxSize = 1UL << kMaxSizeLog;
static const uptr kNumClasses =
- kMidClass + ((kMaxSizeLog - kMidSizeLog) << S) + 1;
+ kMidClass + ((kMaxSizeLog - kMidSizeLog) << S) + 1 + 1;
static const uptr kLargestClassID = kNumClasses - 2;
+ static const uptr kBatchClassID = kNumClasses - 1;
COMPILER_CHECK(kNumClasses >= 16 && kNumClasses <= 256);
static const uptr kNumClassesRounded =
kNumClasses <= 32 ? 32 :
@@ -143,6 +144,11 @@ class SizeClassMap {
kNumClasses <= 128 ? 128 : 256;
static uptr Size(uptr class_id) {
+ // Estimate the result for kBatchClassID because this class does not know
+ // the exact size of TransferBatch. It's OK since we are using the actual
+ // sizeof(TransferBatch) where it matters.
+ if (UNLIKELY(class_id == kBatchClassID))
+ return kMaxNumCachedHint * sizeof(uptr);
if (class_id <= kMidClass)
return kMinSize * class_id;
class_id -= kMidClass;
@@ -151,9 +157,10 @@ class SizeClassMap {
}
static uptr ClassID(uptr size) {
+ if (UNLIKELY(size > kMaxSize))
+ return 0;
if (size <= kMidSize)
return (size + kMinSize - 1) >> kMinSizeLog;
- if (size > kMaxSize) return 0;
uptr l = MostSignificantSetBitIndex(size);
uptr hbits = (size >> (l - S)) & M;
uptr lbits = size & ((1 << (l - S)) - 1);
@@ -162,7 +169,13 @@ class SizeClassMap {
}
static uptr MaxCachedHint(uptr class_id) {
- if (class_id == 0) return 0;
+ // Estimate the result for kBatchClassID because this class does not know
+ // the exact size of TransferBatch. We need to cache fewer batches than user
+ // chunks, so this number can be small.
+ if (UNLIKELY(class_id == kBatchClassID))
+ return 16;
+ if (UNLIKELY(class_id == 0))
+ return 0;
uptr n = (1UL << kMaxBytesCachedLog) / Size(class_id);
return Max<uptr>(1, Min(kMaxNumCachedHint, n));
}
@@ -178,6 +191,8 @@ class SizeClassMap {
uptr p = prev_s ? (d * 100 / prev_s) : 0;
uptr l = s ? MostSignificantSetBitIndex(s) : 0;
uptr cached = MaxCachedHint(i) * s;
+ if (i == kBatchClassID)
+ d = p = l = 0;
Printf("c%02zd => s: %zd diff: +%zd %02zd%% l %zd "
"cached: %zd %zd; id %zd\n",
i, Size(i), d, p, l, MaxCachedHint(i), cached, ClassID(s));
@@ -192,12 +207,13 @@ class SizeClassMap {
// Printf("Validate: c%zd\n", c);
uptr s = Size(c);
CHECK_NE(s, 0U);
+ if (c == kBatchClassID)
+ continue;
CHECK_EQ(ClassID(s), c);
- if (c != kNumClasses - 1)
+ if (c < kLargestClassID)
CHECK_EQ(ClassID(s + 1), c + 1);
CHECK_EQ(ClassID(s - 1), c);
- if (c)
- CHECK_GT(Size(c), Size(c-1));
+ CHECK_GT(Size(c), Size(c - 1));
}
CHECK_EQ(ClassID(kMaxSize + 1), 0);
@@ -207,7 +223,7 @@ class SizeClassMap {
CHECK_LT(c, kNumClasses);
CHECK_GE(Size(c), s);
if (c > 0)
- CHECK_LT(Size(c-1), s);
+ CHECK_LT(Size(c - 1), s);
}
}
};
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h
index 47c2b12a2049..76a0bf71425d 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h
@@ -47,12 +47,12 @@
# define ASM_HIDDEN(symbol) .hidden symbol
# define ASM_TYPE_FUNCTION(symbol) .type symbol, @function
# define ASM_SIZE(symbol) .size symbol, .-symbol
-# define ASM_TSAN_SYMBOL(symbol) symbol
-# define ASM_TSAN_SYMBOL_INTERCEPTOR(symbol) symbol
+# define ASM_SYMBOL(symbol) symbol
+# define ASM_SYMBOL_INTERCEPTOR(symbol) symbol
#else
# define ASM_HIDDEN(symbol)
# define ASM_TYPE_FUNCTION(symbol)
# define ASM_SIZE(symbol)
-# define ASM_TSAN_SYMBOL(symbol) _##symbol
-# define ASM_TSAN_SYMBOL_INTERCEPTOR(symbol) _wrap_##symbol
+# define ASM_SYMBOL(symbol) _##symbol
+# define ASM_SYMBOL_INTERCEPTOR(symbol) _wrap_##symbol
#endif
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h
index d8472732ff21..0f65814c398f 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bitvector.h
@@ -22,7 +22,7 @@ namespace __sanitizer {
template <class basic_int_t = uptr>
class BasicBitVector {
public:
- enum SizeEnum { kSize = sizeof(basic_int_t) * 8 };
+ enum SizeEnum : uptr { kSize = sizeof(basic_int_t) * 8 };
uptr size() const { return kSize; }
// No CTOR.
@@ -115,7 +115,7 @@ class TwoLevelBitVector {
// This structure allows O(kLevel1Size) time for clear() and empty(),
// as well fast handling of sparse BVs.
public:
- enum SizeEnum { kSize = BV::kSize * BV::kSize * kLevel1Size };
+ enum SizeEnum : uptr { kSize = BV::kSize * BV::kSize * kLevel1Size };
// No CTOR.
uptr size() const { return kSize; }
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bvgraph.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bvgraph.h
index df72f1c2d471..9c2db9ae4954 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bvgraph.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_bvgraph.h
@@ -25,7 +25,7 @@ namespace __sanitizer {
template<class BV>
class BVGraph {
public:
- enum SizeEnum { kSize = BV::kSize };
+ enum SizeEnum : uptr { kSize = BV::kSize };
uptr size() const { return kSize; }
// No CTOR.
void clear() {
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc
index 7e6f8fce76d2..ae26d5efc24f 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.cc
@@ -26,72 +26,7 @@ const char *SanitizerToolName = "SanitizerTool";
atomic_uint32_t current_verbosity;
uptr PageSizeCached;
-
-StaticSpinMutex report_file_mu;
-ReportFile report_file = {&report_file_mu, kStderrFd, "", "", 0};
-
-void RawWrite(const char *buffer) {
- report_file.Write(buffer, internal_strlen(buffer));
-}
-
-void ReportFile::ReopenIfNecessary() {
- mu->CheckLocked();
- if (fd == kStdoutFd || fd == kStderrFd) return;
-
- uptr pid = internal_getpid();
- // If in tracer, use the parent's file.
- if (pid == stoptheworld_tracer_pid)
- pid = stoptheworld_tracer_ppid;
- if (fd != kInvalidFd) {
- // If the report file is already opened by the current process,
- // do nothing. Otherwise the report file was opened by the parent
- // process, close it now.
- if (fd_pid == pid)
- return;
- else
- CloseFile(fd);
- }
-
- const char *exe_name = GetProcessName();
- if (common_flags()->log_exe_name && exe_name) {
- internal_snprintf(full_path, kMaxPathLength, "%s.%s.%zu", path_prefix,
- exe_name, pid);
- } else {
- internal_snprintf(full_path, kMaxPathLength, "%s.%zu", path_prefix, pid);
- }
- fd = OpenFile(full_path, WrOnly);
- if (fd == kInvalidFd) {
- const char *ErrorMsgPrefix = "ERROR: Can't open file: ";
- WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
- WriteToFile(kStderrFd, full_path, internal_strlen(full_path));
- Die();
- }
- fd_pid = pid;
-}
-
-void ReportFile::SetReportPath(const char *path) {
- if (!path)
- return;
- uptr len = internal_strlen(path);
- if (len > sizeof(path_prefix) - 100) {
- Report("ERROR: Path is too long: %c%c%c%c%c%c%c%c...\n",
- path[0], path[1], path[2], path[3],
- path[4], path[5], path[6], path[7]);
- Die();
- }
-
- SpinMutexLock l(mu);
- if (fd != kStdoutFd && fd != kStderrFd && fd != kInvalidFd)
- CloseFile(fd);
- fd = kInvalidFd;
- if (internal_strcmp(path, "stdout") == 0) {
- fd = kStdoutFd;
- } else if (internal_strcmp(path, "stderr") == 0) {
- fd = kStderrFd;
- } else {
- internal_snprintf(path_prefix, kMaxPathLength, "%s", path);
- }
-}
+u32 NumberOfCPUsCached;
// PID of the tracer task in StopTheWorld. It shares the address space with the
// main process, but has a different PID and thus requires special handling.
@@ -120,42 +55,6 @@ void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
UNREACHABLE("unable to mmap");
}
-bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
- uptr *read_len, uptr max_len, error_t *errno_p) {
- uptr PageSize = GetPageSizeCached();
- uptr kMinFileLen = PageSize;
- *buff = nullptr;
- *buff_size = 0;
- *read_len = 0;
- // The files we usually open are not seekable, so try different buffer sizes.
- for (uptr size = kMinFileLen; size <= max_len; size *= 2) {
- fd_t fd = OpenFile(file_name, RdOnly, errno_p);
- if (fd == kInvalidFd) return false;
- UnmapOrDie(*buff, *buff_size);
- *buff = (char*)MmapOrDie(size, __func__);
- *buff_size = size;
- *read_len = 0;
- // Read up to one page at a time.
- bool reached_eof = false;
- while (*read_len + PageSize <= size) {
- uptr just_read;
- if (!ReadFromFile(fd, *buff + *read_len, PageSize, &just_read, errno_p)) {
- UnmapOrDie(*buff, *buff_size);
- return false;
- }
- if (just_read == 0) {
- reached_eof = true;
- break;
- }
- *read_len += just_read;
- }
- CloseFile(fd);
- if (reached_eof) // We've read the whole file.
- break;
- }
- return true;
-}
-
typedef bool UptrComparisonFunction(const uptr &a, const uptr &b);
typedef bool U32ComparisonFunction(const u32 &a, const u32 &b);
@@ -285,9 +184,10 @@ void LoadedModule::clear() {
}
void LoadedModule::addAddressRange(uptr beg, uptr end, bool executable,
- bool writable) {
+ bool writable, const char *name) {
void *mem = InternalAlloc(sizeof(AddressRange));
- AddressRange *r = new(mem) AddressRange(beg, end, executable, writable);
+ AddressRange *r =
+ new(mem) AddressRange(beg, end, executable, writable, name);
ranges_.push_back(r);
if (executable && end > max_executable_address_)
max_executable_address_ = end;
@@ -359,36 +259,6 @@ bool TemplateMatch(const char *templ, const char *str) {
return true;
}
-static const char kPathSeparator = SANITIZER_WINDOWS ? ';' : ':';
-
-char *FindPathToBinary(const char *name) {
- if (FileExists(name)) {
- return internal_strdup(name);
- }
-
- const char *path = GetEnv("PATH");
- if (!path)
- return nullptr;
- uptr name_len = internal_strlen(name);
- InternalScopedBuffer<char> buffer(kMaxPathLength);
- const char *beg = path;
- while (true) {
- const char *end = internal_strchrnul(beg, kPathSeparator);
- uptr prefix_len = end - beg;
- if (prefix_len + name_len + 2 <= kMaxPathLength) {
- internal_memcpy(buffer.data(), beg, prefix_len);
- buffer[prefix_len] = '/';
- internal_memcpy(&buffer[prefix_len + 1], name, name_len);
- buffer[prefix_len + 1 + name_len] = '\0';
- if (FileExists(buffer.data()))
- return internal_strdup(buffer.data());
- }
- if (*end == '\0') break;
- beg = end + 1;
- }
- return nullptr;
-}
-
static char binary_name_cache_str[kMaxPathLength];
static char process_name_cache_str[kMaxPathLength];
@@ -482,15 +352,6 @@ static int InstallMallocFreeHooks(void (*malloc_hook)(const void *, uptr),
using namespace __sanitizer; // NOLINT
extern "C" {
-void __sanitizer_set_report_path(const char *path) {
- report_file.SetReportPath(path);
-}
-
-void __sanitizer_set_report_fd(void *fd) {
- report_file.fd = (fd_t)reinterpret_cast<uptr>(fd);
- report_file.fd_pid = internal_getpid();
-}
-
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_report_error_summary,
const char *error_summary) {
Printf("%s\n", error_summary);
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h
index 89aae579856a..1fbaee7e39a1 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h
@@ -29,8 +29,11 @@ extern "C" void _ReadWriteBarrier();
#endif
namespace __sanitizer {
-struct StackTrace;
+
struct AddressInfo;
+struct BufferedStackTrace;
+struct SignalContext;
+struct StackTrace;
// Constants.
const uptr kWordSize = SANITIZER_WORDSIZE / 8;
@@ -71,6 +74,7 @@ INLINE uptr GetPageSizeCached() {
}
uptr GetMmapGranularity();
uptr GetMaxVirtualAddress();
+uptr GetMaxUserVirtualAddress();
// Threads
tid_t GetTid();
uptr GetThreadSelf();
@@ -125,6 +129,30 @@ void CheckVMASize();
void RunMallocHooks(const void *ptr, uptr size);
void RunFreeHooks(const void *ptr);
+class ReservedAddressRange {
+ public:
+ uptr Init(uptr size, const char *name = nullptr, uptr fixed_addr = 0);
+ uptr Map(uptr fixed_addr, uptr size);
+ uptr MapOrDie(uptr fixed_addr, uptr size);
+ void Unmap(uptr addr, uptr size);
+ void *base() const { return base_; }
+ uptr size() const { return size_; }
+
+ private:
+ void* base_;
+ uptr size_;
+ const char* name_;
+ uptr os_handle_;
+};
+
+typedef void (*fill_profile_f)(uptr start, uptr rss, bool file,
+ /*out*/uptr *stats, uptr stats_size);
+
+// Parse the contents of /proc/self/smaps and generate a memory profile.
+// |cb| is a tool-specific callback that fills the |stats| array containing
+// |stats_size| elements.
+void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size);
+
// InternalScopedBuffer can be used instead of large stack arrays to
// keep frame size low.
// FIXME: use InternalAlloc instead of MmapOrDie once
@@ -179,12 +207,15 @@ class LowLevelAllocator {
char *allocated_end_;
char *allocated_current_;
};
+// Set the min alignment of LowLevelAllocator to at least alignment.
+void SetLowLevelAllocateMinAlignment(uptr alignment);
typedef void (*LowLevelAllocateCallback)(uptr ptr, uptr size);
// Allows to register tool-specific callbacks for LowLevelAllocator.
// Passing NULL removes the callback.
void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback);
// IO
+void CatastrophicErrorWrite(const char *buffer, uptr length);
void RawWrite(const char *buffer);
bool ColorizeReports();
void RemoveANSIEscapeSequencesFromString(char *buffer);
@@ -200,67 +231,18 @@ void SetPrintfAndReportCallback(void (*callback)(const char *));
if ((uptr)Verbosity() >= (level)) Printf(__VA_ARGS__); \
} while (0)
-// Can be used to prevent mixing error reports from different sanitizers.
-extern StaticSpinMutex CommonSanitizerReportMutex;
-
-struct ReportFile {
- void Write(const char *buffer, uptr length);
- bool SupportsColors();
- void SetReportPath(const char *path);
-
- // Don't use fields directly. They are only declared public to allow
- // aggregate initialization.
-
- // Protects fields below.
- StaticSpinMutex *mu;
- // Opened file descriptor. Defaults to stderr. It may be equal to
- // kInvalidFd, in which case new file will be opened when necessary.
- fd_t fd;
- // Path prefix of report file, set via __sanitizer_set_report_path.
- char path_prefix[kMaxPathLength];
- // Full path to report, obtained as <path_prefix>.PID
- char full_path[kMaxPathLength];
- // PID of the process that opened fd. If a fork() occurs,
- // the PID of child will be different from fd_pid.
- uptr fd_pid;
+// Lock sanitizer error reporting and protects against nested errors.
+class ScopedErrorReportLock {
+ public:
+ ScopedErrorReportLock();
+ ~ScopedErrorReportLock();
- private:
- void ReopenIfNecessary();
+ static void CheckLocked();
};
-extern ReportFile report_file;
extern uptr stoptheworld_tracer_pid;
extern uptr stoptheworld_tracer_ppid;
-enum FileAccessMode {
- RdOnly,
- WrOnly,
- RdWr
-};
-
-// Returns kInvalidFd on error.
-fd_t OpenFile(const char *filename, FileAccessMode mode,
- error_t *errno_p = nullptr);
-void CloseFile(fd_t);
-
-// Return true on success, false on error.
-bool ReadFromFile(fd_t fd, void *buff, uptr buff_size,
- uptr *bytes_read = nullptr, error_t *error_p = nullptr);
-bool WriteToFile(fd_t fd, const void *buff, uptr buff_size,
- uptr *bytes_written = nullptr, error_t *error_p = nullptr);
-
-bool RenameFile(const char *oldpath, const char *newpath,
- error_t *error_p = nullptr);
-
-// Scoped file handle closer.
-struct FileCloser {
- explicit FileCloser(fd_t fd) : fd(fd) {}
- ~FileCloser() { CloseFile(fd); }
- fd_t fd;
-};
-
-bool SupportsColoredOutput(fd_t fd);
-
// Opens the file 'file_name" and reads up to 'max_len' bytes.
// The resulting buffer is mmaped and stored in '*buff'.
// The size of the mmaped region is stored in '*buff_size'.
@@ -269,11 +251,6 @@ bool SupportsColoredOutput(fd_t fd);
bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
uptr *read_len, uptr max_len = 1 << 26,
error_t *errno_p = nullptr);
-// Maps given file to virtual memory, and returns pointer to it
-// (or NULL if mapping fails). Stores the size of mmaped region
-// in '*buff_size'.
-void *MapFileToMemory(const char *file_name, uptr *buff_size);
-void *MapWritableFileToMemory(void *addr, uptr size, fd_t fd, OFF_T offset);
bool IsAccessibleMemoryRange(uptr beg, uptr size);
@@ -293,27 +270,8 @@ void CacheBinaryName();
void DisableCoreDumperIfNecessary();
void DumpProcessMap();
void PrintModuleMap();
-bool FileExists(const char *filename);
const char *GetEnv(const char *name);
bool SetEnv(const char *name, const char *value);
-const char *GetPwd();
-char *FindPathToBinary(const char *name);
-bool IsPathSeparator(const char c);
-bool IsAbsolutePath(const char *path);
-// Starts a subprocess and returs its pid.
-// If *_fd parameters are not kInvalidFd their corresponding input/output
-// streams will be redirect to the file. The files will always be closed
-// in parent process even in case of an error.
-// The child process will close all fds after STDERR_FILENO
-// before passing control to a program.
-pid_t StartSubprocess(const char *filename, const char *const argv[],
- fd_t stdin_fd = kInvalidFd, fd_t stdout_fd = kInvalidFd,
- fd_t stderr_fd = kInvalidFd);
-// Checks if specified process is still running
-bool IsProcessRunning(pid_t pid);
-// Waits for the process to finish and returns its exit code.
-// Returns -1 in case of an error.
-int WaitForProcess(pid_t pid);
u32 GetUid();
void ReExec();
@@ -337,6 +295,7 @@ uptr GetTlsSize();
void SleepForSeconds(int seconds);
void SleepForMillis(int millis);
u64 NanoTime();
+u64 MonotonicNanoTime();
int Atexit(void (*function)(void));
void SortArray(uptr *array, uptr size);
void SortArray(u32 *array, uptr size);
@@ -385,7 +344,24 @@ void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded));
typedef void (*SignalHandlerType)(int, void *, void *);
HandleSignalMode GetHandleSignalMode(int signum);
void InstallDeadlySignalHandlers(SignalHandlerType handler);
-const char *DescribeSignalOrException(int signo);
+
+// Signal reporting.
+// Each sanitizer uses slightly different implementation of stack unwinding.
+typedef void (*UnwindSignalStackCallbackType)(const SignalContext &sig,
+ const void *callback_context,
+ BufferedStackTrace *stack);
+// Print deadly signal report and die.
+void HandleDeadlySignal(void *siginfo, void *context, u32 tid,
+ UnwindSignalStackCallbackType unwind,
+ const void *unwind_context);
+
+// Part of HandleDeadlySignal, exposed for asan.
+void StartReportDeadlySignal();
+// Part of HandleDeadlySignal, exposed for asan.
+void ReportDeadlySignal(const SignalContext &sig, u32 tid,
+ UnwindSignalStackCallbackType unwind,
+ const void *unwind_context);
+
// Alternative signal stack (POSIX-only).
void SetAlternateSignalStack();
void UnsetAlternateSignalStack();
@@ -702,6 +678,7 @@ inline const char *ModuleArchToString(ModuleArch arch) {
}
const uptr kModuleUUIDSize = 16;
+const uptr kMaxSegName = 16;
// Represents a binary loaded into virtual memory (e.g. this can be an
// executable or a shared object).
@@ -720,7 +697,8 @@ class LoadedModule {
void set(const char *module_name, uptr base_address, ModuleArch arch,
u8 uuid[kModuleUUIDSize], bool instrumented);
void clear();
- void addAddressRange(uptr beg, uptr end, bool executable, bool writable);
+ void addAddressRange(uptr beg, uptr end, bool executable, bool writable,
+ const char *name = nullptr);
bool containsAddress(uptr address) const;
const char *full_name() const { return full_name_; }
@@ -736,13 +714,17 @@ class LoadedModule {
uptr end;
bool executable;
bool writable;
+ char name[kMaxSegName];
- AddressRange(uptr beg, uptr end, bool executable, bool writable)
+ AddressRange(uptr beg, uptr end, bool executable, bool writable,
+ const char *name)
: next(nullptr),
beg(beg),
end(end),
executable(executable),
- writable(writable) {}
+ writable(writable) {
+ internal_strncpy(this->name, (name ? name : ""), ARRAY_SIZE(this->name));
+ }
};
const IntrusiveList<AddressRange> &ranges() const { return ranges_; }
@@ -761,9 +743,10 @@ class LoadedModule {
// filling this information.
class ListOfModules {
public:
- ListOfModules() : modules_(kInitialCapacity) {}
+ ListOfModules() : initialized(false) {}
~ListOfModules() { clear(); }
void init();
+ void fallbackInit(); // Uses fallback init if available, otherwise clears
const LoadedModule *begin() const { return modules_.begin(); }
LoadedModule *begin() { return modules_.begin(); }
const LoadedModule *end() const { return modules_.end(); }
@@ -779,10 +762,15 @@ class ListOfModules {
for (auto &module : modules_) module.clear();
modules_.clear();
}
+ void clearOrInit() {
+ initialized ? clear() : modules_.Initialize(kInitialCapacity);
+ initialized = true;
+ }
- InternalMmapVector<LoadedModule> modules_;
+ InternalMmapVectorNoCtor<LoadedModule> modules_;
// We rarely have more than 16K loaded modules.
static const uptr kInitialCapacity = 1 << 14;
+ bool initialized;
};
// Callback type for iterating over a set of memory ranges.
@@ -858,35 +846,49 @@ static inline void SanitizerBreakOptimization(void *arg) {
}
struct SignalContext {
+ void *siginfo;
void *context;
uptr addr;
uptr pc;
uptr sp;
uptr bp;
bool is_memory_access;
-
enum WriteFlag { UNKNOWN, READ, WRITE } write_flag;
- SignalContext(void *context, uptr addr, uptr pc, uptr sp, uptr bp,
- bool is_memory_access, WriteFlag write_flag)
- : context(context),
- addr(addr),
- pc(pc),
- sp(sp),
- bp(bp),
- is_memory_access(is_memory_access),
- write_flag(write_flag) {}
+ // VS2013 doesn't implement unrestricted unions, so we need a trivial default
+ // constructor
+ SignalContext() = default;
+
+ // Creates signal context in a platform-specific manner.
+ // SignalContext is going to keep pointers to siginfo and context without
+ // owning them.
+ SignalContext(void *siginfo, void *context)
+ : siginfo(siginfo),
+ context(context),
+ addr(GetAddress()),
+ is_memory_access(IsMemoryAccess()),
+ write_flag(GetWriteFlag()) {
+ InitPcSpBp();
+ }
static void DumpAllRegisters(void *context);
- // Creates signal context in a platform-specific manner.
- static SignalContext Create(void *siginfo, void *context);
+ // Type of signal e.g. SIGSEGV or EXCEPTION_ACCESS_VIOLATION.
+ int GetType() const;
- // Returns true if the "context" indicates a memory write.
- static WriteFlag GetWriteFlag(void *context);
-};
+ // String description of the signal.
+ const char *Describe() const;
-void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp);
+ // Returns true if signal is stack overflow.
+ bool IsStackOverflow() const;
+
+ private:
+ // Platform specific initialization.
+ void InitPcSpBp();
+ uptr GetAddress() const;
+ WriteFlag GetWriteFlag() const;
+ bool IsMemoryAccess() const;
+};
void MaybeReexec();
@@ -929,8 +931,17 @@ const s32 kReleaseToOSIntervalNever = -1;
void CheckNoDeepBind(const char *filename, int flag);
// Returns the requested amount of random data (up to 256 bytes) that can then
-// be used to seed a PRNG.
-bool GetRandom(void *buffer, uptr length);
+// be used to seed a PRNG. Defaults to blocking like the underlying syscall.
+bool GetRandom(void *buffer, uptr length, bool blocking = true);
+
+// Returns the number of logical processors on the system.
+u32 GetNumberOfCPUs();
+extern u32 NumberOfCPUsCached;
+INLINE u32 GetNumberOfCPUsCached() {
+ if (!NumberOfCPUsCached)
+ NumberOfCPUsCached = GetNumberOfCPUs();
+ return NumberOfCPUsCached;
+}
} // namespace __sanitizer
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index 8607bf44902d..592a8c7b032a 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -43,6 +43,7 @@
#include "sanitizer_errno.h"
#include "sanitizer_placement_new.h"
#include "sanitizer_platform_interceptors.h"
+#include "sanitizer_symbolizer.h"
#include "sanitizer_tls_get_addr.h"
#include <stdarg.h>
@@ -68,6 +69,46 @@
#define iconv __bsd_iconv
#endif
+#if SANITIZER_NETBSD
+#define clock_getres __clock_getres50
+#define clock_gettime __clock_gettime50
+#define clock_settime __clock_settime50
+#define ctime __ctime50
+#define ctime_r __ctime_r50
+#define getitimer __getitimer50
+#define getpwent __getpwent50
+#define getpwnam __getpwnam50
+#define getpwnam_r __getpwnam_r50
+#define getpwuid __getpwuid50
+#define getpwuid_r __getpwuid_r50
+#define getutent __getutent50
+#define getutxent __getutxent50
+#define getutxid __getutxid50
+#define getutxline __getutxline50
+#define glob __glob30
+#define gmtime __gmtime50
+#define gmtime_r __gmtime_r50
+#define localtime_r __localtime_r50
+#define mktime __mktime50
+#define opendir __opendir30
+#define readdir __readdir30
+#define readdir_r __readdir_r30
+#define scandir __scandir30
+#define setitimer __setitimer50
+#define setlocale __setlocale50
+#define shmctl __shmctl50
+#define sigemptyset __sigemptyset14
+#define sigfillset __sigfillset14
+#define sigpending __sigpending14
+#define sigprocmask __sigprocmask14
+#define sigtimedwait __sigtimedwait50
+#define stat __stat50
+#define time __time50
+#define times __times13
+#define wait3 __wait350
+#define wait4 __wait450
+#endif
+
// Platform-specific options.
#if SANITIZER_MAC
namespace __sanitizer {
@@ -258,7 +299,7 @@ typedef AddrHashMap<CommonInterceptorMetadata, 31051> MetadataHashMap;
static MetadataHashMap *interceptor_metadata_map;
-#if SI_NOT_WINDOWS
+#if SI_POSIX
UNUSED static void SetInterceptorMetadata(__sanitizer_FILE *addr,
const FileMetadata &file) {
MetadataHashMap::Handle h(interceptor_metadata_map, (uptr)addr);
@@ -272,7 +313,7 @@ UNUSED static const FileMetadata *GetInterceptorMetadata(
MetadataHashMap::Handle h(interceptor_metadata_map, (uptr)addr,
/* remove */ false,
/* create */ false);
- if (h.exists()) {
+ if (addr && h.exists()) {
CHECK(!h.created());
CHECK(h->type == CommonInterceptorMetadata::CIMT_FILE);
return &h->file;
@@ -285,7 +326,7 @@ UNUSED static void DeleteInterceptorMetadata(void *addr) {
MetadataHashMap::Handle h(interceptor_metadata_map, (uptr)addr, true);
CHECK(h.exists());
}
-#endif // SI_NOT_WINDOWS
+#endif // SI_POSIX
#if SANITIZER_INTERCEPT_STRLEN
INTERCEPTOR(SIZE_T, strlen, const char *s) {
@@ -886,7 +927,7 @@ INTERCEPTOR(long double, frexpl, long double x, int *exp) {
#define INIT_FREXPF_FREXPL
#endif // SANITIZER_INTERCEPT_FREXPF_FREXPL
-#if SI_NOT_WINDOWS
+#if SI_POSIX
static void write_iovec(void *ctx, struct __sanitizer_iovec *iovec,
SIZE_T iovlen, SIZE_T maxlen) {
for (SIZE_T i = 0; i < iovlen && maxlen; ++i) {
@@ -1176,12 +1217,14 @@ INTERCEPTOR(unsigned long, time, unsigned long *t) {
#if SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS
static void unpoison_tm(void *ctx, __sanitizer_tm *tm) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tm, sizeof(*tm));
+#if !SANITIZER_SOLARIS
if (tm->tm_zone) {
// Can not use COMMON_INTERCEPTOR_WRITE_RANGE here, because tm->tm_zone
// can point to shared memory and tsan would report a data race.
COMMON_INTERCEPTOR_INITIALIZE_RANGE(tm->tm_zone,
REAL(strlen(tm->tm_zone)) + 1);
}
+#endif
}
INTERCEPTOR(__sanitizer_tm *, localtime, unsigned long *timep) {
void *ctx;
@@ -1509,6 +1552,12 @@ INTERCEPTOR(int, vsnprintf, char *str, SIZE_T size, const char *format,
va_list ap)
VSNPRINTF_INTERCEPTOR_IMPL(vsnprintf, str, size, format, ap)
+#if SANITIZER_INTERCEPT___PRINTF_CHK
+INTERCEPTOR(int, __vsnprintf_chk, char *str, SIZE_T size, int flag,
+ SIZE_T size_to, const char *format, va_list ap)
+VSNPRINTF_INTERCEPTOR_IMPL(vsnprintf, str, size, format, ap)
+#endif
+
#if SANITIZER_INTERCEPT_PRINTF_L
INTERCEPTOR(int, vsnprintf_l, char *str, SIZE_T size, void *loc,
const char *format, va_list ap)
@@ -1522,6 +1571,12 @@ FORMAT_INTERCEPTOR_IMPL(snprintf_l, vsnprintf_l, str, size, loc, format)
INTERCEPTOR(int, vsprintf, char *str, const char *format, va_list ap)
VSPRINTF_INTERCEPTOR_IMPL(vsprintf, str, format, ap)
+#if SANITIZER_INTERCEPT___PRINTF_CHK
+INTERCEPTOR(int, __vsprintf_chk, char *str, int flag, SIZE_T size_to,
+ const char *format, va_list ap)
+VSPRINTF_INTERCEPTOR_IMPL(vsprintf, str, format, ap)
+#endif
+
INTERCEPTOR(int, vasprintf, char **strp, const char *format, va_list ap)
VASPRINTF_INTERCEPTOR_IMPL(vasprintf, strp, format, ap)
@@ -1550,12 +1605,30 @@ FORMAT_INTERCEPTOR_IMPL(printf, vprintf, format)
INTERCEPTOR(int, fprintf, __sanitizer_FILE *stream, const char *format, ...)
FORMAT_INTERCEPTOR_IMPL(fprintf, vfprintf, stream, format)
+#if SANITIZER_INTERCEPT___PRINTF_CHK
+INTERCEPTOR(int, __fprintf_chk, __sanitizer_FILE *stream, SIZE_T size,
+ const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(__fprintf_chk, vfprintf, stream, format)
+#endif
+
INTERCEPTOR(int, sprintf, char *str, const char *format, ...) // NOLINT
FORMAT_INTERCEPTOR_IMPL(sprintf, vsprintf, str, format) // NOLINT
+#if SANITIZER_INTERCEPT___PRINTF_CHK
+INTERCEPTOR(int, __sprintf_chk, char *str, int flag, SIZE_T size_to,
+ const char *format, ...) // NOLINT
+FORMAT_INTERCEPTOR_IMPL(__sprintf_chk, vsprintf, str, format) // NOLINT
+#endif
+
INTERCEPTOR(int, snprintf, char *str, SIZE_T size, const char *format, ...)
FORMAT_INTERCEPTOR_IMPL(snprintf, vsnprintf, str, size, format)
+#if SANITIZER_INTERCEPT___PRINTF_CHK
+INTERCEPTOR(int, __snprintf_chk, char *str, SIZE_T size, int flag,
+ SIZE_T size_to, const char *format, ...) // NOLINT
+FORMAT_INTERCEPTOR_IMPL(__snprintf_chk, vsnprintf, str, size, format) // NOLINT
+#endif
+
INTERCEPTOR(int, asprintf, char **strp, const char *format, ...)
FORMAT_INTERCEPTOR_IMPL(asprintf, vasprintf, strp, format)
@@ -1595,6 +1668,17 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_snprintf, __isoc99_vsnprintf, str, size,
#define INIT_PRINTF
#endif
+#if SANITIZER_INTERCEPT___PRINTF_CHK
+#define INIT___PRINTF_CHK \
+ COMMON_INTERCEPT_FUNCTION(__sprintf_chk); \
+ COMMON_INTERCEPT_FUNCTION(__snprintf_chk); \
+ COMMON_INTERCEPT_FUNCTION(__vsprintf_chk); \
+ COMMON_INTERCEPT_FUNCTION(__vsnprintf_chk); \
+ COMMON_INTERCEPT_FUNCTION(__fprintf_chk);
+#else
+#define INIT___PRINTF_CHK
+#endif
+
#if SANITIZER_INTERCEPT_PRINTF_L
#define INIT_PRINTF_L \
COMMON_INTERCEPT_FUNCTION(snprintf_l); \
@@ -1723,7 +1807,8 @@ static void unpoison_group(void *ctx, __sanitizer_group *grp) {
INTERCEPTOR(__sanitizer_passwd *, getpwnam, const char *name) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, getpwnam, name);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ if (name)
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
__sanitizer_passwd *res = REAL(getpwnam)(name);
if (res) unpoison_passwd(ctx, res);
return res;
@@ -2003,6 +2088,13 @@ INTERCEPTOR(int, clock_gettime, u32 clk_id, void *tp) {
}
return res;
}
+namespace __sanitizer {
+extern "C" {
+int real_clock_gettime(u32 clk_id, void *tp) {
+ return REAL(clock_gettime)(clk_id, tp);
+}
+} // extern "C"
+} // namespace __sanitizer
INTERCEPTOR(int, clock_settime, u32 clk_id, const void *tp) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, clock_settime, clk_id, tp);
@@ -2064,6 +2156,18 @@ static void unpoison_glob_t(void *ctx, __sanitizer_glob_t *pglob) {
}
}
+#if SANITIZER_SOLARIS
+INTERCEPTOR(int, glob, const char *pattern, int flags,
+ int (*errfunc)(const char *epath, int eerrno),
+ __sanitizer_glob_t *pglob) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, glob, pattern, flags, errfunc, pglob);
+ COMMON_INTERCEPTOR_READ_STRING(ctx, pattern, 0);
+ int res = REAL(glob)(pattern, flags, errfunc, pglob);
+ if ((!res || res == glob_nomatch) && pglob) unpoison_glob_t(ctx, pglob);
+ return res;
+}
+#else
static THREADLOCAL __sanitizer_glob_t *pglob_copy;
static void wrapped_gl_closedir(void *dir) {
@@ -2127,7 +2231,14 @@ INTERCEPTOR(int, glob, const char *pattern, int flags,
if ((!res || res == glob_nomatch) && pglob) unpoison_glob_t(ctx, pglob);
return res;
}
+#endif // SANITIZER_SOLARIS
+#define INIT_GLOB \
+ COMMON_INTERCEPT_FUNCTION(glob);
+#else // SANITIZER_INTERCEPT_GLOB
+#define INIT_GLOB
+#endif // SANITIZER_INTERCEPT_GLOB
+#if SANITIZER_INTERCEPT_GLOB64
INTERCEPTOR(int, glob64, const char *pattern, int flags,
int (*errfunc)(const char *epath, int eerrno),
__sanitizer_glob_t *pglob) {
@@ -2156,12 +2267,11 @@ INTERCEPTOR(int, glob64, const char *pattern, int flags,
if ((!res || res == glob_nomatch) && pglob) unpoison_glob_t(ctx, pglob);
return res;
}
-#define INIT_GLOB \
- COMMON_INTERCEPT_FUNCTION(glob); \
+#define INIT_GLOB64 \
COMMON_INTERCEPT_FUNCTION(glob64);
-#else // SANITIZER_INTERCEPT_GLOB
-#define INIT_GLOB
-#endif // SANITIZER_INTERCEPT_GLOB
+#else // SANITIZER_INTERCEPT_GLOB64
+#define INIT_GLOB64
+#endif // SANITIZER_INTERCEPT_GLOB64
#if SANITIZER_INTERCEPT_WAIT
// According to sys/wait.h, wait(), waitid(), waitpid() may have symbol version
@@ -2466,7 +2576,15 @@ INTERCEPTOR(struct __sanitizer_hostent *, gethostent, int fake) {
if (res) write_hostent(ctx, res);
return res;
}
+#define INIT_GETHOSTBYNAME \
+ COMMON_INTERCEPT_FUNCTION(gethostent); \
+ COMMON_INTERCEPT_FUNCTION(gethostbyaddr); \
+ COMMON_INTERCEPT_FUNCTION(gethostbyname);
+#else
+#define INIT_GETHOSTBYNAME
+#endif // SANITIZER_INTERCEPT_GETHOSTBYNAME
+#if SANITIZER_INTERCEPT_GETHOSTBYNAME2
INTERCEPTOR(struct __sanitizer_hostent *, gethostbyname2, char *name, int af) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, gethostbyname2, name, af);
@@ -2474,14 +2592,10 @@ INTERCEPTOR(struct __sanitizer_hostent *, gethostbyname2, char *name, int af) {
if (res) write_hostent(ctx, res);
return res;
}
-#define INIT_GETHOSTBYNAME \
- COMMON_INTERCEPT_FUNCTION(gethostent); \
- COMMON_INTERCEPT_FUNCTION(gethostbyaddr); \
- COMMON_INTERCEPT_FUNCTION(gethostbyname); \
- COMMON_INTERCEPT_FUNCTION(gethostbyname2);
+#define INIT_GETHOSTBYNAME2 COMMON_INTERCEPT_FUNCTION(gethostbyname2);
#else
-#define INIT_GETHOSTBYNAME
-#endif
+#define INIT_GETHOSTBYNAME2
+#endif // SANITIZER_INTERCEPT_GETHOSTBYNAME2
#if SANITIZER_INTERCEPT_GETHOSTBYNAME_R
INTERCEPTOR(int, gethostbyname_r, char *name, struct __sanitizer_hostent *ret,
@@ -3371,7 +3485,7 @@ INTERCEPTOR(char *, strerror, int errnum) {
// * GNU version returns message pointer, which points to either buf or some
// static storage.
#if ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE) || \
- SANITIZER_MAC || SANITIZER_ANDROID
+ SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD
// POSIX version. Spec is not clear on whether buf is NULL-terminated.
// At least on OSX, buf contents are valid even when the call fails.
INTERCEPTOR(int, strerror_r, int errnum, char *buf, SIZE_T buflen) {
@@ -3589,7 +3703,7 @@ INTERCEPTOR(int, ppoll, __sanitizer_pollfd *fds, __sanitizer_nfds_t nfds,
if (fds && nfds) read_pollfd(ctx, fds, nfds);
if (timeout_ts)
COMMON_INTERCEPTOR_READ_RANGE(ctx, timeout_ts, struct_timespec_sz);
- // FIXME: read sigmask when all of sigemptyset, etc are intercepted.
+ if (sigmask) COMMON_INTERCEPTOR_READ_RANGE(ctx, sigmask, sizeof(*sigmask));
int res =
COMMON_INTERCEPTOR_BLOCK_REAL(ppoll)(fds, nfds, timeout_ts, sigmask);
if (fds && nfds) write_pollfd(ctx, fds, nfds);
@@ -3630,7 +3744,7 @@ INTERCEPTOR(int, wordexp, char *s, __sanitizer_wordexp_t *p, int flags) {
INTERCEPTOR(int, sigwait, __sanitizer_sigset_t *set, int *sig) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, sigwait, set, sig);
- // FIXME: read sigset_t when all of sigemptyset, etc are intercepted
+ if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
@@ -3647,7 +3761,7 @@ INTERCEPTOR(int, sigwait, __sanitizer_sigset_t *set, int *sig) {
INTERCEPTOR(int, sigwaitinfo, __sanitizer_sigset_t *set, void *info) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, sigwaitinfo, set, info);
- // FIXME: read sigset_t when all of sigemptyset, etc are intercepted
+ if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
@@ -3666,7 +3780,7 @@ INTERCEPTOR(int, sigtimedwait, __sanitizer_sigset_t *set, void *info,
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, sigtimedwait, set, info, timeout);
if (timeout) COMMON_INTERCEPTOR_READ_RANGE(ctx, timeout, struct_timespec_sz);
- // FIXME: read sigset_t when all of sigemptyset, etc are intercepted
+ if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
@@ -3729,7 +3843,7 @@ INTERCEPTOR(int, sigprocmask, int how, __sanitizer_sigset_t *set,
__sanitizer_sigset_t *oldset) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, sigprocmask, how, set, oldset);
- // FIXME: read sigset_t when all of sigemptyset, etc are intercepted
+ if (set) COMMON_INTERCEPTOR_READ_RANGE(ctx, set, sizeof(*set));
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
@@ -3827,6 +3941,15 @@ INTERCEPTOR(int, pthread_mutex_unlock, void *m) {
#define INIT_PTHREAD_MUTEX_UNLOCK
#endif
+#if SANITIZER_NETBSD
+INTERCEPTOR(int, __libc_mutex_lock, void *m) \
+ ALIAS(WRAPPER_NAME(pthread_mutex_lock));
+INTERCEPTOR(int, __libc_mutex_unlock, void *m) \
+ ALIAS(WRAPPER_NAME(pthread_mutex_unlock));
+INTERCEPTOR(int, __libc_thr_setcancelstate, int state, int *oldstate) \
+ ALIAS(WRAPPER_NAME(pthread_setcancelstate));
+#endif
+
#if SANITIZER_INTERCEPT_GETMNTENT || SANITIZER_INTERCEPT_GETMNTENT_R
static void write_mntent(void *ctx, __sanitizer_mntent *mnt) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt, sizeof(*mnt));
@@ -4404,7 +4527,7 @@ INTERCEPTOR(char *, tempnam, char *dir, char *pfx) {
#define INIT_TEMPNAM
#endif
-#if SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP
+#if SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP && !SANITIZER_NETBSD
INTERCEPTOR(int, pthread_setname_np, uptr thread, const char *name) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, pthread_setname_np, thread, name);
@@ -4413,6 +4536,17 @@ INTERCEPTOR(int, pthread_setname_np, uptr thread, const char *name) {
return REAL(pthread_setname_np)(thread, name);
}
#define INIT_PTHREAD_SETNAME_NP COMMON_INTERCEPT_FUNCTION(pthread_setname_np);
+#elif SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP && SANITIZER_NETBSD
+INTERCEPTOR(int, pthread_setname_np, uptr thread, const char *name, void *arg) {
+ void *ctx;
+ char newname[32]; // PTHREAD_MAX_NAMELEN_NP=32
+ COMMON_INTERCEPTOR_ENTER(ctx, pthread_setname_np, thread, name, arg);
+ COMMON_INTERCEPTOR_READ_STRING(ctx, name, 0);
+ internal_snprintf(newname, sizeof(newname), name, arg);
+ COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, newname);
+ return REAL(pthread_setname_np)(thread, name, arg);
+}
+#define INIT_PTHREAD_SETNAME_NP COMMON_INTERCEPT_FUNCTION(pthread_setname_np);
#else
#define INIT_PTHREAD_SETNAME_NP
#endif
@@ -5575,6 +5709,7 @@ INTERCEPTOR(void*, dlopen, const char *filename, int flag) {
if (filename) COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0);
COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag);
void *res = REAL(dlopen)(filename, flag);
+ Symbolizer::GetOrInit()->InvalidateModuleList();
COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res);
return res;
}
@@ -5583,6 +5718,7 @@ INTERCEPTOR(int, dlclose, void *handle) {
void *ctx;
COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlclose, handle);
int res = REAL(dlclose)(handle);
+ Symbolizer::GetOrInit()->InvalidateModuleList();
COMMON_INTERCEPTOR_LIBRARY_UNLOADED();
return res;
}
@@ -5821,7 +5957,7 @@ INTERCEPTOR(int, pthread_setcancelstate, int state, int *oldstate) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, pthread_setcancelstate, state, oldstate);
int res = REAL(pthread_setcancelstate)(state, oldstate);
- if (res == 0)
+ if (res == 0 && oldstate != nullptr)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldstate, sizeof(*oldstate));
return res;
}
@@ -5830,7 +5966,7 @@ INTERCEPTOR(int, pthread_setcanceltype, int type, int *oldtype) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, pthread_setcanceltype, type, oldtype);
int res = REAL(pthread_setcanceltype)(type, oldtype);
- if (res == 0)
+ if (res == 0 && oldtype != nullptr)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldtype, sizeof(*oldtype));
return res;
}
@@ -6313,6 +6449,7 @@ static void InitializeCommonInterceptors() {
INIT_GETITIMER;
INIT_TIME;
INIT_GLOB;
+ INIT_GLOB64;
INIT_WAIT;
INIT_WAIT4;
INIT_INET;
@@ -6321,6 +6458,7 @@ static void InitializeCommonInterceptors() {
INIT_GETNAMEINFO;
INIT_GETSOCKNAME;
INIT_GETHOSTBYNAME;
+ INIT_GETHOSTBYNAME2;
INIT_GETHOSTBYNAME_R;
INIT_GETHOSTBYNAME2_R;
INIT_GETHOSTBYADDR_R;
@@ -6457,4 +6595,12 @@ static void InitializeCommonInterceptors() {
INIT_GETLOADAVG;
INIT_WCSLEN;
INIT_WCSCAT;
+
+#if SANITIZER_NETBSD
+ COMMON_INTERCEPT_FUNCTION(__libc_mutex_lock);
+ COMMON_INTERCEPT_FUNCTION(__libc_mutex_unlock);
+ COMMON_INTERCEPT_FUNCTION(__libc_thr_setcancelstate);
+#endif
+
+ INIT___PRINTF_CHK;
}
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc
index 4ed9afedf84a..24e7548a597e 100755
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc
@@ -57,7 +57,9 @@ static void ioctl_table_fill() {
_(SIOCGIFCONF, CUSTOM, 0);
_(SIOCGPGRP, WRITE, sizeof(int));
_(SIOCSPGRP, READ, sizeof(int));
+#if !SANITIZER_SOLARIS
_(TIOCCONS, NONE, 0);
+#endif
_(TIOCEXCL, NONE, 0);
_(TIOCGETD, WRITE, sizeof(int));
_(TIOCGPGRP, WRITE, pid_t_sz);
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc
index 550427c906a6..2568df3a5bff 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc
@@ -34,6 +34,7 @@ INTERFACE_FUNCTION(__sanitizer_get_heap_size)
INTERFACE_FUNCTION(__sanitizer_get_ownership)
INTERFACE_FUNCTION(__sanitizer_get_unmapped_bytes)
INTERFACE_FUNCTION(__sanitizer_install_malloc_and_free_hooks)
+INTERFACE_FUNCTION(__sanitizer_purge_allocator)
INTERFACE_FUNCTION(__sanitizer_print_memory_profile)
INTERFACE_WEAK_FUNCTION(__sanitizer_free_hook)
INTERFACE_WEAK_FUNCTION(__sanitizer_malloc_hook)
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc
index cf200512de84..5cdfbbb275f7 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cc
@@ -14,7 +14,10 @@
#include "sanitizer_common.h"
#include "sanitizer_allocator_interface.h"
+#include "sanitizer_file.h"
#include "sanitizer_flags.h"
+#include "sanitizer_procmaps.h"
+#include "sanitizer_report_decorator.h"
#include "sanitizer_stackdepot.h"
#include "sanitizer_stacktrace.h"
#include "sanitizer_symbolizer.h"
@@ -25,12 +28,25 @@
namespace __sanitizer {
+#if !SANITIZER_FUCHSIA
+
bool ReportFile::SupportsColors() {
SpinMutexLock l(mu);
ReopenIfNecessary();
return SupportsColoredOutput(fd);
}
+static INLINE bool ReportSupportsColors() {
+ return report_file.SupportsColors();
+}
+
+#else // SANITIZER_FUCHSIA
+
+// Fuchsia's logs always go through post-processing that handles colorization.
+static INLINE bool ReportSupportsColors() { return true; }
+
+#endif // !SANITIZER_FUCHSIA
+
bool ColorizeReports() {
// FIXME: Add proper Windows support to AnsiColorDecorator and re-enable color
// printing on Windows.
@@ -39,7 +55,7 @@ bool ColorizeReports() {
const char *flag = common_flags()->color;
return internal_strcmp(flag, "always") == 0 ||
- (internal_strcmp(flag, "auto") == 0 && report_file.SupportsColors());
+ (internal_strcmp(flag, "auto") == 0 && ReportSupportsColors());
}
static void (*sandboxing_callback)();
@@ -131,6 +147,127 @@ void BackgroundThread(void *arg) {
}
#endif
+#if !SANITIZER_FUCHSIA && !SANITIZER_GO
+void StartReportDeadlySignal() {
+ // Write the first message using fd=2, just in case.
+ // It may actually fail to write in case stderr is closed.
+ CatastrophicErrorWrite(SanitizerToolName, internal_strlen(SanitizerToolName));
+ static const char kDeadlySignal[] = ":DEADLYSIGNAL\n";
+ CatastrophicErrorWrite(kDeadlySignal, sizeof(kDeadlySignal) - 1);
+}
+
+static void MaybeReportNonExecRegion(uptr pc) {
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
+ MemoryMappingLayout proc_maps(/*cache_enabled*/ true);
+ MemoryMappedSegment segment;
+ while (proc_maps.Next(&segment)) {
+ if (pc >= segment.start && pc < segment.end && !segment.IsExecutable())
+ Report("Hint: PC is at a non-executable region. Maybe a wild jump?\n");
+ }
+#endif
+}
+
+static void PrintMemoryByte(InternalScopedString *str, const char *before,
+ u8 byte) {
+ SanitizerCommonDecorator d;
+ str->append("%s%s%x%x%s ", before, d.MemoryByte(), byte >> 4, byte & 15,
+ d.Default());
+}
+
+static void MaybeDumpInstructionBytes(uptr pc) {
+ if (!common_flags()->dump_instruction_bytes || (pc < GetPageSizeCached()))
+ return;
+ InternalScopedString str(1024);
+ str.append("First 16 instruction bytes at pc: ");
+ if (IsAccessibleMemoryRange(pc, 16)) {
+ for (int i = 0; i < 16; ++i) {
+ PrintMemoryByte(&str, "", ((u8 *)pc)[i]);
+ }
+ str.append("\n");
+ } else {
+ str.append("unaccessible\n");
+ }
+ Report("%s", str.data());
+}
+
+static void MaybeDumpRegisters(void *context) {
+ if (!common_flags()->dump_registers) return;
+ SignalContext::DumpAllRegisters(context);
+}
+
+static void ReportStackOverflowImpl(const SignalContext &sig, u32 tid,
+ UnwindSignalStackCallbackType unwind,
+ const void *unwind_context) {
+ SanitizerCommonDecorator d;
+ Printf("%s", d.Warning());
+ static const char kDescription[] = "stack-overflow";
+ Report("ERROR: %s: %s on address %p (pc %p bp %p sp %p T%d)\n",
+ SanitizerToolName, kDescription, (void *)sig.addr, (void *)sig.pc,
+ (void *)sig.bp, (void *)sig.sp, tid);
+ Printf("%s", d.Default());
+ InternalScopedBuffer<BufferedStackTrace> stack_buffer(1);
+ BufferedStackTrace *stack = stack_buffer.data();
+ stack->Reset();
+ unwind(sig, unwind_context, stack);
+ stack->Print();
+ ReportErrorSummary(kDescription, stack);
+}
+
+static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid,
+ UnwindSignalStackCallbackType unwind,
+ const void *unwind_context) {
+ SanitizerCommonDecorator d;
+ Printf("%s", d.Warning());
+ const char *description = sig.Describe();
+ Report("ERROR: %s: %s on unknown address %p (pc %p bp %p sp %p T%d)\n",
+ SanitizerToolName, description, (void *)sig.addr, (void *)sig.pc,
+ (void *)sig.bp, (void *)sig.sp, tid);
+ Printf("%s", d.Default());
+ if (sig.pc < GetPageSizeCached())
+ Report("Hint: pc points to the zero page.\n");
+ if (sig.is_memory_access) {
+ const char *access_type =
+ sig.write_flag == SignalContext::WRITE
+ ? "WRITE"
+ : (sig.write_flag == SignalContext::READ ? "READ" : "UNKNOWN");
+ Report("The signal is caused by a %s memory access.\n", access_type);
+ if (sig.addr < GetPageSizeCached())
+ Report("Hint: address points to the zero page.\n");
+ }
+ MaybeReportNonExecRegion(sig.pc);
+ InternalScopedBuffer<BufferedStackTrace> stack_buffer(1);
+ BufferedStackTrace *stack = stack_buffer.data();
+ stack->Reset();
+ unwind(sig, unwind_context, stack);
+ stack->Print();
+ MaybeDumpInstructionBytes(sig.pc);
+ MaybeDumpRegisters(sig.context);
+ Printf("%s can not provide additional info.\n", SanitizerToolName);
+ ReportErrorSummary(description, stack);
+}
+
+void ReportDeadlySignal(const SignalContext &sig, u32 tid,
+ UnwindSignalStackCallbackType unwind,
+ const void *unwind_context) {
+ if (sig.IsStackOverflow())
+ ReportStackOverflowImpl(sig, tid, unwind, unwind_context);
+ else
+ ReportDeadlySignalImpl(sig, tid, unwind, unwind_context);
+}
+
+void HandleDeadlySignal(void *siginfo, void *context, u32 tid,
+ UnwindSignalStackCallbackType unwind,
+ const void *unwind_context) {
+ StartReportDeadlySignal();
+ ScopedErrorReportLock rl;
+ SignalContext sig(siginfo, context);
+ ReportDeadlySignal(sig, tid, unwind, unwind_context);
+ Report("ABORTING\n");
+ Die();
+}
+
+#endif // !SANITIZER_FUCHSIA && !SANITIZER_GO
+
void WriteToSyslog(const char *msg) {
InternalScopedString msg_copy(kErrorMessageBufferSize);
msg_copy.append("%s", msg);
@@ -161,6 +298,47 @@ void MaybeStartBackgroudThread() {
#endif
}
+static atomic_uintptr_t reporting_thread = {0};
+static StaticSpinMutex CommonSanitizerReportMutex;
+
+ScopedErrorReportLock::ScopedErrorReportLock() {
+ uptr current = GetThreadSelf();
+ for (;;) {
+ uptr expected = 0;
+ if (atomic_compare_exchange_strong(&reporting_thread, &expected, current,
+ memory_order_relaxed)) {
+ // We've claimed reporting_thread so proceed.
+ CommonSanitizerReportMutex.Lock();
+ return;
+ }
+
+ if (expected == current) {
+ // This is either asynch signal or nested error during error reporting.
+ // Fail simple to avoid deadlocks in Report().
+
+ // Can't use Report() here because of potential deadlocks in nested
+ // signal handlers.
+ CatastrophicErrorWrite(SanitizerToolName,
+ internal_strlen(SanitizerToolName));
+ static const char msg[] = ": nested bug in the same thread, aborting.\n";
+ CatastrophicErrorWrite(msg, sizeof(msg) - 1);
+
+ internal__exit(common_flags()->exitcode);
+ }
+
+ internal_sched_yield();
+ }
+}
+
+ScopedErrorReportLock::~ScopedErrorReportLock() {
+ CommonSanitizerReportMutex.Unlock();
+ atomic_store_relaxed(&reporting_thread, 0);
+}
+
+void ScopedErrorReportLock::CheckLocked() {
+ CommonSanitizerReportMutex.CheckLocked();
+}
+
} // namespace __sanitizer
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify,
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc
new file mode 100644
index 000000000000..c5be48bceace
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_fuchsia.cc
@@ -0,0 +1,240 @@
+//===-- sanitizer_coverage_fuchsia.cc ------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// Sanitizer Coverage Controller for Trace PC Guard, Fuchsia-specific version.
+//
+// This Fuchsia-specific implementation uses the same basic scheme and the
+// same simple '.sancov' file format as the generic implementation. The
+// difference is that we just produce a single blob of output for the whole
+// program, not a separate one per DSO. We do not sort the PC table and do
+// not prune the zeros, so the resulting file is always as large as it
+// would be to report 100% coverage. Implicit tracing information about
+// the address ranges of DSOs allows offline tools to split the one big
+// blob into separate files that the 'sancov' tool can understand.
+//
+// Unlike the traditional implementation that uses an atexit hook to write
+// out data files at the end, the results on Fuchsia do not go into a file
+// per se. The 'coverage_dir' option is ignored. Instead, they are stored
+// directly into a shared memory object (a Zircon VMO). At exit, that VMO
+// is handed over to a system service that's responsible for getting the
+// data out to somewhere that it can be fed into the sancov tool (where and
+// how is not our problem).
+
+#include "sanitizer_platform.h"
+#if SANITIZER_FUCHSIA
+#include "sanitizer_atomic.h"
+#include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+
+#include <zircon/process.h>
+#include <zircon/sanitizer.h>
+#include <zircon/syscalls.h>
+
+using namespace __sanitizer; // NOLINT
+
+namespace __sancov {
+namespace {
+
+// TODO(mcgrathr): Move the constant into a header shared with other impls.
+constexpr u64 Magic64 = 0xC0BFFFFFFFFFFF64ULL;
+static_assert(SANITIZER_WORDSIZE == 64, "Fuchsia is always LP64");
+
+constexpr const char kSancovSinkName[] = "sancov";
+
+// Collects trace-pc guard coverage.
+// This class relies on zero-initialization.
+class TracePcGuardController {
+ public:
+ // For each PC location being tracked, there is a u32 reserved in global
+ // data called the "guard". At startup, we assign each guard slot a
+ // unique index into the big results array. Later during runtime, the
+ // first call to TracePcGuard (below) will store the corresponding PC at
+ // that index in the array. (Each later call with the same guard slot is
+ // presumed to be from the same PC.) Then it clears the guard slot back
+ // to zero, which tells the compiler not to bother calling in again. At
+ // the end of the run, we have a big array where each element is either
+ // zero or is a tracked PC location that was hit in the trace.
+
+ // This is called from global constructors. Each translation unit has a
+ // contiguous array of guard slots, and a constructor that calls here
+ // with the bounds of its array. Those constructors are allowed to call
+ // here more than once for the same array. Usually all of these
+ // constructors run in the initial thread, but it's possible that a
+ // dlopen call on a secondary thread will run constructors that get here.
+ void InitTracePcGuard(u32 *start, u32 *end) {
+ if (end > start && *start == 0 && common_flags()->coverage) {
+ // Complete the setup before filling in any guards with indices.
+ // This avoids the possibility of code called from Setup reentering
+ // TracePcGuard.
+ u32 idx = Setup(end - start);
+ for (u32 *p = start; p < end; ++p) {
+ *p = idx++;
+ }
+ }
+ }
+
+ void TracePcGuard(u32 *guard, uptr pc) {
+ atomic_uint32_t *guard_ptr = reinterpret_cast<atomic_uint32_t *>(guard);
+ u32 idx = atomic_exchange(guard_ptr, 0, memory_order_relaxed);
+ if (idx > 0) array_[idx] = pc;
+ }
+
+ void Dump() {
+ BlockingMutexLock locked(&setup_lock_);
+ if (array_) {
+ CHECK_NE(vmo_, ZX_HANDLE_INVALID);
+
+ // Publish the VMO to the system, where it can be collected and
+ // analyzed after this process exits. This always consumes the VMO
+ // handle. Any failure is just logged and not indicated to us.
+ __sanitizer_publish_data(kSancovSinkName, vmo_);
+ vmo_ = ZX_HANDLE_INVALID;
+
+ // This will route to __sanitizer_log_write, which will ensure that
+ // information about shared libraries is written out. This message
+ // uses the `dumpfile` symbolizer markup element to highlight the
+ // dump. See the explanation for this in:
+ // https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md
+ Printf("SanitizerCoverage: {{{dumpfile:%s:%s}}} with up to %u PCs\n",
+ kSancovSinkName, vmo_name_, next_index_ - 1);
+ }
+ }
+
+ private:
+ // We map in the largest possible view into the VMO: one word
+ // for every possible 32-bit index value. This avoids the need
+ // to change the mapping when increasing the size of the VMO.
+ // We can always spare the 32G of address space.
+ static constexpr size_t MappingSize = sizeof(uptr) << 32;
+
+ BlockingMutex setup_lock_;
+ uptr *array_;
+ u32 next_index_;
+ zx_handle_t vmo_;
+ char vmo_name_[ZX_MAX_NAME_LEN];
+
+ size_t DataSize() const { return next_index_ * sizeof(uintptr_t); }
+
+ u32 Setup(u32 num_guards) {
+ BlockingMutexLock locked(&setup_lock_);
+ DCHECK(common_flags()->coverage);
+
+ if (next_index_ == 0) {
+ CHECK_EQ(vmo_, ZX_HANDLE_INVALID);
+ CHECK_EQ(array_, nullptr);
+
+ // The first sample goes at [1] to reserve [0] for the magic number.
+ next_index_ = 1 + num_guards;
+
+ zx_status_t status = _zx_vmo_create(DataSize(), 0, &vmo_);
+ CHECK_EQ(status, ZX_OK);
+
+ // Give the VMO a name including our process KOID so it's easy to spot.
+ internal_snprintf(vmo_name_, sizeof(vmo_name_), "%s.%zu", kSancovSinkName,
+ internal_getpid());
+ _zx_object_set_property(vmo_, ZX_PROP_NAME, vmo_name_,
+ internal_strlen(vmo_name_));
+
+ // Map the largest possible view we might need into the VMO. Later
+ // we might need to increase the VMO's size before we can use larger
+ // indices, but we'll never move the mapping address so we don't have
+ // any multi-thread synchronization issues with that.
+ uintptr_t mapping;
+ status =
+ _zx_vmar_map(_zx_vmar_root_self(), 0, vmo_, 0, MappingSize,
+ ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &mapping);
+ CHECK_EQ(status, ZX_OK);
+
+ // Hereafter other threads are free to start storing into
+ // elements [1, next_index_) of the big array.
+ array_ = reinterpret_cast<uptr *>(mapping);
+
+ // Store the magic number.
+ // Hereafter, the VMO serves as the contents of the '.sancov' file.
+ array_[0] = Magic64;
+
+ return 1;
+ } else {
+ // The VMO is already mapped in, but it's not big enough to use the
+ // new indices. So increase the size to cover the new maximum index.
+
+ CHECK_NE(vmo_, ZX_HANDLE_INVALID);
+ CHECK_NE(array_, nullptr);
+
+ uint32_t first_index = next_index_;
+ next_index_ += num_guards;
+
+ zx_status_t status = _zx_vmo_set_size(vmo_, DataSize());
+ CHECK_EQ(status, ZX_OK);
+
+ return first_index;
+ }
+ }
+};
+
+static TracePcGuardController pc_guard_controller;
+
+} // namespace
+} // namespace __sancov
+
+namespace __sanitizer {
+void InitializeCoverage(bool enabled, const char *dir) {
+ CHECK_EQ(enabled, common_flags()->coverage);
+ CHECK_EQ(dir, common_flags()->coverage_dir);
+
+ static bool coverage_enabled = false;
+ if (!coverage_enabled) {
+ coverage_enabled = enabled;
+ Atexit(__sanitizer_cov_dump);
+ AddDieCallback(__sanitizer_cov_dump);
+ }
+}
+} // namespace __sanitizer
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage( // NOLINT
+ const uptr *pcs, uptr len) {
+ UNIMPLEMENTED();
+}
+
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard, u32 *guard) {
+ if (!*guard) return;
+ __sancov::pc_guard_controller.TracePcGuard(guard, GET_CALLER_PC() - 1);
+}
+
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard_init,
+ u32 *start, u32 *end) {
+ if (start == end || *start) return;
+ __sancov::pc_guard_controller.InitTracePcGuard(start, end);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage() {
+ __sancov::pc_guard_controller.Dump();
+}
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() {
+ __sanitizer_dump_trace_pc_guard_coverage();
+}
+// Default empty implementations (weak). Users should redefine them.
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp1, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp2, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp4, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp8, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp1, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp2, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp4, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp8, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_switch, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div4, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div8, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_gep, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_indir, void) {}
+} // extern "C"
+
+#endif // !SANITIZER_FUCHSIA
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc
index d4749000d80a..f909b74c9752 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc
@@ -9,6 +9,7 @@
// Sanitizer Coverage interface list.
//===----------------------------------------------------------------------===//
INTERFACE_FUNCTION(__sanitizer_cov_dump)
+INTERFACE_FUNCTION(__sanitizer_cov_reset)
INTERFACE_FUNCTION(__sanitizer_dump_coverage)
INTERFACE_FUNCTION(__sanitizer_dump_trace_pc_guard_coverage)
INTERFACE_WEAK_FUNCTION(__sancov_default_options)
@@ -17,6 +18,10 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp1)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp2)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp4)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp8)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp1)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp2)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp4)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp8)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_div4)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_div8)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_gep)
@@ -24,3 +29,5 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard_init)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_indir)
INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_switch)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_8bit_counters_init)
+INTERFACE_WEAK_FUNCTION(__sanitizer_cov_pcs_init)
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc
index 24433356c63d..3c5f29b2899b 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cc
@@ -8,10 +8,14 @@
//===----------------------------------------------------------------------===//
// Sanitizer Coverage Controller for Trace PC Guard.
+#include "sanitizer_platform.h"
+
+#if !SANITIZER_FUCHSIA
#include "sancov_flags.h"
#include "sanitizer_allocator_internal.h"
#include "sanitizer_atomic.h"
#include "sanitizer_common.h"
+#include "sanitizer_file.h"
#include "sanitizer_symbolizer.h"
using namespace __sanitizer;
@@ -124,11 +128,17 @@ class TracePcGuardController {
}
void TracePcGuard(u32* guard, uptr pc) {
- atomic_uint32_t* guard_ptr = reinterpret_cast<atomic_uint32_t*>(guard);
- u32 idx = atomic_exchange(guard_ptr, 0, memory_order_relaxed);
+ u32 idx = *guard;
if (!idx) return;
// we start indices from 1.
- pc_vector[idx - 1] = pc;
+ atomic_uintptr_t* pc_ptr =
+ reinterpret_cast<atomic_uintptr_t*>(&pc_vector[idx - 1]);
+ if (atomic_load(pc_ptr, memory_order_relaxed) == 0)
+ atomic_store(pc_ptr, pc, memory_order_relaxed);
+ }
+
+ void Reset() {
+ internal_memset(&pc_vector[0], 0, sizeof(pc_vector[0]) * pc_vector.size());
}
void Dump() {
@@ -180,15 +190,31 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage() {
SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() {
__sanitizer_dump_trace_pc_guard_coverage();
}
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_reset() {
+ __sancov::pc_guard_controller.Reset();
+}
// Default empty implementations (weak). Users should redefine them.
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp1, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp2, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp4, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_cmp8, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp1, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp2, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp4, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_const_cmp8, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_switch, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div4, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div8, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_gep, void) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_indir, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_8bit_counters_init, void) {}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_pcs_init, void) {}
} // extern "C"
+// Weak definition for code instrumented with -fsanitize-coverage=stack-depth
+// and later linked with code containing a strong definition.
+// E.g., -fsanitize=fuzzer-no-link
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE uptr __sancov_lowest_stack;
+
+#endif // !SANITIZER_FUCHSIA
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h
index 7872b89c227c..42cc290502bf 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_errno.h
@@ -24,8 +24,10 @@
#if SANITIZER_FREEBSD || SANITIZER_MAC
# define __errno_location __error
-#elif SANITIZER_ANDROID
+#elif SANITIZER_ANDROID || SANITIZER_NETBSD
# define __errno_location __errno
+#elif SANITIZER_SOLARIS
+# define __errno_location ___errno
#elif SANITIZER_WINDOWS
# define __errno_location _errno
#endif
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.cc
new file mode 100644
index 000000000000..cde54bfde804
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.cc
@@ -0,0 +1,177 @@
+//===-- sanitizer_file.cc ------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries. It defines filesystem-related interfaces. This
+// is separate from sanitizer_common.cc so that it's simpler to disable
+// all the filesystem support code for a port that doesn't use it.
+//
+//===---------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if !SANITIZER_FUCHSIA
+
+#include "sanitizer_common.h"
+#include "sanitizer_file.h"
+
+namespace __sanitizer {
+
+void CatastrophicErrorWrite(const char *buffer, uptr length) {
+ WriteToFile(kStderrFd, buffer, length);
+}
+
+StaticSpinMutex report_file_mu;
+ReportFile report_file = {&report_file_mu, kStderrFd, "", "", 0};
+
+void RawWrite(const char *buffer) {
+ report_file.Write(buffer, internal_strlen(buffer));
+}
+
+void ReportFile::ReopenIfNecessary() {
+ mu->CheckLocked();
+ if (fd == kStdoutFd || fd == kStderrFd) return;
+
+ uptr pid = internal_getpid();
+ // If in tracer, use the parent's file.
+ if (pid == stoptheworld_tracer_pid)
+ pid = stoptheworld_tracer_ppid;
+ if (fd != kInvalidFd) {
+ // If the report file is already opened by the current process,
+ // do nothing. Otherwise the report file was opened by the parent
+ // process, close it now.
+ if (fd_pid == pid)
+ return;
+ else
+ CloseFile(fd);
+ }
+
+ const char *exe_name = GetProcessName();
+ if (common_flags()->log_exe_name && exe_name) {
+ internal_snprintf(full_path, kMaxPathLength, "%s.%s.%zu", path_prefix,
+ exe_name, pid);
+ } else {
+ internal_snprintf(full_path, kMaxPathLength, "%s.%zu", path_prefix, pid);
+ }
+ fd = OpenFile(full_path, WrOnly);
+ if (fd == kInvalidFd) {
+ const char *ErrorMsgPrefix = "ERROR: Can't open file: ";
+ WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
+ WriteToFile(kStderrFd, full_path, internal_strlen(full_path));
+ Die();
+ }
+ fd_pid = pid;
+}
+
+void ReportFile::SetReportPath(const char *path) {
+ if (!path)
+ return;
+ uptr len = internal_strlen(path);
+ if (len > sizeof(path_prefix) - 100) {
+ Report("ERROR: Path is too long: %c%c%c%c%c%c%c%c...\n",
+ path[0], path[1], path[2], path[3],
+ path[4], path[5], path[6], path[7]);
+ Die();
+ }
+
+ SpinMutexLock l(mu);
+ if (fd != kStdoutFd && fd != kStderrFd && fd != kInvalidFd)
+ CloseFile(fd);
+ fd = kInvalidFd;
+ if (internal_strcmp(path, "stdout") == 0) {
+ fd = kStdoutFd;
+ } else if (internal_strcmp(path, "stderr") == 0) {
+ fd = kStderrFd;
+ } else {
+ internal_snprintf(path_prefix, kMaxPathLength, "%s", path);
+ }
+}
+
+bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
+ uptr *read_len, uptr max_len, error_t *errno_p) {
+ uptr PageSize = GetPageSizeCached();
+ uptr kMinFileLen = PageSize;
+ *buff = nullptr;
+ *buff_size = 0;
+ *read_len = 0;
+ // The files we usually open are not seekable, so try different buffer sizes.
+ for (uptr size = kMinFileLen; size <= max_len; size *= 2) {
+ fd_t fd = OpenFile(file_name, RdOnly, errno_p);
+ if (fd == kInvalidFd) return false;
+ UnmapOrDie(*buff, *buff_size);
+ *buff = (char*)MmapOrDie(size, __func__);
+ *buff_size = size;
+ *read_len = 0;
+ // Read up to one page at a time.
+ bool reached_eof = false;
+ while (*read_len + PageSize <= size) {
+ uptr just_read;
+ if (!ReadFromFile(fd, *buff + *read_len, PageSize, &just_read, errno_p)) {
+ UnmapOrDie(*buff, *buff_size);
+ return false;
+ }
+ if (just_read == 0) {
+ reached_eof = true;
+ break;
+ }
+ *read_len += just_read;
+ }
+ CloseFile(fd);
+ if (reached_eof) // We've read the whole file.
+ break;
+ }
+ return true;
+}
+
+static const char kPathSeparator = SANITIZER_WINDOWS ? ';' : ':';
+
+char *FindPathToBinary(const char *name) {
+ if (FileExists(name)) {
+ return internal_strdup(name);
+ }
+
+ const char *path = GetEnv("PATH");
+ if (!path)
+ return nullptr;
+ uptr name_len = internal_strlen(name);
+ InternalScopedBuffer<char> buffer(kMaxPathLength);
+ const char *beg = path;
+ while (true) {
+ const char *end = internal_strchrnul(beg, kPathSeparator);
+ uptr prefix_len = end - beg;
+ if (prefix_len + name_len + 2 <= kMaxPathLength) {
+ internal_memcpy(buffer.data(), beg, prefix_len);
+ buffer[prefix_len] = '/';
+ internal_memcpy(&buffer[prefix_len + 1], name, name_len);
+ buffer[prefix_len + 1 + name_len] = '\0';
+ if (FileExists(buffer.data()))
+ return internal_strdup(buffer.data());
+ }
+ if (*end == '\0') break;
+ beg = end + 1;
+ }
+ return nullptr;
+}
+
+} // namespace __sanitizer
+
+using namespace __sanitizer; // NOLINT
+
+extern "C" {
+void __sanitizer_set_report_path(const char *path) {
+ report_file.SetReportPath(path);
+}
+
+void __sanitizer_set_report_fd(void *fd) {
+ report_file.fd = (fd_t)reinterpret_cast<uptr>(fd);
+ report_file.fd_pid = internal_getpid();
+}
+} // extern "C"
+
+#endif // !SANITIZER_FUCHSIA
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h
new file mode 100644
index 000000000000..9a12ab7342c7
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_file.h
@@ -0,0 +1,110 @@
+//===-- sanitizer_file.h ---------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is shared between run-time libraries of sanitizers.
+// It declares filesystem-related interfaces. This is separate from
+// sanitizer_common.h so that it's simpler to disable all the filesystem
+// support code for a port that doesn't use it.
+//
+//===---------------------------------------------------------------------===//
+#ifndef SANITIZER_FILE_H
+#define SANITIZER_FILE_H
+
+#include "sanitizer_interface_internal.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_mutex.h"
+
+namespace __sanitizer {
+
+struct ReportFile {
+ void Write(const char *buffer, uptr length);
+ bool SupportsColors();
+ void SetReportPath(const char *path);
+
+ // Don't use fields directly. They are only declared public to allow
+ // aggregate initialization.
+
+ // Protects fields below.
+ StaticSpinMutex *mu;
+ // Opened file descriptor. Defaults to stderr. It may be equal to
+ // kInvalidFd, in which case new file will be opened when necessary.
+ fd_t fd;
+ // Path prefix of report file, set via __sanitizer_set_report_path.
+ char path_prefix[kMaxPathLength];
+ // Full path to report, obtained as <path_prefix>.PID
+ char full_path[kMaxPathLength];
+ // PID of the process that opened fd. If a fork() occurs,
+ // the PID of child will be different from fd_pid.
+ uptr fd_pid;
+
+ private:
+ void ReopenIfNecessary();
+};
+extern ReportFile report_file;
+
+enum FileAccessMode {
+ RdOnly,
+ WrOnly,
+ RdWr
+};
+
+// Returns kInvalidFd on error.
+fd_t OpenFile(const char *filename, FileAccessMode mode,
+ error_t *errno_p = nullptr);
+void CloseFile(fd_t);
+
+// Return true on success, false on error.
+bool ReadFromFile(fd_t fd, void *buff, uptr buff_size,
+ uptr *bytes_read = nullptr, error_t *error_p = nullptr);
+bool WriteToFile(fd_t fd, const void *buff, uptr buff_size,
+ uptr *bytes_written = nullptr, error_t *error_p = nullptr);
+
+bool RenameFile(const char *oldpath, const char *newpath,
+ error_t *error_p = nullptr);
+
+// Scoped file handle closer.
+struct FileCloser {
+ explicit FileCloser(fd_t fd) : fd(fd) {}
+ ~FileCloser() { CloseFile(fd); }
+ fd_t fd;
+};
+
+bool SupportsColoredOutput(fd_t fd);
+
+// OS
+const char *GetPwd();
+bool FileExists(const char *filename);
+char *FindPathToBinary(const char *name);
+bool IsPathSeparator(const char c);
+bool IsAbsolutePath(const char *path);
+// Starts a subprocess and returs its pid.
+// If *_fd parameters are not kInvalidFd their corresponding input/output
+// streams will be redirect to the file. The files will always be closed
+// in parent process even in case of an error.
+// The child process will close all fds after STDERR_FILENO
+// before passing control to a program.
+pid_t StartSubprocess(const char *filename, const char *const argv[],
+ fd_t stdin_fd = kInvalidFd, fd_t stdout_fd = kInvalidFd,
+ fd_t stderr_fd = kInvalidFd);
+// Checks if specified process is still running
+bool IsProcessRunning(pid_t pid);
+// Waits for the process to finish and returns its exit code.
+// Returns -1 in case of an error.
+int WaitForProcess(pid_t pid);
+
+// Maps given file to virtual memory, and returns pointer to it
+// (or NULL if mapping fails). Stores the size of mmaped region
+// in '*buff_size'.
+void *MapFileToMemory(const char *file_name, uptr *buff_size);
+void *MapWritableFileToMemory(void *addr, uptr size, fd_t fd, OFF_T offset);
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_FILE_H
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h
index 4988fbb7a86d..f649f5bffea7 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h
@@ -75,7 +75,7 @@ inline bool FlagHandler<HandleSignalMode>::Parse(const char *value) {
template <>
inline bool FlagHandler<const char *>::Parse(const char *value) {
- *t_ = internal_strdup(value);
+ *t_ = value;
return true;
}
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc
index 9f71861efcdb..445e25f9df70 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc
@@ -62,6 +62,9 @@ COMMON_FLAG(
COMMON_FLAG(
int, verbosity, 0,
"Verbosity level (0 - silent, 1 - a bit of output, 2+ - more output).")
+COMMON_FLAG(bool, strip_env, 1,
+ "Whether to remove the sanitizer from DYLD_INSERT_LIBRARIES to "
+ "avoid passing it to children. Default is true.")
COMMON_FLAG(bool, detect_leaks, !SANITIZER_MAC, "Enable memory leak detection.")
COMMON_FLAG(
bool, leak_check_at_exit, true,
@@ -129,11 +132,11 @@ COMMON_FLAG(uptr, soft_rss_limit_mb, 0,
" This limit does not affect memory allocations other than"
" malloc/new.")
COMMON_FLAG(bool, heap_profile, false, "Experimental heap profiler, asan-only")
-COMMON_FLAG(s32, allocator_release_to_os_interval_ms, kReleaseToOSIntervalNever,
- "Experimental. Only affects a 64-bit allocator. If set, tries to "
- "release unused memory to the OS, but not more often than this "
- "interval (in milliseconds). Negative values mean do not attempt "
- "to release memory to the OS.\n")
+COMMON_FLAG(s32, allocator_release_to_os_interval_ms, 5000,
+ "Only affects a 64-bit allocator. If set, tries to release unused "
+ "memory to the OS, but not more often than this interval (in "
+ "milliseconds). Negative values mean do not attempt to release "
+ "memory to the OS.\n")
COMMON_FLAG(bool, can_use_proc_maps_statm, true,
"If false, do not attempt to read /proc/maps/statm."
" Mostly useful for testing sanitizers.")
@@ -229,3 +232,8 @@ COMMON_FLAG(bool, print_cmdline, false, "Print command line on crash "
"(asan only).")
COMMON_FLAG(bool, html_cov_report, false, "Generate html coverage report.")
COMMON_FLAG(const char *, sancov_path, "sancov", "Sancov tool location.")
+COMMON_FLAG(bool, dump_instruction_bytes, false,
+ "If true, dump 16 bytes starting at the instruction that caused SEGV")
+COMMON_FLAG(bool, dump_registers, true,
+ "If true, dump values of CPU registers when SEGV happens. Only "
+ "available on OS X for now.")
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc
new file mode 100644
index 000000000000..936ec794b8e8
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc
@@ -0,0 +1,539 @@
+//===-- sanitizer_fuchsia.cc ---------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and other sanitizer
+// run-time libraries and implements Fuchsia-specific functions from
+// sanitizer_common.h.
+//===---------------------------------------------------------------------===//
+
+#include "sanitizer_fuchsia.h"
+#if SANITIZER_FUCHSIA
+
+#include "sanitizer_common.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_mutex.h"
+#include "sanitizer_stacktrace.h"
+
+#include <limits.h>
+#include <pthread.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <unwind.h>
+#include <zircon/errors.h>
+#include <zircon/process.h>
+#include <zircon/syscalls.h>
+
+namespace __sanitizer {
+
+void NORETURN internal__exit(int exitcode) { _zx_process_exit(exitcode); }
+
+uptr internal_sched_yield() {
+ zx_status_t status = _zx_nanosleep(0);
+ CHECK_EQ(status, ZX_OK);
+ return 0; // Why doesn't this return void?
+}
+
+static void internal_nanosleep(zx_time_t ns) {
+ zx_status_t status = _zx_nanosleep(_zx_deadline_after(ns));
+ CHECK_EQ(status, ZX_OK);
+}
+
+unsigned int internal_sleep(unsigned int seconds) {
+ internal_nanosleep(ZX_SEC(seconds));
+ return 0;
+}
+
+u64 NanoTime() { return _zx_time_get(ZX_CLOCK_UTC); }
+
+u64 MonotonicNanoTime() { return _zx_time_get(ZX_CLOCK_MONOTONIC); }
+
+uptr internal_getpid() {
+ zx_info_handle_basic_t info;
+ zx_status_t status =
+ _zx_object_get_info(_zx_process_self(), ZX_INFO_HANDLE_BASIC, &info,
+ sizeof(info), NULL, NULL);
+ CHECK_EQ(status, ZX_OK);
+ uptr pid = static_cast<uptr>(info.koid);
+ CHECK_EQ(pid, info.koid);
+ return pid;
+}
+
+uptr GetThreadSelf() { return reinterpret_cast<uptr>(thrd_current()); }
+
+uptr GetTid() { return GetThreadSelf(); }
+
+void Abort() { abort(); }
+
+int Atexit(void (*function)(void)) { return atexit(function); }
+
+void SleepForSeconds(int seconds) { internal_sleep(seconds); }
+
+void SleepForMillis(int millis) { internal_nanosleep(ZX_MSEC(millis)); }
+
+void GetThreadStackTopAndBottom(bool, uptr *stack_top, uptr *stack_bottom) {
+ pthread_attr_t attr;
+ CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
+ void *base;
+ size_t size;
+ CHECK_EQ(pthread_attr_getstack(&attr, &base, &size), 0);
+ CHECK_EQ(pthread_attr_destroy(&attr), 0);
+
+ *stack_bottom = reinterpret_cast<uptr>(base);
+ *stack_top = *stack_bottom + size;
+}
+
+void MaybeReexec() {}
+void PrepareForSandboxing(__sanitizer_sandbox_arguments *args) {}
+void DisableCoreDumperIfNecessary() {}
+void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
+void StartReportDeadlySignal() {}
+void ReportDeadlySignal(const SignalContext &sig, u32 tid,
+ UnwindSignalStackCallbackType unwind,
+ const void *unwind_context) {}
+void SetAlternateSignalStack() {}
+void UnsetAlternateSignalStack() {}
+void InitTlsSize() {}
+
+void PrintModuleMap() {}
+
+bool SignalContext::IsStackOverflow() const { return false; }
+void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); }
+const char *SignalContext::Describe() const { UNIMPLEMENTED(); }
+
+struct UnwindTraceArg {
+ BufferedStackTrace *stack;
+ u32 max_depth;
+};
+
+_Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) {
+ UnwindTraceArg *arg = static_cast<UnwindTraceArg *>(param);
+ CHECK_LT(arg->stack->size, arg->max_depth);
+ uptr pc = _Unwind_GetIP(ctx);
+ if (pc < PAGE_SIZE) return _URC_NORMAL_STOP;
+ arg->stack->trace_buffer[arg->stack->size++] = pc;
+ return (arg->stack->size == arg->max_depth ? _URC_NORMAL_STOP
+ : _URC_NO_REASON);
+}
+
+void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) {
+ CHECK_GE(max_depth, 2);
+ size = 0;
+ UnwindTraceArg arg = {this, Min(max_depth + 1, kStackTraceMax)};
+ _Unwind_Backtrace(Unwind_Trace, &arg);
+ CHECK_GT(size, 0);
+ // We need to pop a few frames so that pc is on top.
+ uptr to_pop = LocatePcInTrace(pc);
+ // trace_buffer[0] belongs to the current function so we always pop it,
+ // unless there is only 1 frame in the stack trace (1 frame is always better
+ // than 0!).
+ PopStackFrames(Min(to_pop, static_cast<uptr>(1)));
+ trace_buffer[0] = pc;
+}
+
+void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context,
+ u32 max_depth) {
+ CHECK_NE(context, nullptr);
+ UNREACHABLE("signal context doesn't exist");
+}
+
+enum MutexState : int { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 };
+
+BlockingMutex::BlockingMutex() {
+ // NOTE! It's important that this use internal_memset, because plain
+ // memset might be intercepted (e.g., actually be __asan_memset).
+ // Defining this so the compiler initializes each field, e.g.:
+ // BlockingMutex::BlockingMutex() : BlockingMutex(LINKER_INITIALIZED) {}
+ // might result in the compiler generating a call to memset, which would
+ // have the same problem.
+ internal_memset(this, 0, sizeof(*this));
+}
+
+void BlockingMutex::Lock() {
+ CHECK_EQ(owner_, 0);
+ atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+ if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked)
+ return;
+ while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) {
+ zx_status_t status = _zx_futex_wait(reinterpret_cast<zx_futex_t *>(m),
+ MtxSleeping, ZX_TIME_INFINITE);
+ if (status != ZX_ERR_BAD_STATE) // Normal race.
+ CHECK_EQ(status, ZX_OK);
+ }
+}
+
+void BlockingMutex::Unlock() {
+ atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+ u32 v = atomic_exchange(m, MtxUnlocked, memory_order_release);
+ CHECK_NE(v, MtxUnlocked);
+ if (v == MtxSleeping) {
+ zx_status_t status = _zx_futex_wake(reinterpret_cast<zx_futex_t *>(m), 1);
+ CHECK_EQ(status, ZX_OK);
+ }
+}
+
+void BlockingMutex::CheckLocked() {
+ atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+ CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
+}
+
+uptr GetPageSize() { return PAGE_SIZE; }
+
+uptr GetMmapGranularity() { return PAGE_SIZE; }
+
+sanitizer_shadow_bounds_t ShadowBounds;
+
+uptr GetMaxUserVirtualAddress() {
+ ShadowBounds = __sanitizer_shadow_bounds();
+ return ShadowBounds.memory_limit - 1;
+}
+
+uptr GetMaxVirtualAddress() { return GetMaxUserVirtualAddress(); }
+
+static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type,
+ bool raw_report, bool die_for_nomem) {
+ size = RoundUpTo(size, PAGE_SIZE);
+
+ zx_handle_t vmo;
+ zx_status_t status = _zx_vmo_create(size, 0, &vmo);
+ if (status != ZX_OK) {
+ if (status != ZX_ERR_NO_MEMORY || die_for_nomem)
+ ReportMmapFailureAndDie(size, mem_type, "zx_vmo_create", status,
+ raw_report);
+ return nullptr;
+ }
+ _zx_object_set_property(vmo, ZX_PROP_NAME, mem_type,
+ internal_strlen(mem_type));
+
+ // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that?
+ uintptr_t addr;
+ status = _zx_vmar_map(_zx_vmar_root_self(), 0, vmo, 0, size,
+ ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &addr);
+ _zx_handle_close(vmo);
+
+ if (status != ZX_OK) {
+ if (status != ZX_ERR_NO_MEMORY || die_for_nomem)
+ ReportMmapFailureAndDie(size, mem_type, "zx_vmar_map", status,
+ raw_report);
+ return nullptr;
+ }
+
+ IncreaseTotalMmap(size);
+
+ return reinterpret_cast<void *>(addr);
+}
+
+void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
+ return DoAnonymousMmapOrDie(size, mem_type, raw_report, true);
+}
+
+void *MmapNoReserveOrDie(uptr size, const char *mem_type) {
+ return MmapOrDie(size, mem_type);
+}
+
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
+ return DoAnonymousMmapOrDie(size, mem_type, false, false);
+}
+
+uptr ReservedAddressRange::Init(uptr init_size, const char *name,
+ uptr fixed_addr) {
+ init_size = RoundUpTo(init_size, PAGE_SIZE);
+ DCHECK_EQ(os_handle_, ZX_HANDLE_INVALID);
+ uintptr_t base;
+ zx_handle_t vmar;
+ zx_status_t status =
+ _zx_vmar_allocate(_zx_vmar_root_self(), 0, init_size,
+ ZX_VM_FLAG_CAN_MAP_READ | ZX_VM_FLAG_CAN_MAP_WRITE |
+ ZX_VM_FLAG_CAN_MAP_SPECIFIC,
+ &vmar, &base);
+ if (status != ZX_OK)
+ ReportMmapFailureAndDie(init_size, name, "zx_vmar_allocate", status);
+ base_ = reinterpret_cast<void *>(base);
+ size_ = init_size;
+ name_ = name;
+ os_handle_ = vmar;
+
+ return reinterpret_cast<uptr>(base_);
+}
+
+static uptr DoMmapFixedOrDie(zx_handle_t vmar, uptr fixed_addr, uptr map_size,
+ void *base, const char *name, bool die_for_nomem) {
+ uptr offset = fixed_addr - reinterpret_cast<uptr>(base);
+ map_size = RoundUpTo(map_size, PAGE_SIZE);
+ zx_handle_t vmo;
+ zx_status_t status = _zx_vmo_create(map_size, 0, &vmo);
+ if (status != ZX_OK) {
+ if (status != ZX_ERR_NO_MEMORY || die_for_nomem)
+ ReportMmapFailureAndDie(map_size, name, "zx_vmo_create", status);
+ return 0;
+ }
+ _zx_object_set_property(vmo, ZX_PROP_NAME, name, sizeof(name) - 1);
+ DCHECK_GE(base + size_, map_size + offset);
+ uintptr_t addr;
+
+ status = _zx_vmar_map(
+ vmar, offset, vmo, 0, map_size,
+ ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE | ZX_VM_FLAG_SPECIFIC,
+ &addr);
+ _zx_handle_close(vmo);
+ if (status != ZX_OK) {
+ if (status != ZX_ERR_NO_MEMORY || die_for_nomem) {
+ ReportMmapFailureAndDie(map_size, name, "zx_vmar_map", status);
+ }
+ return 0;
+ }
+ IncreaseTotalMmap(map_size);
+ return addr;
+}
+
+uptr ReservedAddressRange::Map(uptr fixed_addr, uptr map_size) {
+ return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_,
+ name_, false);
+}
+
+uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr map_size) {
+ return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_,
+ name_, true);
+}
+
+void UnmapOrDieVmar(void *addr, uptr size, zx_handle_t target_vmar) {
+ if (!addr || !size) return;
+ size = RoundUpTo(size, PAGE_SIZE);
+
+ zx_status_t status =
+ _zx_vmar_unmap(target_vmar, reinterpret_cast<uintptr_t>(addr), size);
+ if (status != ZX_OK) {
+ Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n",
+ SanitizerToolName, size, size, addr);
+ CHECK("unable to unmap" && 0);
+ }
+
+ DecreaseTotalMmap(size);
+}
+
+void ReservedAddressRange::Unmap(uptr fixed_addr, uptr size) {
+ uptr offset = fixed_addr - reinterpret_cast<uptr>(base_);
+ uptr addr = reinterpret_cast<uptr>(base_) + offset;
+ void *addr_as_void = reinterpret_cast<void *>(addr);
+ uptr base_as_uptr = reinterpret_cast<uptr>(base_);
+ // Only unmap at the beginning or end of the range.
+ CHECK((addr_as_void == base_) || (addr + size == base_as_uptr + size_));
+ CHECK_LE(size, size_);
+ UnmapOrDieVmar(reinterpret_cast<void *>(addr), size,
+ static_cast<zx_handle_t>(os_handle_));
+ if (addr_as_void == base_) {
+ base_ = reinterpret_cast<void *>(addr + size);
+ }
+ size_ = size_ - size;
+}
+
+// This should never be called.
+void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) {
+ UNIMPLEMENTED();
+}
+
+void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
+ const char *mem_type) {
+ CHECK_GE(size, PAGE_SIZE);
+ CHECK(IsPowerOfTwo(size));
+ CHECK(IsPowerOfTwo(alignment));
+
+ zx_handle_t vmo;
+ zx_status_t status = _zx_vmo_create(size, 0, &vmo);
+ if (status != ZX_OK) {
+ if (status != ZX_ERR_NO_MEMORY)
+ ReportMmapFailureAndDie(size, mem_type, "zx_vmo_create", status, false);
+ return nullptr;
+ }
+ _zx_object_set_property(vmo, ZX_PROP_NAME, mem_type,
+ internal_strlen(mem_type));
+
+ // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that?
+
+ // Map a larger size to get a chunk of address space big enough that
+ // it surely contains an aligned region of the requested size. Then
+ // overwrite the aligned middle portion with a mapping from the
+ // beginning of the VMO, and unmap the excess before and after.
+ size_t map_size = size + alignment;
+ uintptr_t addr;
+ status = _zx_vmar_map(_zx_vmar_root_self(), 0, vmo, 0, map_size,
+ ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, &addr);
+ if (status == ZX_OK) {
+ uintptr_t map_addr = addr;
+ uintptr_t map_end = map_addr + map_size;
+ addr = RoundUpTo(map_addr, alignment);
+ uintptr_t end = addr + size;
+ if (addr != map_addr) {
+ zx_info_vmar_t info;
+ status = _zx_object_get_info(_zx_vmar_root_self(), ZX_INFO_VMAR, &info,
+ sizeof(info), NULL, NULL);
+ if (status == ZX_OK) {
+ uintptr_t new_addr;
+ status =
+ _zx_vmar_map(_zx_vmar_root_self(), addr - info.base, vmo, 0, size,
+ ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE |
+ ZX_VM_FLAG_SPECIFIC_OVERWRITE,
+ &new_addr);
+ if (status == ZX_OK) CHECK_EQ(new_addr, addr);
+ }
+ }
+ if (status == ZX_OK && addr != map_addr)
+ status = _zx_vmar_unmap(_zx_vmar_root_self(), map_addr, addr - map_addr);
+ if (status == ZX_OK && end != map_end)
+ status = _zx_vmar_unmap(_zx_vmar_root_self(), end, map_end - end);
+ }
+ _zx_handle_close(vmo);
+
+ if (status != ZX_OK) {
+ if (status != ZX_ERR_NO_MEMORY)
+ ReportMmapFailureAndDie(size, mem_type, "zx_vmar_map", status, false);
+ return nullptr;
+ }
+
+ IncreaseTotalMmap(size);
+
+ return reinterpret_cast<void *>(addr);
+}
+
+void UnmapOrDie(void *addr, uptr size) {
+ UnmapOrDieVmar(addr, size, _zx_vmar_root_self());
+}
+
+// This is used on the shadow mapping, which cannot be changed.
+// Zircon doesn't have anything like MADV_DONTNEED.
+void ReleaseMemoryPagesToOS(uptr beg, uptr end) {}
+
+void DumpProcessMap() {
+ // TODO(mcgrathr): write it
+ return;
+}
+
+bool IsAccessibleMemoryRange(uptr beg, uptr size) {
+ // TODO(mcgrathr): Figure out a better way.
+ zx_handle_t vmo;
+ zx_status_t status = _zx_vmo_create(size, 0, &vmo);
+ if (status == ZX_OK) {
+ while (size > 0) {
+ size_t wrote;
+ status = _zx_vmo_write(vmo, reinterpret_cast<const void *>(beg), 0, size,
+ &wrote);
+ if (status != ZX_OK) break;
+ CHECK_GT(wrote, 0);
+ CHECK_LE(wrote, size);
+ beg += wrote;
+ size -= wrote;
+ }
+ _zx_handle_close(vmo);
+ }
+ return status == ZX_OK;
+}
+
+// FIXME implement on this platform.
+void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {}
+
+bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
+ uptr *read_len, uptr max_len, error_t *errno_p) {
+ zx_handle_t vmo;
+ zx_status_t status = __sanitizer_get_configuration(file_name, &vmo);
+ if (status == ZX_OK) {
+ uint64_t vmo_size;
+ status = _zx_vmo_get_size(vmo, &vmo_size);
+ if (status == ZX_OK) {
+ if (vmo_size < max_len) max_len = vmo_size;
+ size_t map_size = RoundUpTo(max_len, PAGE_SIZE);
+ uintptr_t addr;
+ status = _zx_vmar_map(_zx_vmar_root_self(), 0, vmo, 0, map_size,
+ ZX_VM_FLAG_PERM_READ, &addr);
+ if (status == ZX_OK) {
+ *buff = reinterpret_cast<char *>(addr);
+ *buff_size = map_size;
+ *read_len = max_len;
+ }
+ }
+ _zx_handle_close(vmo);
+ }
+ if (status != ZX_OK && errno_p) *errno_p = status;
+ return status == ZX_OK;
+}
+
+void RawWrite(const char *buffer) {
+ __sanitizer_log_write(buffer, internal_strlen(buffer));
+}
+
+void CatastrophicErrorWrite(const char *buffer, uptr length) {
+ __sanitizer_log_write(buffer, length);
+}
+
+char **StoredArgv;
+char **StoredEnviron;
+
+char **GetArgv() { return StoredArgv; }
+
+const char *GetEnv(const char *name) {
+ if (StoredEnviron) {
+ uptr NameLen = internal_strlen(name);
+ for (char **Env = StoredEnviron; *Env != 0; Env++) {
+ if (internal_strncmp(*Env, name, NameLen) == 0 && (*Env)[NameLen] == '=')
+ return (*Env) + NameLen + 1;
+ }
+ }
+ return nullptr;
+}
+
+uptr ReadBinaryName(/*out*/ char *buf, uptr buf_len) {
+ const char *argv0 = StoredArgv[0];
+ if (!argv0) argv0 = "<UNKNOWN>";
+ internal_strncpy(buf, argv0, buf_len);
+ return internal_strlen(buf);
+}
+
+uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) {
+ return ReadBinaryName(buf, buf_len);
+}
+
+uptr MainThreadStackBase, MainThreadStackSize;
+
+bool GetRandom(void *buffer, uptr length, bool blocking) {
+ CHECK_LE(length, ZX_CPRNG_DRAW_MAX_LEN);
+ size_t size;
+ CHECK_EQ(_zx_cprng_draw(buffer, length, &size), ZX_OK);
+ CHECK_EQ(size, length);
+ return true;
+}
+
+u32 GetNumberOfCPUs() {
+ return zx_system_get_num_cpus();
+}
+
+uptr GetRSS() { UNIMPLEMENTED(); }
+
+} // namespace __sanitizer
+
+using namespace __sanitizer; // NOLINT
+
+extern "C" {
+void __sanitizer_startup_hook(int argc, char **argv, char **envp,
+ void *stack_base, size_t stack_size) {
+ __sanitizer::StoredArgv = argv;
+ __sanitizer::StoredEnviron = envp;
+ __sanitizer::MainThreadStackBase = reinterpret_cast<uintptr_t>(stack_base);
+ __sanitizer::MainThreadStackSize = stack_size;
+}
+
+void __sanitizer_set_report_path(const char *path) {
+ // Handle the initialization code in each sanitizer, but no other calls.
+ // This setting is never consulted on Fuchsia.
+ DCHECK_EQ(path, common_flags()->log_path);
+}
+
+void __sanitizer_set_report_fd(void *fd) {
+ UNREACHABLE("not available on Fuchsia");
+}
+} // extern "C"
+
+#endif // SANITIZER_FUCHSIA
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.h
new file mode 100644
index 000000000000..18821b4fd9f1
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.h
@@ -0,0 +1,31 @@
+//===-- sanitizer_fuchsia.h ------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// Fuchsia-specific sanitizer support.
+//
+//===---------------------------------------------------------------------===//
+#ifndef SANITIZER_FUCHSIA_H
+#define SANITIZER_FUCHSIA_H
+
+#include "sanitizer_platform.h"
+#if SANITIZER_FUCHSIA
+
+#include "sanitizer_common.h"
+
+#include <zircon/sanitizer.h>
+
+namespace __sanitizer {
+
+extern uptr MainThreadStackBase, MainThreadStackSize;
+extern sanitizer_shadow_bounds_t ShadowBounds;
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_FUCHSIA
+#endif // SANITIZER_FUCHSIA_H
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h
new file mode 100644
index 000000000000..934e311a7851
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_getauxval.h
@@ -0,0 +1,47 @@
+//===-- sanitizer_getauxval.h -----------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Common getauxval() guards and definitions.
+// getauxval() is not defined until glibc version 2.16, or until API level 21
+// for Android.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_GETAUXVAL_H
+#define SANITIZER_GETAUXVAL_H
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_LINUX
+
+# include <features.h>
+
+# ifndef __GLIBC_PREREQ
+# define __GLIBC_PREREQ(x, y) 0
+# endif
+
+# if __GLIBC_PREREQ(2, 16) || (SANITIZER_ANDROID && __ANDROID_API__ >= 21)
+# define SANITIZER_USE_GETAUXVAL 1
+# else
+# define SANITIZER_USE_GETAUXVAL 0
+# endif
+
+# if SANITIZER_USE_GETAUXVAL
+# include <sys/auxv.h>
+# else
+// The weak getauxval definition allows to check for the function at runtime.
+// This is useful for Android, when compiled at a lower API level yet running
+// on a more recent platform that offers the function.
+extern "C" SANITIZER_WEAK_ATTRIBUTE
+unsigned long getauxval(unsigned long type); // NOLINT
+# endif
+
+#endif // SANITIZER_LINUX
+
+#endif // SANITIZER_GETAUXVAL_H
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h
index b28d8f08e7a3..942f0b1cb586 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h
@@ -46,7 +46,6 @@ extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
void __sanitizer_report_error_summary(const char *error_summary);
- SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init();
SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump();
SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage(
const __sanitizer::uptr *pcs, const __sanitizer::uptr len);
@@ -81,6 +80,14 @@ extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
void __sanitizer_cov_trace_cmp8();
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+ void __sanitizer_cov_trace_const_cmp1();
+ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+ void __sanitizer_cov_trace_const_cmp2();
+ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+ void __sanitizer_cov_trace_const_cmp4();
+ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+ void __sanitizer_cov_trace_const_cmp8();
+ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
void __sanitizer_cov_trace_switch();
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
void __sanitizer_cov_trace_div4();
@@ -95,6 +102,10 @@ extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
void __sanitizer_cov_trace_pc_guard_init(__sanitizer::u32*,
__sanitizer::u32*);
+ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+ void __sanitizer_cov_8bit_counters_init();
+ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+ void __sanitizer_cov_pcs_init();
} // extern "C"
#endif // SANITIZER_INTERFACE_INTERNAL_H
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h
index f35b095ee94e..dc480e75f98d 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h
@@ -35,6 +35,14 @@
# define SANITIZER_WEAK_ATTRIBUTE __attribute__((weak))
#endif
+// TLS is handled differently on different platforms
+#if SANITIZER_LINUX || SANITIZER_NETBSD
+# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE \
+ __attribute__((tls_model("initial-exec"))) thread_local
+#else
+# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE
+#endif
+
//--------------------------- WEAK FUNCTIONS ---------------------------------//
// When working with weak functions, to simplify the code and make it more
// portable, when possible define a default implementation using this macro:
@@ -56,11 +64,13 @@
// SANITIZER_SUPPORTS_WEAK_HOOKS means that we support real weak functions that
// will evaluate to a null pointer when not defined.
-#if (SANITIZER_LINUX || SANITIZER_MAC) && !SANITIZER_GO
+#ifndef SANITIZER_SUPPORTS_WEAK_HOOKS
+#if (SANITIZER_LINUX || SANITIZER_MAC || SANITIZER_SOLARIS) && !SANITIZER_GO
# define SANITIZER_SUPPORTS_WEAK_HOOKS 1
#else
# define SANITIZER_SUPPORTS_WEAK_HOOKS 0
#endif
+#endif // SANITIZER_SUPPORTS_WEAK_HOOKS
// For some weak hooks that will be called very often and we want to avoid the
// overhead of executing the default implementation when it is not necessary,
// we can use the flag SANITIZER_SUPPORTS_WEAK_HOOKS to only define the default
@@ -81,6 +91,10 @@
// FIXME: do we have anything like this on Mac?
#if SANITIZER_LINUX && !SANITIZER_ANDROID && !defined(PIC)
# define SANITIZER_CAN_USE_PREINIT_ARRAY 1
+// Before Solaris 11.4, .preinit_array is fully supported only with GNU ld.
+// FIXME: Check for those conditions.
+#elif SANITIZER_SOLARIS && !defined(PIC)
+# define SANITIZER_CAN_USE_PREINIT_ARRAY 1
#else
# define SANITIZER_CAN_USE_PREINIT_ARRAY 0
#endif
@@ -127,14 +141,14 @@ typedef unsigned error_t;
typedef int fd_t;
typedef int error_t;
#endif
+#if SANITIZER_SOLARIS && !defined(_LP64)
+typedef long pid_t;
+#else
typedef int pid_t;
+#endif
-// WARNING: OFF_T may be different from OS type off_t, depending on the value of
-// _FILE_OFFSET_BITS. This definition of OFF_T matches the ABI of system calls
-// like pread and mmap, as opposed to pread64 and mmap64.
-// FreeBSD, Mac and Linux/x86-64 are special.
-#if SANITIZER_FREEBSD || SANITIZER_MAC || \
- (SANITIZER_LINUX && defined(__x86_64__))
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC || \
+ (SANITIZER_LINUX && defined(__x86_64__))
typedef u64 OFF_T;
#else
typedef uptr OFF_T;
@@ -257,8 +271,8 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond,
#define CHECK_IMPL(c1, op, c2) \
do { \
- __sanitizer::u64 v1 = (u64)(c1); \
- __sanitizer::u64 v2 = (u64)(c2); \
+ __sanitizer::u64 v1 = (__sanitizer::u64)(c1); \
+ __sanitizer::u64 v2 = (__sanitizer::u64)(c2); \
if (UNLIKELY(!(v1 op v2))) \
__sanitizer::CheckFailed(__FILE__, __LINE__, \
"(" #c1 ") " #op " (" #c2 ")", v1, v2); \
@@ -384,6 +398,7 @@ namespace __dfsan { using namespace __sanitizer; } // NOLINT
namespace __esan { using namespace __sanitizer; } // NOLINT
namespace __lsan { using namespace __sanitizer; } // NOLINT
namespace __msan { using namespace __sanitizer; } // NOLINT
+namespace __hwasan { using namespace __sanitizer; } // NOLINT
namespace __tsan { using namespace __sanitizer; } // NOLINT
namespace __scudo { using namespace __sanitizer; } // NOLINT
namespace __ubsan { using namespace __sanitizer; } // NOLINT
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc
index aa4fa88efffa..0df055edae97 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc
@@ -9,7 +9,7 @@
#include "sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || SANITIZER_NETBSD
#include "sanitizer_libignore.h"
#include "sanitizer_flags.h"
@@ -125,4 +125,5 @@ void LibIgnore::OnLibraryUnloaded() {
} // namespace __sanitizer
-#endif // #if SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC ||
+ // SANITIZER_NETBSD
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc
index 8c3c1e5d6a5d..6c83e8db42a5 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc
@@ -14,10 +14,12 @@
#include "sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS
#include "sanitizer_common.h"
#include "sanitizer_flags.h"
+#include "sanitizer_getauxval.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_libc.h"
#include "sanitizer_linux.h"
@@ -27,10 +29,14 @@
#include "sanitizer_stacktrace.h"
#include "sanitizer_symbolizer.h"
-#if !SANITIZER_FREEBSD
+#if SANITIZER_LINUX
#include <asm/param.h>
#endif
+#if SANITIZER_NETBSD
+#include <lwp.h>
+#endif
+
// For mips64, syscall(__NR_stat) fills the buffer in the 'struct kernel_stat'
// format. Struct kernel_stat is defined as 'struct stat' in asm/stat.h. To
// access stat from asm/stat.h, without conflicting with definition in
@@ -50,7 +56,9 @@
#include <pthread.h>
#include <sched.h>
#include <sys/mman.h>
+#if !SANITIZER_SOLARIS
#include <sys/ptrace.h>
+#endif
#include <sys/resource.h>
#include <sys/stat.h>
#include <sys/syscall.h>
@@ -79,22 +87,24 @@ extern "C" {
extern char **environ; // provided by crt1
#endif // SANITIZER_FREEBSD
-#if !SANITIZER_ANDROID
-#include <sys/signal.h>
-#endif
+#if SANITIZER_NETBSD
+#include <limits.h> // For NAME_MAX
+#include <sys/sysctl.h>
+extern char **environ; // provided by crt1
+#include <sys/exec.h>
+extern struct ps_strings *__ps_strings;
+#endif // SANITIZER_NETBSD
-#ifndef __GLIBC_PREREQ
-#define __GLIBC_PREREQ(x, y) 0
-#endif
+#if SANITIZER_SOLARIS
+#include <stdlib.h>
+#include <thread.h>
-#if SANITIZER_LINUX && __GLIBC_PREREQ(2, 16)
-# define SANITIZER_USE_GETAUXVAL 1
-#else
-# define SANITIZER_USE_GETAUXVAL 0
+extern char **_environ;
+#define environ _environ
#endif
-#if SANITIZER_USE_GETAUXVAL
-#include <sys/auxv.h>
+#if !SANITIZER_ANDROID
+#include <sys/signal.h>
#endif
#if SANITIZER_LINUX
@@ -125,21 +135,36 @@ extern void internal_sigreturn();
}
#endif
+#if SANITIZER_LINUX && defined(__NR_getrandom)
+# if !defined(GRND_NONBLOCK)
+# define GRND_NONBLOCK 1
+# endif
+# define SANITIZER_USE_GETRANDOM 1
+#else
+# define SANITIZER_USE_GETRANDOM 0
+#endif // SANITIZER_LINUX && defined(__NR_getrandom)
+
namespace __sanitizer {
#if SANITIZER_LINUX && defined(__x86_64__)
#include "sanitizer_syscall_linux_x86_64.inc"
#elif SANITIZER_LINUX && defined(__aarch64__)
#include "sanitizer_syscall_linux_aarch64.inc"
+#elif SANITIZER_LINUX && defined(__arm__)
+#include "sanitizer_syscall_linux_arm.inc"
#else
#include "sanitizer_syscall_generic.inc"
#endif
// --------------- sanitizer_libc.h
+#if !SANITIZER_SOLARIS
#if !SANITIZER_S390
uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
OFF_T offset) {
-#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS
+#if SANITIZER_NETBSD
+ return internal_syscall_ptr(SYSCALL(mmap), addr, length, prot, flags, fd,
+ (long)0, offset);
+#elif SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS
return internal_syscall(SYSCALL(mmap), (uptr)addr, length, prot, flags, fd,
offset);
#else
@@ -152,11 +177,11 @@ uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
#endif // !SANITIZER_S390
uptr internal_munmap(void *addr, uptr length) {
- return internal_syscall(SYSCALL(munmap), (uptr)addr, length);
+ return internal_syscall_ptr(SYSCALL(munmap), (uptr)addr, length);
}
int internal_mprotect(void *addr, uptr length, int prot) {
- return internal_syscall(SYSCALL(mprotect), (uptr)addr, length, prot);
+ return internal_syscall_ptr(SYSCALL(mprotect), (uptr)addr, length, prot);
}
uptr internal_close(fd_t fd) {
@@ -167,7 +192,7 @@ uptr internal_open(const char *filename, int flags) {
#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags);
#else
- return internal_syscall(SYSCALL(open), (uptr)filename, flags);
+ return internal_syscall_ptr(SYSCALL(open), (uptr)filename, flags);
#endif
}
@@ -176,32 +201,36 @@ uptr internal_open(const char *filename, int flags, u32 mode) {
return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags,
mode);
#else
- return internal_syscall(SYSCALL(open), (uptr)filename, flags, mode);
+ return internal_syscall_ptr(SYSCALL(open), (uptr)filename, flags, mode);
#endif
}
uptr internal_read(fd_t fd, void *buf, uptr count) {
sptr res;
- HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(read), fd, (uptr)buf,
+ HANDLE_EINTR(res, (sptr)internal_syscall_ptr(SYSCALL(read), fd, (uptr)buf,
count));
return res;
}
uptr internal_write(fd_t fd, const void *buf, uptr count) {
sptr res;
- HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(write), fd, (uptr)buf,
+ HANDLE_EINTR(res, (sptr)internal_syscall_ptr(SYSCALL(write), fd, (uptr)buf,
count));
return res;
}
uptr internal_ftruncate(fd_t fd, uptr size) {
sptr res;
+#if SANITIZER_NETBSD
+ HANDLE_EINTR(res, internal_syscall64(SYSCALL(ftruncate), fd, 0, (s64)size));
+#else
HANDLE_EINTR(res, (sptr)internal_syscall(SYSCALL(ftruncate), fd,
(OFF_T)size));
+#endif
return res;
}
-#if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && !SANITIZER_FREEBSD
+#if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && SANITIZER_LINUX
static void stat64_to_stat(struct stat64 *in, struct stat *out) {
internal_memset(out, 0, sizeof(*out));
out->st_dev = in->st_dev;
@@ -221,6 +250,21 @@ static void stat64_to_stat(struct stat64 *in, struct stat *out) {
#endif
#if defined(__mips64)
+// Undefine compatibility macros from <sys/stat.h>
+// so that they would not clash with the kernel_stat
+// st_[a|m|c]time fields
+#undef st_atime
+#undef st_mtime
+#undef st_ctime
+#if defined(SANITIZER_ANDROID)
+// Bionic sys/stat.h defines additional macros
+// for compatibility with the old NDKs and
+// they clash with the kernel_stat structure
+// st_[a|m|c]time_nsec fields.
+#undef st_atime_nsec
+#undef st_mtime_nsec
+#undef st_ctime_nsec
+#endif
static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) {
internal_memset(out, 0, sizeof(*out));
out->st_dev = in->st_dev;
@@ -233,15 +277,29 @@ static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) {
out->st_size = in->st_size;
out->st_blksize = in->st_blksize;
out->st_blocks = in->st_blocks;
- out->st_atime = in->st_atime_nsec;
- out->st_mtime = in->st_mtime_nsec;
- out->st_ctime = in->st_ctime_nsec;
+#if defined(__USE_MISC) || \
+ defined(__USE_XOPEN2K8) || \
+ defined(SANITIZER_ANDROID)
+ out->st_atim.tv_sec = in->st_atime;
+ out->st_atim.tv_nsec = in->st_atime_nsec;
+ out->st_mtim.tv_sec = in->st_mtime;
+ out->st_mtim.tv_nsec = in->st_mtime_nsec;
+ out->st_ctim.tv_sec = in->st_ctime;
+ out->st_ctim.tv_nsec = in->st_ctime_nsec;
+#else
+ out->st_atime = in->st_atime;
+ out->st_atimensec = in->st_atime_nsec;
+ out->st_mtime = in->st_mtime;
+ out->st_mtimensec = in->st_mtime_nsec;
+ out->st_ctime = in->st_ctime;
+ out->st_atimensec = in->st_ctime_nsec;
+#endif
}
#endif
uptr internal_stat(const char *path, void *buf) {
-#if SANITIZER_FREEBSD
- return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path,
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
+ return internal_syscall_ptr(SYSCALL(fstatat), AT_FDCWD, (uptr)path,
(uptr)buf, 0);
#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path,
@@ -265,7 +323,9 @@ uptr internal_stat(const char *path, void *buf) {
}
uptr internal_lstat(const char *path, void *buf) {
-#if SANITIZER_FREEBSD
+#if SANITIZER_NETBSD
+ return internal_syscall_ptr(SYSCALL(lstat), path, buf);
+#elif SANITIZER_FREEBSD
return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path,
(uptr)buf, AT_SYMLINK_NOFOLLOW);
#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
@@ -290,15 +350,15 @@ uptr internal_lstat(const char *path, void *buf) {
}
uptr internal_fstat(fd_t fd, void *buf) {
-#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS
-# if SANITIZER_MIPS64
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS
+# if SANITIZER_MIPS64 && !SANITIZER_NETBSD
// For mips64, fstat syscall fills buffer in the format of kernel_stat
struct kernel_stat kbuf;
int res = internal_syscall(SYSCALL(fstat), fd, &kbuf);
kernel_stat_to_stat(&kbuf, (struct stat *)buf);
return res;
# else
- return internal_syscall(SYSCALL(fstat), fd, (uptr)buf);
+ return internal_syscall_ptr(SYSCALL(fstat), fd, (uptr)buf);
# endif
#else
struct stat64 buf64;
@@ -328,7 +388,7 @@ uptr internal_readlink(const char *path, char *buf, uptr bufsize) {
return internal_syscall(SYSCALL(readlinkat), AT_FDCWD,
(uptr)path, (uptr)buf, bufsize);
#else
- return internal_syscall(SYSCALL(readlink), (uptr)path, (uptr)buf, bufsize);
+ return internal_syscall_ptr(SYSCALL(readlink), path, buf, bufsize);
#endif
}
@@ -336,7 +396,7 @@ uptr internal_unlink(const char *path) {
#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
return internal_syscall(SYSCALL(unlinkat), AT_FDCWD, (uptr)path, 0);
#else
- return internal_syscall(SYSCALL(unlink), (uptr)path);
+ return internal_syscall_ptr(SYSCALL(unlink), (uptr)path);
#endif
}
@@ -345,7 +405,7 @@ uptr internal_rename(const char *oldpath, const char *newpath) {
return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
(uptr)newpath);
#else
- return internal_syscall(SYSCALL(rename), (uptr)oldpath, (uptr)newpath);
+ return internal_syscall_ptr(SYSCALL(rename), (uptr)oldpath, (uptr)newpath);
#endif
}
@@ -354,7 +414,7 @@ uptr internal_sched_yield() {
}
void internal__exit(int exitcode) {
-#if SANITIZER_FREEBSD
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
internal_syscall(SYSCALL(exit), exitcode);
#else
internal_syscall(SYSCALL(exit_group), exitcode);
@@ -366,16 +426,17 @@ unsigned int internal_sleep(unsigned int seconds) {
struct timespec ts;
ts.tv_sec = 1;
ts.tv_nsec = 0;
- int res = internal_syscall(SYSCALL(nanosleep), &ts, &ts);
+ int res = internal_syscall_ptr(SYSCALL(nanosleep), &ts, &ts);
if (res) return ts.tv_sec;
return 0;
}
uptr internal_execve(const char *filename, char *const argv[],
char *const envp[]) {
- return internal_syscall(SYSCALL(execve), (uptr)filename, (uptr)argv,
+ return internal_syscall_ptr(SYSCALL(execve), (uptr)filename, (uptr)argv,
(uptr)envp);
}
+#endif // !SANITIZER_SOLARIS
// ----------------- sanitizer_common.h
bool FileExists(const char *filename) {
@@ -393,27 +454,37 @@ bool FileExists(const char *filename) {
tid_t GetTid() {
#if SANITIZER_FREEBSD
return (uptr)pthread_self();
+#elif SANITIZER_NETBSD
+ return _lwp_self();
+#elif SANITIZER_SOLARIS
+ return (uptr)thr_self();
#else
return internal_syscall(SYSCALL(gettid));
#endif
}
+#if !SANITIZER_SOLARIS
u64 NanoTime() {
-#if SANITIZER_FREEBSD
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
timeval tv;
#else
kernel_timeval tv;
#endif
internal_memset(&tv, 0, sizeof(tv));
- internal_syscall(SYSCALL(gettimeofday), (uptr)&tv, 0);
+ internal_syscall_ptr(SYSCALL(gettimeofday), &tv, 0);
return (u64)tv.tv_sec * 1000*1000*1000 + tv.tv_usec * 1000;
}
+uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) {
+ return internal_syscall_ptr(SYSCALL(clock_gettime), clk_id, tp);
+}
+#endif // !SANITIZER_SOLARIS
+
// Like getenv, but reads env directly from /proc (on Linux) or parses the
-// 'environ' array (on FreeBSD) and does not use libc. This function should be
-// called first inside __asan_init.
+// 'environ' array (on some others) and does not use libc. This function
+// should be called first inside __asan_init.
const char *GetEnv(const char *name) {
-#if SANITIZER_FREEBSD
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_SOLARIS
if (::environ != 0) {
uptr NameLen = internal_strlen(name);
for (char **Env = ::environ; *Env != 0; Env++) {
@@ -451,13 +522,13 @@ const char *GetEnv(const char *name) {
#endif
}
-#if !SANITIZER_FREEBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
extern "C" {
SANITIZER_WEAK_ATTRIBUTE extern void *__libc_stack_end;
}
#endif
-#if !SANITIZER_GO && !SANITIZER_FREEBSD
+#if !SANITIZER_GO && !SANITIZER_FREEBSD && !SANITIZER_NETBSD
static void ReadNullSepFileToArray(const char *path, char ***arr,
int arr_size) {
char *buff;
@@ -483,7 +554,22 @@ static void ReadNullSepFileToArray(const char *path, char ***arr,
#endif
static void GetArgsAndEnv(char ***argv, char ***envp) {
-#if !SANITIZER_FREEBSD
+#if SANITIZER_FREEBSD
+ // On FreeBSD, retrieving the argument and environment arrays is done via the
+ // kern.ps_strings sysctl, which returns a pointer to a structure containing
+ // this information. See also <sys/exec.h>.
+ ps_strings *pss;
+ size_t sz = sizeof(pss);
+ if (sysctlbyname("kern.ps_strings", &pss, &sz, NULL, 0) == -1) {
+ Printf("sysctl kern.ps_strings failed\n");
+ Die();
+ }
+ *argv = pss->ps_argvstr;
+ *envp = pss->ps_envstr;
+#elif SANITIZER_NETBSD
+ *argv = __ps_strings->ps_argvstr;
+ *argv = __ps_strings->ps_envstr;
+#else
#if !SANITIZER_GO
if (&__libc_stack_end) {
#endif
@@ -498,18 +584,6 @@ static void GetArgsAndEnv(char ***argv, char ***envp) {
ReadNullSepFileToArray("/proc/self/environ", envp, kMaxEnvp);
}
#endif
-#else
- // On FreeBSD, retrieving the argument and environment arrays is done via the
- // kern.ps_strings sysctl, which returns a pointer to a structure containing
- // this information. See also <sys/exec.h>.
- ps_strings *pss;
- size_t sz = sizeof(pss);
- if (sysctlbyname("kern.ps_strings", &pss, &sz, NULL, 0) == -1) {
- Printf("sysctl kern.ps_strings failed\n");
- Die();
- }
- *argv = pss->ps_argvstr;
- *envp = pss->ps_envstr;
#endif
}
@@ -521,14 +595,32 @@ char **GetArgv() {
void ReExec() {
char **argv, **envp;
+ const char *pathname = "/proc/self/exe";
+
+#if SANITIZER_NETBSD
+ static const int name[] = {
+ CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME,
+ };
+ char path[400];
+ size_t len;
+
+ len = sizeof(path);
+ if (sysctl(name, ARRAY_SIZE(name), path, &len, NULL, 0) != -1)
+ pathname = path;
+#elif SANITIZER_SOLARIS
+ pathname = getexecname();
+ CHECK_NE(pathname, NULL);
+#endif
+
GetArgsAndEnv(&argv, &envp);
- uptr rv = internal_execve("/proc/self/exe", argv, envp);
+ uptr rv = internal_execve(pathname, argv, envp);
int rverrno;
CHECK_EQ(internal_iserror(rv, &rverrno), true);
Printf("execve failed, errno %d\n", rverrno);
Die();
}
+#if !SANITIZER_SOLARIS
enum MutexState {
MtxUnlocked = 0,
MtxLocked = 1,
@@ -547,6 +639,8 @@ void BlockingMutex::Lock() {
while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) {
#if SANITIZER_FREEBSD
_umtx_op(m, UMTX_OP_WAIT_UINT, MtxSleeping, 0, 0);
+#elif SANITIZER_NETBSD
+ sched_yield(); /* No userspace futex-like synchronization */
#else
internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAIT, MtxSleeping, 0, 0, 0);
#endif
@@ -560,6 +654,8 @@ void BlockingMutex::Unlock() {
if (v == MtxSleeping) {
#if SANITIZER_FREEBSD
_umtx_op(m, UMTX_OP_WAKE, 1, 0, 0);
+#elif SANITIZER_NETBSD
+ /* No userspace futex-like synchronization */
#else
internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAKE, 1, 0, 0, 0);
#endif
@@ -570,11 +666,23 @@ void BlockingMutex::CheckLocked() {
atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
}
+#endif // !SANITIZER_SOLARIS
// ----------------- sanitizer_linux.h
// The actual size of this structure is specified by d_reclen.
// Note that getdents64 uses a different structure format. We only provide the
// 32-bit syscall here.
+#if SANITIZER_NETBSD
+// struct dirent is different for Linux and us. At this moment, we use only
+// d_fileno (Linux call this d_ino), d_reclen, and d_name.
+struct linux_dirent {
+ u64 d_ino; // d_fileno
+ u16 d_reclen;
+ u16 d_namlen; // not used
+ u8 d_type; // not used
+ char d_name[NAME_MAX + 1];
+};
+#else
struct linux_dirent {
#if SANITIZER_X32 || defined(__aarch64__)
u64 d_ino;
@@ -589,15 +697,29 @@ struct linux_dirent {
#endif
char d_name[256];
};
+#endif
+#if !SANITIZER_SOLARIS
// Syscall wrappers.
uptr internal_ptrace(int request, int pid, void *addr, void *data) {
+#if SANITIZER_NETBSD
+ // XXX We need additional work for ptrace:
+ // - for request, we use PT_FOO whereas Linux uses PTRACE_FOO
+ // - data is int for us, but void * for Linux
+ // - Linux sometimes uses data in the case where we use addr instead
+ // At this moment, this function is used only within
+ // "#if SANITIZER_LINUX && defined(__x86_64__)" block in
+ // sanitizer_stoptheworld_linux_libcdep.cc.
+ return internal_syscall_ptr(SYSCALL(ptrace), request, pid, (uptr)addr,
+ (uptr)data);
+#else
return internal_syscall(SYSCALL(ptrace), request, pid, (uptr)addr,
(uptr)data);
+#endif
}
uptr internal_waitpid(int pid, int *status, int options) {
- return internal_syscall(SYSCALL(wait4), pid, (uptr)status, options,
+ return internal_syscall_ptr(SYSCALL(wait4), pid, (uptr)status, options,
0 /* rusage */);
}
@@ -615,12 +737,16 @@ uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count) {
#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
return internal_syscall(SYSCALL(getdents64), fd, (uptr)dirp, count);
#else
- return internal_syscall(SYSCALL(getdents), fd, (uptr)dirp, count);
+ return internal_syscall_ptr(SYSCALL(getdents), fd, (uptr)dirp, count);
#endif
}
uptr internal_lseek(fd_t fd, OFF_T offset, int whence) {
+#if SANITIZER_NETBSD
+ return internal_syscall64(SYSCALL(lseek), fd, 0, offset, whence);
+#else
return internal_syscall(SYSCALL(lseek), fd, offset, whence);
+#endif
}
#if SANITIZER_LINUX
@@ -630,7 +756,7 @@ uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) {
#endif
uptr internal_sigaltstack(const void *ss, void *oss) {
- return internal_syscall(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss);
+ return internal_syscall_ptr(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss);
}
int internal_fork() {
@@ -711,8 +837,8 @@ int internal_sigaction_syscall(int signum, const void *act, void *oldact) {
uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
__sanitizer_sigset_t *oldset) {
-#if SANITIZER_FREEBSD
- return internal_syscall(SYSCALL(sigprocmask), how, set, oldset);
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
+ return internal_syscall_ptr(SYSCALL(sigprocmask), how, set, oldset);
#else
__sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set;
__sanitizer_kernel_sigset_t *k_oldset = (__sanitizer_kernel_sigset_t *)oldset;
@@ -751,6 +877,7 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {
return k_set->sig[idx] & (1 << bit);
}
#endif // SANITIZER_LINUX
+#endif // !SANITIZER_SOLARIS
// ThreadLister implementation.
ThreadLister::ThreadLister(int pid)
@@ -856,7 +983,9 @@ static uptr GetKernelAreaSize() {
#endif // SANITIZER_WORDSIZE == 32
uptr GetMaxVirtualAddress() {
-#if SANITIZER_WORDSIZE == 64
+#if SANITIZER_NETBSD && defined(__x86_64__)
+ return 0x7f7ffffff000ULL; // (0x00007f8000000000 - PAGE_SIZE)
+#elif SANITIZER_WORDSIZE == 64
# if defined(__powerpc64__) || defined(__aarch64__)
// On PowerPC64 we have two different address space layouts: 44- and 46-bit.
// We somehow need to figure out which one we are using now and choose
@@ -877,15 +1006,21 @@ uptr GetMaxVirtualAddress() {
# if defined(__s390__)
return (1ULL << 31) - 1; // 0x7fffffff;
# else
- uptr res = (1ULL << 32) - 1; // 0xffffffff;
- if (!common_flags()->full_address_space)
- res -= GetKernelAreaSize();
- CHECK_LT(reinterpret_cast<uptr>(&res), res);
- return res;
+ return (1ULL << 32) - 1; // 0xffffffff;
# endif
#endif // SANITIZER_WORDSIZE
}
+uptr GetMaxUserVirtualAddress() {
+ uptr addr = GetMaxVirtualAddress();
+#if SANITIZER_WORDSIZE == 32 && !defined(__s390__)
+ if (!common_flags()->full_address_space)
+ addr -= GetKernelAreaSize();
+ CHECK_LT(reinterpret_cast<uptr>(&addr), addr);
+#endif
+ return addr;
+}
+
uptr GetPageSize() {
// Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array.
#if SANITIZER_ANDROID
@@ -900,8 +1035,17 @@ uptr GetPageSize() {
}
uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
+#if SANITIZER_SOLARIS
+ const char *default_module_name = getexecname();
+ CHECK_NE(default_module_name, NULL);
+ return internal_snprintf(buf, buf_len, "%s", default_module_name);
+#else
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
#if SANITIZER_FREEBSD
- const int Mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 };
+ const int Mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1};
+#else
+ const int Mib[4] = {CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME};
+#endif
const char *default_module_name = "kern.proc.pathname";
size_t Size = buf_len;
bool IsErr = (sysctl(Mib, ARRAY_SIZE(Mib), buf, &Size, NULL, 0) != 0);
@@ -913,7 +1057,7 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
default_module_name, buf, buf_len);
int readlink_error;
bool IsErr = internal_iserror(module_name_len, &readlink_error);
-#endif
+#endif // SANITIZER_SOLARIS
if (IsErr) {
// We can't read binary name for some reason, assume it's unknown.
Report("WARNING: reading executable name failed with errno %d, "
@@ -923,6 +1067,7 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
CHECK_LT(module_name_len, buf_len);
}
return module_name_len;
+#endif
}
uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) {
@@ -1539,12 +1684,17 @@ static bool Aarch64GetESR(ucontext_t *ucontext, u64 *esr) {
}
#endif
-SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) {
+SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
ucontext_t *ucontext = (ucontext_t *)context;
#if defined(__x86_64__) || defined(__i386__)
static const uptr PF_WRITE = 1U << 1;
#if SANITIZER_FREEBSD
uptr err = ucontext->uc_mcontext.mc_err;
+#elif SANITIZER_NETBSD
+ uptr err = ucontext->uc_mcontext.__gregs[_REG_ERR];
+#elif SANITIZER_SOLARIS && defined(__i386__)
+# define ERR 13
+ uptr err = ucontext->uc_mcontext.gregs[ERR];
#else
uptr err = ucontext->uc_mcontext.gregs[REG_ERR];
#endif
@@ -1558,6 +1708,12 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) {
u64 esr;
if (!Aarch64GetESR(ucontext, &esr)) return UNKNOWN;
return esr & ESR_ELx_WNR ? WRITE : READ;
+#elif SANITIZER_SOLARIS && defined(__sparc__)
+ // Decode the instruction to determine the access type.
+ // From OpenSolaris $SRC/uts/sun4/os/trap.c (get_accesstype).
+ uptr pc = ucontext->uc_mcontext.gregs[REG_PC];
+ u32 instr = *(u32 *)pc;
+ return (instr >> 21) & 1 ? WRITE: READ;
#else
(void)ucontext;
return UNKNOWN; // FIXME: Implement.
@@ -1568,7 +1724,7 @@ void SignalContext::DumpAllRegisters(void *context) {
// FIXME: Implement this.
}
-void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
+static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
#if defined(__arm__)
ucontext_t *ucontext = (ucontext_t*)context;
*pc = ucontext->uc_mcontext.arm_pc;
@@ -1591,6 +1747,11 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
*pc = ucontext->uc_mcontext.mc_rip;
*bp = ucontext->uc_mcontext.mc_rbp;
*sp = ucontext->uc_mcontext.mc_rsp;
+#elif SANITIZER_NETBSD
+ ucontext_t *ucontext = (ucontext_t *)context;
+ *pc = ucontext->uc_mcontext.__gregs[_REG_RIP];
+ *bp = ucontext->uc_mcontext.__gregs[_REG_RBP];
+ *sp = ucontext->uc_mcontext.__gregs[_REG_RSP];
# else
ucontext_t *ucontext = (ucontext_t*)context;
*pc = ucontext->uc_mcontext.gregs[REG_RIP];
@@ -1603,8 +1764,26 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
*pc = ucontext->uc_mcontext.mc_eip;
*bp = ucontext->uc_mcontext.mc_ebp;
*sp = ucontext->uc_mcontext.mc_esp;
+#elif SANITIZER_NETBSD
+ ucontext_t *ucontext = (ucontext_t *)context;
+ *pc = ucontext->uc_mcontext.__gregs[_REG_EIP];
+ *bp = ucontext->uc_mcontext.__gregs[_REG_EBP];
+ *sp = ucontext->uc_mcontext.__gregs[_REG_ESP];
# else
ucontext_t *ucontext = (ucontext_t*)context;
+# if SANITIZER_SOLARIS
+ /* Use the numeric values: the symbolic ones are undefined by llvm
+ include/llvm/Support/Solaris.h. */
+# ifndef REG_EIP
+# define REG_EIP 14 // REG_PC
+# endif
+# ifndef REG_EBP
+# define REG_EBP 6 // REG_FP
+# endif
+# ifndef REG_ESP
+# define REG_ESP 17 // REG_SP
+# endif
+# endif
*pc = ucontext->uc_mcontext.gregs[REG_EIP];
*bp = ucontext->uc_mcontext.gregs[REG_EBP];
*sp = ucontext->uc_mcontext.gregs[REG_ESP];
@@ -1619,7 +1798,16 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
#elif defined(__sparc__)
ucontext_t *ucontext = (ucontext_t*)context;
uptr *stk_ptr;
-# if defined (__arch64__)
+# if defined (__sparcv9)
+# ifndef MC_PC
+# define MC_PC REG_PC
+# endif
+# ifndef MC_O6
+# define MC_O6 REG_O6
+# endif
+# ifdef SANITIZER_SOLARIS
+# define mc_gregs gregs
+# endif
*pc = ucontext->uc_mcontext.mc_gregs[MC_PC];
*sp = ucontext->uc_mcontext.mc_gregs[MC_O6];
stk_ptr = (uptr *) (*sp + 2047);
@@ -1649,6 +1837,8 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
#endif
}
+void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); }
+
void MaybeReexec() {
// No need to re-exec on Linux.
}
@@ -1676,25 +1866,27 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
return 0;
}
-bool GetRandom(void *buffer, uptr length) {
+bool GetRandom(void *buffer, uptr length, bool blocking) {
if (!buffer || !length || length > 256)
return false;
-#if defined(__NR_getrandom)
+#if SANITIZER_USE_GETRANDOM
static atomic_uint8_t skip_getrandom_syscall;
if (!atomic_load_relaxed(&skip_getrandom_syscall)) {
// Up to 256 bytes, getrandom will not be interrupted.
- uptr res = internal_syscall(SYSCALL(getrandom), buffer, length, 0);
+ uptr res = internal_syscall(SYSCALL(getrandom), buffer, length,
+ blocking ? 0 : GRND_NONBLOCK);
int rverrno = 0;
if (internal_iserror(res, &rverrno) && rverrno == ENOSYS)
atomic_store_relaxed(&skip_getrandom_syscall, 1);
else if (res == length)
return true;
}
-#endif
+#endif // SANITIZER_USE_GETRANDOM
+ // Up to 256 bytes, a read off /dev/urandom will not be interrupted.
+ // blocking is moot here, O_NONBLOCK has no effect when opening /dev/urandom.
uptr fd = internal_open("/dev/urandom", O_RDONLY);
if (internal_iserror(fd))
return false;
- // internal_read deals with EINTR.
uptr res = internal_read(fd, buffer, length);
if (internal_iserror(res))
return false;
@@ -1704,4 +1896,6 @@ bool GetRandom(void *buffer, uptr length) {
} // namespace __sanitizer
-#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD ||
+ // SANITIZER_SOLARIS
+
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
index 11cad6b80933..2d227f82a99e 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
@@ -14,11 +14,14 @@
#define SANITIZER_LINUX_H
#include "sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS
#include "sanitizer_common.h"
#include "sanitizer_internal_defs.h"
-#include "sanitizer_posix.h"
+#include "sanitizer_platform_limits_netbsd.h"
#include "sanitizer_platform_limits_posix.h"
+#include "sanitizer_platform_limits_solaris.h"
+#include "sanitizer_posix.h"
struct link_map; // Opaque type returned by dlopen().
@@ -27,11 +30,25 @@ namespace __sanitizer {
// the one in <dirent.h>, which is used by readdir().
struct linux_dirent;
+struct ProcSelfMapsBuff {
+ char *data;
+ uptr mmaped_size;
+ uptr len;
+};
+
+struct MemoryMappingLayoutData {
+ ProcSelfMapsBuff proc_self_maps;
+ const char *current;
+};
+
+void ReadProcMaps(ProcSelfMapsBuff *proc_maps);
+
// Syscall wrappers.
uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count);
uptr internal_sigaltstack(const void* ss, void* oss);
uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
__sanitizer_sigset_t *oldset);
+uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp);
// Linux-only syscalls.
#if SANITIZER_LINUX
@@ -128,5 +145,6 @@ ALWAYS_INLINE uptr *get_android_tls_ptr() {
} // namespace __sanitizer
-#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD ||
+ // SANITIZER_SOLARIS
#endif // SANITIZER_LINUX_H
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc
index 52196db12731..56fdfc8705f3 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cc
@@ -14,11 +14,13 @@
#include "sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS
#include "sanitizer_allocator_internal.h"
#include "sanitizer_atomic.h"
#include "sanitizer_common.h"
+#include "sanitizer_file.h"
#include "sanitizer_flags.h"
#include "sanitizer_freebsd.h"
#include "sanitizer_linux.h"
@@ -36,19 +38,35 @@
#if SANITIZER_FREEBSD
#include <pthread_np.h>
#include <osreldate.h>
+#include <sys/sysctl.h>
#define pthread_getattr_np pthread_attr_get_np
#endif
+#if SANITIZER_NETBSD
+#include <sys/sysctl.h>
+#include <sys/tls.h>
+#endif
+
+#if SANITIZER_SOLARIS
+#include <thread.h>
+#endif
+
#if SANITIZER_LINUX
#include <sys/prctl.h>
#endif
#if SANITIZER_ANDROID
#include <android/api-level.h>
+#if !defined(CPU_COUNT) && !defined(__aarch64__)
+#include <dirent.h>
+#include <fcntl.h>
+struct __sanitizer::linux_dirent {
+ long d_ino;
+ off_t d_off;
+ unsigned short d_reclen;
+ char d_name[];
+};
#endif
-
-#if SANITIZER_ANDROID && __ANDROID_API__ < 21
-#include <android/log.h>
#endif
#if !SANITIZER_ANDROID
@@ -102,13 +120,20 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
*stack_bottom = segment.end - stacksize;
return;
}
+ uptr stacksize = 0;
+ void *stackaddr = nullptr;
+#if SANITIZER_SOLARIS
+ stack_t ss;
+ CHECK_EQ(thr_stksegment(&ss), 0);
+ stacksize = ss.ss_size;
+ stackaddr = (char *)ss.ss_sp - stacksize;
+#else // !SANITIZER_SOLARIS
pthread_attr_t attr;
pthread_attr_init(&attr);
CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
- uptr stacksize = 0;
- void *stackaddr = nullptr;
my_pthread_attr_getstack(&attr, &stackaddr, &stacksize);
pthread_attr_destroy(&attr);
+#endif // SANITIZER_SOLARIS
*stack_top = (uptr)stackaddr + stacksize;
*stack_bottom = (uptr)stackaddr;
@@ -148,7 +173,8 @@ bool SanitizerGetThreadName(char *name, int max_len) {
#endif
}
-#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO
+#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO && \
+ !SANITIZER_NETBSD && !SANITIZER_SOLARIS
static uptr g_tls_size;
#ifdef __i386__
@@ -176,7 +202,8 @@ void InitTlsSize() {
}
#else
void InitTlsSize() { }
-#endif // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO
+#endif // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO &&
+ // !SANITIZER_NETBSD && !SANITIZER_SOLARIS
#if (defined(__x86_64__) || defined(__i386__) || defined(__mips__) \
|| defined(__aarch64__) || defined(__powerpc64__) || defined(__s390__) \
@@ -323,7 +350,7 @@ static void **ThreadSelfSegbase() {
// sysarch(AMD64_GET_FSBASE, segbase);
__asm __volatile("movq %%fs:0, %0" : "=r" (segbase));
# else
-# error "unsupported CPU arch for FreeBSD platform"
+# error "unsupported CPU arch"
# endif
return segbase;
}
@@ -333,6 +360,35 @@ uptr ThreadSelf() {
}
#endif // SANITIZER_FREEBSD
+#if SANITIZER_NETBSD
+static struct tls_tcb * ThreadSelfTlsTcb() {
+ struct tls_tcb * tcb;
+# ifdef __HAVE___LWP_GETTCB_FAST
+ tcb = (struct tls_tcb *)__lwp_gettcb_fast();
+# elif defined(__HAVE___LWP_GETPRIVATE_FAST)
+ tcb = (struct tls_tcb *)__lwp_getprivate_fast();
+# endif
+ return tcb;
+}
+
+uptr ThreadSelf() {
+ return (uptr)ThreadSelfTlsTcb()->tcb_pthread;
+}
+
+int GetSizeFromHdr(struct dl_phdr_info *info, size_t size, void *data) {
+ const Elf_Phdr *hdr = info->dlpi_phdr;
+ const Elf_Phdr *last_hdr = hdr + info->dlpi_phnum;
+
+ for (; hdr != last_hdr; ++hdr) {
+ if (hdr->p_type == PT_TLS && info->dlpi_tls_modid == 1) {
+ *(uptr*)data = hdr->p_memsz;
+ break;
+ }
+ }
+ return 0;
+}
+#endif // SANITIZER_NETBSD
+
#if !SANITIZER_GO
static void GetTls(uptr *addr, uptr *size) {
#if SANITIZER_LINUX && !SANITIZER_ANDROID
@@ -362,9 +418,27 @@ static void GetTls(uptr *addr, uptr *size) {
*addr = (uptr) dtv[2];
*size = (*addr == 0) ? 0 : ((uptr) segbase[0] - (uptr) dtv[2]);
}
+#elif SANITIZER_NETBSD
+ struct tls_tcb * const tcb = ThreadSelfTlsTcb();
+ *addr = 0;
+ *size = 0;
+ if (tcb != 0) {
+ // Find size (p_memsz) of dlpi_tls_modid 1 (TLS block of the main program).
+ // ld.elf_so hardcodes the index 1.
+ dl_iterate_phdr(GetSizeFromHdr, size);
+
+ if (*size != 0) {
+ // The block has been found and tcb_dtv[1] contains the base address
+ *addr = (uptr)tcb->tcb_dtv[1];
+ }
+ }
#elif SANITIZER_ANDROID
*addr = 0;
*size = 0;
+#elif SANITIZER_SOLARIS
+ // FIXME
+ *addr = 0;
+ *size = 0;
#else
# error "Unknown OS"
#endif
@@ -373,7 +447,8 @@ static void GetTls(uptr *addr, uptr *size) {
#if !SANITIZER_GO
uptr GetTlsSize() {
-#if SANITIZER_FREEBSD || SANITIZER_ANDROID
+#if SANITIZER_FREEBSD || SANITIZER_ANDROID || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS
uptr addr, size;
GetTls(&addr, &size);
return size;
@@ -419,7 +494,7 @@ typedef ElfW(Phdr) Elf_Phdr;
# endif
struct DlIteratePhdrData {
- InternalMmapVector<LoadedModule> *modules;
+ InternalMmapVectorNoCtor<LoadedModule> *modules;
bool first;
};
@@ -457,21 +532,41 @@ extern "C" __attribute__((weak)) int dl_iterate_phdr(
int (*)(struct dl_phdr_info *, size_t, void *), void *);
#endif
-void ListOfModules::init() {
- clear();
+static bool requiresProcmaps() {
#if SANITIZER_ANDROID && __ANDROID_API__ <= 22
- u32 api_level = AndroidGetApiLevel();
// Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken.
// The runtime check allows the same library to work with
// both K and L (and future) Android releases.
- if (api_level <= ANDROID_LOLLIPOP_MR1) { // L or earlier
- MemoryMappingLayout memory_mapping(false);
- memory_mapping.DumpListOfModules(&modules_);
- return;
- }
+ return AndroidGetApiLevel() <= ANDROID_LOLLIPOP_MR1;
+#else
+ return false;
#endif
- DlIteratePhdrData data = {&modules_, true};
- dl_iterate_phdr(dl_iterate_phdr_cb, &data);
+}
+
+static void procmapsInit(InternalMmapVectorNoCtor<LoadedModule> *modules) {
+ MemoryMappingLayout memory_mapping(/*cache_enabled*/true);
+ memory_mapping.DumpListOfModules(modules);
+}
+
+void ListOfModules::init() {
+ clearOrInit();
+ if (requiresProcmaps()) {
+ procmapsInit(&modules_);
+ } else {
+ DlIteratePhdrData data = {&modules_, true};
+ dl_iterate_phdr(dl_iterate_phdr_cb, &data);
+ }
+}
+
+// When a custom loader is used, dl_iterate_phdr may not contain the full
+// list of modules. Allow callers to fall back to using procmaps.
+void ListOfModules::fallbackInit() {
+ if (!requiresProcmaps()) {
+ clearOrInit();
+ procmapsInit(&modules_);
+ } else {
+ clear();
+ }
}
// getrusage does not give us the current RSS, only the max RSS.
@@ -513,12 +608,65 @@ uptr GetRSS() {
return rss * GetPageSizeCached();
}
-// 64-bit Android targets don't provide the deprecated __android_log_write.
-// Starting with the L release, syslog() works and is preferable to
-// __android_log_write.
+// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used on most platforms as
+// they allocate memory.
+u32 GetNumberOfCPUs() {
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
+ u32 ncpu;
+ int req[2];
+ size_t len = sizeof(ncpu);
+ req[0] = CTL_HW;
+ req[1] = HW_NCPU;
+ CHECK_EQ(sysctl(req, 2, &ncpu, &len, NULL, 0), 0);
+ return ncpu;
+#elif SANITIZER_ANDROID && !defined(CPU_COUNT) && !defined(__aarch64__)
+ // Fall back to /sys/devices/system/cpu on Android when cpu_set_t doesn't
+ // exist in sched.h. That is the case for toolchains generated with older
+ // NDKs.
+ // This code doesn't work on AArch64 because internal_getdents makes use of
+ // the 64bit getdents syscall, but cpu_set_t seems to always exist on AArch64.
+ uptr fd = internal_open("/sys/devices/system/cpu", O_RDONLY | O_DIRECTORY);
+ if (internal_iserror(fd))
+ return 0;
+ InternalScopedBuffer<u8> buffer(4096);
+ uptr bytes_read = buffer.size();
+ uptr n_cpus = 0;
+ u8 *d_type;
+ struct linux_dirent *entry = (struct linux_dirent *)&buffer[bytes_read];
+ while (true) {
+ if ((u8 *)entry >= &buffer[bytes_read]) {
+ bytes_read = internal_getdents(fd, (struct linux_dirent *)buffer.data(),
+ buffer.size());
+ if (internal_iserror(bytes_read) || !bytes_read)
+ break;
+ entry = (struct linux_dirent *)buffer.data();
+ }
+ d_type = (u8 *)entry + entry->d_reclen - 1;
+ if (d_type >= &buffer[bytes_read] ||
+ (u8 *)&entry->d_name[3] >= &buffer[bytes_read])
+ break;
+ if (entry->d_ino != 0 && *d_type == DT_DIR) {
+ if (entry->d_name[0] == 'c' && entry->d_name[1] == 'p' &&
+ entry->d_name[2] == 'u' &&
+ entry->d_name[3] >= '0' && entry->d_name[3] <= '9')
+ n_cpus++;
+ }
+ entry = (struct linux_dirent *)(((u8 *)entry) + entry->d_reclen);
+ }
+ internal_close(fd);
+ return n_cpus;
+#elif SANITIZER_SOLARIS
+ return sysconf(_SC_NPROCESSORS_ONLN);
+#else
+ cpu_set_t CPUs;
+ CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0);
+ return CPU_COUNT(&CPUs);
+#endif
+}
+
#if SANITIZER_LINUX
-#if SANITIZER_ANDROID
+# if SANITIZER_ANDROID
static atomic_uint8_t android_log_initialized;
void AndroidLogInit() {
@@ -529,34 +677,97 @@ void AndroidLogInit() {
static bool ShouldLogAfterPrintf() {
return atomic_load(&android_log_initialized, memory_order_acquire);
}
-#else
-void AndroidLogInit() {}
-static bool ShouldLogAfterPrintf() { return true; }
-#endif // SANITIZER_ANDROID
+extern "C" SANITIZER_WEAK_ATTRIBUTE
+int async_safe_write_log(int pri, const char* tag, const char* msg);
+extern "C" SANITIZER_WEAK_ATTRIBUTE
+int __android_log_write(int prio, const char* tag, const char* msg);
+
+// ANDROID_LOG_INFO is 4, but can't be resolved at runtime.
+#define SANITIZER_ANDROID_LOG_INFO 4
+// async_safe_write_log is a new public version of __libc_write_log that is
+// used behind syslog. It is preferable to syslog as it will not do any dynamic
+// memory allocation or formatting.
+// If the function is not available, syslog is preferred for L+ (it was broken
+// pre-L) as __android_log_write triggers a racey behavior with the strncpy
+// interceptor. Fallback to __android_log_write pre-L.
void WriteOneLineToSyslog(const char *s) {
-#if SANITIZER_ANDROID &&__ANDROID_API__ < 21
- __android_log_write(ANDROID_LOG_INFO, NULL, s);
-#else
- syslog(LOG_INFO, "%s", s);
-#endif
+ if (&async_safe_write_log) {
+ async_safe_write_log(SANITIZER_ANDROID_LOG_INFO, GetProcessName(), s);
+ } else if (AndroidGetApiLevel() > ANDROID_KITKAT) {
+ syslog(LOG_INFO, "%s", s);
+ } else {
+ CHECK(&__android_log_write);
+ __android_log_write(SANITIZER_ANDROID_LOG_INFO, nullptr, s);
+ }
}
+extern "C" SANITIZER_WEAK_ATTRIBUTE
+void android_set_abort_message(const char *);
+
+void SetAbortMessage(const char *str) {
+ if (&android_set_abort_message)
+ android_set_abort_message(str);
+}
+# else
+void AndroidLogInit() {}
+
+static bool ShouldLogAfterPrintf() { return true; }
+
+void WriteOneLineToSyslog(const char *s) { syslog(LOG_INFO, "%s", s); }
+
+void SetAbortMessage(const char *str) {}
+# endif // SANITIZER_ANDROID
+
void LogMessageOnPrintf(const char *str) {
if (common_flags()->log_to_syslog && ShouldLogAfterPrintf())
WriteToSyslog(str);
}
-#if SANITIZER_ANDROID && __ANDROID_API__ >= 21
-extern "C" void android_set_abort_message(const char *msg);
-void SetAbortMessage(const char *str) { android_set_abort_message(str); }
-#else
-void SetAbortMessage(const char *str) {}
-#endif
+#endif // SANITIZER_LINUX
+
+#if SANITIZER_LINUX && !SANITIZER_GO
+// glibc crashes when using clock_gettime from a preinit_array function as the
+// vDSO function pointers haven't been initialized yet. __progname is
+// initialized after the vDSO function pointers, so if it exists, is not null
+// and is not empty, we can use clock_gettime.
+extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname;
+INLINE bool CanUseVDSO() {
+ // Bionic is safe, it checks for the vDSO function pointers to be initialized.
+ if (SANITIZER_ANDROID)
+ return true;
+ if (&__progname && __progname && *__progname)
+ return true;
+ return false;
+}
-#endif // SANITIZER_LINUX
+// MonotonicNanoTime is a timing function that can leverage the vDSO by calling
+// clock_gettime. real_clock_gettime only exists if clock_gettime is
+// intercepted, so define it weakly and use it if available.
+extern "C" SANITIZER_WEAK_ATTRIBUTE
+int real_clock_gettime(u32 clk_id, void *tp);
+u64 MonotonicNanoTime() {
+ timespec ts;
+ if (CanUseVDSO()) {
+ if (&real_clock_gettime)
+ real_clock_gettime(CLOCK_MONOTONIC, &ts);
+ else
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ } else {
+ internal_clock_gettime(CLOCK_MONOTONIC, &ts);
+ }
+ return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
+}
+#else
+// Non-Linux & Go always use the syscall.
+u64 MonotonicNanoTime() {
+ timespec ts;
+ internal_clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
+}
+#endif // SANITIZER_LINUX && !SANITIZER_GO
} // namespace __sanitizer
-#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc
index 1edd4157fd6b..e1c51f580874 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc
@@ -23,6 +23,7 @@
#include <stdio.h>
#include "sanitizer_common.h"
+#include "sanitizer_file.h"
#include "sanitizer_flags.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_libc.h"
@@ -184,7 +185,7 @@ uptr internal_getpid() {
int internal_sigaction(int signum, const void *act, void *oldact) {
return sigaction(signum,
- (struct sigaction *)act, (struct sigaction *)oldact);
+ (const struct sigaction *)act, (struct sigaction *)oldact);
}
void internal_sigfillset(__sanitizer_sigset_t *set) { sigfillset(set); }
@@ -364,6 +365,10 @@ u64 NanoTime() {
return 0;
}
+u64 MonotonicNanoTime() {
+ return 0;
+}
+
uptr GetTlsSize() {
return 0;
}
@@ -410,11 +415,13 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
}
void ListOfModules::init() {
- clear();
+ clearOrInit();
MemoryMappingLayout memory_mapping(false);
memory_mapping.DumpListOfModules(&modules_);
}
+void ListOfModules::fallbackInit() { clear(); }
+
static HandleSignalMode GetHandleSignalModeImpl(int signum) {
switch (signum) {
case SIGABRT:
@@ -573,7 +580,7 @@ void LogFullErrorReport(const char *buffer) {
#endif
}
-SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) {
+SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
#if defined(__x86_64__) || defined(__i386__)
ucontext_t *ucontext = static_cast<ucontext_t*>(context);
return ucontext->uc_mcontext->__es.__err & 2 /*T_PF_WRITE*/ ? WRITE : READ;
@@ -582,7 +589,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag(void *context) {
#endif
}
-void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
+static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
ucontext_t *ucontext = (ucontext_t*)context;
# if defined(__aarch64__)
*pc = ucontext->uc_mcontext->__ss.__pc;
@@ -609,6 +616,8 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) {
# endif
}
+void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); }
+
#if !SANITIZER_GO
static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES";
LowLevelAllocator allocator_for_env;
@@ -736,6 +745,9 @@ void MaybeReexec() {
if (!lib_is_in_env)
return;
+ if (!common_flags()->strip_env)
+ return;
+
// DYLD_INSERT_LIBRARIES is set and contains the runtime library. Let's remove
// the dylib from the environment variable, because interceptors are installed
// and we don't want our children to inherit the variable.
@@ -844,7 +856,7 @@ uptr GetTaskInfoMaxAddress() {
}
#endif
-uptr GetMaxVirtualAddress() {
+uptr GetMaxUserVirtualAddress() {
#if SANITIZER_WORDSIZE == 64
# if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM
// Get the maximum VM address
@@ -859,6 +871,10 @@ uptr GetMaxVirtualAddress() {
#endif // SANITIZER_WORDSIZE
}
+uptr GetMaxVirtualAddress() {
+ return GetMaxUserVirtualAddress();
+}
+
uptr FindAvailableMemoryRange(uptr shadow_size,
uptr alignment,
uptr left_padding,
@@ -881,6 +897,11 @@ uptr FindAvailableMemoryRange(uptr shadow_size,
mach_msg_type_number_t count = kRegionInfoSize;
kr = mach_vm_region_recurse(mach_task_self(), &address, &vmsize, &depth,
(vm_region_info_t)&vminfo, &count);
+ if (kr == KERN_INVALID_ADDRESS) {
+ // No more regions beyond "address", consider the gap at the end of VM.
+ address = GetMaxVirtualAddress() + 1;
+ vmsize = 0;
+ }
if (free_begin != address) {
// We found a free region [free_begin..address-1].
uptr gap_start = RoundUpTo((uptr)free_begin + left_padding, alignment);
@@ -991,7 +1012,12 @@ void CheckNoDeepBind(const char *filename, int flag) {
}
// FIXME: implement on this platform.
-bool GetRandom(void *buffer, uptr length) {
+bool GetRandom(void *buffer, uptr length, bool blocking) {
+ UNIMPLEMENTED();
+}
+
+// FIXME: implement on this platform.
+u32 GetNumberOfCPUs() {
UNIMPLEMENTED();
}
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h
index 3f1c68c8610a..e022a2c0363c 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac.h
@@ -20,6 +20,17 @@
namespace __sanitizer {
+struct MemoryMappingLayoutData {
+ int current_image;
+ u32 current_magic;
+ u32 current_filetype;
+ ModuleArch current_arch;
+ u8 current_uuid[kModuleUUIDSize];
+ int current_load_cmd_count;
+ const char *current_load_cmd_addr;
+ bool current_instrumented;
+};
+
enum MacosVersion {
MACOS_VERSION_UNINITIALIZED = 0,
MACOS_VERSION_UNKNOWN,
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc
index c95daa9372ad..090f17c888b4 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc
@@ -20,7 +20,7 @@
namespace __sanitizer {
void RestrictMemoryToMaxAddress(uptr max_address) {
- uptr size_to_mmap = GetMaxVirtualAddress() + 1 - max_address;
+ uptr size_to_mmap = GetMaxUserVirtualAddress() + 1 - max_address;
void *res = MmapFixedNoAccess(max_address, size_to_mmap, "high gap");
CHECK(res != MAP_FAILED);
}
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h
index 1759bf13f689..4bb878ee19dc 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_mutex.h
@@ -92,8 +92,10 @@ class BlockingMutex {
// checks that the mutex is owned, and assumes callers to be generally
// well-behaved.
void CheckLocked();
+
private:
- uptr opaque_storage_[10];
+ // Solaris mutex_t has a member that requires 64-bit alignment.
+ ALIGNED(8) uptr opaque_storage_[10];
uptr owner_; // for debugging
};
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h
index 396f7c9346d6..334903c26f9f 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform.h
@@ -14,7 +14,8 @@
#define SANITIZER_PLATFORM_H
#if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__NetBSD__) && \
- !defined(__APPLE__) && !defined(_WIN32)
+ !defined(__APPLE__) && !defined(_WIN32) && !defined(__Fuchsia__) && \
+ !(defined(__sun__) && defined(__svr4__))
# error "This operating system is not supported"
#endif
@@ -36,6 +37,12 @@
# define SANITIZER_NETBSD 0
#endif
+#if defined(__sun__) && defined(__svr4__)
+# define SANITIZER_SOLARIS 1
+#else
+# define SANITIZER_SOLARIS 0
+#endif
+
#if defined(__APPLE__)
# define SANITIZER_MAC 1
# include <TargetConditionals.h>
@@ -44,7 +51,7 @@
# else
# define SANITIZER_IOS 0
# endif
-# if TARGET_IPHONE_SIMULATOR
+# if TARGET_OS_SIMULATOR
# define SANITIZER_IOSSIM 1
# else
# define SANITIZER_IOSSIM 0
@@ -85,8 +92,15 @@
# define SANITIZER_ANDROID 0
#endif
+#if defined(__Fuchsia__)
+# define SANITIZER_FUCHSIA 1
+#else
+# define SANITIZER_FUCHSIA 0
+#endif
+
#define SANITIZER_POSIX \
- (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || SANITIZER_NETBSD)
+ (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || \
+ SANITIZER_NETBSD || SANITIZER_SOLARIS)
#if __LP64__ || defined(_WIN64)
# define SANITIZER_WORDSIZE 64
@@ -175,13 +189,19 @@
# define SANITIZER_ARM 0
#endif
+#if SANITIZER_SOLARIS && SANITIZER_WORDSIZE == 32
+# define SANITIZER_SOLARIS32 1
+#else
+# define SANITIZER_SOLARIS32 0
+#endif
+
// By default we allow to use SizeClassAllocator64 on 64-bit platform.
// But in some cases (e.g. AArch64's 39-bit address space) SizeClassAllocator64
// does not work well and we need to fallback to SizeClassAllocator32.
// For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or
// change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here.
#ifndef SANITIZER_CAN_USE_ALLOCATOR64
-# if SANITIZER_ANDROID && defined(__aarch64__)
+# if (SANITIZER_ANDROID && defined(__aarch64__)) || SANITIZER_FUCHSIA
# define SANITIZER_CAN_USE_ALLOCATOR64 1
# elif defined(__mips64) || defined(__aarch64__)
# define SANITIZER_CAN_USE_ALLOCATOR64 0
@@ -276,5 +296,10 @@
# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 0
#endif
+#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD || SANITIZER_SOLARIS
+# define SANITIZER_MADVISE_DONTNEED MADV_FREE
+#else
+# define SANITIZER_MADVISE_DONTNEED MADV_DONTNEED
+#endif
#endif // SANITIZER_PLATFORM_H
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
index 0380cee92a00..7e33fa59b312 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
@@ -16,19 +16,26 @@
#include "sanitizer_internal_defs.h"
+#if SANITIZER_POSIX
+# define SI_POSIX 1
+#else
+# define SI_POSIX 0
+#endif
+
#if !SANITIZER_WINDOWS
# define SI_WINDOWS 0
-# define SI_NOT_WINDOWS 1
-# include "sanitizer_platform_limits_posix.h"
#else
# define SI_WINDOWS 1
-# define SI_NOT_WINDOWS 0
#endif
-#if SANITIZER_POSIX
-# define SI_POSIX 1
-#else
-# define SI_POSIX 0
+#if (SI_POSIX != 0) == (SI_WINDOWS != 0) && !SANITIZER_FUCHSIA
+# error "Windows is not POSIX!"
+#endif
+
+#if SI_POSIX
+# include "sanitizer_platform_limits_netbsd.h"
+# include "sanitizer_platform_limits_posix.h"
+# include "sanitizer_platform_limits_solaris.h"
#endif
#if SANITIZER_LINUX && !SANITIZER_ANDROID
@@ -75,10 +82,28 @@
# define SI_IOS 0
#endif
-#if !SANITIZER_WINDOWS && !SANITIZER_MAC
-# define SI_UNIX_NOT_MAC 1
+#if SANITIZER_FUCHSIA
+# define SI_NOT_FUCHSIA 0
+#else
+# define SI_NOT_FUCHSIA 1
+#endif
+
+#if SANITIZER_SOLARIS
+# define SI_SOLARIS 1
#else
-# define SI_UNIX_NOT_MAC 0
+# define SI_SOLARIS 0
+#endif
+
+#if SANITIZER_SOLARIS32
+# define SI_SOLARIS32 1
+#else
+# define SI_SOLARIS32 0
+#endif
+
+#if SANITIZER_POSIX && !SANITIZER_MAC
+# define SI_POSIX_NOT_MAC 1
+#else
+# define SI_POSIX_NOT_MAC 0
#endif
#if SANITIZER_LINUX && !SANITIZER_FREEBSD
@@ -87,23 +112,23 @@
# define SI_LINUX_NOT_FREEBSD 0
#endif
-#define SANITIZER_INTERCEPT_STRLEN 1
-#define SANITIZER_INTERCEPT_STRNLEN SI_NOT_MAC
-#define SANITIZER_INTERCEPT_STRCMP 1
-#define SANITIZER_INTERCEPT_STRSTR 1
-#define SANITIZER_INTERCEPT_STRCASESTR SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_STRTOK 1
-#define SANITIZER_INTERCEPT_STRCHR 1
-#define SANITIZER_INTERCEPT_STRCHRNUL SI_UNIX_NOT_MAC
-#define SANITIZER_INTERCEPT_STRRCHR 1
-#define SANITIZER_INTERCEPT_STRSPN 1
-#define SANITIZER_INTERCEPT_STRPBRK 1
-#define SANITIZER_INTERCEPT_TEXTDOMAIN SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_STRCASECMP SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_STRLEN SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRNLEN (SI_NOT_MAC && SI_NOT_FUCHSIA)
+#define SANITIZER_INTERCEPT_STRCMP SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRSTR SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRCASESTR SI_POSIX
+#define SANITIZER_INTERCEPT_STRTOK SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRCHR SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRCHRNUL SI_POSIX_NOT_MAC
+#define SANITIZER_INTERCEPT_STRRCHR SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRSPN SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_STRPBRK SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_TEXTDOMAIN SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_STRCASECMP SI_POSIX
#define SANITIZER_INTERCEPT_MEMSET 1
#define SANITIZER_INTERCEPT_MEMMOVE 1
#define SANITIZER_INTERCEPT_MEMCPY 1
-#define SANITIZER_INTERCEPT_MEMCMP 1
+#define SANITIZER_INTERCEPT_MEMCMP SI_NOT_FUCHSIA
#define SANITIZER_INTERCEPT_STRNDUP SI_POSIX
#define SANITIZER_INTERCEPT___STRNDUP SI_LINUX_NOT_FREEBSD
#if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \
@@ -114,24 +139,23 @@
#endif
// memmem on Darwin doesn't exist on 10.6
// FIXME: enable memmem on Windows.
-#define SANITIZER_INTERCEPT_MEMMEM \
- (SI_NOT_WINDOWS && !SI_MAC_DEPLOYMENT_BELOW_10_7)
-#define SANITIZER_INTERCEPT_MEMCHR 1
+#define SANITIZER_INTERCEPT_MEMMEM (SI_POSIX && !SI_MAC_DEPLOYMENT_BELOW_10_7)
+#define SANITIZER_INTERCEPT_MEMCHR SI_NOT_FUCHSIA
#define SANITIZER_INTERCEPT_MEMRCHR (SI_FREEBSD || SI_LINUX || SI_NETBSD)
-#define SANITIZER_INTERCEPT_READ SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_PREAD SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_WRITE SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_PWRITE SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_READ SI_POSIX
+#define SANITIZER_INTERCEPT_PREAD SI_POSIX
+#define SANITIZER_INTERCEPT_WRITE SI_POSIX
+#define SANITIZER_INTERCEPT_PWRITE SI_POSIX
-#define SANITIZER_INTERCEPT_FREAD SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_FWRITE SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_FREAD SI_POSIX
+#define SANITIZER_INTERCEPT_FWRITE SI_POSIX
-#define SANITIZER_INTERCEPT_PREAD64 SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_PWRITE64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_PREAD64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
+#define SANITIZER_INTERCEPT_PWRITE64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
-#define SANITIZER_INTERCEPT_READV SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_WRITEV SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_READV SI_POSIX
+#define SANITIZER_INTERCEPT_WRITEV SI_POSIX
#define SANITIZER_INTERCEPT_PREADV \
(SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
@@ -141,58 +165,68 @@
#define SANITIZER_INTERCEPT_PRCTL SI_LINUX
-#define SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_STRPTIME SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS SI_POSIX
+#define SANITIZER_INTERCEPT_STRPTIME SI_POSIX
-#define SANITIZER_INTERCEPT_SCANF SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_SCANF SI_POSIX
#define SANITIZER_INTERCEPT_ISOC99_SCANF SI_LINUX_NOT_ANDROID
#ifndef SANITIZER_INTERCEPT_PRINTF
-# define SANITIZER_INTERCEPT_PRINTF SI_NOT_WINDOWS
+# define SANITIZER_INTERCEPT_PRINTF SI_POSIX
# define SANITIZER_INTERCEPT_PRINTF_L (SI_FREEBSD || SI_NETBSD)
# define SANITIZER_INTERCEPT_ISOC99_PRINTF SI_LINUX_NOT_ANDROID
#endif
-#define SANITIZER_INTERCEPT_FREXP 1
-#define SANITIZER_INTERCEPT_FREXPF_FREXPL SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT___PRINTF_CHK \
+ (SANITIZER_INTERCEPT_PRINTF && SI_LINUX_NOT_ANDROID)
+
+#define SANITIZER_INTERCEPT_FREXP SI_NOT_FUCHSIA
+#define SANITIZER_INTERCEPT_FREXPF_FREXPL SI_POSIX
-#define SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS SI_POSIX
#define SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS \
- (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
+ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_GETPWENT \
- (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_FGETPWENT SI_LINUX_NOT_ANDROID
+ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_FGETPWENT SI_LINUX_NOT_ANDROID || SI_SOLARIS
#define SANITIZER_INTERCEPT_GETPWENT_R \
- (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_SETPWENT (SI_MAC || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_CLOCK_GETTIME (SI_FREEBSD || SI_NETBSD || SI_LINUX)
-#define SANITIZER_INTERCEPT_GETITIMER SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_TIME SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_GLOB SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_WAIT SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_INET SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_GETADDRINFO SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_GETNAMEINFO SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_GETSOCKNAME SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_GETHOSTBYNAME SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_GETHOSTBYNAME_R (SI_FREEBSD || SI_LINUX)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_SETPWENT \
+ (SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_CLOCK_GETTIME \
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_GETITIMER SI_POSIX
+#define SANITIZER_INTERCEPT_TIME SI_POSIX
+#define SANITIZER_INTERCEPT_GLOB SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_GLOB64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_WAIT SI_POSIX
+#define SANITIZER_INTERCEPT_INET SI_POSIX
+#define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM SI_POSIX
+#define SANITIZER_INTERCEPT_GETADDRINFO SI_POSIX
+#define SANITIZER_INTERCEPT_GETNAMEINFO SI_POSIX
+#define SANITIZER_INTERCEPT_GETSOCKNAME SI_POSIX
+#define SANITIZER_INTERCEPT_GETHOSTBYNAME SI_POSIX
+#define SANITIZER_INTERCEPT_GETHOSTBYNAME2 SI_POSIX && !SI_SOLARIS
+#define SANITIZER_INTERCEPT_GETHOSTBYNAME_R \
+ (SI_FREEBSD || SI_LINUX || SI_SOLARIS)
#define SANITIZER_INTERCEPT_GETHOSTBYNAME2_R \
(SI_FREEBSD || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_GETHOSTBYADDR_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_GETHOSTENT_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_GETSOCKOPT SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_ACCEPT SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_GETHOSTBYADDR_R \
+ (SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_GETHOSTENT_R \
+ (SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_GETSOCKOPT SI_POSIX
+#define SANITIZER_INTERCEPT_ACCEPT SI_POSIX
#define SANITIZER_INTERCEPT_ACCEPT4 SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_MODF SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_RECVMSG SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_SENDMSG SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_GETPEERNAME SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_IOCTL SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_INET_ATON SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_MODF SI_POSIX
+#define SANITIZER_INTERCEPT_RECVMSG SI_POSIX
+#define SANITIZER_INTERCEPT_SENDMSG SI_POSIX
+#define SANITIZER_INTERCEPT_GETPEERNAME SI_POSIX
+#define SANITIZER_INTERCEPT_IOCTL SI_POSIX
+#define SANITIZER_INTERCEPT_INET_ATON SI_POSIX
#define SANITIZER_INTERCEPT_SYSINFO SI_LINUX
-#define SANITIZER_INTERCEPT_READDIR SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_READDIR64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_READDIR SI_POSIX
+#define SANITIZER_INTERCEPT_READDIR64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
#if SI_LINUX_NOT_ANDROID && \
(defined(__i386) || defined(__x86_64) || defined(__mips64) || \
defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
@@ -201,116 +235,125 @@
#else
#define SANITIZER_INTERCEPT_PTRACE 0
#endif
-#define SANITIZER_INTERCEPT_SETLOCALE SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_GETCWD SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_SETLOCALE SI_POSIX
+#define SANITIZER_INTERCEPT_GETCWD SI_POSIX
#define SANITIZER_INTERCEPT_GET_CURRENT_DIR_NAME SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_STRTOIMAX SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_MBSTOWCS SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_MBSNRTOWCS (SI_MAC || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_WCSTOMBS SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_STRTOIMAX SI_POSIX
+#define SANITIZER_INTERCEPT_MBSTOWCS SI_POSIX
+#define SANITIZER_INTERCEPT_MBSNRTOWCS \
+ (SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_WCSTOMBS SI_POSIX
#define SANITIZER_INTERCEPT_WCSNRTOMBS \
- (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
+ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_WCRTOMB \
- (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_TCGETATTR SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_REALPATH SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME SI_LINUX_NOT_ANDROID
+ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_TCGETATTR SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_REALPATH SI_POSIX
+#define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME \
+ (SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_CONFSTR \
- (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
+ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_SCHED_GETAFFINITY SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_SCHED_GETPARAM SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_STRERROR SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_STRERROR_R SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_SCHED_GETPARAM SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_STRERROR SI_POSIX
+#define SANITIZER_INTERCEPT_STRERROR_R SI_POSIX
#define SANITIZER_INTERCEPT_XPG_STRERROR_R SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_SCANDIR \
- (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_SCANDIR64 SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_GETGROUPS SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_POLL SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_SCANDIR64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
+#define SANITIZER_INTERCEPT_GETGROUPS SI_POSIX
+#define SANITIZER_INTERCEPT_POLL SI_POSIX
+#define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID || SI_SOLARIS
#define SANITIZER_INTERCEPT_WORDEXP \
- (SI_FREEBSD || SI_NETBSD || (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_SIGWAIT SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_SIGWAITINFO SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_SIGTIMEDWAIT SI_LINUX_NOT_ANDROID
+ (SI_FREEBSD || SI_NETBSD || (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID || \
+ SI_SOLARIS)
+#define SANITIZER_INTERCEPT_SIGWAIT SI_POSIX
+#define SANITIZER_INTERCEPT_SIGWAITINFO SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_SIGTIMEDWAIT SI_LINUX_NOT_ANDROID || SI_SOLARIS
#define SANITIZER_INTERCEPT_SIGSETOPS \
- (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_SIGPENDING SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_SIGPROCMASK SI_NOT_WINDOWS
+ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_SIGPENDING SI_POSIX
+#define SANITIZER_INTERCEPT_SIGPROCMASK SI_POSIX
#define SANITIZER_INTERCEPT_BACKTRACE \
- (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_GETMNTENT SI_LINUX
#define SANITIZER_INTERCEPT_GETMNTENT_R SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_STATFS \
- (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
+ (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_STATFS64 \
((SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID)
#define SANITIZER_INTERCEPT_STATVFS \
(SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
#define SANITIZER_INTERCEPT_STATVFS64 SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_INITGROUPS SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_ETHER_NTOA_ATON SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_INITGROUPS SI_POSIX
+#define SANITIZER_INTERCEPT_ETHER_NTOA_ATON SI_POSIX
#define SANITIZER_INTERCEPT_ETHER_HOST \
(SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
#define SANITIZER_INTERCEPT_ETHER_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID)
#define SANITIZER_INTERCEPT_SHMCTL \
- (SI_NETBSD || ((SI_FREEBSD || SI_LINUX_NOT_ANDROID) && \
+ (SI_NETBSD || SI_SOLARIS || ((SI_FREEBSD || SI_LINUX_NOT_ANDROID) && \
SANITIZER_WORDSIZE == 64)) // NOLINT
#define SANITIZER_INTERCEPT_RANDOM_R SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_POSIX
#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \
- (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
+ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETTYPE SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED \
+ (SI_POSIX && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETTYPE SI_POSIX
#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPROTOCOL \
- (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID)
+ (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPRIOCEILING \
- (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST SI_LINUX_NOT_ANDROID
+ (SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST \
+ (SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST_NP SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETPSHARED SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETPSHARED \
+ (SI_POSIX && !SI_NETBSD)
#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETKIND_NP SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETPSHARED SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETCLOCK SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_TMPNAM SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_TMPNAM_R SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_TTYNAME_R SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_TEMPNAM SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_SINCOS SI_LINUX
-#define SANITIZER_INTERCEPT_REMQUO SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_LGAMMA SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_LGAMMA_R (SI_FREEBSD || SI_LINUX)
-#define SANITIZER_INTERCEPT_LGAMMAL_R SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETPSHARED \
+ (SI_POSIX && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETCLOCK \
+ (SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED \
+ (SI_LINUX_NOT_ANDROID && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_TMPNAM SI_POSIX
+#define SANITIZER_INTERCEPT_TMPNAM_R SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_TTYNAME_R SI_POSIX
+#define SANITIZER_INTERCEPT_TEMPNAM SI_POSIX
+#define SANITIZER_INTERCEPT_SINCOS SI_LINUX || SI_SOLARIS
+#define SANITIZER_INTERCEPT_REMQUO SI_POSIX
+#define SANITIZER_INTERCEPT_LGAMMA SI_POSIX
+#define SANITIZER_INTERCEPT_LGAMMA_R (SI_FREEBSD || SI_LINUX || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_LGAMMAL_R SI_LINUX_NOT_ANDROID || SI_SOLARIS
#define SANITIZER_INTERCEPT_DRAND48_R SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_RAND_R \
- (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
+ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_ICONV \
- (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_TIMES SI_NOT_WINDOWS
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_TIMES SI_POSIX
// FIXME: getline seems to be available on OSX 10.7
#define SANITIZER_INTERCEPT_GETLINE \
- (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT__EXIT \
- (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_MAC)
+ (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_MAC || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_PHTREAD_MUTEX SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_PHTREAD_MUTEX SI_POSIX
#define SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP \
- (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_TLS_GET_ADDR \
- (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_LISTXATTR SI_LINUX
#define SANITIZER_INTERCEPT_GETXATTR SI_LINUX
#define SANITIZER_INTERCEPT_GETRESID SI_LINUX
#define SANITIZER_INTERCEPT_GETIFADDRS \
- (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC || SI_SOLARIS)
#define SANITIZER_INTERCEPT_IF_INDEXTONAME \
- (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC || SI_SOLARIS)
#define SANITIZER_INTERCEPT_CAPGET SI_LINUX_NOT_ANDROID
#if SI_LINUX && defined(__arm__)
#define SANITIZER_INTERCEPT_AEABI_MEM 1
@@ -318,64 +361,71 @@
#define SANITIZER_INTERCEPT_AEABI_MEM 0
#endif
#define SANITIZER_INTERCEPT___BZERO SI_MAC
-#define SANITIZER_INTERCEPT_FTIME (!SI_FREEBSD && !SI_NETBSD && SI_NOT_WINDOWS)
-#define SANITIZER_INTERCEPT_XDR SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_FTIME (!SI_FREEBSD && !SI_NETBSD && SI_POSIX)
+#define SANITIZER_INTERCEPT_XDR SI_LINUX_NOT_ANDROID || SI_SOLARIS
#define SANITIZER_INTERCEPT_TSEARCH \
- (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD)
+ (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD || SI_SOLARIS)
#define SANITIZER_INTERCEPT_LIBIO_INTERNALS SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_FOPEN SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_FOPEN64 SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_OPEN_MEMSTREAM (SI_LINUX_NOT_ANDROID || SI_NETBSD)
+#define SANITIZER_INTERCEPT_FOPEN SI_POSIX
+#define SANITIZER_INTERCEPT_FOPEN64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
+#define SANITIZER_INTERCEPT_OPEN_MEMSTREAM \
+ (SI_LINUX_NOT_ANDROID || SI_NETBSD || SI_SOLARIS)
#define SANITIZER_INTERCEPT_OBSTACK SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_FFLUSH SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_FCLOSE SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_FFLUSH SI_POSIX
+#define SANITIZER_INTERCEPT_FCLOSE SI_POSIX
#ifndef SANITIZER_INTERCEPT_DLOPEN_DLCLOSE
#define SANITIZER_INTERCEPT_DLOPEN_DLCLOSE \
- (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC || SI_SOLARIS)
#endif
#define SANITIZER_INTERCEPT_GETPASS \
(SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD)
#define SANITIZER_INTERCEPT_TIMERFD SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_MLOCKX SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_MLOCKX SI_POSIX
#define SANITIZER_INTERCEPT_FOPENCOOKIE SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_SEM (SI_LINUX || SI_FREEBSD || SI_NETBSD)
-#define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_MINCORE (SI_LINUX || SI_NETBSD)
+#define SANITIZER_INTERCEPT_SEM \
+ (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL SI_POSIX
+#define SANITIZER_INTERCEPT_MINCORE (SI_LINUX || SI_NETBSD || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PROCESS_VM_READV SI_LINUX
#define SANITIZER_INTERCEPT_CTERMID \
- (SI_LINUX || SI_MAC || SI_FREEBSD || SI_NETBSD)
-#define SANITIZER_INTERCEPT_CTERMID_R (SI_MAC || SI_FREEBSD)
+ (SI_LINUX || SI_MAC || SI_FREEBSD || SI_NETBSD || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_CTERMID_R (SI_MAC || SI_FREEBSD || SI_SOLARIS)
#define SANITIZER_INTERCEPTOR_HOOKS (SI_LINUX || SI_MAC || SI_WINDOWS)
-#define SANITIZER_INTERCEPT_RECV_RECVFROM SI_NOT_WINDOWS
-#define SANITIZER_INTERCEPT_SEND_SENDTO SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_RECV_RECVFROM SI_POSIX
+#define SANITIZER_INTERCEPT_SEND_SENDTO SI_POSIX
#define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE SI_LINUX
#define SANITIZER_INTERCEPT_STAT \
- (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD)
-#define SANITIZER_INTERCEPT___XSTAT \
- (!SANITIZER_INTERCEPT_STAT && SI_NOT_WINDOWS)
+ (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD || SI_SOLARIS)
+#define SANITIZER_INTERCEPT___XSTAT (!SANITIZER_INTERCEPT_STAT && SI_POSIX)
#define SANITIZER_INTERCEPT___XSTAT64 SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT___LXSTAT SANITIZER_INTERCEPT___XSTAT
#define SANITIZER_INTERCEPT___LXSTAT64 SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_UTMP (SI_NOT_WINDOWS && !SI_MAC && !SI_FREEBSD)
-#define SANITIZER_INTERCEPT_UTMPX (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_UTMP \
+ (SI_POSIX && !SI_MAC && !SI_FREEBSD && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_UTMPX \
+ (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD || SI_NETBSD)
#define SANITIZER_INTERCEPT_GETLOADAVG \
(SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD || SI_NETBSD)
#define SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO \
- (!SI_FREEBSD && !SI_MAC && !SI_NETBSD)
+ (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && SI_NOT_FUCHSIA)
#define SANITIZER_INTERCEPT_MEMALIGN (!SI_FREEBSD && !SI_MAC && !SI_NETBSD)
-#define SANITIZER_INTERCEPT_PVALLOC (!SI_FREEBSD && !SI_MAC && !SI_NETBSD)
-#define SANITIZER_INTERCEPT_CFREE (!SI_FREEBSD && !SI_MAC && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_PVALLOC \
+ (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && SI_NOT_FUCHSIA)
+#define SANITIZER_INTERCEPT_CFREE \
+ (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && SI_NOT_FUCHSIA)
#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC)
#define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC)
#define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_WCSCAT SI_NOT_WINDOWS
+#define SANITIZER_INTERCEPT_WCSCAT SI_POSIX
+#define SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION (!SI_WINDOWS && SI_NOT_FUCHSIA)
+#define SANITIZER_INTERCEPT_BSD_SIGNAL SI_ANDROID
#endif // #ifndef SANITIZER_PLATFORM_INTERCEPTORS_H
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc
new file mode 100644
index 000000000000..108e196e7a05
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cc
@@ -0,0 +1,363 @@
+//===-- sanitizer_platform_limits_netbsd.cc -------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific NetBSD data structures.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_NETBSD
+#include <arpa/inet.h>
+#include <dirent.h>
+#include <glob.h>
+#include <grp.h>
+#include <ifaddrs.h>
+#include <limits.h>
+#include <link_elf.h>
+#include <net/if.h>
+#include <net/if_ether.h>
+#include <net/ppp_defs.h>
+#include <net/route.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/ip_mroute.h>
+#include <poll.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stddef.h>
+#include <sys/filio.h>
+#include <sys/ipc.h>
+#include <sys/mman.h>
+#include <sys/mount.h>
+#include <sys/mqueue.h>
+#include <sys/msg.h>
+#include <sys/mtio.h>
+#include <sys/ptrace.h>
+#include <sys/resource.h>
+#include <sys/shm.h>
+#include <sys/signal.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/soundcard.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/timeb.h>
+#include <sys/times.h>
+#include <sys/timespec.h>
+#include <sys/timex.h>
+#include <sys/types.h>
+#include <sys/ucontext.h>
+#include <sys/utsname.h>
+#include <term.h>
+#include <termios.h>
+#include <time.h>
+#include <utime.h>
+#include <utmp.h>
+#include <utmpx.h>
+#include <wchar.h>
+#include <wordexp.h>
+
+// Include these after system headers to avoid name clashes and ambiguities.
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform_limits_netbsd.h"
+
+namespace __sanitizer {
+unsigned struct_utsname_sz = sizeof(struct utsname);
+unsigned struct_stat_sz = sizeof(struct stat);
+unsigned struct_rusage_sz = sizeof(struct rusage);
+unsigned struct_tm_sz = sizeof(struct tm);
+unsigned struct_passwd_sz = sizeof(struct passwd);
+unsigned struct_group_sz = sizeof(struct group);
+unsigned siginfo_t_sz = sizeof(siginfo_t);
+unsigned struct_sigaction_sz = sizeof(struct sigaction);
+unsigned struct_itimerval_sz = sizeof(struct itimerval);
+unsigned pthread_t_sz = sizeof(pthread_t);
+unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t);
+unsigned pthread_cond_t_sz = sizeof(pthread_cond_t);
+unsigned pid_t_sz = sizeof(pid_t);
+unsigned timeval_sz = sizeof(timeval);
+unsigned uid_t_sz = sizeof(uid_t);
+unsigned gid_t_sz = sizeof(gid_t);
+unsigned mbstate_t_sz = sizeof(mbstate_t);
+unsigned sigset_t_sz = sizeof(sigset_t);
+unsigned struct_timezone_sz = sizeof(struct timezone);
+unsigned struct_tms_sz = sizeof(struct tms);
+unsigned struct_sigevent_sz = sizeof(struct sigevent);
+unsigned struct_sched_param_sz = sizeof(struct sched_param);
+unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
+unsigned ucontext_t_sz = sizeof(ucontext_t);
+unsigned struct_rlimit_sz = sizeof(struct rlimit);
+unsigned struct_timespec_sz = sizeof(struct timespec);
+unsigned struct_utimbuf_sz = sizeof(struct utimbuf);
+unsigned struct_itimerspec_sz = sizeof(struct itimerspec);
+unsigned struct_timex_sz = sizeof(struct timex);
+unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds);
+unsigned struct_mq_attr_sz = sizeof(struct mq_attr);
+unsigned struct_statvfs_sz = sizeof(struct statvfs);
+
+const uptr sig_ign = (uptr)SIG_IGN;
+const uptr sig_dfl = (uptr)SIG_DFL;
+const uptr sig_err = (uptr)SIG_ERR;
+const uptr sa_siginfo = (uptr)SA_SIGINFO;
+
+int shmctl_ipc_stat = (int)IPC_STAT;
+
+unsigned struct_utmp_sz = sizeof(struct utmp);
+unsigned struct_utmpx_sz = sizeof(struct utmpx);
+
+int map_fixed = MAP_FIXED;
+
+int af_inet = (int)AF_INET;
+int af_inet6 = (int)AF_INET6;
+
+uptr __sanitizer_in_addr_sz(int af) {
+ if (af == AF_INET)
+ return sizeof(struct in_addr);
+ else if (af == AF_INET6)
+ return sizeof(struct in6_addr);
+ else
+ return 0;
+}
+
+unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
+
+int glob_nomatch = GLOB_NOMATCH;
+int glob_altdirfunc = GLOB_ALTDIRFUNC;
+
+unsigned path_max = PATH_MAX;
+
+// ioctl arguments
+unsigned struct_ifreq_sz = sizeof(struct ifreq);
+unsigned struct_termios_sz = sizeof(struct termios);
+unsigned struct_winsize_sz = sizeof(struct winsize);
+unsigned struct_mtget_sz = sizeof(struct mtget);
+unsigned struct_mtop_sz = sizeof(struct mtop);
+unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info);
+unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats);
+unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
+unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
+
+const unsigned IOCTL_NOT_PRESENT = 0;
+
+unsigned IOCTL_FIOASYNC = FIOASYNC;
+unsigned IOCTL_FIOCLEX = FIOCLEX;
+unsigned IOCTL_FIOGETOWN = FIOGETOWN;
+unsigned IOCTL_FIONBIO = FIONBIO;
+unsigned IOCTL_FIONCLEX = FIONCLEX;
+unsigned IOCTL_FIOSETOWN = FIOSETOWN;
+unsigned IOCTL_SIOCADDMULTI = SIOCADDMULTI;
+unsigned IOCTL_SIOCATMARK = SIOCATMARK;
+unsigned IOCTL_SIOCDELMULTI = SIOCDELMULTI;
+unsigned IOCTL_SIOCGIFADDR = SIOCGIFADDR;
+unsigned IOCTL_SIOCGIFBRDADDR = SIOCGIFBRDADDR;
+unsigned IOCTL_SIOCGIFCONF = SIOCGIFCONF;
+unsigned IOCTL_SIOCGIFDSTADDR = SIOCGIFDSTADDR;
+unsigned IOCTL_SIOCGIFFLAGS = SIOCGIFFLAGS;
+unsigned IOCTL_SIOCGIFMETRIC = SIOCGIFMETRIC;
+unsigned IOCTL_SIOCGIFMTU = SIOCGIFMTU;
+unsigned IOCTL_SIOCGIFNETMASK = SIOCGIFNETMASK;
+unsigned IOCTL_SIOCGPGRP = SIOCGPGRP;
+unsigned IOCTL_SIOCSIFADDR = SIOCSIFADDR;
+unsigned IOCTL_SIOCSIFBRDADDR = SIOCSIFBRDADDR;
+unsigned IOCTL_SIOCSIFDSTADDR = SIOCSIFDSTADDR;
+unsigned IOCTL_SIOCSIFFLAGS = SIOCSIFFLAGS;
+unsigned IOCTL_SIOCSIFMETRIC = SIOCSIFMETRIC;
+unsigned IOCTL_SIOCSIFMTU = SIOCSIFMTU;
+unsigned IOCTL_SIOCSIFNETMASK = SIOCSIFNETMASK;
+unsigned IOCTL_SIOCSPGRP = SIOCSPGRP;
+unsigned IOCTL_TIOCCONS = TIOCCONS;
+unsigned IOCTL_TIOCEXCL = TIOCEXCL;
+unsigned IOCTL_TIOCGETD = TIOCGETD;
+unsigned IOCTL_TIOCGPGRP = TIOCGPGRP;
+unsigned IOCTL_TIOCGWINSZ = TIOCGWINSZ;
+unsigned IOCTL_TIOCMBIC = TIOCMBIC;
+unsigned IOCTL_TIOCMBIS = TIOCMBIS;
+unsigned IOCTL_TIOCMGET = TIOCMGET;
+unsigned IOCTL_TIOCMSET = TIOCMSET;
+unsigned IOCTL_TIOCNOTTY = TIOCNOTTY;
+unsigned IOCTL_TIOCNXCL = TIOCNXCL;
+unsigned IOCTL_TIOCOUTQ = TIOCOUTQ;
+unsigned IOCTL_TIOCPKT = TIOCPKT;
+unsigned IOCTL_TIOCSCTTY = TIOCSCTTY;
+unsigned IOCTL_TIOCSETD = TIOCSETD;
+unsigned IOCTL_TIOCSPGRP = TIOCSPGRP;
+unsigned IOCTL_TIOCSTI = TIOCSTI;
+unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ;
+unsigned IOCTL_SIOCGETSGCNT = SIOCGETSGCNT;
+unsigned IOCTL_SIOCGETVIFCNT = SIOCGETVIFCNT;
+
+const int si_SEGV_MAPERR = SEGV_MAPERR;
+const int si_SEGV_ACCERR = SEGV_ACCERR;
+} // namespace __sanitizer
+
+using namespace __sanitizer;
+
+COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t));
+
+COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned));
+CHECK_TYPE_SIZE(pthread_key_t);
+
+// There are more undocumented fields in dl_phdr_info that we are not interested
+// in.
+COMPILER_CHECK(sizeof(__sanitizer_dl_phdr_info) <= sizeof(dl_phdr_info));
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum);
+
+CHECK_TYPE_SIZE(glob_t);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_offs);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_flags);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_closedir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_readdir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_opendir);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_lstat);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_stat);
+
+CHECK_TYPE_SIZE(addrinfo);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_family);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_socktype);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr);
+
+CHECK_TYPE_SIZE(hostent);
+CHECK_SIZE_AND_OFFSET(hostent, h_name);
+CHECK_SIZE_AND_OFFSET(hostent, h_aliases);
+CHECK_SIZE_AND_OFFSET(hostent, h_addrtype);
+CHECK_SIZE_AND_OFFSET(hostent, h_length);
+CHECK_SIZE_AND_OFFSET(hostent, h_addr_list);
+
+CHECK_TYPE_SIZE(iovec);
+CHECK_SIZE_AND_OFFSET(iovec, iov_base);
+CHECK_SIZE_AND_OFFSET(iovec, iov_len);
+
+CHECK_TYPE_SIZE(msghdr);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_name);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iov);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_control);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_flags);
+
+CHECK_TYPE_SIZE(cmsghdr);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type);
+
+COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent));
+CHECK_SIZE_AND_OFFSET(dirent, d_fileno);
+CHECK_SIZE_AND_OFFSET(dirent, d_reclen);
+
+CHECK_TYPE_SIZE(ifconf);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_len);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_ifcu);
+
+CHECK_TYPE_SIZE(pollfd);
+CHECK_SIZE_AND_OFFSET(pollfd, fd);
+CHECK_SIZE_AND_OFFSET(pollfd, events);
+CHECK_SIZE_AND_OFFSET(pollfd, revents);
+
+CHECK_TYPE_SIZE(nfds_t);
+
+CHECK_TYPE_SIZE(sigset_t);
+
+COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction));
+// Can't write checks for sa_handler and sa_sigaction due to them being
+// preprocessor macros.
+CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask);
+
+CHECK_TYPE_SIZE(wordexp_t);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs);
+
+CHECK_TYPE_SIZE(tm);
+CHECK_SIZE_AND_OFFSET(tm, tm_sec);
+CHECK_SIZE_AND_OFFSET(tm, tm_min);
+CHECK_SIZE_AND_OFFSET(tm, tm_hour);
+CHECK_SIZE_AND_OFFSET(tm, tm_mday);
+CHECK_SIZE_AND_OFFSET(tm, tm_mon);
+CHECK_SIZE_AND_OFFSET(tm, tm_year);
+CHECK_SIZE_AND_OFFSET(tm, tm_wday);
+CHECK_SIZE_AND_OFFSET(tm, tm_yday);
+CHECK_SIZE_AND_OFFSET(tm, tm_isdst);
+CHECK_SIZE_AND_OFFSET(tm, tm_gmtoff);
+CHECK_SIZE_AND_OFFSET(tm, tm_zone);
+
+CHECK_TYPE_SIZE(ether_addr);
+
+CHECK_TYPE_SIZE(ipc_perm);
+CHECK_SIZE_AND_OFFSET(ipc_perm, _key);
+CHECK_SIZE_AND_OFFSET(ipc_perm, _seq);
+CHECK_SIZE_AND_OFFSET(ipc_perm, uid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, gid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cuid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cgid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, mode);
+
+CHECK_TYPE_SIZE(shmid_ds);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_segsz);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_atime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_dtime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_ctime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_cpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_lpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_nattch);
+
+CHECK_TYPE_SIZE(clock_t);
+
+CHECK_TYPE_SIZE(ifaddrs);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_addr);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask);
+// Compare against the union, because we can't reach into the union in a
+// compliant way.
+#ifdef ifa_dstaddr
+#undef ifa_dstaddr
+#endif
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_dstaddr);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data);
+
+CHECK_TYPE_SIZE(timeb);
+CHECK_SIZE_AND_OFFSET(timeb, time);
+CHECK_SIZE_AND_OFFSET(timeb, millitm);
+CHECK_SIZE_AND_OFFSET(timeb, timezone);
+CHECK_SIZE_AND_OFFSET(timeb, dstflag);
+
+CHECK_TYPE_SIZE(passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_name);
+CHECK_SIZE_AND_OFFSET(passwd, pw_passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_uid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_gid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_dir);
+CHECK_SIZE_AND_OFFSET(passwd, pw_shell);
+
+CHECK_SIZE_AND_OFFSET(passwd, pw_gecos);
+
+CHECK_TYPE_SIZE(group);
+CHECK_SIZE_AND_OFFSET(group, gr_name);
+CHECK_SIZE_AND_OFFSET(group, gr_passwd);
+CHECK_SIZE_AND_OFFSET(group, gr_gid);
+CHECK_SIZE_AND_OFFSET(group, gr_mem);
+
+#endif // SANITIZER_NETBSD
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h
new file mode 100644
index 000000000000..6823004d7b41
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h
@@ -0,0 +1,582 @@
+//===-- sanitizer_platform_limits_netbsd.h --------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific NetBSD data structures.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_PLATFORM_LIMITS_NETBSD_H
+#define SANITIZER_PLATFORM_LIMITS_NETBSD_H
+
+#if SANITIZER_NETBSD
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform.h"
+
+#define _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, shift) \
+ ((link_map *)((handle) == nullptr ? nullptr : ((char *)(handle) + (shift))))
+
+#if defined(__x86_64__)
+#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
+ _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 312)
+#elif defined(__i386__)
+#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
+ _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 164)
+#endif
+
+namespace __sanitizer {
+extern unsigned struct_utsname_sz;
+extern unsigned struct_stat_sz;
+extern unsigned struct_rusage_sz;
+extern unsigned siginfo_t_sz;
+extern unsigned struct_itimerval_sz;
+extern unsigned pthread_t_sz;
+extern unsigned pthread_mutex_t_sz;
+extern unsigned pthread_cond_t_sz;
+extern unsigned pid_t_sz;
+extern unsigned timeval_sz;
+extern unsigned uid_t_sz;
+extern unsigned gid_t_sz;
+extern unsigned mbstate_t_sz;
+extern unsigned struct_timezone_sz;
+extern unsigned struct_tms_sz;
+extern unsigned struct_itimerspec_sz;
+extern unsigned struct_sigevent_sz;
+extern unsigned struct_sched_param_sz;
+extern unsigned struct_statfs_sz;
+extern unsigned struct_sockaddr_sz;
+extern unsigned ucontext_t_sz;
+
+extern unsigned struct_rlimit_sz;
+extern unsigned struct_utimbuf_sz;
+extern unsigned struct_timespec_sz;
+
+struct __sanitizer_iocb {
+ u64 aio_offset;
+ uptr aio_buf;
+ long aio_nbytes;
+ u32 aio_fildes;
+ u32 aio_lio_opcode;
+ long aio_reqprio;
+#if SANITIZER_WORDSIZE == 64
+ u8 aio_sigevent[32];
+#else
+ u8 aio_sigevent[20];
+#endif
+ u32 _state;
+ u32 _errno;
+ long _retval;
+};
+
+struct __sanitizer___sysctl_args {
+ int *name;
+ int nlen;
+ void *oldval;
+ uptr *oldlenp;
+ void *newval;
+ uptr newlen;
+};
+
+struct __sanitizer_sem_t {
+ uptr data[5];
+};
+
+struct __sanitizer_ipc_perm {
+ u32 uid;
+ u32 gid;
+ u32 cuid;
+ u32 cgid;
+ u32 mode;
+ unsigned short _seq;
+ long _key;
+};
+
+struct __sanitizer_shmid_ds {
+ __sanitizer_ipc_perm shm_perm;
+ unsigned long shm_segsz;
+ u32 shm_lpid;
+ u32 shm_cpid;
+ unsigned int shm_nattch;
+ u64 shm_atime;
+ u64 shm_dtime;
+ u64 shm_ctime;
+ void *_shm_internal;
+};
+
+extern unsigned struct_msqid_ds_sz;
+extern unsigned struct_mq_attr_sz;
+extern unsigned struct_timex_sz;
+extern unsigned struct_statvfs_sz;
+
+struct __sanitizer_iovec {
+ void *iov_base;
+ uptr iov_len;
+};
+
+struct __sanitizer_ifaddrs {
+ struct __sanitizer_ifaddrs *ifa_next;
+ char *ifa_name;
+ unsigned int ifa_flags;
+ void *ifa_addr; // (struct sockaddr *)
+ void *ifa_netmask; // (struct sockaddr *)
+ void *ifa_dstaddr; // (struct sockaddr *)
+ void *ifa_data;
+ unsigned int ifa_addrflags;
+};
+
+typedef unsigned __sanitizer_pthread_key_t;
+
+typedef long long __sanitizer_time_t;
+
+struct __sanitizer_passwd {
+ char *pw_name;
+ char *pw_passwd;
+ int pw_uid;
+ int pw_gid;
+ __sanitizer_time_t pw_change;
+ char *pw_class;
+ char *pw_gecos;
+ char *pw_dir;
+ char *pw_shell;
+ __sanitizer_time_t pw_expire;
+};
+
+struct __sanitizer_group {
+ char *gr_name;
+ char *gr_passwd;
+ int gr_gid;
+ char **gr_mem;
+};
+
+struct __sanitizer_timeb {
+ __sanitizer_time_t time;
+ unsigned short millitm;
+ short timezone;
+ short dstflag;
+};
+
+struct __sanitizer_ether_addr {
+ u8 octet[6];
+};
+
+struct __sanitizer_tm {
+ int tm_sec;
+ int tm_min;
+ int tm_hour;
+ int tm_mday;
+ int tm_mon;
+ int tm_year;
+ int tm_wday;
+ int tm_yday;
+ int tm_isdst;
+ long int tm_gmtoff;
+ const char *tm_zone;
+};
+
+struct __sanitizer_msghdr {
+ void *msg_name;
+ unsigned msg_namelen;
+ struct __sanitizer_iovec *msg_iov;
+ unsigned msg_iovlen;
+ void *msg_control;
+ unsigned msg_controllen;
+ int msg_flags;
+};
+struct __sanitizer_cmsghdr {
+ unsigned cmsg_len;
+ int cmsg_level;
+ int cmsg_type;
+};
+
+struct __sanitizer_dirent {
+ u64 d_fileno;
+ u16 d_reclen;
+ // more fields that we don't care about
+};
+
+typedef int __sanitizer_clock_t;
+typedef int __sanitizer_clockid_t;
+
+typedef u32 __sanitizer___kernel_uid_t;
+typedef u32 __sanitizer___kernel_gid_t;
+typedef u64 __sanitizer___kernel_off_t;
+typedef struct {
+ u32 fds_bits[8];
+} __sanitizer___kernel_fd_set;
+
+typedef struct {
+ unsigned int pta_magic;
+ int pta_flags;
+ void *pta_private;
+} __sanitizer_pthread_attr_t;
+
+struct __sanitizer_sigset_t {
+ // uint32_t * 4
+ unsigned int __bits[4];
+};
+
+struct __sanitizer_siginfo {
+ // The size is determined by looking at sizeof of real siginfo_t on linux.
+ u64 opaque[128 / sizeof(u64)];
+};
+
+using __sanitizer_sighandler_ptr = void (*)(int sig);
+using __sanitizer_sigactionhandler_ptr = void (*)(int sig,
+ __sanitizer_siginfo *siginfo,
+ void *uctx);
+
+struct __sanitizer_sigaction {
+ union {
+ __sanitizer_sighandler_ptr handler;
+ __sanitizer_sigactionhandler_ptr sigaction;
+ };
+ __sanitizer_sigset_t sa_mask;
+ int sa_flags;
+};
+
+typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t;
+
+struct __sanitizer_kernel_sigaction_t {
+ union {
+ void (*handler)(int signo);
+ void (*sigaction)(int signo, void *info, void *ctx);
+ };
+ unsigned long sa_flags;
+ void (*sa_restorer)(void);
+ __sanitizer_kernel_sigset_t sa_mask;
+};
+
+extern const uptr sig_ign;
+extern const uptr sig_dfl;
+extern const uptr sig_err;
+extern const uptr sa_siginfo;
+
+extern int af_inet;
+extern int af_inet6;
+uptr __sanitizer_in_addr_sz(int af);
+
+struct __sanitizer_dl_phdr_info {
+ uptr dlpi_addr;
+ const char *dlpi_name;
+ const void *dlpi_phdr;
+ short dlpi_phnum;
+};
+
+extern unsigned struct_ElfW_Phdr_sz;
+
+struct __sanitizer_addrinfo {
+ int ai_flags;
+ int ai_family;
+ int ai_socktype;
+ int ai_protocol;
+ unsigned ai_addrlen;
+ char *ai_canonname;
+ void *ai_addr;
+ struct __sanitizer_addrinfo *ai_next;
+};
+
+struct __sanitizer_hostent {
+ char *h_name;
+ char **h_aliases;
+ int h_addrtype;
+ int h_length;
+ char **h_addr_list;
+};
+
+struct __sanitizer_pollfd {
+ int fd;
+ short events;
+ short revents;
+};
+
+typedef unsigned __sanitizer_nfds_t;
+
+struct __sanitizer_glob_t {
+ uptr gl_pathc;
+ uptr gl_matchc;
+ uptr gl_offs;
+ int gl_flags;
+ char **gl_pathv;
+ int (*gl_errfunc)(const char *, int);
+ void (*gl_closedir)(void *dirp);
+ struct dirent *(*gl_readdir)(void *dirp);
+ void *(*gl_opendir)(const char *);
+ int (*gl_lstat)(const char *, void * /* struct stat* */);
+ int (*gl_stat)(const char *, void * /* struct stat* */);
+};
+
+extern int glob_nomatch;
+extern int glob_altdirfunc;
+
+extern unsigned path_max;
+
+struct __sanitizer_wordexp_t {
+ uptr we_wordc;
+ char **we_wordv;
+ uptr we_offs;
+ char *we_strings;
+ uptr we_nbytes;
+};
+
+typedef char __sanitizer_FILE;
+#define SANITIZER_HAS_STRUCT_FILE 0
+
+extern int shmctl_ipc_stat;
+
+// This simplifies generic code
+#define struct_shminfo_sz -1
+#define struct_shm_info_sz -1
+#define shmctl_shm_stat -1
+#define shmctl_ipc_info -1
+#define shmctl_shm_info -1
+
+extern unsigned struct_utmp_sz;
+extern unsigned struct_utmpx_sz;
+
+extern int map_fixed;
+
+// ioctl arguments
+struct __sanitizer_ifconf {
+ int ifc_len;
+ union {
+ void *ifcu_req;
+ } ifc_ifcu;
+};
+
+#define IOC_NRBITS 8
+#define IOC_TYPEBITS 8
+#define IOC_SIZEBITS 14
+#define IOC_DIRBITS 2
+#define IOC_NONE 0U
+#define IOC_WRITE 1U
+#define IOC_READ 2U
+#define IOC_NRMASK ((1 << IOC_NRBITS) - 1)
+#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1)
+#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1)
+#undef IOC_DIRMASK
+#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1)
+#define IOC_NRSHIFT 0
+#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS)
+#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS)
+#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS)
+#define EVIOC_EV_MAX 0x1f
+#define EVIOC_ABS_MAX 0x3f
+
+#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK)
+#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK)
+#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK)
+#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK)
+
+extern unsigned struct_ifreq_sz;
+extern unsigned struct_termios_sz;
+extern unsigned struct_winsize_sz;
+
+extern unsigned struct_arpreq_sz;
+
+extern unsigned struct_mtget_sz;
+extern unsigned struct_mtop_sz;
+extern unsigned struct_rtentry_sz;
+extern unsigned struct_sbi_instrument_sz;
+extern unsigned struct_seq_event_rec_sz;
+extern unsigned struct_synth_info_sz;
+extern unsigned struct_vt_mode_sz;
+extern unsigned struct_audio_buf_info_sz;
+extern unsigned struct_ppp_stats_sz;
+extern unsigned struct_sioc_sg_req_sz;
+extern unsigned struct_sioc_vif_req_sz;
+
+// ioctl request identifiers
+
+// A special value to mark ioctls that are not present on the target platform,
+// when it can not be determined without including any system headers.
+extern const unsigned IOCTL_NOT_PRESENT;
+
+extern unsigned IOCTL_FIOASYNC;
+extern unsigned IOCTL_FIOCLEX;
+extern unsigned IOCTL_FIOGETOWN;
+extern unsigned IOCTL_FIONBIO;
+extern unsigned IOCTL_FIONCLEX;
+extern unsigned IOCTL_FIOSETOWN;
+extern unsigned IOCTL_SIOCADDMULTI;
+extern unsigned IOCTL_SIOCATMARK;
+extern unsigned IOCTL_SIOCDELMULTI;
+extern unsigned IOCTL_SIOCGIFADDR;
+extern unsigned IOCTL_SIOCGIFBRDADDR;
+extern unsigned IOCTL_SIOCGIFCONF;
+extern unsigned IOCTL_SIOCGIFDSTADDR;
+extern unsigned IOCTL_SIOCGIFFLAGS;
+extern unsigned IOCTL_SIOCGIFMETRIC;
+extern unsigned IOCTL_SIOCGIFMTU;
+extern unsigned IOCTL_SIOCGIFNETMASK;
+extern unsigned IOCTL_SIOCGPGRP;
+extern unsigned IOCTL_SIOCSIFADDR;
+extern unsigned IOCTL_SIOCSIFBRDADDR;
+extern unsigned IOCTL_SIOCSIFDSTADDR;
+extern unsigned IOCTL_SIOCSIFFLAGS;
+extern unsigned IOCTL_SIOCSIFMETRIC;
+extern unsigned IOCTL_SIOCSIFMTU;
+extern unsigned IOCTL_SIOCSIFNETMASK;
+extern unsigned IOCTL_SIOCSPGRP;
+extern unsigned IOCTL_TIOCCONS;
+extern unsigned IOCTL_TIOCEXCL;
+extern unsigned IOCTL_TIOCGETD;
+extern unsigned IOCTL_TIOCGPGRP;
+extern unsigned IOCTL_TIOCGWINSZ;
+extern unsigned IOCTL_TIOCMBIC;
+extern unsigned IOCTL_TIOCMBIS;
+extern unsigned IOCTL_TIOCMGET;
+extern unsigned IOCTL_TIOCMSET;
+extern unsigned IOCTL_TIOCNOTTY;
+extern unsigned IOCTL_TIOCNXCL;
+extern unsigned IOCTL_TIOCOUTQ;
+extern unsigned IOCTL_TIOCPKT;
+extern unsigned IOCTL_TIOCSCTTY;
+extern unsigned IOCTL_TIOCSETD;
+extern unsigned IOCTL_TIOCSPGRP;
+extern unsigned IOCTL_TIOCSTI;
+extern unsigned IOCTL_TIOCSWINSZ;
+extern unsigned IOCTL_SIOCGETSGCNT;
+extern unsigned IOCTL_SIOCGETVIFCNT;
+extern unsigned IOCTL_MTIOCGET;
+extern unsigned IOCTL_MTIOCTOP;
+extern unsigned IOCTL_SIOCADDRT;
+extern unsigned IOCTL_SIOCDELRT;
+extern unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE;
+extern unsigned IOCTL_SNDCTL_DSP_GETFMTS;
+extern unsigned IOCTL_SNDCTL_DSP_NONBLOCK;
+extern unsigned IOCTL_SNDCTL_DSP_POST;
+extern unsigned IOCTL_SNDCTL_DSP_RESET;
+extern unsigned IOCTL_SNDCTL_DSP_SETFMT;
+extern unsigned IOCTL_SNDCTL_DSP_SETFRAGMENT;
+extern unsigned IOCTL_SNDCTL_DSP_SPEED;
+extern unsigned IOCTL_SNDCTL_DSP_STEREO;
+extern unsigned IOCTL_SNDCTL_DSP_SUBDIVIDE;
+extern unsigned IOCTL_SNDCTL_DSP_SYNC;
+extern unsigned IOCTL_SNDCTL_FM_4OP_ENABLE;
+extern unsigned IOCTL_SNDCTL_FM_LOAD_INSTR;
+extern unsigned IOCTL_SNDCTL_MIDI_INFO;
+extern unsigned IOCTL_SNDCTL_MIDI_PRETIME;
+extern unsigned IOCTL_SNDCTL_SEQ_CTRLRATE;
+extern unsigned IOCTL_SNDCTL_SEQ_GETINCOUNT;
+extern unsigned IOCTL_SNDCTL_SEQ_GETOUTCOUNT;
+extern unsigned IOCTL_SNDCTL_SEQ_NRMIDIS;
+extern unsigned IOCTL_SNDCTL_SEQ_NRSYNTHS;
+extern unsigned IOCTL_SNDCTL_SEQ_OUTOFBAND;
+extern unsigned IOCTL_SNDCTL_SEQ_PANIC;
+extern unsigned IOCTL_SNDCTL_SEQ_PERCMODE;
+extern unsigned IOCTL_SNDCTL_SEQ_RESET;
+extern unsigned IOCTL_SNDCTL_SEQ_RESETSAMPLES;
+extern unsigned IOCTL_SNDCTL_SEQ_SYNC;
+extern unsigned IOCTL_SNDCTL_SEQ_TESTMIDI;
+extern unsigned IOCTL_SNDCTL_SEQ_THRESHOLD;
+extern unsigned IOCTL_SNDCTL_SYNTH_INFO;
+extern unsigned IOCTL_SNDCTL_SYNTH_MEMAVL;
+extern unsigned IOCTL_SNDCTL_TMR_CONTINUE;
+extern unsigned IOCTL_SNDCTL_TMR_METRONOME;
+extern unsigned IOCTL_SNDCTL_TMR_SELECT;
+extern unsigned IOCTL_SNDCTL_TMR_SOURCE;
+extern unsigned IOCTL_SNDCTL_TMR_START;
+extern unsigned IOCTL_SNDCTL_TMR_STOP;
+extern unsigned IOCTL_SNDCTL_TMR_TEMPO;
+extern unsigned IOCTL_SNDCTL_TMR_TIMEBASE;
+extern unsigned IOCTL_SOUND_MIXER_READ_ALTPCM;
+extern unsigned IOCTL_SOUND_MIXER_READ_BASS;
+extern unsigned IOCTL_SOUND_MIXER_READ_CAPS;
+extern unsigned IOCTL_SOUND_MIXER_READ_CD;
+extern unsigned IOCTL_SOUND_MIXER_READ_DEVMASK;
+extern unsigned IOCTL_SOUND_MIXER_READ_ENHANCE;
+extern unsigned IOCTL_SOUND_MIXER_READ_IGAIN;
+extern unsigned IOCTL_SOUND_MIXER_READ_IMIX;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE1;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE2;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE3;
+extern unsigned IOCTL_SOUND_MIXER_READ_LINE;
+extern unsigned IOCTL_SOUND_MIXER_READ_LOUD;
+extern unsigned IOCTL_SOUND_MIXER_READ_MIC;
+extern unsigned IOCTL_SOUND_MIXER_READ_MUTE;
+extern unsigned IOCTL_SOUND_MIXER_READ_OGAIN;
+extern unsigned IOCTL_SOUND_MIXER_READ_PCM;
+extern unsigned IOCTL_SOUND_MIXER_READ_RECLEV;
+extern unsigned IOCTL_SOUND_MIXER_READ_RECMASK;
+extern unsigned IOCTL_SOUND_MIXER_READ_RECSRC;
+extern unsigned IOCTL_SOUND_MIXER_READ_SPEAKER;
+extern unsigned IOCTL_SOUND_MIXER_READ_STEREODEVS;
+extern unsigned IOCTL_SOUND_MIXER_READ_SYNTH;
+extern unsigned IOCTL_SOUND_MIXER_READ_TREBLE;
+extern unsigned IOCTL_SOUND_MIXER_READ_VOLUME;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_ALTPCM;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_BASS;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_CD;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_ENHANCE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_IGAIN;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_IMIX;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE1;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE2;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE3;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_LOUD;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_MIC;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_MUTE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_OGAIN;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_PCM;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_RECLEV;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_RECSRC;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_SPEAKER;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_SYNTH;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_TREBLE;
+extern unsigned IOCTL_SOUND_MIXER_WRITE_VOLUME;
+extern unsigned IOCTL_SOUND_PCM_READ_BITS;
+extern unsigned IOCTL_SOUND_PCM_READ_CHANNELS;
+extern unsigned IOCTL_SOUND_PCM_READ_FILTER;
+extern unsigned IOCTL_SOUND_PCM_READ_RATE;
+extern unsigned IOCTL_SOUND_PCM_WRITE_CHANNELS;
+extern unsigned IOCTL_SOUND_PCM_WRITE_FILTER;
+extern unsigned IOCTL_VT_ACTIVATE;
+extern unsigned IOCTL_VT_GETMODE;
+extern unsigned IOCTL_VT_OPENQRY;
+extern unsigned IOCTL_VT_RELDISP;
+extern unsigned IOCTL_VT_SETMODE;
+extern unsigned IOCTL_VT_WAITACTIVE;
+extern unsigned IOCTL_KDDISABIO;
+extern unsigned IOCTL_KDENABIO;
+extern unsigned IOCTL_KDGETLED;
+extern unsigned IOCTL_KDGKBMODE;
+extern unsigned IOCTL_KDGKBTYPE;
+extern unsigned IOCTL_KDMKTONE;
+extern unsigned IOCTL_KDSETLED;
+extern unsigned IOCTL_KDSETMODE;
+extern unsigned IOCTL_KDSKBMODE;
+
+extern const int si_SEGV_MAPERR;
+extern const int si_SEGV_ACCERR;
+} // namespace __sanitizer
+
+#define CHECK_TYPE_SIZE(TYPE) \
+ COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE))
+
+#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \
+ COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *)NULL)->MEMBER) == \
+ sizeof(((CLASS *)NULL)->MEMBER)); \
+ COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \
+ offsetof(CLASS, MEMBER))
+
+// For sigaction, which is a function and struct at the same time,
+// and thus requires explicit "struct" in sizeof() expression.
+#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \
+ COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *)NULL)->MEMBER) == \
+ sizeof(((struct CLASS *)NULL)->MEMBER)); \
+ COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \
+ offsetof(struct CLASS, MEMBER))
+
+#define SIGACTION_SYMNAME __sigaction14
+
+#endif // SANITIZER_NETBSD
+
+#endif
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc
index 83f4fd22f623..f12e8206abe6 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc
@@ -210,6 +210,7 @@ namespace __sanitizer {
unsigned struct_sigaction_sz = sizeof(struct sigaction);
unsigned struct_itimerval_sz = sizeof(struct itimerval);
unsigned pthread_t_sz = sizeof(pthread_t);
+ unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t);
unsigned pthread_cond_t_sz = sizeof(pthread_cond_t);
unsigned pid_t_sz = sizeof(pid_t);
unsigned timeval_sz = sizeof(timeval);
@@ -264,9 +265,10 @@ namespace __sanitizer {
unsigned struct_statvfs_sz = sizeof(struct statvfs);
#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID
- uptr sig_ign = (uptr)SIG_IGN;
- uptr sig_dfl = (uptr)SIG_DFL;
- uptr sa_siginfo = (uptr)SA_SIGINFO;
+ const uptr sig_ign = (uptr)SIG_IGN;
+ const uptr sig_dfl = (uptr)SIG_DFL;
+ const uptr sig_err = (uptr)SIG_ERR;
+ const uptr sa_siginfo = (uptr)SA_SIGINFO;
#if SANITIZER_LINUX
int e_tabsz = (int)E_TABSZ;
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
index 808418012d75..2853a84d43ae 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
@@ -15,6 +15,8 @@
#ifndef SANITIZER_PLATFORM_LIMITS_POSIX_H
#define SANITIZER_PLATFORM_LIMITS_POSIX_H
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC
+
#include "sanitizer_internal_defs.h"
#include "sanitizer_platform.h"
@@ -44,6 +46,7 @@ namespace __sanitizer {
extern unsigned siginfo_t_sz;
extern unsigned struct_itimerval_sz;
extern unsigned pthread_t_sz;
+ extern unsigned pthread_mutex_t_sz;
extern unsigned pthread_cond_t_sz;
extern unsigned pid_t_sz;
extern unsigned timeval_sz;
@@ -535,7 +538,7 @@ namespace __sanitizer {
typedef long __sanitizer_clock_t;
#endif
-#if SANITIZER_LINUX
+#if SANITIZER_LINUX || SANITIZER_FREEBSD
typedef int __sanitizer_clockid_t;
#endif
@@ -596,13 +599,22 @@ namespace __sanitizer {
};
#endif
+ struct __sanitizer_siginfo {
+ // The size is determined by looking at sizeof of real siginfo_t on linux.
+ u64 opaque[128 / sizeof(u64)];
+ };
+
+ using __sanitizer_sighandler_ptr = void (*)(int sig);
+ using __sanitizer_sigactionhandler_ptr =
+ void (*)(int sig, __sanitizer_siginfo *siginfo, void *uctx);
+
// Linux system headers define the 'sa_handler' and 'sa_sigaction' macros.
#if SANITIZER_ANDROID && (SANITIZER_WORDSIZE == 64)
struct __sanitizer_sigaction {
unsigned sa_flags;
union {
- void (*sigaction)(int sig, void *siginfo, void *uctx);
- void (*handler)(int sig);
+ __sanitizer_sigactionhandler_ptr sigaction;
+ __sanitizer_sighandler_ptr handler;
};
__sanitizer_sigset_t sa_mask;
void (*sa_restorer)();
@@ -611,16 +623,16 @@ namespace __sanitizer {
struct __sanitizer_sigaction {
unsigned sa_flags;
union {
- void (*sigaction)(int sig, void *siginfo, void *uctx);
- void (*handler)(int sig);
+ __sanitizer_sigactionhandler_ptr sigaction;
+ __sanitizer_sighandler_ptr handler;
};
__sanitizer_sigset_t sa_mask;
};
#elif SANITIZER_ANDROID && (SANITIZER_WORDSIZE == 32)
struct __sanitizer_sigaction {
union {
- void (*sigaction)(int sig, void *siginfo, void *uctx);
- void (*handler)(int sig);
+ __sanitizer_sigactionhandler_ptr sigaction;
+ __sanitizer_sighandler_ptr handler;
};
__sanitizer_sigset_t sa_mask;
uptr sa_flags;
@@ -632,8 +644,8 @@ namespace __sanitizer {
unsigned int sa_flags;
#endif
union {
- void (*sigaction)(int sig, void *siginfo, void *uctx);
- void (*handler)(int sig);
+ __sanitizer_sigactionhandler_ptr sigaction;
+ __sanitizer_sighandler_ptr handler;
};
#if SANITIZER_FREEBSD
int sa_flags;
@@ -692,7 +704,7 @@ namespace __sanitizer {
unsigned int sa_flags;
union {
void (*handler)(int signo);
- void (*sigaction)(int signo, void *info, void *ctx);
+ void (*sigaction)(int signo, __sanitizer_siginfo *info, void *ctx);
};
__sanitizer_kernel_sigset_t sa_mask;
void (*sa_restorer)(void);
@@ -701,7 +713,7 @@ namespace __sanitizer {
struct __sanitizer_kernel_sigaction_t {
union {
void (*handler)(int signo);
- void (*sigaction)(int signo, void *info, void *ctx);
+ void (*sigaction)(int signo, __sanitizer_siginfo *info, void *ctx);
};
unsigned long sa_flags;
void (*sa_restorer)(void);
@@ -709,9 +721,10 @@ namespace __sanitizer {
};
#endif
- extern uptr sig_ign;
- extern uptr sig_dfl;
- extern uptr sa_siginfo;
+ extern const uptr sig_ign;
+ extern const uptr sig_dfl;
+ extern const uptr sig_err;
+ extern const uptr sa_siginfo;
#if SANITIZER_LINUX
extern int e_tabsz;
@@ -1489,4 +1502,8 @@ struct __sanitizer_cookie_io_functions_t {
COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \
offsetof(struct CLASS, MEMBER))
+#define SIGACTION_SYMNAME sigaction
+
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC
+
#endif
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc
new file mode 100644
index 000000000000..15b80a81ae3b
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.cc
@@ -0,0 +1,366 @@
+//===-- sanitizer_platform_limits_solaris.cc ------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific Solaris data structures.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+
+#if SANITIZER_SOLARIS
+#include <arpa/inet.h>
+#include <dirent.h>
+#include <glob.h>
+#include <grp.h>
+#include <ifaddrs.h>
+#include <limits.h>
+#include <link.h>
+#include <net/if.h>
+#include <net/route.h>
+#include <netdb.h>
+#include <netinet/ip_mroute.h>
+#include <poll.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <rpc/xdr.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stddef.h>
+#include <sys/ethernet.h>
+#include <sys/filio.h>
+#include <sys/ipc.h>
+#include <sys/mman.h>
+#include <sys/mount.h>
+#include <sys/mtio.h>
+#include <sys/ptyvar.h>
+#include <sys/resource.h>
+#include <sys/shm.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/stat.h>
+#include <sys/statfs.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/timeb.h>
+#include <sys/times.h>
+#include <sys/types.h>
+#include <sys/utsname.h>
+#include <termios.h>
+#include <time.h>
+#include <utmp.h>
+#include <utmpx.h>
+#include <wchar.h>
+#include <wordexp.h>
+
+// Include these after system headers to avoid name clashes and ambiguities.
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform_limits_solaris.h"
+
+namespace __sanitizer {
+ unsigned struct_utsname_sz = sizeof(struct utsname);
+ unsigned struct_stat_sz = sizeof(struct stat);
+ unsigned struct_stat64_sz = sizeof(struct stat64);
+ unsigned struct_rusage_sz = sizeof(struct rusage);
+ unsigned struct_tm_sz = sizeof(struct tm);
+ unsigned struct_passwd_sz = sizeof(struct passwd);
+ unsigned struct_group_sz = sizeof(struct group);
+ unsigned siginfo_t_sz = sizeof(siginfo_t);
+ unsigned struct_sigaction_sz = sizeof(struct sigaction);
+ unsigned struct_itimerval_sz = sizeof(struct itimerval);
+ unsigned pthread_t_sz = sizeof(pthread_t);
+ unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t);
+ unsigned pthread_cond_t_sz = sizeof(pthread_cond_t);
+ unsigned pid_t_sz = sizeof(pid_t);
+ unsigned timeval_sz = sizeof(timeval);
+ unsigned uid_t_sz = sizeof(uid_t);
+ unsigned gid_t_sz = sizeof(gid_t);
+ unsigned mbstate_t_sz = sizeof(mbstate_t);
+ unsigned sigset_t_sz = sizeof(sigset_t);
+ unsigned struct_timezone_sz = sizeof(struct timezone);
+ unsigned struct_tms_sz = sizeof(struct tms);
+ unsigned struct_sigevent_sz = sizeof(struct sigevent);
+ unsigned struct_sched_param_sz = sizeof(struct sched_param);
+ unsigned struct_statfs_sz = sizeof(struct statfs);
+ unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
+ unsigned ucontext_t_sz = sizeof(ucontext_t);
+ unsigned struct_timespec_sz = sizeof(struct timespec);
+#if SANITIZER_SOLARIS32
+ unsigned struct_statvfs64_sz = sizeof(struct statvfs64);
+#endif
+ unsigned struct_statvfs_sz = sizeof(struct statvfs);
+
+ const uptr sig_ign = (uptr)SIG_IGN;
+ const uptr sig_dfl = (uptr)SIG_DFL;
+ const uptr sig_err = (uptr)SIG_ERR;
+ const uptr sa_siginfo = (uptr)SA_SIGINFO;
+
+ int shmctl_ipc_stat = (int)IPC_STAT;
+
+ unsigned struct_utmp_sz = sizeof(struct utmp);
+ unsigned struct_utmpx_sz = sizeof(struct utmpx);
+
+ int map_fixed = MAP_FIXED;
+
+ int af_inet = (int)AF_INET;
+ int af_inet6 = (int)AF_INET6;
+
+ uptr __sanitizer_in_addr_sz(int af) {
+ if (af == AF_INET)
+ return sizeof(struct in_addr);
+ else if (af == AF_INET6)
+ return sizeof(struct in6_addr);
+ else
+ return 0;
+ }
+
+ unsigned struct_ElfW_Phdr_sz = sizeof(ElfW(Phdr));
+
+ int glob_nomatch = GLOB_NOMATCH;
+
+ unsigned path_max = PATH_MAX;
+
+ // ioctl arguments
+ unsigned struct_ifreq_sz = sizeof(struct ifreq);
+ unsigned struct_termios_sz = sizeof(struct termios);
+ unsigned struct_winsize_sz = sizeof(struct winsize);
+
+ unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
+ unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
+
+ const unsigned IOCTL_NOT_PRESENT = 0;
+
+ unsigned IOCTL_FIOASYNC = FIOASYNC;
+ unsigned IOCTL_FIOCLEX = FIOCLEX;
+ unsigned IOCTL_FIOGETOWN = FIOGETOWN;
+ unsigned IOCTL_FIONBIO = FIONBIO;
+ unsigned IOCTL_FIONCLEX = FIONCLEX;
+ unsigned IOCTL_FIOSETOWN = FIOSETOWN;
+ unsigned IOCTL_SIOCADDMULTI = SIOCADDMULTI;
+ unsigned IOCTL_SIOCATMARK = SIOCATMARK;
+ unsigned IOCTL_SIOCDELMULTI = SIOCDELMULTI;
+ unsigned IOCTL_SIOCGIFADDR = SIOCGIFADDR;
+ unsigned IOCTL_SIOCGIFBRDADDR = SIOCGIFBRDADDR;
+ unsigned IOCTL_SIOCGIFCONF = SIOCGIFCONF;
+ unsigned IOCTL_SIOCGIFDSTADDR = SIOCGIFDSTADDR;
+ unsigned IOCTL_SIOCGIFFLAGS = SIOCGIFFLAGS;
+ unsigned IOCTL_SIOCGIFMETRIC = SIOCGIFMETRIC;
+ unsigned IOCTL_SIOCGIFMTU = SIOCGIFMTU;
+ unsigned IOCTL_SIOCGIFNETMASK = SIOCGIFNETMASK;
+ unsigned IOCTL_SIOCGPGRP = SIOCGPGRP;
+ unsigned IOCTL_SIOCSIFADDR = SIOCSIFADDR;
+ unsigned IOCTL_SIOCSIFBRDADDR = SIOCSIFBRDADDR;
+ unsigned IOCTL_SIOCSIFDSTADDR = SIOCSIFDSTADDR;
+ unsigned IOCTL_SIOCSIFFLAGS = SIOCSIFFLAGS;
+ unsigned IOCTL_SIOCSIFMETRIC = SIOCSIFMETRIC;
+ unsigned IOCTL_SIOCSIFMTU = SIOCSIFMTU;
+ unsigned IOCTL_SIOCSIFNETMASK = SIOCSIFNETMASK;
+ unsigned IOCTL_SIOCSPGRP = SIOCSPGRP;
+ unsigned IOCTL_TIOCEXCL = TIOCEXCL;
+ unsigned IOCTL_TIOCGETD = TIOCGETD;
+ unsigned IOCTL_TIOCGPGRP = TIOCGPGRP;
+ unsigned IOCTL_TIOCGWINSZ = TIOCGWINSZ;
+ unsigned IOCTL_TIOCMBIC = TIOCMBIC;
+ unsigned IOCTL_TIOCMBIS = TIOCMBIS;
+ unsigned IOCTL_TIOCMGET = TIOCMGET;
+ unsigned IOCTL_TIOCMSET = TIOCMSET;
+ unsigned IOCTL_TIOCNOTTY = TIOCNOTTY;
+ unsigned IOCTL_TIOCNXCL = TIOCNXCL;
+ unsigned IOCTL_TIOCOUTQ = TIOCOUTQ;
+ unsigned IOCTL_TIOCPKT = TIOCPKT;
+ unsigned IOCTL_TIOCSCTTY = TIOCSCTTY;
+ unsigned IOCTL_TIOCSETD = TIOCSETD;
+ unsigned IOCTL_TIOCSPGRP = TIOCSPGRP;
+ unsigned IOCTL_TIOCSTI = TIOCSTI;
+ unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ;
+
+ unsigned IOCTL_MTIOCGET = MTIOCGET;
+ unsigned IOCTL_MTIOCTOP = MTIOCTOP;
+
+ const int si_SEGV_MAPERR = SEGV_MAPERR;
+ const int si_SEGV_ACCERR = SEGV_ACCERR;
+} // namespace __sanitizer
+
+using namespace __sanitizer;
+
+COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t));
+
+COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned));
+CHECK_TYPE_SIZE(pthread_key_t);
+
+// There are more undocumented fields in dl_phdr_info that we are not interested
+// in.
+COMPILER_CHECK(sizeof(__sanitizer_dl_phdr_info) <= sizeof(dl_phdr_info));
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr);
+CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum);
+
+CHECK_TYPE_SIZE(glob_t);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv);
+CHECK_SIZE_AND_OFFSET(glob_t, gl_offs);
+
+CHECK_TYPE_SIZE(addrinfo);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_family);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_socktype);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname);
+CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr);
+
+CHECK_TYPE_SIZE(hostent);
+CHECK_SIZE_AND_OFFSET(hostent, h_name);
+CHECK_SIZE_AND_OFFSET(hostent, h_aliases);
+CHECK_SIZE_AND_OFFSET(hostent, h_addrtype);
+CHECK_SIZE_AND_OFFSET(hostent, h_length);
+CHECK_SIZE_AND_OFFSET(hostent, h_addr_list);
+
+CHECK_TYPE_SIZE(iovec);
+CHECK_SIZE_AND_OFFSET(iovec, iov_base);
+CHECK_SIZE_AND_OFFSET(iovec, iov_len);
+
+CHECK_TYPE_SIZE(msghdr);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_name);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iov);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_control);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen);
+CHECK_SIZE_AND_OFFSET(msghdr, msg_flags);
+
+CHECK_TYPE_SIZE(cmsghdr);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level);
+CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type);
+
+COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent));
+CHECK_SIZE_AND_OFFSET(dirent, d_ino);
+CHECK_SIZE_AND_OFFSET(dirent, d_off);
+CHECK_SIZE_AND_OFFSET(dirent, d_reclen);
+
+#if SANITIZER_SOLARIS32
+COMPILER_CHECK(sizeof(__sanitizer_dirent64) <= sizeof(dirent64));
+CHECK_SIZE_AND_OFFSET(dirent64, d_ino);
+CHECK_SIZE_AND_OFFSET(dirent64, d_off);
+CHECK_SIZE_AND_OFFSET(dirent64, d_reclen);
+#endif
+
+CHECK_TYPE_SIZE(ifconf);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_len);
+CHECK_SIZE_AND_OFFSET(ifconf, ifc_ifcu);
+
+CHECK_TYPE_SIZE(pollfd);
+CHECK_SIZE_AND_OFFSET(pollfd, fd);
+CHECK_SIZE_AND_OFFSET(pollfd, events);
+CHECK_SIZE_AND_OFFSET(pollfd, revents);
+
+CHECK_TYPE_SIZE(nfds_t);
+
+CHECK_TYPE_SIZE(sigset_t);
+
+COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction));
+// Can't write checks for sa_handler and sa_sigaction due to them being
+// preprocessor macros.
+CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask);
+CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_flags);
+
+CHECK_TYPE_SIZE(wordexp_t);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv);
+CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs);
+
+CHECK_TYPE_SIZE(tm);
+CHECK_SIZE_AND_OFFSET(tm, tm_sec);
+CHECK_SIZE_AND_OFFSET(tm, tm_min);
+CHECK_SIZE_AND_OFFSET(tm, tm_hour);
+CHECK_SIZE_AND_OFFSET(tm, tm_mday);
+CHECK_SIZE_AND_OFFSET(tm, tm_mon);
+CHECK_SIZE_AND_OFFSET(tm, tm_year);
+CHECK_SIZE_AND_OFFSET(tm, tm_wday);
+CHECK_SIZE_AND_OFFSET(tm, tm_yday);
+CHECK_SIZE_AND_OFFSET(tm, tm_isdst);
+
+CHECK_TYPE_SIZE(ether_addr);
+
+CHECK_TYPE_SIZE(ipc_perm);
+CHECK_SIZE_AND_OFFSET(ipc_perm, key);
+CHECK_SIZE_AND_OFFSET(ipc_perm, seq);
+CHECK_SIZE_AND_OFFSET(ipc_perm, uid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, gid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cuid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, cgid);
+CHECK_SIZE_AND_OFFSET(ipc_perm, mode);
+
+CHECK_TYPE_SIZE(shmid_ds);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_segsz);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_atime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_dtime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_ctime);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_cpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_lpid);
+CHECK_SIZE_AND_OFFSET(shmid_ds, shm_nattch);
+
+CHECK_TYPE_SIZE(clock_t);
+
+CHECK_TYPE_SIZE(ifaddrs);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_addr);
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask);
+// Compare against the union, because we can't reach into the union in a
+// compliant way.
+#ifdef ifa_dstaddr
+#undef ifa_dstaddr
+#endif
+COMPILER_CHECK(sizeof(((__sanitizer_ifaddrs *)nullptr)->ifa_dstaddr) ==
+ sizeof(((ifaddrs *)nullptr)->ifa_ifu));
+COMPILER_CHECK(offsetof(__sanitizer_ifaddrs, ifa_dstaddr) ==
+ offsetof(ifaddrs, ifa_ifu));
+CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data);
+
+CHECK_TYPE_SIZE(timeb);
+CHECK_SIZE_AND_OFFSET(timeb, time);
+CHECK_SIZE_AND_OFFSET(timeb, millitm);
+CHECK_SIZE_AND_OFFSET(timeb, timezone);
+CHECK_SIZE_AND_OFFSET(timeb, dstflag);
+
+CHECK_TYPE_SIZE(passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_name);
+CHECK_SIZE_AND_OFFSET(passwd, pw_passwd);
+CHECK_SIZE_AND_OFFSET(passwd, pw_uid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_gid);
+CHECK_SIZE_AND_OFFSET(passwd, pw_dir);
+CHECK_SIZE_AND_OFFSET(passwd, pw_shell);
+
+CHECK_SIZE_AND_OFFSET(passwd, pw_gecos);
+
+CHECK_TYPE_SIZE(group);
+CHECK_SIZE_AND_OFFSET(group, gr_name);
+CHECK_SIZE_AND_OFFSET(group, gr_passwd);
+CHECK_SIZE_AND_OFFSET(group, gr_gid);
+CHECK_SIZE_AND_OFFSET(group, gr_mem);
+
+CHECK_TYPE_SIZE(XDR);
+CHECK_SIZE_AND_OFFSET(XDR, x_op);
+CHECK_SIZE_AND_OFFSET(XDR, x_ops);
+CHECK_SIZE_AND_OFFSET(XDR, x_public);
+CHECK_SIZE_AND_OFFSET(XDR, x_private);
+CHECK_SIZE_AND_OFFSET(XDR, x_base);
+CHECK_SIZE_AND_OFFSET(XDR, x_handy);
+COMPILER_CHECK(__sanitizer_XDR_ENCODE == XDR_ENCODE);
+COMPILER_CHECK(__sanitizer_XDR_DECODE == XDR_DECODE);
+COMPILER_CHECK(__sanitizer_XDR_FREE == XDR_FREE);
+
+CHECK_TYPE_SIZE(sem_t);
+
+#endif // SANITIZER_SOLARIS
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h
new file mode 100644
index 000000000000..a9db71b99c81
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.h
@@ -0,0 +1,485 @@
+//===-- sanitizer_platform_limits_solaris.h -------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Sizes and layouts of platform-specific Solaris data structures.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_PLATFORM_LIMITS_SOLARIS_H
+#define SANITIZER_PLATFORM_LIMITS_SOLARIS_H
+
+#if SANITIZER_SOLARIS
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform.h"
+
+namespace __sanitizer {
+extern unsigned struct_utsname_sz;
+extern unsigned struct_stat_sz;
+extern unsigned struct_stat64_sz;
+extern unsigned struct_rusage_sz;
+extern unsigned siginfo_t_sz;
+extern unsigned struct_itimerval_sz;
+extern unsigned pthread_t_sz;
+extern unsigned pthread_mutex_t_sz;
+extern unsigned pthread_cond_t_sz;
+extern unsigned pid_t_sz;
+extern unsigned timeval_sz;
+extern unsigned uid_t_sz;
+extern unsigned gid_t_sz;
+extern unsigned mbstate_t_sz;
+extern unsigned struct_timezone_sz;
+extern unsigned struct_tms_sz;
+extern unsigned struct_itimerspec_sz;
+extern unsigned struct_sigevent_sz;
+extern unsigned struct_sched_param_sz;
+extern unsigned struct_statfs64_sz;
+extern unsigned struct_statfs_sz;
+extern unsigned struct_sockaddr_sz;
+extern unsigned ucontext_t_sz;
+
+extern unsigned struct_timespec_sz;
+extern unsigned struct_rlimit_sz;
+extern unsigned struct_utimbuf_sz;
+
+struct __sanitizer_sem_t {
+ //u64 data[6];
+ u32 sem_count;
+ u16 sem_type;
+ u16 sem_magic;
+ u64 sem_pad1[3];
+ u64 sem_pad2[2];
+};
+
+struct __sanitizer_ipc_perm {
+ unsigned int uid; // uid_t
+ unsigned int gid; // gid_t
+ unsigned int cuid; // uid_t
+ unsigned int cgid; // gid_t
+ unsigned int mode; // mode_t
+ unsigned int seq; // uint_t
+ int key; // key_t
+#if !defined(_LP64)
+ int pad[4];
+#endif
+ };
+
+struct __sanitizer_shmid_ds {
+ __sanitizer_ipc_perm shm_perm;
+ unsigned long shm_segsz; // size_t
+ unsigned long shm_flags; // uintptr_t
+ unsigned short shm_lkcnt; // ushort_t
+ int shm_lpid; // pid_t
+ int shm_cpid; // pid_t
+ unsigned long shm_nattch; // shmatt_t
+ unsigned long shm_cnattch; // ulong_t
+#if defined(_LP64)
+ long shm_atime; // time_t
+ long shm_dtime;
+ long shm_ctime;
+ void *shm_amp;
+ u64 shm_gransize; // uint64_t
+ u64 shm_allocated; // uint64_t
+ u64 shm_pad4[1]; // int64_t
+#else
+ long shm_atime; // time_t
+ int shm_pad1; // int32_t
+ long shm_dtime; // time_t
+ int shm_pad2; // int32_t
+ long shm_ctime; // time_t
+ void *shm_amp;
+ u64 shm_gransize; // uint64_t
+ u64 shm_allocated; // uint64_t
+#endif
+};
+
+extern unsigned struct_statvfs_sz;
+#if SANITIZER_SOLARIS32
+extern unsigned struct_statvfs64_sz;
+#endif
+
+struct __sanitizer_iovec {
+ void *iov_base;
+ uptr iov_len;
+};
+
+struct __sanitizer_ifaddrs {
+ struct __sanitizer_ifaddrs *ifa_next;
+ char *ifa_name;
+ u64 ifa_flags; // uint64_t
+ void *ifa_addr; // (struct sockaddr *)
+ void *ifa_netmask; // (struct sockaddr *)
+ // This is a union on Linux.
+# ifdef ifa_dstaddr
+# undef ifa_dstaddr
+# endif
+ void *ifa_dstaddr; // (struct sockaddr *)
+ void *ifa_data;
+};
+
+typedef unsigned __sanitizer_pthread_key_t;
+
+struct __sanitizer_XDR {
+ int x_op;
+ void *x_ops;
+ uptr x_public;
+ uptr x_private;
+ uptr x_base;
+ unsigned x_handy;
+};
+
+const int __sanitizer_XDR_ENCODE = 0;
+const int __sanitizer_XDR_DECODE = 1;
+const int __sanitizer_XDR_FREE = 2;
+
+struct __sanitizer_passwd {
+ char *pw_name;
+ char *pw_passwd;
+ unsigned int pw_uid; // uid_t
+ unsigned int pw_gid; // gid_t
+ char *pw_age;
+ char *pw_comment;
+ char *pw_gecos;
+ char *pw_dir;
+ char *pw_shell;
+};
+
+struct __sanitizer_group {
+ char *gr_name;
+ char *gr_passwd;
+ int gr_gid;
+ char **gr_mem;
+};
+
+typedef long __sanitizer_time_t;
+
+struct __sanitizer_timeb {
+ __sanitizer_time_t time;
+ unsigned short millitm;
+ short timezone;
+ short dstflag;
+};
+
+struct __sanitizer_ether_addr {
+ u8 octet[6];
+};
+
+struct __sanitizer_tm {
+ int tm_sec;
+ int tm_min;
+ int tm_hour;
+ int tm_mday;
+ int tm_mon;
+ int tm_year;
+ int tm_wday;
+ int tm_yday;
+ int tm_isdst;
+};
+
+struct __sanitizer_msghdr {
+ void *msg_name;
+ unsigned msg_namelen;
+ struct __sanitizer_iovec *msg_iov;
+ unsigned msg_iovlen;
+ void *msg_control;
+ unsigned msg_controllen;
+ int msg_flags;
+};
+struct __sanitizer_cmsghdr {
+ unsigned cmsg_len;
+ int cmsg_level;
+ int cmsg_type;
+};
+
+#if SANITIZER_SOLARIS32 && 0
+// FIXME: need to deal with large file and non-large file cases
+struct __sanitizer_dirent {
+ unsigned long long d_ino;
+ long long d_off;
+ unsigned short d_reclen;
+ // more fields that we don't care about
+};
+#else
+struct __sanitizer_dirent {
+ unsigned long d_ino;
+ long d_off;
+ unsigned short d_reclen;
+ // more fields that we don't care about
+};
+#endif
+
+struct __sanitizer_dirent64 {
+ unsigned long long d_ino;
+ unsigned long long d_off;
+ unsigned short d_reclen;
+ // more fields that we don't care about
+};
+
+typedef long __sanitizer_clock_t;
+typedef int __sanitizer_clockid_t;
+
+// This thing depends on the platform. We are only interested in the upper
+// limit. Verified with a compiler assert in .cc.
+const int pthread_attr_t_max_sz = 128;
+union __sanitizer_pthread_attr_t {
+ char size[pthread_attr_t_max_sz]; // NOLINT
+ void *align;
+};
+
+struct __sanitizer_sigset_t {
+ // uint32_t * 4
+ unsigned int __bits[4];
+};
+
+struct __sanitizer_siginfo {
+ // The size is determined by looking at sizeof of real siginfo_t on linux.
+ u64 opaque[128 / sizeof(u64)];
+};
+
+using __sanitizer_sighandler_ptr = void (*)(int sig);
+using __sanitizer_sigactionhandler_ptr =
+ void (*)(int sig, __sanitizer_siginfo *siginfo, void *uctx);
+
+struct __sanitizer_sigaction {
+ int sa_flags;
+ union {
+ __sanitizer_sigactionhandler_ptr sigaction;
+ __sanitizer_sighandler_ptr handler;
+ };
+ __sanitizer_sigset_t sa_mask;
+#if !defined(_LP64)
+ int sa_resv[2];
+#endif
+};
+
+struct __sanitizer_kernel_sigset_t {
+ u8 sig[8];
+};
+
+struct __sanitizer_kernel_sigaction_t {
+ union {
+ void (*handler)(int signo);
+ void (*sigaction)(int signo, __sanitizer_siginfo *info, void *ctx);
+ };
+ unsigned long sa_flags;
+ void (*sa_restorer)(void);
+ __sanitizer_kernel_sigset_t sa_mask;
+};
+
+extern const uptr sig_ign;
+extern const uptr sig_dfl;
+extern const uptr sig_err;
+extern const uptr sa_siginfo;
+
+extern int af_inet;
+extern int af_inet6;
+uptr __sanitizer_in_addr_sz(int af);
+
+struct __sanitizer_dl_phdr_info {
+ uptr dlpi_addr;
+ const char *dlpi_name;
+ const void *dlpi_phdr;
+ short dlpi_phnum;
+};
+
+extern unsigned struct_ElfW_Phdr_sz;
+
+struct __sanitizer_addrinfo {
+ int ai_flags;
+ int ai_family;
+ int ai_socktype;
+ int ai_protocol;
+#if defined(__sparcv9)
+ int _ai_pad;
+#endif
+ unsigned ai_addrlen;
+ char *ai_canonname;
+ void *ai_addr;
+ struct __sanitizer_addrinfo *ai_next;
+};
+
+struct __sanitizer_hostent {
+ char *h_name;
+ char **h_aliases;
+ int h_addrtype;
+ int h_length;
+ char **h_addr_list;
+};
+
+struct __sanitizer_pollfd {
+ int fd;
+ short events;
+ short revents;
+};
+
+typedef unsigned long __sanitizer_nfds_t;
+
+struct __sanitizer_glob_t {
+ uptr gl_pathc;
+ char **gl_pathv;
+ uptr gl_offs;
+ char **gl_pathp;
+ int gl_pathn;
+};
+
+extern int glob_nomatch;
+extern int glob_altdirfunc;
+
+extern unsigned path_max;
+
+struct __sanitizer_wordexp_t {
+ uptr we_wordc;
+ char **we_wordv;
+ uptr we_offs;
+ char **we_wordp;
+ int we_wordn;
+};
+
+typedef void __sanitizer_FILE;
+#define SANITIZER_HAS_STRUCT_FILE 0
+
+// This simplifies generic code
+#define struct_shminfo_sz -1
+#define struct_shm_info_sz -1
+#define shmctl_shm_stat -1
+#define shmctl_ipc_info -1
+#define shmctl_shm_info -1
+
+extern int shmctl_ipc_stat;
+
+extern unsigned struct_utmp_sz;
+extern unsigned struct_utmpx_sz;
+
+extern int map_fixed;
+
+// ioctl arguments
+struct __sanitizer_ifconf {
+ int ifc_len;
+ union {
+ void *ifcu_req;
+ } ifc_ifcu;
+};
+
+// <sys/ioccom.h>
+#define IOC_NRBITS 8
+#define IOC_TYPEBITS 8
+#define IOC_SIZEBITS 12
+#define IOC_DIRBITS 4
+#undef IOC_NONE
+#define IOC_NONE 2U // IOC_VOID
+#define IOC_READ 4U // IOC_OUT
+#define IOC_WRITE 8U // IOC_IN
+
+#define IOC_NRMASK ((1 << IOC_NRBITS) - 1)
+#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1)
+#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1)
+#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1)
+#define IOC_NRSHIFT 0
+#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS)
+#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS)
+#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS)
+
+#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK)
+#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK)
+#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK)
+
+#if defined(__sparc__)
+// In sparc the 14 bits SIZE field overlaps with the
+// least significant bit of DIR, so either IOC_READ or
+// IOC_WRITE shall be 1 in order to get a non-zero SIZE.
+#define IOC_SIZE(nr) \
+ ((((((nr) >> 29) & 0x7) & (4U | 2U)) == 0) ? 0 : (((nr) >> 16) & 0x3fff))
+#else
+#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK)
+#endif
+
+extern unsigned struct_ifreq_sz;
+extern unsigned struct_termios_sz;
+extern unsigned struct_winsize_sz;
+
+extern unsigned struct_sioc_sg_req_sz;
+extern unsigned struct_sioc_vif_req_sz;
+
+// ioctl request identifiers
+
+// A special value to mark ioctls that are not present on the target platform,
+// when it can not be determined without including any system headers.
+extern const unsigned IOCTL_NOT_PRESENT;
+
+extern unsigned IOCTL_FIOASYNC;
+extern unsigned IOCTL_FIOCLEX;
+extern unsigned IOCTL_FIOGETOWN;
+extern unsigned IOCTL_FIONBIO;
+extern unsigned IOCTL_FIONCLEX;
+extern unsigned IOCTL_FIOSETOWN;
+extern unsigned IOCTL_SIOCADDMULTI;
+extern unsigned IOCTL_SIOCATMARK;
+extern unsigned IOCTL_SIOCDELMULTI;
+extern unsigned IOCTL_SIOCGIFADDR;
+extern unsigned IOCTL_SIOCGIFBRDADDR;
+extern unsigned IOCTL_SIOCGIFCONF;
+extern unsigned IOCTL_SIOCGIFDSTADDR;
+extern unsigned IOCTL_SIOCGIFFLAGS;
+extern unsigned IOCTL_SIOCGIFMETRIC;
+extern unsigned IOCTL_SIOCGIFMTU;
+extern unsigned IOCTL_SIOCGIFNETMASK;
+extern unsigned IOCTL_SIOCGPGRP;
+extern unsigned IOCTL_SIOCSIFADDR;
+extern unsigned IOCTL_SIOCSIFBRDADDR;
+extern unsigned IOCTL_SIOCSIFDSTADDR;
+extern unsigned IOCTL_SIOCSIFFLAGS;
+extern unsigned IOCTL_SIOCSIFMETRIC;
+extern unsigned IOCTL_SIOCSIFMTU;
+extern unsigned IOCTL_SIOCSIFNETMASK;
+extern unsigned IOCTL_SIOCSPGRP;
+extern unsigned IOCTL_TIOCEXCL;
+extern unsigned IOCTL_TIOCGETD;
+extern unsigned IOCTL_TIOCGPGRP;
+extern unsigned IOCTL_TIOCGWINSZ;
+extern unsigned IOCTL_TIOCMBIC;
+extern unsigned IOCTL_TIOCMBIS;
+extern unsigned IOCTL_TIOCMGET;
+extern unsigned IOCTL_TIOCMSET;
+extern unsigned IOCTL_TIOCNOTTY;
+extern unsigned IOCTL_TIOCNXCL;
+extern unsigned IOCTL_TIOCOUTQ;
+extern unsigned IOCTL_TIOCPKT;
+extern unsigned IOCTL_TIOCSCTTY;
+extern unsigned IOCTL_TIOCSETD;
+extern unsigned IOCTL_TIOCSPGRP;
+extern unsigned IOCTL_TIOCSTI;
+extern unsigned IOCTL_TIOCSWINSZ;
+extern unsigned IOCTL_MTIOCGET;
+extern unsigned IOCTL_MTIOCTOP;
+
+extern const int si_SEGV_MAPERR;
+extern const int si_SEGV_ACCERR;
+} // namespace __sanitizer
+
+#define CHECK_TYPE_SIZE(TYPE) \
+ COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE))
+
+#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \
+ COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *) NULL)->MEMBER) == \
+ sizeof(((CLASS *) NULL)->MEMBER)); \
+ COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \
+ offsetof(CLASS, MEMBER))
+
+// For sigaction, which is a function and struct at the same time,
+// and thus requires explicit "struct" in sizeof() expression.
+#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \
+ COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *) NULL)->MEMBER) == \
+ sizeof(((struct CLASS *) NULL)->MEMBER)); \
+ COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \
+ offsetof(struct CLASS, MEMBER))
+
+#endif // SANITIZER_SOLARIS
+
+#endif
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc
index 8d3128ae199d..1fad71f3582b 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cc
@@ -17,6 +17,7 @@
#if SANITIZER_POSIX
#include "sanitizer_common.h"
+#include "sanitizer_file.h"
#include "sanitizer_libc.h"
#include "sanitizer_posix.h"
#include "sanitizer_procmaps.h"
@@ -294,18 +295,22 @@ bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end) {
return false;
}
-SignalContext SignalContext::Create(void *siginfo, void *context) {
- auto si = (siginfo_t *)siginfo;
- uptr addr = (uptr)si->si_addr;
- uptr pc, sp, bp;
- GetPcSpBp(context, &pc, &sp, &bp);
- WriteFlag write_flag = GetWriteFlag(context);
- bool is_memory_access = si->si_signo == SIGSEGV;
- return SignalContext(context, addr, pc, sp, bp, is_memory_access, write_flag);
+uptr SignalContext::GetAddress() const {
+ auto si = static_cast<const siginfo_t *>(siginfo);
+ return (uptr)si->si_addr;
}
-const char *DescribeSignalOrException(int signo) {
- switch (signo) {
+bool SignalContext::IsMemoryAccess() const {
+ auto si = static_cast<const siginfo_t *>(siginfo);
+ return si->si_signo == SIGSEGV;
+}
+
+int SignalContext::GetType() const {
+ return static_cast<const siginfo_t *>(siginfo)->si_signo;
+}
+
+const char *SignalContext::Describe() const {
+ switch (GetType()) {
case SIGFPE:
return "FPE";
case SIGILL:
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h
index e7d37cbf0882..adef08248004 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h
@@ -16,7 +16,9 @@
// ----------- ATTENTION -------------
// This header should NOT include any other headers from sanitizer runtime.
#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform_limits_netbsd.h"
#include "sanitizer_platform_limits_posix.h"
+#include "sanitizer_platform_limits_solaris.h"
#if !SANITIZER_POSIX
// Make it hard to accidentally use any of functions declared in this file:
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc
index e113fb1093d4..db41cad6c318 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc
@@ -18,7 +18,9 @@
#include "sanitizer_common.h"
#include "sanitizer_flags.h"
+#include "sanitizer_platform_limits_netbsd.h"
#include "sanitizer_platform_limits_posix.h"
+#include "sanitizer_platform_limits_solaris.h"
#include "sanitizer_posix.h"
#include "sanitizer_procmaps.h"
#include "sanitizer_stacktrace.h"
@@ -61,7 +63,11 @@ void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
uptr beg_aligned = RoundUpTo(beg, page_size);
uptr end_aligned = RoundDownTo(end, page_size);
if (beg_aligned < end_aligned)
- madvise((void*)beg_aligned, end_aligned - beg_aligned, MADV_DONTNEED);
+ // In the default Solaris compilation environment, madvise() is declared
+ // to take a caddr_t arg; casting it to void * results in an invalid
+ // conversion error, so use char * instead.
+ madvise((char *)beg_aligned, end_aligned - beg_aligned,
+ SANITIZER_MADVISE_DONTNEED);
}
void NoHugePagesInRegion(uptr addr, uptr size) {
@@ -213,6 +219,53 @@ void InstallDeadlySignalHandlers(SignalHandlerType handler) {
MaybeInstallSigaction(SIGFPE, handler);
MaybeInstallSigaction(SIGILL, handler);
}
+
+bool SignalContext::IsStackOverflow() const {
+ // Access at a reasonable offset above SP, or slightly below it (to account
+ // for x86_64 or PowerPC redzone, ARM push of multiple registers, etc) is
+ // probably a stack overflow.
+#ifdef __s390__
+ // On s390, the fault address in siginfo points to start of the page, not
+ // to the precise word that was accessed. Mask off the low bits of sp to
+ // take it into account.
+ bool IsStackAccess = addr >= (sp & ~0xFFF) && addr < sp + 0xFFFF;
+#else
+ bool IsStackAccess = addr + 512 > sp && addr < sp + 0xFFFF;
+#endif
+
+#if __powerpc__
+ // Large stack frames can be allocated with e.g.
+ // lis r0,-10000
+ // stdux r1,r1,r0 # store sp to [sp-10000] and update sp by -10000
+ // If the store faults then sp will not have been updated, so test above
+ // will not work, because the fault address will be more than just "slightly"
+ // below sp.
+ if (!IsStackAccess && IsAccessibleMemoryRange(pc, 4)) {
+ u32 inst = *(unsigned *)pc;
+ u32 ra = (inst >> 16) & 0x1F;
+ u32 opcd = inst >> 26;
+ u32 xo = (inst >> 1) & 0x3FF;
+ // Check for store-with-update to sp. The instructions we accept are:
+ // stbu rs,d(ra) stbux rs,ra,rb
+ // sthu rs,d(ra) sthux rs,ra,rb
+ // stwu rs,d(ra) stwux rs,ra,rb
+ // stdu rs,ds(ra) stdux rs,ra,rb
+ // where ra is r1 (the stack pointer).
+ if (ra == 1 &&
+ (opcd == 39 || opcd == 45 || opcd == 37 || opcd == 62 ||
+ (opcd == 31 && (xo == 247 || xo == 439 || xo == 183 || xo == 181))))
+ IsStackAccess = true;
+ }
+#endif // __powerpc__
+
+ // We also check si_code to filter out SEGV caused by something else other
+ // then hitting the guard page or unmapped memory, like, for example,
+ // unaligned memory access.
+ auto si = static_cast<const siginfo_t *>(siginfo);
+ return IsStackAccess &&
+ (si->si_code == si_SEGV_MAPERR || si->si_code == si_SEGV_ACCERR);
+}
+
#endif // SANITIZER_GO
bool IsAccessibleMemoryRange(uptr beg, uptr size) {
@@ -289,6 +342,46 @@ void *MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) {
return (void *)p;
}
+uptr ReservedAddressRange::Init(uptr size, const char *name, uptr fixed_addr) {
+ // We don't pass `name` along because, when you enable `decorate_proc_maps`
+ // AND actually use a named mapping AND are using a sanitizer intercepting
+ // `open` (e.g. TSAN, ESAN), then you'll get a failure during initialization.
+ // TODO(flowerhack): Fix the implementation of GetNamedMappingFd to solve
+ // this problem.
+ if (fixed_addr) {
+ base_ = MmapFixedNoAccess(fixed_addr, size);
+ } else {
+ base_ = MmapNoAccess(size);
+ }
+ size_ = size;
+ name_ = name;
+ (void)os_handle_; // unsupported
+ return reinterpret_cast<uptr>(base_);
+}
+
+// Uses fixed_addr for now.
+// Will use offset instead once we've implemented this function for real.
+uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size) {
+ return reinterpret_cast<uptr>(MmapFixedOrDieOnFatalError(fixed_addr, size));
+}
+
+uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size) {
+ return reinterpret_cast<uptr>(MmapFixedOrDie(fixed_addr, size));
+}
+
+void ReservedAddressRange::Unmap(uptr addr, uptr size) {
+ void* addr_as_void = reinterpret_cast<void*>(addr);
+ uptr base_as_uptr = reinterpret_cast<uptr>(base_);
+ // Only unmap at the beginning or end of the range.
+ CHECK((addr_as_void == base_) || (addr + size == base_as_uptr + size_));
+ CHECK_LE(size, size_);
+ UnmapOrDie(reinterpret_cast<void*>(addr), size);
+ if (addr_as_void == base_) {
+ base_ = reinterpret_cast<void*>(addr + size);
+ }
+ size_ = size_ - size;
+}
+
void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) {
int fd = name ? GetNamedMappingFd(name, size) : -1;
unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE;
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_printf.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_printf.cc
index 99b7ff1b55cf..5c2360043b07 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_printf.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_printf.cc
@@ -28,8 +28,6 @@
namespace __sanitizer {
-StaticSpinMutex CommonSanitizerReportMutex;
-
static int AppendChar(char **buff, const char *buff_end, char c) {
if (*buff < buff_end) {
**buff = c;
@@ -229,19 +227,16 @@ static void CallPrintfAndReportCallback(const char *str) {
PrintfAndReportCallback(str);
}
-static void SharedPrintfCode(bool append_pid, const char *format,
- va_list args) {
+static void NOINLINE SharedPrintfCodeNoBuffer(bool append_pid,
+ char *local_buffer,
+ int buffer_size,
+ const char *format,
+ va_list args) {
va_list args2;
va_copy(args2, args);
const int kLen = 16 * 1024;
- // |local_buffer| is small enough not to overflow the stack and/or violate
- // the stack limit enforced by TSan (-Wframe-larger-than=512). On the other
- // hand, the bigger the buffer is, the more the chance the error report will
- // fit into it.
- char local_buffer[400];
int needed_length;
char *buffer = local_buffer;
- int buffer_size = ARRAY_SIZE(local_buffer);
// First try to print a message using a local buffer, and then fall back to
// mmaped buffer.
for (int use_mmap = 0; use_mmap < 2; use_mmap++) {
@@ -259,7 +254,9 @@ static void SharedPrintfCode(bool append_pid, const char *format,
RAW_CHECK_MSG(needed_length < kLen, \
"Buffer in Report is too short!\n"); \
}
- if (append_pid) {
+ // Fuchsia's logging infrastructure always keeps track of the logging
+ // process, thread, and timestamp, so never prepend such information.
+ if (!SANITIZER_FUCHSIA && append_pid) {
int pid = internal_getpid();
const char *exe_name = GetProcessName();
if (common_flags()->log_exe_name && exe_name) {
@@ -267,9 +264,8 @@ static void SharedPrintfCode(bool append_pid, const char *format,
"==%s", exe_name);
CHECK_NEEDED_LENGTH
}
- needed_length += internal_snprintf(buffer + needed_length,
- buffer_size - needed_length,
- "==%d==", pid);
+ needed_length += internal_snprintf(
+ buffer + needed_length, buffer_size - needed_length, "==%d==", pid);
CHECK_NEEDED_LENGTH
}
needed_length += VSNPrintf(buffer + needed_length,
@@ -292,6 +288,17 @@ static void SharedPrintfCode(bool append_pid, const char *format,
va_end(args2);
}
+static void NOINLINE SharedPrintfCode(bool append_pid, const char *format,
+ va_list args) {
+ // |local_buffer| is small enough not to overflow the stack and/or violate
+ // the stack limit enforced by TSan (-Wframe-larger-than=512). On the other
+ // hand, the bigger the buffer is, the more the chance the error report will
+ // fit into it.
+ char local_buffer[400];
+ SharedPrintfCodeNoBuffer(append_pid, local_buffer, ARRAY_SIZE(local_buffer),
+ format, args);
+}
+
FORMAT(1, 2)
void Printf(const char *format, ...) {
va_list args;
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h
index 06d072b4dc77..ea2cb7a5c68b 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h
@@ -14,22 +14,19 @@
#ifndef SANITIZER_PROCMAPS_H
#define SANITIZER_PROCMAPS_H
+#include "sanitizer_platform.h"
+
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
+ SANITIZER_MAC || SANITIZER_SOLARIS
+
#include "sanitizer_common.h"
#include "sanitizer_internal_defs.h"
+#include "sanitizer_linux.h"
+#include "sanitizer_mac.h"
#include "sanitizer_mutex.h"
namespace __sanitizer {
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
-struct ProcSelfMapsBuff {
- char *data;
- uptr mmaped_size;
- uptr len;
-};
-
-// Reads process memory map in an OS-specific way.
-void ReadProcMaps(ProcSelfMapsBuff *proc_maps);
-#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
// Memory protection masks.
static const uptr kProtectionRead = 1;
@@ -37,15 +34,20 @@ static const uptr kProtectionWrite = 2;
static const uptr kProtectionExecute = 4;
static const uptr kProtectionShared = 8;
-struct MemoryMappedSegment {
+struct MemoryMappedSegmentData;
+
+class MemoryMappedSegment {
+ public:
MemoryMappedSegment(char *buff = nullptr, uptr size = 0)
- : filename(buff), filename_size(size) {}
+ : filename(buff), filename_size(size), data_(nullptr) {}
~MemoryMappedSegment() {}
- bool IsReadable() { return protection & kProtectionRead; }
- bool IsWritable() { return protection & kProtectionWrite; }
- bool IsExecutable() { return protection & kProtectionExecute; }
- bool IsShared() { return protection & kProtectionShared; }
+ bool IsReadable() const { return protection & kProtectionRead; }
+ bool IsWritable() const { return protection & kProtectionWrite; }
+ bool IsExecutable() const { return protection & kProtectionExecute; }
+ bool IsShared() const { return protection & kProtectionShared; }
+
+ void AddAddressRanges(LoadedModule *module);
uptr start;
uptr end;
@@ -55,6 +57,12 @@ struct MemoryMappedSegment {
uptr protection;
ModuleArch arch;
u8 uuid[kModuleUUIDSize];
+
+ private:
+ friend class MemoryMappingLayout;
+
+ // This field is assigned and owned by MemoryMappingLayout if needed
+ MemoryMappedSegmentData *data_;
};
class MemoryMappingLayout {
@@ -69,42 +77,14 @@ class MemoryMappingLayout {
static void CacheMemoryMappings();
// Adds all mapped objects into a vector.
- void DumpListOfModules(InternalMmapVector<LoadedModule> *modules);
+ void DumpListOfModules(InternalMmapVectorNoCtor<LoadedModule> *modules);
private:
void LoadFromCache();
- // FIXME: Hide implementation details for different platforms in
- // platform-specific files.
-# if SANITIZER_FREEBSD || SANITIZER_LINUX
- ProcSelfMapsBuff proc_self_maps_;
- const char *current_;
-
- // Static mappings cache.
- static ProcSelfMapsBuff cached_proc_self_maps_;
- static StaticSpinMutex cache_lock_; // protects cached_proc_self_maps_.
-# elif SANITIZER_MAC
- template <u32 kLCSegment, typename SegmentCommand>
- bool NextSegmentLoad(MemoryMappedSegment *segment);
- int current_image_;
- u32 current_magic_;
- u32 current_filetype_;
- ModuleArch current_arch_;
- u8 current_uuid_[kModuleUUIDSize];
- int current_load_cmd_count_;
- char *current_load_cmd_addr_;
- bool current_instrumented_;
-# endif
+ MemoryMappingLayoutData data_;
};
-typedef void (*fill_profile_f)(uptr start, uptr rss, bool file,
- /*out*/uptr *stats, uptr stats_size);
-
-// Parse the contents of /proc/self/smaps and generate a memory profile.
-// |cb| is a tool-specific callback that fills the |stats| array containing
-// |stats_size| elements.
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size);
-
// Returns code range for the specified module.
bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end);
@@ -115,4 +95,6 @@ uptr ParseHex(const char **p);
} // namespace __sanitizer
+#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD ||
+ // SANITIZER_MAC || SANITIZER_SOLARIS
#endif // SANITIZER_PROCMAPS_H
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc
index b95f301a437d..0cd3e2458d87 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_common.cc
@@ -12,7 +12,8 @@
#include "sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS
#include "sanitizer_common.h"
#include "sanitizer_placement_new.h"
@@ -20,9 +21,8 @@
namespace __sanitizer {
-// Linker initialized.
-ProcSelfMapsBuff MemoryMappingLayout::cached_proc_self_maps_;
-StaticSpinMutex MemoryMappingLayout::cache_lock_; // Linker initialized.
+static ProcSelfMapsBuff cached_proc_self_maps;
+static StaticSpinMutex cache_lock;
static int TranslateDigit(char c) {
if (c >= '0' && c <= '9')
@@ -64,60 +64,60 @@ uptr ParseHex(const char **p) {
return ParseNumber(p, 16);
}
+void MemoryMappedSegment::AddAddressRanges(LoadedModule *module) {
+ // data_ should be unused on this platform
+ CHECK(!data_);
+ module->addAddressRange(start, end, IsExecutable(), IsWritable());
+}
+
MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) {
- ReadProcMaps(&proc_self_maps_);
- if (cache_enabled) {
- if (proc_self_maps_.mmaped_size == 0) {
- LoadFromCache();
- CHECK_GT(proc_self_maps_.len, 0);
- }
- } else {
- CHECK_GT(proc_self_maps_.mmaped_size, 0);
- }
- Reset();
// FIXME: in the future we may want to cache the mappings on demand only.
if (cache_enabled)
CacheMemoryMappings();
+
+ // Read maps after the cache update to capture the maps/unmaps happening in
+ // the process of updating.
+ ReadProcMaps(&data_.proc_self_maps);
+ if (cache_enabled && data_.proc_self_maps.mmaped_size == 0)
+ LoadFromCache();
+ CHECK_GT(data_.proc_self_maps.mmaped_size, 0);
+ CHECK_GT(data_.proc_self_maps.len, 0);
+
+ Reset();
}
MemoryMappingLayout::~MemoryMappingLayout() {
// Only unmap the buffer if it is different from the cached one. Otherwise
// it will be unmapped when the cache is refreshed.
- if (proc_self_maps_.data != cached_proc_self_maps_.data) {
- UnmapOrDie(proc_self_maps_.data, proc_self_maps_.mmaped_size);
- }
+ if (data_.proc_self_maps.data != cached_proc_self_maps.data)
+ UnmapOrDie(data_.proc_self_maps.data, data_.proc_self_maps.mmaped_size);
}
void MemoryMappingLayout::Reset() {
- current_ = proc_self_maps_.data;
+ data_.current = data_.proc_self_maps.data;
}
// static
void MemoryMappingLayout::CacheMemoryMappings() {
- SpinMutexLock l(&cache_lock_);
+ ProcSelfMapsBuff new_proc_self_maps;
+ ReadProcMaps(&new_proc_self_maps);
// Don't invalidate the cache if the mappings are unavailable.
- ProcSelfMapsBuff old_proc_self_maps;
- old_proc_self_maps = cached_proc_self_maps_;
- ReadProcMaps(&cached_proc_self_maps_);
- if (cached_proc_self_maps_.mmaped_size == 0) {
- cached_proc_self_maps_ = old_proc_self_maps;
- } else {
- if (old_proc_self_maps.mmaped_size) {
- UnmapOrDie(old_proc_self_maps.data,
- old_proc_self_maps.mmaped_size);
- }
- }
+ if (new_proc_self_maps.mmaped_size == 0)
+ return;
+ SpinMutexLock l(&cache_lock);
+ if (cached_proc_self_maps.mmaped_size)
+ UnmapOrDie(cached_proc_self_maps.data, cached_proc_self_maps.mmaped_size);
+ cached_proc_self_maps = new_proc_self_maps;
}
void MemoryMappingLayout::LoadFromCache() {
- SpinMutexLock l(&cache_lock_);
- if (cached_proc_self_maps_.data) {
- proc_self_maps_ = cached_proc_self_maps_;
- }
+ SpinMutexLock l(&cache_lock);
+ if (cached_proc_self_maps.data)
+ data_.proc_self_maps = cached_proc_self_maps;
}
void MemoryMappingLayout::DumpListOfModules(
- InternalMmapVector<LoadedModule> *modules) {
+ InternalMmapVectorNoCtor<LoadedModule> *modules) {
Reset();
InternalScopedString module_name(kMaxPathLength);
MemoryMappedSegment segment(module_name.data(), module_name.size());
@@ -139,8 +139,7 @@ void MemoryMappingLayout::DumpListOfModules(
uptr base_address = (i ? segment.start : 0) - segment.offset;
LoadedModule cur_module;
cur_module.set(cur_name, base_address);
- cur_module.addAddressRange(segment.start, segment.end,
- segment.IsExecutable(), segment.IsWritable());
+ segment.AddAddressRanges(&cur_module);
modules->push_back(cur_module);
}
}
@@ -171,4 +170,5 @@ void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {
} // namespace __sanitizer
-#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD ||
+ // SANITIZER_SOLARIS
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc
index f0cdbeb4483a..c26a6d47ea75 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc
@@ -7,18 +7,22 @@
//
//===----------------------------------------------------------------------===//
//
-// Information about the process mappings (FreeBSD-specific parts).
+// Information about the process mappings (FreeBSD and NetBSD-specific parts).
//===----------------------------------------------------------------------===//
#include "sanitizer_platform.h"
-#if SANITIZER_FREEBSD
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
#include "sanitizer_common.h"
+#if SANITIZER_FREEBSD
#include "sanitizer_freebsd.h"
+#endif
#include "sanitizer_procmaps.h"
#include <unistd.h>
#include <sys/sysctl.h>
+#if SANITIZER_FREEBSD
#include <sys/user.h>
+#endif
// Fix 'kinfo_vmentry' definition on FreeBSD prior v9.2 in 32-bit mode.
#if SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32)
@@ -31,16 +35,30 @@
namespace __sanitizer {
void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
- const int Mib[4] = { CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid() };
+ const int Mib[] = {
+#if SANITIZER_FREEBSD
+ CTL_KERN,
+ KERN_PROC,
+ KERN_PROC_VMMAP,
+ getpid()
+#else
+ CTL_VM,
+ VM_PROC,
+ VM_PROC_MAP,
+ getpid(),
+ sizeof(struct kinfo_vmentry)
+#endif
+ };
+
size_t Size = 0;
- int Err = sysctl(Mib, 4, NULL, &Size, NULL, 0);
+ int Err = sysctl(Mib, ARRAY_SIZE(Mib), NULL, &Size, NULL, 0);
CHECK_EQ(Err, 0);
CHECK_GT(Size, 0);
size_t MmapedSize = Size * 4 / 3;
void *VmMap = MmapOrDie(MmapedSize, "ReadProcMaps()");
Size = MmapedSize;
- Err = sysctl(Mib, 4, VmMap, &Size, NULL, 0);
+ Err = sysctl(Mib, ARRAY_SIZE(Mib), VmMap, &Size, NULL, 0);
CHECK_EQ(Err, 0);
proc_maps->data = (char*)VmMap;
@@ -49,9 +67,9 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
}
bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
- char *last = proc_self_maps_.data + proc_self_maps_.len;
- if (current_ >= last) return false;
- struct kinfo_vmentry *VmEntry = (struct kinfo_vmentry*)current_;
+ char *last = data_.proc_self_maps.data + data_.proc_self_maps.len;
+ if (data_.current >= last) return false;
+ struct kinfo_vmentry *VmEntry = (struct kinfo_vmentry *)data_.current;
segment->start = (uptr)VmEntry->kve_start;
segment->end = (uptr)VmEntry->kve_end;
@@ -71,11 +89,15 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
VmEntry->kve_path);
}
- current_ += VmEntry->kve_structsize;
+#if SANITIZER_FREEBSD
+ data_.current += VmEntry->kve_structsize;
+#else
+ data_.current += sizeof(*VmEntry);
+#endif
return true;
}
} // namespace __sanitizer
-#endif // SANITIZER_FREEBSD
+#endif // SANITIZER_FREEBSD || SANITIZER_NETBSD
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc
index 1bcad2bf70e6..633e9393cce0 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_linux.cc
@@ -18,8 +18,12 @@
namespace __sanitizer {
void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
- ReadFileToBuffer("/proc/self/maps", &proc_maps->data, &proc_maps->mmaped_size,
- &proc_maps->len);
+ if (!ReadFileToBuffer("/proc/self/maps", &proc_maps->data,
+ &proc_maps->mmaped_size, &proc_maps->len)) {
+ proc_maps->data = nullptr;
+ proc_maps->mmaped_size = 0;
+ proc_maps->len = 0;
+ }
}
static bool IsOneOf(char c, char c1, char c2) {
@@ -27,48 +31,48 @@ static bool IsOneOf(char c, char c1, char c2) {
}
bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
- char *last = proc_self_maps_.data + proc_self_maps_.len;
- if (current_ >= last) return false;
- char *next_line = (char*)internal_memchr(current_, '\n', last - current_);
+ char *last = data_.proc_self_maps.data + data_.proc_self_maps.len;
+ if (data_.current >= last) return false;
+ char *next_line =
+ (char *)internal_memchr(data_.current, '\n', last - data_.current);
if (next_line == 0)
next_line = last;
// Example: 08048000-08056000 r-xp 00000000 03:0c 64593 /foo/bar
- segment->start = ParseHex(&current_);
- CHECK_EQ(*current_++, '-');
- segment->end = ParseHex(&current_);
- CHECK_EQ(*current_++, ' ');
- CHECK(IsOneOf(*current_, '-', 'r'));
+ segment->start = ParseHex(&data_.current);
+ CHECK_EQ(*data_.current++, '-');
+ segment->end = ParseHex(&data_.current);
+ CHECK_EQ(*data_.current++, ' ');
+ CHECK(IsOneOf(*data_.current, '-', 'r'));
segment->protection = 0;
- if (*current_++ == 'r') segment->protection |= kProtectionRead;
- CHECK(IsOneOf(*current_, '-', 'w'));
- if (*current_++ == 'w') segment->protection |= kProtectionWrite;
- CHECK(IsOneOf(*current_, '-', 'x'));
- if (*current_++ == 'x') segment->protection |= kProtectionExecute;
- CHECK(IsOneOf(*current_, 's', 'p'));
- if (*current_++ == 's') segment->protection |= kProtectionShared;
- CHECK_EQ(*current_++, ' ');
- segment->offset = ParseHex(&current_);
- CHECK_EQ(*current_++, ' ');
- ParseHex(&current_);
- CHECK_EQ(*current_++, ':');
- ParseHex(&current_);
- CHECK_EQ(*current_++, ' ');
- while (IsDecimal(*current_))
- current_++;
+ if (*data_.current++ == 'r') segment->protection |= kProtectionRead;
+ CHECK(IsOneOf(*data_.current, '-', 'w'));
+ if (*data_.current++ == 'w') segment->protection |= kProtectionWrite;
+ CHECK(IsOneOf(*data_.current, '-', 'x'));
+ if (*data_.current++ == 'x') segment->protection |= kProtectionExecute;
+ CHECK(IsOneOf(*data_.current, 's', 'p'));
+ if (*data_.current++ == 's') segment->protection |= kProtectionShared;
+ CHECK_EQ(*data_.current++, ' ');
+ segment->offset = ParseHex(&data_.current);
+ CHECK_EQ(*data_.current++, ' ');
+ ParseHex(&data_.current);
+ CHECK_EQ(*data_.current++, ':');
+ ParseHex(&data_.current);
+ CHECK_EQ(*data_.current++, ' ');
+ while (IsDecimal(*data_.current)) data_.current++;
// Qemu may lack the trailing space.
// https://github.com/google/sanitizers/issues/160
- // CHECK_EQ(*current_++, ' ');
+ // CHECK_EQ(*data_.current++, ' ');
// Skip spaces.
- while (current_ < next_line && *current_ == ' ')
- current_++;
+ while (data_.current < next_line && *data_.current == ' ') data_.current++;
// Fill in the filename.
if (segment->filename) {
- uptr len = Min((uptr)(next_line - current_), segment->filename_size - 1);
- internal_strncpy(segment->filename, current_, len);
+ uptr len =
+ Min((uptr)(next_line - data_.current), segment->filename_size - 1);
+ internal_strncpy(segment->filename, data_.current, len);
segment->filename[len] = 0;
}
- current_ = next_line + 1;
+ data_.current = next_line + 1;
return true;
}
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc
index 560451a16d90..0167ab18ba13 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cc
@@ -36,6 +36,51 @@
namespace __sanitizer {
+// Contains information used to iterate through sections.
+struct MemoryMappedSegmentData {
+ char name[kMaxSegName];
+ uptr nsects;
+ const char *current_load_cmd_addr;
+ u32 lc_type;
+ uptr base_virt_addr;
+ uptr addr_mask;
+};
+
+template <typename Section>
+static void NextSectionLoad(LoadedModule *module, MemoryMappedSegmentData *data,
+ bool isWritable) {
+ const Section *sc = (const Section *)data->current_load_cmd_addr;
+ data->current_load_cmd_addr += sizeof(Section);
+
+ uptr sec_start = (sc->addr & data->addr_mask) + data->base_virt_addr;
+ uptr sec_end = sec_start + sc->size;
+ module->addAddressRange(sec_start, sec_end, /*executable=*/false, isWritable,
+ sc->sectname);
+}
+
+void MemoryMappedSegment::AddAddressRanges(LoadedModule *module) {
+ // Don't iterate over sections when the caller hasn't set up the
+ // data pointer, when there are no sections, or when the segment
+ // is executable. Avoid iterating over executable sections because
+ // it will confuse libignore, and because the extra granularity
+ // of information is not needed by any sanitizers.
+ if (!data_ || !data_->nsects || IsExecutable()) {
+ module->addAddressRange(start, end, IsExecutable(), IsWritable(),
+ data_ ? data_->name : nullptr);
+ return;
+ }
+
+ do {
+ if (data_->lc_type == LC_SEGMENT) {
+ NextSectionLoad<struct section>(module, data_, IsWritable());
+#ifdef MH_MAGIC_64
+ } else if (data_->lc_type == LC_SEGMENT_64) {
+ NextSectionLoad<struct section_64>(module, data_, IsWritable());
+#endif
+ }
+ } while (--data_->nsects);
+}
+
MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) {
Reset();
}
@@ -63,13 +108,13 @@ void MemoryMappingLayout::Reset() {
// _dyld_image_count is thread-unsafe. We need to register callbacks for
// adding and removing images which will invalidate the MemoryMappingLayout
// state.
- current_image_ = _dyld_image_count();
- current_load_cmd_count_ = -1;
- current_load_cmd_addr_ = 0;
- current_magic_ = 0;
- current_filetype_ = 0;
- current_arch_ = kModuleArchUnknown;
- internal_memset(current_uuid_, 0, kModuleUUIDSize);
+ data_.current_image = _dyld_image_count();
+ data_.current_load_cmd_count = -1;
+ data_.current_load_cmd_addr = 0;
+ data_.current_magic = 0;
+ data_.current_filetype = 0;
+ data_.current_arch = kModuleArchUnknown;
+ internal_memset(data_.current_uuid, 0, kModuleUUIDSize);
}
// The dyld load address should be unchanged throughout process execution,
@@ -138,39 +183,57 @@ const mach_header *get_dyld_hdr() {
// segment.
// Note that the segment addresses are not necessarily sorted.
template <u32 kLCSegment, typename SegmentCommand>
-bool MemoryMappingLayout::NextSegmentLoad(MemoryMappedSegment *segment) {
- const char *lc = current_load_cmd_addr_;
- current_load_cmd_addr_ += ((const load_command *)lc)->cmdsize;
+static bool NextSegmentLoad(MemoryMappedSegment *segment,
+MemoryMappedSegmentData *seg_data, MemoryMappingLayoutData &layout_data) {
+ const char *lc = layout_data.current_load_cmd_addr;
+ layout_data.current_load_cmd_addr += ((const load_command *)lc)->cmdsize;
if (((const load_command *)lc)->cmd == kLCSegment) {
const SegmentCommand* sc = (const SegmentCommand *)lc;
-
- if (current_image_ == kDyldImageIdx) {
+ uptr base_virt_addr, addr_mask;
+ if (layout_data.current_image == kDyldImageIdx) {
+ base_virt_addr = (uptr)get_dyld_hdr();
// vmaddr is masked with 0xfffff because on macOS versions < 10.12,
// it contains an absolute address rather than an offset for dyld.
// To make matters even more complicated, this absolute address
// isn't actually the absolute segment address, but the offset portion
// of the address is accurate when combined with the dyld base address,
// and the mask will give just this offset.
- segment->start = (sc->vmaddr & 0xfffff) + (uptr)get_dyld_hdr();
- segment->end = (sc->vmaddr & 0xfffff) + sc->vmsize + (uptr)get_dyld_hdr();
+ addr_mask = 0xfffff;
} else {
- const sptr dlloff = _dyld_get_image_vmaddr_slide(current_image_);
- segment->start = sc->vmaddr + dlloff;
- segment->end = sc->vmaddr + sc->vmsize + dlloff;
+ base_virt_addr =
+ (uptr)_dyld_get_image_vmaddr_slide(layout_data.current_image);
+ addr_mask = ~0;
+ }
+
+ segment->start = (sc->vmaddr & addr_mask) + base_virt_addr;
+ segment->end = segment->start + sc->vmsize;
+ // Most callers don't need section information, so only fill this struct
+ // when required.
+ if (seg_data) {
+ seg_data->nsects = sc->nsects;
+ seg_data->current_load_cmd_addr =
+ (const char *)lc + sizeof(SegmentCommand);
+ seg_data->lc_type = kLCSegment;
+ seg_data->base_virt_addr = base_virt_addr;
+ seg_data->addr_mask = addr_mask;
+ internal_strncpy(seg_data->name, sc->segname,
+ ARRAY_SIZE(seg_data->name));
}
// Return the initial protection.
segment->protection = sc->initprot;
- segment->offset =
- (current_filetype_ == /*MH_EXECUTE*/ 0x2) ? sc->vmaddr : sc->fileoff;
+ segment->offset = (layout_data.current_filetype ==
+ /*MH_EXECUTE*/ 0x2)
+ ? sc->vmaddr
+ : sc->fileoff;
if (segment->filename) {
- const char *src = (current_image_ == kDyldImageIdx)
+ const char *src = (layout_data.current_image == kDyldImageIdx)
? kDyldPath
- : _dyld_get_image_name(current_image_);
+ : _dyld_get_image_name(layout_data.current_image);
internal_strncpy(segment->filename, src, segment->filename_size);
}
- segment->arch = current_arch_;
- internal_memcpy(segment->uuid, current_uuid_, kModuleUUIDSize);
+ segment->arch = layout_data.current_arch;
+ internal_memcpy(segment->uuid, layout_data.current_uuid, kModuleUUIDSize);
return true;
}
return false;
@@ -202,7 +265,7 @@ ModuleArch ModuleArchFromCpuType(cpu_type_t cputype, cpu_subtype_t cpusubtype) {
}
static const load_command *NextCommand(const load_command *lc) {
- return (const load_command *)((char *)lc + lc->cmdsize);
+ return (const load_command *)((const char *)lc + lc->cmdsize);
}
static void FindUUID(const load_command *first_lc, u8 *uuid_output) {
@@ -232,50 +295,53 @@ static bool IsModuleInstrumented(const load_command *first_lc) {
}
bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
- for (; current_image_ >= kDyldImageIdx; current_image_--) {
- const mach_header *hdr = (current_image_ == kDyldImageIdx)
+ for (; data_.current_image >= kDyldImageIdx; data_.current_image--) {
+ const mach_header *hdr = (data_.current_image == kDyldImageIdx)
? get_dyld_hdr()
- : _dyld_get_image_header(current_image_);
+ : _dyld_get_image_header(data_.current_image);
if (!hdr) continue;
- if (current_load_cmd_count_ < 0) {
+ if (data_.current_load_cmd_count < 0) {
// Set up for this image;
- current_load_cmd_count_ = hdr->ncmds;
- current_magic_ = hdr->magic;
- current_filetype_ = hdr->filetype;
- current_arch_ = ModuleArchFromCpuType(hdr->cputype, hdr->cpusubtype);
- switch (current_magic_) {
+ data_.current_load_cmd_count = hdr->ncmds;
+ data_.current_magic = hdr->magic;
+ data_.current_filetype = hdr->filetype;
+ data_.current_arch = ModuleArchFromCpuType(hdr->cputype, hdr->cpusubtype);
+ switch (data_.current_magic) {
#ifdef MH_MAGIC_64
case MH_MAGIC_64: {
- current_load_cmd_addr_ = (char*)hdr + sizeof(mach_header_64);
+ data_.current_load_cmd_addr =
+ (const char *)hdr + sizeof(mach_header_64);
break;
}
#endif
case MH_MAGIC: {
- current_load_cmd_addr_ = (char*)hdr + sizeof(mach_header);
+ data_.current_load_cmd_addr = (const char *)hdr + sizeof(mach_header);
break;
}
default: {
continue;
}
}
- FindUUID((const load_command *)current_load_cmd_addr_, &current_uuid_[0]);
- current_instrumented_ =
- IsModuleInstrumented((const load_command *)current_load_cmd_addr_);
+ FindUUID((const load_command *)data_.current_load_cmd_addr,
+ data_.current_uuid);
+ data_.current_instrumented = IsModuleInstrumented(
+ (const load_command *)data_.current_load_cmd_addr);
}
- for (; current_load_cmd_count_ >= 0; current_load_cmd_count_--) {
- switch (current_magic_) {
- // current_magic_ may be only one of MH_MAGIC, MH_MAGIC_64.
+ for (; data_.current_load_cmd_count >= 0; data_.current_load_cmd_count--) {
+ switch (data_.current_magic) {
+ // data_.current_magic may be only one of MH_MAGIC, MH_MAGIC_64.
#ifdef MH_MAGIC_64
case MH_MAGIC_64: {
if (NextSegmentLoad<LC_SEGMENT_64, struct segment_command_64>(
- segment))
+ segment, segment->data_, data_))
return true;
break;
}
#endif
case MH_MAGIC: {
- if (NextSegmentLoad<LC_SEGMENT, struct segment_command>(segment))
+ if (NextSegmentLoad<LC_SEGMENT, struct segment_command>(
+ segment, segment->data_, data_))
return true;
break;
}
@@ -288,11 +354,13 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
}
void MemoryMappingLayout::DumpListOfModules(
- InternalMmapVector<LoadedModule> *modules) {
+ InternalMmapVectorNoCtor<LoadedModule> *modules) {
Reset();
InternalScopedString module_name(kMaxPathLength);
MemoryMappedSegment segment(module_name.data(), kMaxPathLength);
- for (uptr i = 0; Next(&segment); i++) {
+ MemoryMappedSegmentData data;
+ segment.data_ = &data;
+ while (Next(&segment)) {
if (segment.filename[0] == '\0') continue;
LoadedModule *cur_module = nullptr;
if (!modules->empty() &&
@@ -302,10 +370,9 @@ void MemoryMappingLayout::DumpListOfModules(
modules->push_back(LoadedModule());
cur_module = &modules->back();
cur_module->set(segment.filename, segment.start, segment.arch,
- segment.uuid, current_instrumented_);
+ segment.uuid, data_.current_instrumented);
}
- cur_module->addAddressRange(segment.start, segment.end,
- segment.IsExecutable(), segment.IsWritable());
+ segment.AddAddressRanges(cur_module);
}
}
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc
new file mode 100644
index 000000000000..bfe83170f4e2
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cc
@@ -0,0 +1,57 @@
+//===-- sanitizer_procmaps_solaris.cc -------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Information about the process mappings (Solaris-specific parts).
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_SOLARIS
+#include "sanitizer_common.h"
+#include "sanitizer_procmaps.h"
+
+#include <procfs.h>
+#include <limits.h>
+
+namespace __sanitizer {
+
+void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
+ ReadFileToBuffer("/proc/self/xmap", &proc_maps->data, &proc_maps->mmaped_size,
+ &proc_maps->len);
+}
+
+bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
+ char *last = data_.proc_self_maps.data + data_.proc_self_maps.len;
+ if (data_.current >= last) return false;
+
+ prxmap_t *xmapentry = (prxmap_t*)data_.current;
+
+ segment->start = (uptr)xmapentry->pr_vaddr;
+ segment->end = (uptr)(xmapentry->pr_vaddr + xmapentry->pr_size);
+ segment->offset = (uptr)xmapentry->pr_offset;
+
+ segment->protection = 0;
+ if ((xmapentry->pr_mflags & MA_READ) != 0)
+ segment->protection |= kProtectionRead;
+ if ((xmapentry->pr_mflags & MA_WRITE) != 0)
+ segment->protection |= kProtectionWrite;
+ if ((xmapentry->pr_mflags & MA_EXEC) != 0)
+ segment->protection |= kProtectionExecute;
+
+ if (segment->filename != NULL && segment->filename_size > 0) {
+ internal_snprintf(segment->filename,
+ Min(segment->filename_size, (uptr)PATH_MAX), "%s",
+ xmapentry->pr_mapname);
+ }
+
+ data_.current += sizeof(prxmap_t);
+
+ return true;
+}
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_SOLARIS
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h
index db38867ced28..886445a27322 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h
@@ -87,15 +87,14 @@ class Quarantine {
// is zero (it allows us to perform just one atomic read per Put() call).
CHECK((size == 0 && cache_size == 0) || cache_size != 0);
- atomic_store(&max_size_, size, memory_order_relaxed);
- atomic_store(&min_size_, size / 10 * 9,
- memory_order_relaxed); // 90% of max size.
- atomic_store(&max_cache_size_, cache_size, memory_order_relaxed);
+ atomic_store_relaxed(&max_size_, size);
+ atomic_store_relaxed(&min_size_, size / 10 * 9); // 90% of max size.
+ atomic_store_relaxed(&max_cache_size_, cache_size);
}
- uptr GetSize() const { return atomic_load(&max_size_, memory_order_relaxed); }
+ uptr GetSize() const { return atomic_load_relaxed(&max_size_); }
uptr GetCacheSize() const {
- return atomic_load(&max_cache_size_, memory_order_relaxed);
+ return atomic_load_relaxed(&max_cache_size_);
}
void Put(Cache *c, Callback cb, Node *ptr, uptr size) {
@@ -117,7 +116,16 @@ class Quarantine {
cache_.Transfer(c);
}
if (cache_.Size() > GetSize() && recycle_mutex_.TryLock())
- Recycle(cb);
+ Recycle(atomic_load_relaxed(&min_size_), cb);
+ }
+
+ void NOINLINE DrainAndRecycle(Cache *c, Callback cb) {
+ {
+ SpinMutexLock l(&cache_mutex_);
+ cache_.Transfer(c);
+ }
+ recycle_mutex_.Lock();
+ Recycle(0, cb);
}
void PrintStats() const {
@@ -139,9 +147,8 @@ class Quarantine {
Cache cache_;
char pad2_[kCacheLineSize];
- void NOINLINE Recycle(Callback cb) {
+ void NOINLINE Recycle(uptr min_size, Callback cb) {
Cache tmp;
- uptr min_size = atomic_load(&min_size_, memory_order_relaxed);
{
SpinMutexLock l(&cache_mutex_);
// Go over the batches and merge partially filled ones to
@@ -201,7 +208,7 @@ class QuarantineCache {
// Total memory used, including internal accounting.
uptr Size() const {
- return atomic_load(&size_, memory_order_relaxed);
+ return atomic_load_relaxed(&size_);
}
// Memory used for internal accounting.
@@ -225,7 +232,7 @@ class QuarantineCache {
list_.append_back(&from_cache->list_);
SizeAdd(from_cache->Size());
- atomic_store(&from_cache->size_, 0, memory_order_relaxed);
+ atomic_store_relaxed(&from_cache->size_, 0);
}
void EnqueueBatch(QuarantineBatch *b) {
@@ -296,10 +303,10 @@ class QuarantineCache {
atomic_uintptr_t size_;
void SizeAdd(uptr add) {
- atomic_store(&size_, Size() + add, memory_order_relaxed);
+ atomic_store_relaxed(&size_, Size() + add);
}
void SizeSub(uptr sub) {
- atomic_store(&size_, Size() - sub, memory_order_relaxed);
+ atomic_store_relaxed(&size_, Size() - sub);
}
};
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_report_decorator.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_report_decorator.h
index 86536aa19658..060b58d3f2d3 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_report_decorator.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_report_decorator.h
@@ -27,8 +27,9 @@ class SanitizerCommonDecorator {
SanitizerCommonDecorator() : ansi_(ColorizeReports()) {}
const char *Bold() const { return ansi_ ? "\033[1m" : ""; }
const char *Default() const { return ansi_ ? "\033[1m\033[0m" : ""; }
- const char *Warning() { return Red(); }
- const char *EndWarning() { return Default(); }
+ const char *Warning() const { return Red(); }
+ const char *MemoryByte() { return Magenta(); }
+
protected:
const char *Black() const { return ansi_ ? "\033[1m\033[30m" : ""; }
const char *Red() const { return ansi_ ? "\033[1m\033[31m" : ""; }
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc
new file mode 100644
index 000000000000..f51fc049e327
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc
@@ -0,0 +1,87 @@
+//===-- sanitizer_signal_interceptors.inc -----------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Signal interceptors for sanitizers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "interception/interception.h"
+#include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform_interceptors.h"
+
+using namespace __sanitizer;
+
+#if SANITIZER_NETBSD
+#define sigaction_symname __sigaction14
+#else
+#define sigaction_symname sigaction
+#endif
+
+#ifndef SIGNAL_INTERCEPTOR_SIGNAL_IMPL
+#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signum, handler) \
+ { return REAL(func)(signum, handler); }
+#endif
+
+#ifndef SIGNAL_INTERCEPTOR_SIGACTION_IMPL
+#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact) \
+ { return REAL(sigaction_symname)(signum, act, oldact); }
+#endif
+
+#if SANITIZER_INTERCEPT_BSD_SIGNAL
+INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) {
+ if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0;
+ SIGNAL_INTERCEPTOR_SIGNAL_IMPL(bsd_signal, signum, handler);
+}
+#define INIT_BSD_SIGNAL COMMON_INTERCEPT_FUNCTION(bsd_signal)
+#else // SANITIZER_INTERCEPT_BSD_SIGNAL
+#define INIT_BSD_SIGNAL
+#endif // SANITIZER_INTERCEPT_BSD_SIGNAL
+
+#if SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION
+INTERCEPTOR(uptr, signal, int signum, uptr handler) {
+ if (GetHandleSignalMode(signum) == kHandleSignalExclusive)
+ return (uptr) nullptr;
+ SIGNAL_INTERCEPTOR_SIGNAL_IMPL(signal, signum, handler);
+}
+#define INIT_SIGNAL COMMON_INTERCEPT_FUNCTION(signal)
+
+INTERCEPTOR(int, sigaction_symname, int signum,
+ const __sanitizer_sigaction *act, __sanitizer_sigaction *oldact) {
+ if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0;
+ SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact);
+}
+#define INIT_SIGACTION COMMON_INTERCEPT_FUNCTION(sigaction_symname)
+
+namespace __sanitizer {
+int real_sigaction(int signum, const void *act, void *oldact) {
+ return REAL(sigaction_symname)(signum, (const __sanitizer_sigaction *)act,
+ (__sanitizer_sigaction *)oldact);
+}
+} // namespace __sanitizer
+#else // SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION
+#define INIT_SIGNAL
+#define INIT_SIGACTION
+// We need to have defined REAL(sigaction) on other systems.
+namespace __sanitizer {
+struct __sanitizer_sigaction;
+}
+DEFINE_REAL(int, sigaction, int signum, const __sanitizer_sigaction *act,
+ __sanitizer_sigaction *oldact)
+#endif // SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION
+
+static void InitializeSignalInterceptors() {
+ static bool was_called_once;
+ CHECK(!was_called_once);
+ was_called_once = true;
+
+ INIT_BSD_SIGNAL;
+ INIT_SIGNAL;
+ INIT_SIGACTION;
+}
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc
new file mode 100644
index 000000000000..a5db22994e0a
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_solaris.cc
@@ -0,0 +1,219 @@
+//===-- sanitizer_solaris.cc ----------------------------------------------===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries and
+// implements Solaris-specific functions.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_SOLARIS
+
+#include <stdio.h>
+
+#include "sanitizer_common.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_placement_new.h"
+#include "sanitizer_platform_limits_posix.h"
+#include "sanitizer_procmaps.h"
+
+#include <fcntl.h>
+#include <pthread.h>
+#include <sched.h>
+#include <thread.h>
+#include <synch.h>
+#include <signal.h>
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdlib.h>
+
+namespace __sanitizer {
+
+//#include "sanitizer_syscall_generic.inc"
+
+#define _REAL(func) _ ## func
+#define DECLARE__REAL(ret_type, func, ...) \
+ extern "C" ret_type _REAL(func)(__VA_ARGS__)
+#define DECLARE__REAL_AND_INTERNAL(ret_type, func, ...) \
+ DECLARE__REAL(ret_type, func, __VA_ARGS__); \
+ ret_type internal_ ## func(__VA_ARGS__)
+
+// ---------------------- sanitizer_libc.h
+DECLARE__REAL_AND_INTERNAL(uptr, mmap, void *addr, uptr /*size_t*/ length,
+ int prot, int flags, int fd, OFF_T offset) {
+ return (uptr)_REAL(mmap)(addr, length, prot, flags, fd, offset);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, munmap, void *addr, uptr length) {
+ return _REAL(munmap)(addr, length);
+}
+
+DECLARE__REAL_AND_INTERNAL(int, mprotect, void *addr, uptr length, int prot) {
+ return _REAL(mprotect)(addr, length, prot);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, close, fd_t fd) {
+ return _REAL(close)(fd);
+}
+
+extern "C" int _REAL(open)(const char *, int, ...);
+
+uptr internal_open(const char *filename, int flags) {
+ return _REAL(open)(filename, flags);
+}
+
+uptr internal_open(const char *filename, int flags, u32 mode) {
+ return _REAL(open)(filename, flags, mode);
+}
+
+uptr OpenFile(const char *filename, bool write) {
+ return internal_open(filename,
+ write ? O_WRONLY | O_CREAT : O_RDONLY, 0660);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, read, fd_t fd, void *buf, uptr count) {
+ return _REAL(read)(fd, buf, count);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, write, fd_t fd, const void *buf, uptr count) {
+ return _REAL(write)(fd, buf, count);
+}
+
+// FIXME: There's only _ftruncate64 beginning with Solaris 11.
+DECLARE__REAL_AND_INTERNAL(uptr, ftruncate, fd_t fd, uptr size) {
+ return ftruncate(fd, size);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, stat, const char *path, void *buf) {
+ return _REAL(stat)(path, (struct stat *)buf);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, lstat, const char *path, void *buf) {
+ return _REAL(lstat)(path, (struct stat *)buf);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, fstat, fd_t fd, void *buf) {
+ return _REAL(fstat)(fd, (struct stat *)buf);
+}
+
+uptr internal_filesize(fd_t fd) {
+ struct stat st;
+ if (internal_fstat(fd, &st))
+ return -1;
+ return (uptr)st.st_size;
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, dup2, int oldfd, int newfd) {
+ return _REAL(dup2)(oldfd, newfd);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, readlink, const char *path, char *buf,
+ uptr bufsize) {
+ return _REAL(readlink)(path, buf, bufsize);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, unlink, const char *path) {
+ return _REAL(unlink)(path);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, rename, const char *oldpath,
+ const char *newpath) {
+ return _REAL(rename)(oldpath, newpath);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, sched_yield, void) {
+ return sched_yield();
+}
+
+DECLARE__REAL_AND_INTERNAL(void, _exit, int exitcode) {
+ _exit(exitcode);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, execve, const char *filename,
+ char *const argv[], char *const envp[]) {
+ return _REAL(execve)(filename, argv, envp);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, waitpid, int pid, int *status, int options) {
+ return _REAL(waitpid)(pid, status, options);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, getpid, void) {
+ return _REAL(getpid)();
+}
+
+// FIXME: This might be wrong: _getdents doesn't take a struct linux_dirent *.
+DECLARE__REAL_AND_INTERNAL(uptr, getdents, fd_t fd, struct linux_dirent *dirp,
+ unsigned int count) {
+ return _REAL(getdents)(fd, dirp, count);
+}
+
+DECLARE__REAL_AND_INTERNAL(uptr, lseek, fd_t fd, OFF_T offset, int whence) {
+ return _REAL(lseek)(fd, offset, whence);
+}
+
+// FIXME: This might be wrong: _sigfillset doesn't take a
+// __sanitizer_sigset_t *.
+DECLARE__REAL_AND_INTERNAL(void, sigfillset, __sanitizer_sigset_t *set) {
+ _REAL(sigfillset)(set);
+}
+
+// FIXME: This might be wrong: _sigprocmask doesn't take __sanitizer_sigset_t *.
+DECLARE__REAL_AND_INTERNAL(uptr, sigprocmask, int how,
+ __sanitizer_sigset_t *set,
+ __sanitizer_sigset_t *oldset) {
+ return _REAL(sigprocmask)(how, set, oldset);
+}
+
+DECLARE__REAL_AND_INTERNAL(int, fork, void) {
+ // TODO(glider): this may call user's pthread_atfork() handlers which is bad.
+ return _REAL(fork)();
+}
+
+u64 NanoTime() {
+ return gethrtime();
+}
+
+uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) {
+ // FIXME: No internal variant.
+ return clock_gettime(clk_id, (timespec *)tp);
+}
+
+// ----------------- sanitizer_common.h
+BlockingMutex::BlockingMutex() {
+ CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_));
+ internal_memset(this, 0, sizeof(*this));
+ CHECK_EQ(mutex_init((mutex_t *)&opaque_storage_, USYNC_THREAD, NULL), 0);
+}
+
+void BlockingMutex::Lock() {
+ CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_));
+ CHECK_NE(owner_, (uptr)thr_self());
+ CHECK_EQ(mutex_lock((mutex_t *)&opaque_storage_), 0);
+ CHECK(!owner_);
+ owner_ = (uptr)thr_self();
+}
+
+void BlockingMutex::Unlock() {
+ CHECK(owner_ == (uptr)thr_self());
+ owner_ = 0;
+ CHECK_EQ(mutex_unlock((mutex_t *)&opaque_storage_), 0);
+}
+
+void BlockingMutex::CheckLocked() {
+ CHECK_EQ((uptr)thr_self(), owner_);
+}
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_SOLARIS
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h
index 90142dffd5e4..f15196e4b725 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h
@@ -97,6 +97,11 @@ struct BufferedStackTrace : public StackTrace {
void Unwind(u32 max_depth, uptr pc, uptr bp, void *context, uptr stack_top,
uptr stack_bottom, bool request_fast_unwind);
+ void Reset() {
+ *static_cast<StackTrace *>(this) = StackTrace(trace_buffer, 0);
+ top_frame_bp = 0;
+ }
+
private:
void FastUnwindStack(uptr pc, uptr bp, uptr stack_top, uptr stack_bottom,
u32 max_depth);
@@ -106,8 +111,8 @@ struct BufferedStackTrace : public StackTrace {
void PopStackFrames(uptr count);
uptr LocatePcInTrace(uptr pc);
- BufferedStackTrace(const BufferedStackTrace &);
- void operator=(const BufferedStackTrace &);
+ BufferedStackTrace(const BufferedStackTrace &) = delete;
+ void operator=(const BufferedStackTrace &) = delete;
};
// Check if given pointer points into allocated stack area.
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc
index 377f1ce755be..a271302708be 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cc
@@ -12,9 +12,14 @@
//===----------------------------------------------------------------------===//
#include "sanitizer_stacktrace_printer.h"
+#include "sanitizer_file.h"
+#include "sanitizer_fuchsia.h"
namespace __sanitizer {
+// sanitizer_symbolizer_fuchsia.cc implements these differently for Fuchsia.
+#if !SANITIZER_FUCHSIA
+
static const char *StripFunctionName(const char *function, const char *prefix) {
if (!function) return nullptr;
if (!prefix) return function;
@@ -24,6 +29,80 @@ static const char *StripFunctionName(const char *function, const char *prefix) {
return function;
}
+static const char *DemangleFunctionName(const char *function) {
+ if (!function) return nullptr;
+
+ // NetBSD uses indirection for old threading functions for historical reasons
+ // The mangled names are internal implementation detail and should not be
+ // exposed even in backtraces.
+#if SANITIZER_NETBSD
+ if (!internal_strcmp(function, "__libc_mutex_init"))
+ return "pthread_mutex_init";
+ if (!internal_strcmp(function, "__libc_mutex_lock"))
+ return "pthread_mutex_lock";
+ if (!internal_strcmp(function, "__libc_mutex_trylock"))
+ return "pthread_mutex_trylock";
+ if (!internal_strcmp(function, "__libc_mutex_unlock"))
+ return "pthread_mutex_unlock";
+ if (!internal_strcmp(function, "__libc_mutex_destroy"))
+ return "pthread_mutex_destroy";
+ if (!internal_strcmp(function, "__libc_mutexattr_init"))
+ return "pthread_mutexattr_init";
+ if (!internal_strcmp(function, "__libc_mutexattr_settype"))
+ return "pthread_mutexattr_settype";
+ if (!internal_strcmp(function, "__libc_mutexattr_destroy"))
+ return "pthread_mutexattr_destroy";
+ if (!internal_strcmp(function, "__libc_cond_init"))
+ return "pthread_cond_init";
+ if (!internal_strcmp(function, "__libc_cond_signal"))
+ return "pthread_cond_signal";
+ if (!internal_strcmp(function, "__libc_cond_broadcast"))
+ return "pthread_cond_broadcast";
+ if (!internal_strcmp(function, "__libc_cond_wait"))
+ return "pthread_cond_wait";
+ if (!internal_strcmp(function, "__libc_cond_timedwait"))
+ return "pthread_cond_timedwait";
+ if (!internal_strcmp(function, "__libc_cond_destroy"))
+ return "pthread_cond_destroy";
+ if (!internal_strcmp(function, "__libc_rwlock_init"))
+ return "pthread_rwlock_init";
+ if (!internal_strcmp(function, "__libc_rwlock_rdlock"))
+ return "pthread_rwlock_rdlock";
+ if (!internal_strcmp(function, "__libc_rwlock_wrlock"))
+ return "pthread_rwlock_wrlock";
+ if (!internal_strcmp(function, "__libc_rwlock_tryrdlock"))
+ return "pthread_rwlock_tryrdlock";
+ if (!internal_strcmp(function, "__libc_rwlock_trywrlock"))
+ return "pthread_rwlock_trywrlock";
+ if (!internal_strcmp(function, "__libc_rwlock_unlock"))
+ return "pthread_rwlock_unlock";
+ if (!internal_strcmp(function, "__libc_rwlock_destroy"))
+ return "pthread_rwlock_destroy";
+ if (!internal_strcmp(function, "__libc_thr_keycreate"))
+ return "pthread_key_create";
+ if (!internal_strcmp(function, "__libc_thr_setspecific"))
+ return "pthread_setspecific";
+ if (!internal_strcmp(function, "__libc_thr_getspecific"))
+ return "pthread_getspecific";
+ if (!internal_strcmp(function, "__libc_thr_keydelete"))
+ return "pthread_key_delete";
+ if (!internal_strcmp(function, "__libc_thr_once"))
+ return "pthread_once";
+ if (!internal_strcmp(function, "__libc_thr_self"))
+ return "pthread_self";
+ if (!internal_strcmp(function, "__libc_thr_exit"))
+ return "pthread_exit";
+ if (!internal_strcmp(function, "__libc_thr_setcancelstate"))
+ return "pthread_setcancelstate";
+ if (!internal_strcmp(function, "__libc_thr_equal"))
+ return "pthread_equal";
+ if (!internal_strcmp(function, "__libc_thr_curcpu"))
+ return "pthread_curcpu_np";
+#endif
+
+ return function;
+}
+
static const char kDefaultFormat[] = " #%n %p %F %L";
void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
@@ -55,7 +134,9 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
buffer->append("0x%zx", info.module_offset);
break;
case 'f':
- buffer->append("%s", StripFunctionName(info.function, strip_func_prefix));
+ buffer->append("%s",
+ DemangleFunctionName(
+ StripFunctionName(info.function, strip_func_prefix)));
break;
case 'q':
buffer->append("0x%zx", info.function_offset != AddressInfo::kUnknown
@@ -76,7 +157,8 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
// Function name and offset, if file is unknown.
if (info.function) {
buffer->append("in %s",
- StripFunctionName(info.function, strip_func_prefix));
+ DemangleFunctionName(
+ StripFunctionName(info.function, strip_func_prefix)));
if (!info.file && info.function_offset != AddressInfo::kUnknown)
buffer->append("+0x%zx", info.function_offset);
}
@@ -146,6 +228,8 @@ void RenderData(InternalScopedString *buffer, const char *format,
}
}
+#endif // !SANITIZER_FUCHSIA
+
void RenderSourceLocation(InternalScopedString *buffer, const char *file,
int line, int column, bool vs_style,
const char *strip_path_prefix) {
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc
index d7fa5f6451d1..0f543d2d7c87 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc
@@ -57,6 +57,14 @@
#include "sanitizer_mutex.h"
#include "sanitizer_placement_new.h"
+// Sufficiently old kernel headers don't provide this value, but we can still
+// call prctl with it. If the runtime kernel is new enough, the prctl call will
+// have the desired effect; if the kernel is too old, the call will error and we
+// can ignore said error.
+#ifndef PR_SET_PTRACER
+#define PR_SET_PTRACER 0x59616d61
+#endif
+
// This module works by spawning a Linux task which then attaches to every
// thread in the caller process with ptrace. This suspends the threads, and
// PTRACE_GETREGS can then be used to obtain their register state. The callback
@@ -245,8 +253,9 @@ static void TracerThreadDieCallback() {
}
// Signal handler to wake up suspended threads when the tracer thread dies.
-static void TracerThreadSignalHandler(int signum, void *siginfo, void *uctx) {
- SignalContext ctx = SignalContext::Create(siginfo, uctx);
+static void TracerThreadSignalHandler(int signum, __sanitizer_siginfo *siginfo,
+ void *uctx) {
+ SignalContext ctx(siginfo, uctx);
Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum,
ctx.addr, ctx.pc, ctx.sp);
ThreadSuspender *inst = thread_suspender_instance;
@@ -263,7 +272,7 @@ static void TracerThreadSignalHandler(int signum, void *siginfo, void *uctx) {
}
// Size of alternative stack for signal handlers in the tracer thread.
-static const int kHandlerStackSize = 4096;
+static const int kHandlerStackSize = 8192;
// This function will be run as a cloned task.
static int TracerThread(void* argument) {
@@ -433,9 +442,7 @@ void StopTheWorld(StopTheWorldCallback callback, void *argument) {
ScopedSetTracerPID scoped_set_tracer_pid(tracer_pid);
// On some systems we have to explicitly declare that we want to be traced
// by the tracer thread.
-#ifdef PR_SET_PTRACER
internal_prctl(PR_SET_PTRACER, tracer_pid, 0, 0, 0);
-#endif
// Allow the tracer thread to start.
tracer_thread_argument.mutex.Unlock();
// NOTE: errno is shared between this thread and the tracer thread.
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc
index 0c27c472f02e..d84ebef3b7f9 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cc
@@ -55,17 +55,9 @@ void RunThread(void *arg) {
struct RunThreadArgs *run_args = (struct RunThreadArgs *)arg;
SuspendedThreadsListMac suspended_threads_list;
- mach_port_t task;
- kern_return_t err = task_for_pid(mach_task_self(), internal_getpid(), &task);
- if (err != KERN_SUCCESS) {
- VReport(1, "Getting task from pid failed (errno %d).\n", err);
- return;
- }
-
thread_array_t threads;
mach_msg_type_number_t num_threads;
-
- err = task_threads(task, &threads, &num_threads);
+ kern_return_t err = task_threads(mach_task_self(), &threads, &num_threads);
if (err != KERN_SUCCESS) {
VReport(1, "Failed to get threads for task (errno %d).\n", err);
return;
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc
index f0f2c9c725a1..89ddfddd1108 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cc
@@ -16,6 +16,7 @@
#include "sanitizer_allocator_internal.h"
#include "sanitizer_common.h"
#include "sanitizer_flags.h"
+#include "sanitizer_file.h"
#include "sanitizer_libc.h"
#include "sanitizer_placement_new.h"
@@ -50,6 +51,7 @@ void SuppressionContext::ParseFromFile(const char *filename) {
if (filename[0] == '\0')
return;
+#if !SANITIZER_FUCHSIA
// If we cannot find the file, check if its location is relative to
// the location of the executable.
InternalScopedString new_file_path(kMaxPathLength);
@@ -58,6 +60,7 @@ void SuppressionContext::ParseFromFile(const char *filename) {
new_file_path.size())) {
filename = new_file_path.data();
}
+#endif // !SANITIZER_FUCHSIA
// Read the file.
VPrintf(1, "%s: reading suppressions file at %s\n",
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cc
index 1cd5b6ee24c0..672c936684be 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.cc
@@ -71,6 +71,10 @@ Symbolizer *Symbolizer::symbolizer_;
StaticSpinMutex Symbolizer::init_mu_;
LowLevelAllocator Symbolizer::symbolizer_allocator_;
+void Symbolizer::InvalidateModuleList() {
+ modules_fresh_ = false;
+}
+
void Symbolizer::AddHooks(Symbolizer::StartSymbolizationHook start_hook,
Symbolizer::EndSymbolizationHook end_hook) {
CHECK(start_hook_ == 0 && end_hook_ == 0);
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h
index 4fc7742a21e3..208362d2f478 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h
@@ -119,8 +119,11 @@ class Symbolizer final {
void AddHooks(StartSymbolizationHook start_hook,
EndSymbolizationHook end_hook);
+ void RefreshModules();
const LoadedModule *FindModuleForAddress(uptr address);
+ void InvalidateModuleList();
+
private:
// GetModuleNameAndOffsetForPC has to return a string to the caller.
// Since the corresponding module might get unloaded later, we should create
@@ -149,6 +152,7 @@ class Symbolizer final {
uptr *module_offset,
ModuleArch *module_arch);
ListOfModules modules_;
+ ListOfModules fallback_modules_;
// If stale, need to reload the modules before looking up addresses.
bool modules_fresh_;
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.cc
new file mode 100644
index 000000000000..3d1117d9d3ca
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_fuchsia.cc
@@ -0,0 +1,107 @@
+//===-- sanitizer_symbolizer_fuchsia.cc -----------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries.
+//
+// Implementation of Fuchsia-specific symbolizer.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_FUCHSIA
+
+#include "sanitizer_fuchsia.h"
+#include "sanitizer_symbolizer.h"
+
+namespace __sanitizer {
+
+// For Fuchsia we don't do any actual symbolization per se.
+// Instead, we emit text containing raw addresses and raw linkage
+// symbol names, embedded in Fuchsia's symbolization markup format.
+// Fuchsia's logging infrastructure emits enough information about
+// process memory layout that a post-processing filter can do the
+// symbolization and pretty-print the markup. See the spec at:
+// https://fuchsia.googlesource.com/zircon/+/master/docs/symbolizer_markup.md
+
+// This is used by UBSan for type names, and by ASan for global variable names.
+constexpr const char *kFormatDemangle = "{{{symbol:%s}}}";
+constexpr uptr kFormatDemangleMax = 1024; // Arbitrary.
+
+// Function name or equivalent from PC location.
+constexpr const char *kFormatFunction = "{{{pc:%p}}}";
+constexpr uptr kFormatFunctionMax = 64; // More than big enough for 64-bit hex.
+
+// Global variable name or equivalent from data memory address.
+constexpr const char *kFormatData = "{{{data:%p}}}";
+
+// One frame in a backtrace (printed on a line by itself).
+constexpr const char *kFormatFrame = "{{{bt:%u:%p}}}";
+
+// This is used by UBSan for type names, and by ASan for global variable names.
+// It's expected to return a static buffer that will be reused on each call.
+const char *Symbolizer::Demangle(const char *name) {
+ static char buffer[kFormatDemangleMax];
+ internal_snprintf(buffer, sizeof(buffer), kFormatDemangle, name);
+ return buffer;
+}
+
+// This is used mostly for suppression matching. Making it work
+// would enable "interceptor_via_lib" suppressions. It's also used
+// once in UBSan to say "in module ..." in a message that also
+// includes an address in the module, so post-processing can already
+// pretty-print that so as to indicate the module.
+bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
+ uptr *module_address) {
+ return false;
+}
+
+// This is used in some places for suppression checking, which we
+// don't really support for Fuchsia. It's also used in UBSan to
+// identify a PC location to a function name, so we always fill in
+// the function member with a string containing markup around the PC
+// value.
+// TODO(mcgrathr): Under SANITIZER_GO, it's currently used by TSan
+// to render stack frames, but that should be changed to use
+// RenderStackFrame.
+SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
+ SymbolizedStack *s = SymbolizedStack::New(addr);
+ char buffer[kFormatFunctionMax];
+ internal_snprintf(buffer, sizeof(buffer), kFormatFunction, addr);
+ s->info.function = internal_strdup(buffer);
+ return s;
+}
+
+// Always claim we succeeded, so that RenderDataInfo will be called.
+bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
+ info->Clear();
+ info->start = addr;
+ return true;
+}
+
+// We ignore the format argument to __sanitizer_symbolize_global.
+void RenderData(InternalScopedString *buffer, const char *format,
+ const DataInfo *DI, const char *strip_path_prefix) {
+ buffer->append(kFormatData, DI->start);
+}
+
+// We don't support the stack_trace_format flag at all.
+void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
+ const AddressInfo &info, bool vs_style,
+ const char *strip_path_prefix, const char *strip_func_prefix) {
+ buffer->append(kFormatFrame, frame_no, info.address);
+}
+
+Symbolizer *Symbolizer::PlatformInit() {
+ return new (symbolizer_allocator_) Symbolizer({});
+}
+
+void Symbolizer::LateInitialize() { Symbolizer::GetOrInit(); }
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_FUCHSIA
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h
index 2ae42b33868b..a2ac1188241d 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h
@@ -15,6 +15,7 @@
#define SANITIZER_SYMBOLIZER_INTERNAL_H
#include "sanitizer_symbolizer.h"
+#include "sanitizer_file.h"
namespace __sanitizer {
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc
index 57354668bfbb..e98fab01d6bb 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cc
@@ -95,7 +95,8 @@ struct SymbolizeCodeCallbackArg {
if (frames_symbolized > 0) {
SymbolizedStack *cur = SymbolizedStack::New(addr);
AddressInfo *info = &cur->info;
- info->FillModuleInfo(first->info.module, first->info.module_offset);
+ info->FillModuleInfo(first->info.module, first->info.module_offset,
+ first->info.module_arch);
last->next = cur;
last = cur;
}
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc
index 614470a633d0..a4bab668b27a 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc
@@ -17,6 +17,18 @@
namespace __sanitizer {
+Symbolizer *Symbolizer::GetOrInit() {
+ SpinMutexLock l(&init_mu_);
+ if (symbolizer_)
+ return symbolizer_;
+ symbolizer_ = PlatformInit();
+ CHECK(symbolizer_);
+ return symbolizer_;
+}
+
+// See sanitizer_symbolizer_fuchsia.cc.
+#if !SANITIZER_FUCHSIA
+
const char *ExtractToken(const char *str, const char *delims, char **result) {
uptr prefix_len = internal_strcspn(str, delims);
*result = (char*)InternalAlloc(prefix_len + 1);
@@ -151,37 +163,47 @@ bool Symbolizer::FindModuleNameAndOffsetForAddress(uptr address,
return true;
}
+void Symbolizer::RefreshModules() {
+ modules_.init();
+ fallback_modules_.fallbackInit();
+ RAW_CHECK(modules_.size() > 0);
+ modules_fresh_ = true;
+}
+
+static const LoadedModule *SearchForModule(const ListOfModules &modules,
+ uptr address) {
+ for (uptr i = 0; i < modules.size(); i++) {
+ if (modules[i].containsAddress(address)) {
+ return &modules[i];
+ }
+ }
+ return nullptr;
+}
+
const LoadedModule *Symbolizer::FindModuleForAddress(uptr address) {
bool modules_were_reloaded = false;
if (!modules_fresh_) {
- modules_.init();
- RAW_CHECK(modules_.size() > 0);
- modules_fresh_ = true;
+ RefreshModules();
modules_were_reloaded = true;
}
- for (uptr i = 0; i < modules_.size(); i++) {
- if (modules_[i].containsAddress(address)) {
- return &modules_[i];
- }
- }
- // Reload the modules and look up again, if we haven't tried it yet.
+ const LoadedModule *module = SearchForModule(modules_, address);
+ if (module) return module;
+
+ // dlopen/dlclose interceptors invalidate the module list, but when
+ // interception is disabled, we need to retry if the lookup fails in
+ // case the module list changed.
+#if !SANITIZER_INTERCEPT_DLOPEN_DLCLOSE
if (!modules_were_reloaded) {
- // FIXME: set modules_fresh_ from dlopen()/dlclose() interceptors.
- // It's too aggressive to reload the list of modules each time we fail
- // to find a module for a given address.
- modules_fresh_ = false;
- return FindModuleForAddress(address);
+ RefreshModules();
+ module = SearchForModule(modules_, address);
+ if (module) return module;
}
- return 0;
-}
+#endif
-Symbolizer *Symbolizer::GetOrInit() {
- SpinMutexLock l(&init_mu_);
- if (symbolizer_)
- return symbolizer_;
- symbolizer_ = PlatformInit();
- CHECK(symbolizer_);
- return symbolizer_;
+ if (fallback_modules_.size()) {
+ module = SearchForModule(fallback_modules_, address);
+ }
+ return module;
}
// For now we assume the following protocol:
@@ -451,7 +473,7 @@ bool SymbolizerProcess::ReadFromSymbolizer(char *buffer, uptr max_length) {
if (ReachedEndOfOutput(buffer, read_len))
break;
if (read_len + 1 == max_length) {
- Report("WARNING: Symbolizer buffer too small");
+ Report("WARNING: Symbolizer buffer too small\n");
read_len = 0;
break;
}
@@ -472,4 +494,6 @@ bool SymbolizerProcess::WriteToSymbolizer(const char *buffer, uptr length) {
return true;
}
+#endif // !SANITIZER_FUCHSIA
+
} // namespace __sanitizer
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc
index 60ec7506c312..71d748d0515f 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc
@@ -16,6 +16,7 @@
#if SANITIZER_POSIX
#include "sanitizer_allocator_internal.h"
#include "sanitizer_common.h"
+#include "sanitizer_file.h"
#include "sanitizer_flags.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_linux.h"
@@ -76,6 +77,7 @@ static swift_demangle_ft swift_demangle_f;
// symbolication.
static void InitializeSwiftDemangler() {
swift_demangle_f = (swift_demangle_ft)dlsym(RTLD_DEFAULT, "swift_demangle");
+ (void)dlerror(); // Cleanup error message in case of failure
}
// Attempts to demangle a Swift name. The demangler will return nullptr if a
@@ -271,6 +273,10 @@ class Addr2LineProcess : public SymbolizerProcess {
bool ReadFromSymbolizer(char *buffer, uptr max_length) override {
if (!SymbolizerProcess::ReadFromSymbolizer(buffer, max_length))
return false;
+ // The returned buffer is empty when output is valid, but exceeds
+ // max_length.
+ if (*buffer == '\0')
+ return true;
// We should cut out output_terminator_ at the end of given buffer,
// appended by addr2line to mark the end of its meaningful output.
// We cannot scan buffer from it's beginning, because it is legal for it
@@ -470,16 +476,16 @@ static SymbolizerTool *ChooseExternalSymbolizer(LowLevelAllocator *allocator) {
// Otherwise symbolizer program is unknown, let's search $PATH
CHECK(path == nullptr);
- if (const char *found_path = FindPathToBinary("llvm-symbolizer")) {
- VReport(2, "Using llvm-symbolizer found at: %s\n", found_path);
- return new(*allocator) LLVMSymbolizer(found_path, allocator);
- }
#if SANITIZER_MAC
if (const char *found_path = FindPathToBinary("atos")) {
VReport(2, "Using atos found at: %s\n", found_path);
return new(*allocator) AtosSymbolizer(found_path, allocator);
}
#endif // SANITIZER_MAC
+ if (const char *found_path = FindPathToBinary("llvm-symbolizer")) {
+ VReport(2, "Using llvm-symbolizer found at: %s\n", found_path);
+ return new(*allocator) LLVMSymbolizer(found_path, allocator);
+ }
if (common_flags()->allow_addr2line) {
if (const char *found_path = FindPathToBinary("addr2line")) {
VReport(2, "Using addr2line found at: %s\n", found_path);
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc
index 15cf05f06087..3c8e77867ce5 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_generic.inc
@@ -7,20 +7,49 @@
//
//===----------------------------------------------------------------------===//
//
-// Generic implementations of internal_syscall and internal_iserror.
+// Generic implementations of internal_syscall* and internal_iserror.
//
//===----------------------------------------------------------------------===//
-#if SANITIZER_FREEBSD || SANITIZER_MAC
+#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD || SANITIZER_SOLARIS
# define SYSCALL(name) SYS_ ## name
#else
# define SYSCALL(name) __NR_ ## name
#endif
-#if (SANITIZER_FREEBSD || SANITIZER_MAC) && defined(__x86_64__)
+#if SANITIZER_NETBSD
+// We use 3 kinds of internal_syscall's for different types of retval in order
+// to address differences in calling conventions (e.g. registers to place the
+// return value in).
+// - internal_syscall for 32-bit length (int, pid_t)
+// - internal_syscall64 for 64-bit length (off_t)
+// - internal_syscall_ptr for pointer and (s)size_t
+# define internal_syscall syscall
+# define internal_syscall64 __syscall
+// Handle syscall renames manually
+# define SYS_stat SYS___stat50
+# define SYS_lstat SYS___lstat50
+# define SYS_fstat SYS___fstat50
+# define SYS_gettimeofday SYS___gettimeofday50
+# define SYS_wait4 SYS___wait450
+# define SYS_getdents SYS___getdents30
+# define SYS_sigaltstack SYS___sigaltstack14
+# define SYS_sigprocmask SYS___sigprocmask14
+# define SYS_nanosleep SYS___nanosleep50
+# define SYS_clock_gettime SYS___clock_gettime50
+# if SANITIZER_WORDSIZE == 64
+# define internal_syscall_ptr __syscall
+# else
+# define internal_syscall_ptr syscall
+# endif
+#elif defined(__x86_64__) && (SANITIZER_FREEBSD || SANITIZER_MAC)
# define internal_syscall __syscall
+# define internal_syscall64 __syscall
+# define internal_syscall_ptr __syscall
# else
# define internal_syscall syscall
+# define internal_syscall64 syscall
+# define internal_syscall_ptr syscall
#endif
bool internal_iserror(uptr retval, int *rverrno) {
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc
index 7ab1d7641449..1f05ed9b6c1c 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc
@@ -127,6 +127,9 @@ static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3,
#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__)
+#define internal_syscall_ptr internal_syscall
+#define internal_syscall64 internal_syscall
+
// Helper function used to avoid cobbler errno.
bool internal_iserror(uptr retval, int *rverrno) {
if (retval >= (uptr)-4095) {
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc
new file mode 100644
index 000000000000..a3fdb9e60d5d
--- /dev/null
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_arm.inc
@@ -0,0 +1,141 @@
+//===-- sanitizer_syscall_linux_arm.inc -------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementations of internal_syscall and internal_iserror for Linux/arm.
+//
+//===----------------------------------------------------------------------===//
+
+#define SYSCALL(name) __NR_ ## name
+
+static uptr __internal_syscall(u32 nr) {
+ register u32 r8 asm("r7") = nr;
+ register u32 r0 asm("r0");
+ asm volatile("swi #0"
+ : "=r"(r0)
+ : "r"(r8)
+ : "memory", "cc");
+ return r0;
+}
+#define __internal_syscall0(n) \
+ (__internal_syscall)(n)
+
+static uptr __internal_syscall(u32 nr, u32 arg1) {
+ register u32 r8 asm("r7") = nr;
+ register u32 r0 asm("r0") = arg1;
+ asm volatile("swi #0"
+ : "=r"(r0)
+ : "r"(r8), "0"(r0)
+ : "memory", "cc");
+ return r0;
+}
+#define __internal_syscall1(n, a1) \
+ (__internal_syscall)(n, (u32)(a1))
+
+static uptr __internal_syscall(u32 nr, u32 arg1, long arg2) {
+ register u32 r8 asm("r7") = nr;
+ register u32 r0 asm("r0") = arg1;
+ register u32 r1 asm("r1") = arg2;
+ asm volatile("swi #0"
+ : "=r"(r0)
+ : "r"(r8), "0"(r0), "r"(r1)
+ : "memory", "cc");
+ return r0;
+}
+#define __internal_syscall2(n, a1, a2) \
+ (__internal_syscall)(n, (u32)(a1), (long)(a2))
+
+static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3) {
+ register u32 r8 asm("r7") = nr;
+ register u32 r0 asm("r0") = arg1;
+ register u32 r1 asm("r1") = arg2;
+ register u32 r2 asm("r2") = arg3;
+ asm volatile("swi #0"
+ : "=r"(r0)
+ : "r"(r8), "0"(r0), "r"(r1), "r"(r2)
+ : "memory", "cc");
+ return r0;
+}
+#define __internal_syscall3(n, a1, a2, a3) \
+ (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3))
+
+static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3,
+ u32 arg4) {
+ register u32 r8 asm("r7") = nr;
+ register u32 r0 asm("r0") = arg1;
+ register u32 r1 asm("r1") = arg2;
+ register u32 r2 asm("r2") = arg3;
+ register u32 r3 asm("r3") = arg4;
+ asm volatile("swi #0"
+ : "=r"(r0)
+ : "r"(r8), "0"(r0), "r"(r1), "r"(r2), "r"(r3)
+ : "memory", "cc");
+ return r0;
+}
+#define __internal_syscall4(n, a1, a2, a3, a4) \
+ (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3), (long)(a4))
+
+static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3,
+ u32 arg4, long arg5) {
+ register u32 r8 asm("r7") = nr;
+ register u32 r0 asm("r0") = arg1;
+ register u32 r1 asm("r1") = arg2;
+ register u32 r2 asm("r2") = arg3;
+ register u32 r3 asm("r3") = arg4;
+ register u32 r4 asm("r4") = arg5;
+ asm volatile("swi #0"
+ : "=r"(r0)
+ : "r"(r8), "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4)
+ : "memory", "cc");
+ return r0;
+}
+#define __internal_syscall5(n, a1, a2, a3, a4, a5) \
+ (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3), (long)(a4), \
+ (u32)(a5))
+
+static uptr __internal_syscall(u32 nr, u32 arg1, long arg2, long arg3,
+ u32 arg4, long arg5, long arg6) {
+ register u32 r8 asm("r7") = nr;
+ register u32 r0 asm("r0") = arg1;
+ register u32 r1 asm("r1") = arg2;
+ register u32 r2 asm("r2") = arg3;
+ register u32 r3 asm("r3") = arg4;
+ register u32 r4 asm("r4") = arg5;
+ register u32 r5 asm("r5") = arg6;
+ asm volatile("swi #0"
+ : "=r"(r0)
+ : "r"(r8), "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5)
+ : "memory", "cc");
+ return r0;
+}
+#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \
+ (__internal_syscall)(n, (u32)(a1), (long)(a2), (long)(a3), (long)(a4), \
+ (u32)(a5), (long)(a6))
+
+#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n
+#define __SYSCALL_NARGS(...) \
+ __SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, )
+#define __SYSCALL_CONCAT_X(a, b) a##b
+#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b)
+#define __SYSCALL_DISP(b, ...) \
+ __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__)
+
+#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__)
+
+#define internal_syscall_ptr internal_syscall
+#define internal_syscall64 internal_syscall
+
+// Helper function used to avoid cobbler errno.
+bool internal_iserror(uptr retval, int *rverrno) {
+ if (retval >= (uptr)-4095) {
+ if (rverrno)
+ *rverrno = -retval;
+ return true;
+ }
+ return false;
+}
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc
index 9853a6a675d3..327aaa80a674 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_x86_64.inc
@@ -20,6 +20,9 @@ static uptr internal_syscall(u64 nr) {
return retval;
}
+#define internal_syscall_ptr internal_syscall
+#define internal_syscall64 internal_syscall
+
template <typename T1>
static uptr internal_syscall(u64 nr, T1 arg1) {
u64 retval;
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc
index 439e33a08d01..d9fd6549ba48 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cc
@@ -21,6 +21,7 @@ ThreadContextBase::ThreadContextBase(u32 tid)
status(ThreadStatusInvalid),
detached(false), workerthread(false), parent_tid(0), next(0) {
name[0] = '\0';
+ atomic_store(&thread_destroyed, 0, memory_order_release);
}
ThreadContextBase::~ThreadContextBase() {
@@ -44,6 +45,14 @@ void ThreadContextBase::SetDead() {
OnDead();
}
+void ThreadContextBase::SetDestroyed() {
+ atomic_store(&thread_destroyed, 1, memory_order_release);
+}
+
+bool ThreadContextBase::GetDestroyed() {
+ return !!atomic_load(&thread_destroyed, memory_order_acquire);
+}
+
void ThreadContextBase::SetJoined(void *arg) {
// FIXME(dvyukov): print message and continue (it's user error).
CHECK_EQ(false, detached);
@@ -54,8 +63,11 @@ void ThreadContextBase::SetJoined(void *arg) {
}
void ThreadContextBase::SetFinished() {
- if (!detached)
- status = ThreadStatusFinished;
+ // ThreadRegistry::FinishThread calls here in ThreadStatusCreated state
+ // for a thread that never actually started. In that case the thread
+ // should go to ThreadStatusFinished regardless of whether it was created
+ // as detached.
+ if (!detached || status == ThreadStatusCreated) status = ThreadStatusFinished;
OnFinished();
}
@@ -82,6 +94,7 @@ void ThreadContextBase::SetCreated(uptr _user_id, u64 _unique_id,
void ThreadContextBase::Reset() {
status = ThreadStatusInvalid;
SetName(0);
+ atomic_store(&thread_destroyed, 0, memory_order_release);
OnReset();
}
@@ -204,7 +217,8 @@ void ThreadRegistry::SetThreadName(u32 tid, const char *name) {
CHECK_LT(tid, n_contexts_);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
- CHECK_EQ(ThreadStatusRunning, tctx->status);
+ CHECK_EQ(SANITIZER_FUCHSIA ? ThreadStatusCreated : ThreadStatusRunning,
+ tctx->status);
tctx->SetName(name);
}
@@ -239,33 +253,54 @@ void ThreadRegistry::DetachThread(u32 tid, void *arg) {
}
void ThreadRegistry::JoinThread(u32 tid, void *arg) {
- BlockingMutexLock l(&mtx_);
- CHECK_LT(tid, n_contexts_);
- ThreadContextBase *tctx = threads_[tid];
- CHECK_NE(tctx, 0);
- if (tctx->status == ThreadStatusInvalid) {
- Report("%s: Join of non-existent thread\n", SanitizerToolName);
- return;
- }
- tctx->SetJoined(arg);
- QuarantinePush(tctx);
+ bool destroyed = false;
+ do {
+ {
+ BlockingMutexLock l(&mtx_);
+ CHECK_LT(tid, n_contexts_);
+ ThreadContextBase *tctx = threads_[tid];
+ CHECK_NE(tctx, 0);
+ if (tctx->status == ThreadStatusInvalid) {
+ Report("%s: Join of non-existent thread\n", SanitizerToolName);
+ return;
+ }
+ if ((destroyed = tctx->GetDestroyed())) {
+ tctx->SetJoined(arg);
+ QuarantinePush(tctx);
+ }
+ }
+ if (!destroyed)
+ internal_sched_yield();
+ } while (!destroyed);
}
+// Normally this is called when the thread is about to exit. If
+// called in ThreadStatusCreated state, then this thread was never
+// really started. We just did CreateThread for a prospective new
+// thread before trying to create it, and then failed to actually
+// create it, and so never called StartThread.
void ThreadRegistry::FinishThread(u32 tid) {
BlockingMutexLock l(&mtx_);
CHECK_GT(alive_threads_, 0);
alive_threads_--;
- CHECK_GT(running_threads_, 0);
- running_threads_--;
CHECK_LT(tid, n_contexts_);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
- CHECK_EQ(ThreadStatusRunning, tctx->status);
+ bool dead = tctx->detached;
+ if (tctx->status == ThreadStatusRunning) {
+ CHECK_GT(running_threads_, 0);
+ running_threads_--;
+ } else {
+ // The thread never really existed.
+ CHECK_EQ(tctx->status, ThreadStatusCreated);
+ dead = true;
+ }
tctx->SetFinished();
- if (tctx->detached) {
+ if (dead) {
tctx->SetDead();
QuarantinePush(tctx);
}
+ tctx->SetDestroyed();
}
void ThreadRegistry::StartThread(u32 tid, tid_t os_id, bool workerthread,
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h
index 9aae875c7360..b203be2f4dbc 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h
@@ -50,6 +50,8 @@ class ThreadContextBase {
u32 parent_tid;
ThreadContextBase *next; // For storing thread contexts in a list.
+ atomic_uint32_t thread_destroyed; // To address race of Joined vs Finished
+
void SetName(const char *new_name);
void SetDead();
@@ -60,6 +62,9 @@ class ThreadContextBase {
u32 _parent_tid, void *arg);
void Reset();
+ void SetDestroyed();
+ bool GetDestroyed();
+
// The following methods may be overriden by subclasses.
// Some of them take opaque arg that may be optionally be used
// by subclasses.
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cc
index 29db37b8a464..6fa0e9298b31 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cc
@@ -45,7 +45,7 @@ static const uptr kDestroyedThread = -1;
static inline void DTLS_Deallocate(DTLS::DTV *dtv, uptr size) {
if (!size) return;
- VPrintf(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", dtv, size);
+ VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", dtv, size);
UnmapOrDie(dtv, size * sizeof(DTLS::DTV));
atomic_fetch_sub(&number_of_live_dtls, 1, memory_order_relaxed);
}
@@ -58,7 +58,7 @@ static inline void DTLS_Resize(uptr new_size) {
(DTLS::DTV *)MmapOrDie(new_size * sizeof(DTLS::DTV), "DTLS_Resize");
uptr num_live_dtls =
atomic_fetch_add(&number_of_live_dtls, 1, memory_order_relaxed);
- VPrintf(2, "__tls_get_addr: DTLS_Resize %p %zd\n", &dtls, num_live_dtls);
+ VReport(2, "__tls_get_addr: DTLS_Resize %p %zd\n", &dtls, num_live_dtls);
CHECK_LT(num_live_dtls, 1 << 20);
uptr old_dtv_size = dtls.dtv_size;
DTLS::DTV *old_dtv = dtls.dtv;
@@ -72,7 +72,7 @@ static inline void DTLS_Resize(uptr new_size) {
void DTLS_Destroy() {
if (!common_flags()->intercept_tls_get_addr) return;
- VPrintf(2, "__tls_get_addr: DTLS_Destroy %p %zd\n", &dtls, dtls.dtv_size);
+ VReport(2, "__tls_get_addr: DTLS_Destroy %p %zd\n", &dtls, dtls.dtv_size);
uptr s = dtls.dtv_size;
dtls.dtv_size = kDestroyedThread; // Do this before unmap for AS-safety.
DTLS_Deallocate(dtls.dtv, s);
@@ -97,28 +97,28 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
if (dtls.dtv[dso_id].beg) return 0;
uptr tls_size = 0;
uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
- VPrintf(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p "
+ VReport(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p "
"num_live_dtls %zd\n",
arg, arg->dso_id, arg->offset, res, tls_beg, &tls_beg,
atomic_load(&number_of_live_dtls, memory_order_relaxed));
if (dtls.last_memalign_ptr == tls_beg) {
tls_size = dtls.last_memalign_size;
- VPrintf(2, "__tls_get_addr: glibc <=2.18 suspected; tls={%p,%p}\n",
+ VReport(2, "__tls_get_addr: glibc <=2.18 suspected; tls={%p,%p}\n",
tls_beg, tls_size);
} else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) {
// This is the static TLS block which was initialized / unpoisoned at thread
// creation.
- VPrintf(2, "__tls_get_addr: static tls: %p\n", tls_beg);
+ VReport(2, "__tls_get_addr: static tls: %p\n", tls_beg);
tls_size = 0;
} else if ((tls_beg % 4096) == sizeof(Glibc_2_19_tls_header)) {
// We may want to check gnu_get_libc_version().
Glibc_2_19_tls_header *header = (Glibc_2_19_tls_header *)tls_beg - 1;
tls_size = header->size;
tls_beg = header->start;
- VPrintf(2, "__tls_get_addr: glibc >=2.19 suspected; tls={%p %p}\n",
+ VReport(2, "__tls_get_addr: glibc >=2.19 suspected; tls={%p %p}\n",
tls_beg, tls_size);
} else {
- VPrintf(2, "__tls_get_addr: Can't guess glibc version\n");
+ VReport(2, "__tls_get_addr: Can't guess glibc version\n");
// This may happen inside the DTOR of main thread, so just ignore it.
tls_size = 0;
}
@@ -129,7 +129,7 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
void DTLS_on_libc_memalign(void *ptr, uptr size) {
if (!common_flags()->intercept_tls_get_addr) return;
- VPrintf(2, "DTLS_on_libc_memalign: %p %p\n", ptr, size);
+ VReport(2, "DTLS_on_libc_memalign: %p %p\n", ptr, size);
dtls.last_memalign_ptr = reinterpret_cast<uptr>(ptr);
dtls.last_memalign_size = size;
}
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc
index 5943125c898c..24c94700173d 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cc
@@ -8,11 +8,12 @@
//===----------------------------------------------------------------------===//
//
// This file contains the unwind.h-based (aka "slow") stack unwinding routines
-// available to the tools on Linux, Android, and FreeBSD.
+// available to the tools on Linux, Android, NetBSD, FreeBSD, and Solaris.
//===----------------------------------------------------------------------===//
#include "sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS
#include "sanitizer_common.h"
#include "sanitizer_stacktrace.h"
@@ -78,7 +79,8 @@ void SanitizerInitializeUnwinder() {
}
#endif
-#ifdef __arm__
+#if defined(__arm__) && !SANITIZER_NETBSD
+// NetBSD uses dwarf EH
#define UNWIND_STOP _URC_END_OF_STACK
#define UNWIND_CONTINUE _URC_NO_REASON
#else
@@ -165,4 +167,5 @@ void BufferedStackTrace::SlowUnwindStackWithContext(uptr pc, void *context,
} // namespace __sanitizer
-#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD ||
+ // SANITIZER_SOLARIS
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_vector.h b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_vector.h
index a7fb3fa58d54..25cfeed35f22 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_vector.h
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_vector.h
@@ -1,4 +1,4 @@
-//===-- tsan_vector.h -------------------------------------------*- C++ -*-===//
+//===-- sanitizer_vector.h -------------------------------------*- C++ -*-===//
//
// The LLVM Compiler Infrastructure
//
@@ -7,38 +7,37 @@
//
//===----------------------------------------------------------------------===//
//
-// This file is a part of ThreadSanitizer (TSan), a race detector.
+// This file is shared between sanitizers run-time libraries.
//
//===----------------------------------------------------------------------===//
// Low-fat STL-like vector container.
-#ifndef TSAN_VECTOR_H
-#define TSAN_VECTOR_H
+#ifndef SANITIZER_VECTOR_H
+#define SANITIZER_VECTOR_H
-#include "tsan_defs.h"
-#include "tsan_mman.h"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_libc.h"
-namespace __tsan {
+namespace __sanitizer {
template<typename T>
class Vector {
public:
- explicit Vector(MBlockType typ)
- : typ_(typ)
- , begin_()
+ explicit Vector()
+ : begin_()
, end_()
, last_() {
}
~Vector() {
if (begin_)
- internal_free(begin_);
+ InternalFree(begin_);
}
void Reset() {
if (begin_)
- internal_free(begin_);
+ InternalFree(begin_);
begin_ = 0;
end_ = 0;
last_ = 0;
@@ -91,7 +90,6 @@ class Vector {
}
private:
- const MBlockType typ_;
T *begin_;
T *end_;
T *last_;
@@ -109,10 +107,10 @@ class Vector {
cap = 16;
if (cap < size)
cap = size;
- T *p = (T*)internal_alloc(typ_, cap * sizeof(T));
+ T *p = (T*)InternalAlloc(cap * sizeof(T));
if (cap0) {
internal_memcpy(p, begin_, cap0 * sizeof(T));
- internal_free(begin_);
+ InternalFree(begin_);
}
begin_ = p;
end_ = begin_ + size;
@@ -122,6 +120,6 @@ class Vector {
Vector(const Vector&);
void operator=(const Vector&);
};
-} // namespace __tsan
+} // namespace __sanitizer
-#endif // #ifndef TSAN_VECTOR_H
+#endif // #ifndef SANITIZER_VECTOR_H
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc
index de01e8d119a1..34bf1812d605 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win.cc
@@ -24,10 +24,10 @@
#include "sanitizer_common.h"
#include "sanitizer_dbghelp.h"
+#include "sanitizer_file.h"
#include "sanitizer_libc.h"
#include "sanitizer_mutex.h"
#include "sanitizer_placement_new.h"
-#include "sanitizer_procmaps.h"
#include "sanitizer_stacktrace.h"
#include "sanitizer_symbolizer.h"
#include "sanitizer_win_defs.h"
@@ -64,12 +64,16 @@ uptr GetMmapGranularity() {
return si.dwAllocationGranularity;
}
-uptr GetMaxVirtualAddress() {
+uptr GetMaxUserVirtualAddress() {
SYSTEM_INFO si;
GetSystemInfo(&si);
return (uptr)si.lpMaximumApplicationAddress;
}
+uptr GetMaxVirtualAddress() {
+ return GetMaxUserVirtualAddress();
+}
+
bool FileExists(const char *filename) {
return ::GetFileAttributesA(filename) != INVALID_FILE_ATTRIBUTES;
}
@@ -235,6 +239,28 @@ void *MmapFixedOrDie(uptr fixed_addr, uptr size) {
return p;
}
+// Uses fixed_addr for now.
+// Will use offset instead once we've implemented this function for real.
+uptr ReservedAddressRange::Map(uptr fixed_addr, uptr size) {
+ return reinterpret_cast<uptr>(MmapFixedOrDieOnFatalError(fixed_addr, size));
+}
+
+uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr size) {
+ return reinterpret_cast<uptr>(MmapFixedOrDie(fixed_addr, size));
+}
+
+void ReservedAddressRange::Unmap(uptr addr, uptr size) {
+ void* addr_as_void = reinterpret_cast<void*>(addr);
+ uptr base_as_uptr = reinterpret_cast<uptr>(base_);
+ // Only unmap if it covers the entire range.
+ CHECK((addr == base_as_uptr) && (size == size_));
+ UnmapOrDie(addr_as_void, size);
+ if (addr_as_void == base_) {
+ base_ = reinterpret_cast<void*>(addr + size);
+ }
+ size_ = size_ - size;
+}
+
void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size) {
void *p = VirtualAlloc((LPVOID)fixed_addr, size,
MEM_COMMIT, PAGE_READWRITE);
@@ -252,6 +278,19 @@ void *MmapNoReserveOrDie(uptr size, const char *mem_type) {
return MmapOrDie(size, mem_type);
}
+uptr ReservedAddressRange::Init(uptr size, const char *name, uptr fixed_addr) {
+ if (fixed_addr) {
+ base_ = MmapFixedNoAccess(fixed_addr, size, name);
+ } else {
+ base_ = MmapNoAccess(size);
+ }
+ size_ = size;
+ name_ = name;
+ (void)os_handle_; // unsupported
+ return reinterpret_cast<uptr>(base_);
+}
+
+
void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) {
(void)name; // unsupported
void *res = VirtualAlloc((LPVOID)fixed_addr, size,
@@ -379,7 +418,7 @@ struct ModuleInfo {
#if !SANITIZER_GO
int CompareModulesBase(const void *pl, const void *pr) {
- const ModuleInfo *l = (ModuleInfo *)pl, *r = (ModuleInfo *)pr;
+ const ModuleInfo *l = (const ModuleInfo *)pl, *r = (const ModuleInfo *)pr;
if (l->base_address < r->base_address)
return -1;
return l->base_address > r->base_address;
@@ -466,6 +505,10 @@ u64 NanoTime() {
return 0;
}
+u64 MonotonicNanoTime() {
+ return 0;
+}
+
void Abort() {
internal__exit(3);
}
@@ -524,7 +567,7 @@ static uptr GetPreferredBase(const char *modname) {
}
void ListOfModules::init() {
- clear();
+ clearOrInit();
HANDLE cur_process = GetCurrentProcess();
// Query the list of modules. Start by assuming there are no more than 256
@@ -583,7 +626,9 @@ void ListOfModules::init() {
modules_.push_back(cur_module);
}
UnmapOrDie(hmodules, modules_buffer_size);
-};
+}
+
+void ListOfModules::fallbackInit() { clear(); }
// We can't use atexit() directly at __asan_init time as the CRT is not fully
// initialized at this point. Place the functions into a vector and use
@@ -792,7 +837,7 @@ void BufferedStackTrace::SlowUnwindStack(uptr pc, u32 max_depth) {
// FIXME: Compare with StackWalk64.
// FIXME: Look at LLVMUnhandledExceptionFilter in Signals.inc
size = CaptureStackBackTrace(1, Min(max_depth, kStackTraceMax),
- (void**)trace, 0);
+ (void **)&trace_buffer[0], 0);
if (size == 0)
return;
@@ -889,32 +934,6 @@ bool IsHandledDeadlyException(DWORD exceptionCode) {
return false;
}
-const char *DescribeSignalOrException(int signo) {
- unsigned code = signo;
- // Get the string description of the exception if this is a known deadly
- // exception.
- switch (code) {
- case EXCEPTION_ACCESS_VIOLATION: return "access-violation";
- case EXCEPTION_ARRAY_BOUNDS_EXCEEDED: return "array-bounds-exceeded";
- case EXCEPTION_STACK_OVERFLOW: return "stack-overflow";
- case EXCEPTION_DATATYPE_MISALIGNMENT: return "datatype-misalignment";
- case EXCEPTION_IN_PAGE_ERROR: return "in-page-error";
- case EXCEPTION_ILLEGAL_INSTRUCTION: return "illegal-instruction";
- case EXCEPTION_PRIV_INSTRUCTION: return "priv-instruction";
- case EXCEPTION_BREAKPOINT: return "breakpoint";
- case EXCEPTION_FLT_DENORMAL_OPERAND: return "flt-denormal-operand";
- case EXCEPTION_FLT_DIVIDE_BY_ZERO: return "flt-divide-by-zero";
- case EXCEPTION_FLT_INEXACT_RESULT: return "flt-inexact-result";
- case EXCEPTION_FLT_INVALID_OPERATION: return "flt-invalid-operation";
- case EXCEPTION_FLT_OVERFLOW: return "flt-overflow";
- case EXCEPTION_FLT_STACK_CHECK: return "flt-stack-check";
- case EXCEPTION_FLT_UNDERFLOW: return "flt-underflow";
- case EXCEPTION_INT_DIVIDE_BY_ZERO: return "int-divide-by-zero";
- case EXCEPTION_INT_OVERFLOW: return "int-overflow";
- }
- return "unknown exception";
-}
-
bool IsAccessibleMemoryRange(uptr beg, uptr size) {
SYSTEM_INFO si;
GetNativeSystemInfo(&si);
@@ -940,39 +959,101 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) {
return true;
}
-SignalContext SignalContext::Create(void *siginfo, void *context) {
+bool SignalContext::IsStackOverflow() const {
+ return (DWORD)GetType() == EXCEPTION_STACK_OVERFLOW;
+}
+
+void SignalContext::InitPcSpBp() {
EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo;
CONTEXT *context_record = (CONTEXT *)context;
- uptr pc = (uptr)exception_record->ExceptionAddress;
+ pc = (uptr)exception_record->ExceptionAddress;
#ifdef _WIN64
- uptr bp = (uptr)context_record->Rbp;
- uptr sp = (uptr)context_record->Rsp;
+ bp = (uptr)context_record->Rbp;
+ sp = (uptr)context_record->Rsp;
#else
- uptr bp = (uptr)context_record->Ebp;
- uptr sp = (uptr)context_record->Esp;
+ bp = (uptr)context_record->Ebp;
+ sp = (uptr)context_record->Esp;
#endif
- uptr access_addr = exception_record->ExceptionInformation[1];
+}
+
+uptr SignalContext::GetAddress() const {
+ EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo;
+ return exception_record->ExceptionInformation[1];
+}
+
+bool SignalContext::IsMemoryAccess() const {
+ return GetWriteFlag() != SignalContext::UNKNOWN;
+}
+SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
+ EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo;
// The contents of this array are documented at
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa363082(v=vs.85).aspx
// The first element indicates read as 0, write as 1, or execute as 8. The
// second element is the faulting address.
- WriteFlag write_flag = SignalContext::UNKNOWN;
switch (exception_record->ExceptionInformation[0]) {
- case 0: write_flag = SignalContext::READ; break;
- case 1: write_flag = SignalContext::WRITE; break;
- case 8: write_flag = SignalContext::UNKNOWN; break;
+ case 0:
+ return SignalContext::READ;
+ case 1:
+ return SignalContext::WRITE;
+ case 8:
+ return SignalContext::UNKNOWN;
}
- bool is_memory_access = write_flag != SignalContext::UNKNOWN;
- return SignalContext(context, access_addr, pc, sp, bp, is_memory_access,
- write_flag);
+ return SignalContext::UNKNOWN;
}
void SignalContext::DumpAllRegisters(void *context) {
// FIXME: Implement this.
}
+int SignalContext::GetType() const {
+ return static_cast<const EXCEPTION_RECORD *>(siginfo)->ExceptionCode;
+}
+
+const char *SignalContext::Describe() const {
+ unsigned code = GetType();
+ // Get the string description of the exception if this is a known deadly
+ // exception.
+ switch (code) {
+ case EXCEPTION_ACCESS_VIOLATION:
+ return "access-violation";
+ case EXCEPTION_ARRAY_BOUNDS_EXCEEDED:
+ return "array-bounds-exceeded";
+ case EXCEPTION_STACK_OVERFLOW:
+ return "stack-overflow";
+ case EXCEPTION_DATATYPE_MISALIGNMENT:
+ return "datatype-misalignment";
+ case EXCEPTION_IN_PAGE_ERROR:
+ return "in-page-error";
+ case EXCEPTION_ILLEGAL_INSTRUCTION:
+ return "illegal-instruction";
+ case EXCEPTION_PRIV_INSTRUCTION:
+ return "priv-instruction";
+ case EXCEPTION_BREAKPOINT:
+ return "breakpoint";
+ case EXCEPTION_FLT_DENORMAL_OPERAND:
+ return "flt-denormal-operand";
+ case EXCEPTION_FLT_DIVIDE_BY_ZERO:
+ return "flt-divide-by-zero";
+ case EXCEPTION_FLT_INEXACT_RESULT:
+ return "flt-inexact-result";
+ case EXCEPTION_FLT_INVALID_OPERATION:
+ return "flt-invalid-operation";
+ case EXCEPTION_FLT_OVERFLOW:
+ return "flt-overflow";
+ case EXCEPTION_FLT_STACK_CHECK:
+ return "flt-stack-check";
+ case EXCEPTION_FLT_UNDERFLOW:
+ return "flt-underflow";
+ case EXCEPTION_INT_DIVIDE_BY_ZERO:
+ return "int-divide-by-zero";
+ case EXCEPTION_INT_OVERFLOW:
+ return "int-overflow";
+ }
+ return "unknown exception";
+}
+
uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
// FIXME: Actually implement this function.
CHECK_GT(buf_len, 0);
@@ -1021,7 +1102,12 @@ void CheckNoDeepBind(const char *filename, int flag) {
}
// FIXME: implement on this platform.
-bool GetRandom(void *buffer, uptr length) {
+bool GetRandom(void *buffer, uptr length, bool blocking) {
+ UNIMPLEMENTED();
+}
+
+// FIXME: implement on this platform.
+u32 GetNumberOfCPUs() {
UNIMPLEMENTED();
}
diff --git a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cc b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cc
index 364319398198..5711f5dc8b6a 100644
--- a/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cc
+++ b/contrib/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cc
@@ -12,7 +12,7 @@
// definition is provided.
//===----------------------------------------------------------------------===//
-#include "sanitizer_common/sanitizer_platform.h"
+#include "sanitizer_platform.h"
#if SANITIZER_WINDOWS && SANITIZER_DYNAMIC
#include "sanitizer_win_weak_interception.h"
#include "sanitizer_allocator_interface.h"
diff --git a/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh b/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
index c5865ecfee6c..0559a2e7eb05 100755
--- a/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
+++ b/contrib/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
@@ -110,12 +110,12 @@ cd ${LIBCXX_BUILD}
ninja cxx cxxabi
FLAGS="${FLAGS} -fno-rtti -fno-exceptions"
+LLVM_FLAGS="${FLAGS} -nostdinc++ -I${ZLIB_BUILD} -I${LIBCXX_BUILD}/include/c++/v1"
# Build LLVM.
if [[ ! -d ${LLVM_BUILD} ]]; then
mkdir -p ${LLVM_BUILD}
cd ${LLVM_BUILD}
- LLVM_FLAGS="${FLAGS} -I${ZLIB_BUILD} -I${LIBCXX_BUILD}/include/c++/v1"
cmake -GNinja \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER=$CC \
@@ -137,7 +137,7 @@ mkdir ${SYMBOLIZER_BUILD}
cd ${SYMBOLIZER_BUILD}
echo "Compiling..."
-SYMBOLIZER_FLAGS="$FLAGS -std=c++11 -I${LLVM_SRC}/include -I${LLVM_BUILD}/include -I${LIBCXX_BUILD}/include/c++/v1"
+SYMBOLIZER_FLAGS="$LLVM_FLAGS -I${LLVM_SRC}/include -I${LLVM_BUILD}/include -std=c++11"
$CXX $SYMBOLIZER_FLAGS ${SRC_DIR}/sanitizer_symbolize.cc ${SRC_DIR}/sanitizer_wrappers.cc -c
$AR rc symbolizer.a sanitizer_symbolize.o sanitizer_wrappers.o
diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp b/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp
index 6f30ee987513..e5a4d714c66e 100644
--- a/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp
+++ b/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp
@@ -16,26 +16,27 @@
#include "scudo_allocator.h"
#include "scudo_crc32.h"
-#include "scudo_tls.h"
+#include "scudo_flags.h"
+#include "scudo_tsd.h"
#include "scudo_utils.h"
#include "sanitizer_common/sanitizer_allocator_checks.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
-#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_quarantine.h"
+#include <errno.h>
#include <string.h>
namespace __scudo {
// Global static cookie, initialized at start-up.
-static uptr Cookie;
+static u32 Cookie;
// We default to software CRC32 if the alternatives are not supported, either
// at compilation or at runtime.
static atomic_uint8_t HashAlgorithm = { CRC32Software };
-INLINE u32 computeCRC32(uptr Crc, uptr Value, uptr *Array, uptr ArraySize) {
+INLINE u32 computeCRC32(u32 Crc, uptr Value, uptr *Array, uptr ArraySize) {
// If the hardware CRC32 feature is defined here, it was enabled everywhere,
// as opposed to only for scudo_crc32.cpp. This means that other hardware
// specific instructions were likely emitted at other places, and as a
@@ -61,46 +62,60 @@ INLINE u32 computeCRC32(uptr Crc, uptr Value, uptr *Array, uptr ArraySize) {
static ScudoBackendAllocator &getBackendAllocator();
-struct ScudoChunk : UnpackedHeader {
+namespace Chunk {
// We can't use the offset member of the chunk itself, as we would double
// fetch it without any warranty that it wouldn't have been tampered. To
// prevent this, we work with a local copy of the header.
- void *getAllocBeg(UnpackedHeader *Header) {
- return reinterpret_cast<void *>(
- reinterpret_cast<uptr>(this) - (Header->Offset << MinAlignmentLog));
+ static INLINE void *getBackendPtr(const void *Ptr, UnpackedHeader *Header) {
+ return reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
+ AlignedChunkHeaderSize -
+ (Header->Offset << MinAlignmentLog));
+ }
+
+ static INLINE AtomicPackedHeader *getAtomicHeader(void *Ptr) {
+ return reinterpret_cast<AtomicPackedHeader *>(reinterpret_cast<uptr>(Ptr) -
+ AlignedChunkHeaderSize);
+ }
+ static INLINE
+ const AtomicPackedHeader *getConstAtomicHeader(const void *Ptr) {
+ return reinterpret_cast<const AtomicPackedHeader *>(
+ reinterpret_cast<uptr>(Ptr) - AlignedChunkHeaderSize);
+ }
+
+ static INLINE bool isAligned(const void *Ptr) {
+ return IsAligned(reinterpret_cast<uptr>(Ptr), MinAlignment);
}
// Returns the usable size for a chunk, meaning the amount of bytes from the
// beginning of the user data to the end of the backend allocated chunk.
- uptr getUsableSize(UnpackedHeader *Header) {
- uptr Size =
- getBackendAllocator().getActuallyAllocatedSize(getAllocBeg(Header),
- Header->FromPrimary);
+ static INLINE uptr getUsableSize(const void *Ptr, UnpackedHeader *Header) {
+ const uptr Size = getBackendAllocator().getActuallyAllocatedSize(
+ getBackendPtr(Ptr, Header), Header->ClassId);
if (Size == 0)
return 0;
return Size - AlignedChunkHeaderSize - (Header->Offset << MinAlignmentLog);
}
- // Compute the checksum of the Chunk pointer and its ChunkHeader.
- u16 computeChecksum(UnpackedHeader *Header) const {
+ // Compute the checksum of the chunk pointer and its header.
+ static INLINE u16 computeChecksum(const void *Ptr, UnpackedHeader *Header) {
UnpackedHeader ZeroChecksumHeader = *Header;
ZeroChecksumHeader.Checksum = 0;
uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)];
memcpy(&HeaderHolder, &ZeroChecksumHeader, sizeof(HeaderHolder));
- u32 Crc = computeCRC32(Cookie, reinterpret_cast<uptr>(this), HeaderHolder,
- ARRAY_SIZE(HeaderHolder));
+ const u32 Crc = computeCRC32(Cookie, reinterpret_cast<uptr>(Ptr),
+ HeaderHolder, ARRAY_SIZE(HeaderHolder));
return static_cast<u16>(Crc);
}
// Checks the validity of a chunk by verifying its checksum. It doesn't
// incur termination in the event of an invalid chunk.
- bool isValid() {
- UnpackedHeader NewUnpackedHeader;
- const AtomicPackedHeader *AtomicHeader =
- reinterpret_cast<const AtomicPackedHeader *>(this);
- PackedHeader NewPackedHeader = atomic_load_relaxed(AtomicHeader);
- NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader);
- return (NewUnpackedHeader.Checksum == computeChecksum(&NewUnpackedHeader));
+ static INLINE bool isValid(const void *Ptr) {
+ PackedHeader NewPackedHeader =
+ atomic_load_relaxed(getConstAtomicHeader(Ptr));
+ UnpackedHeader NewUnpackedHeader =
+ bit_cast<UnpackedHeader>(NewPackedHeader);
+ return (NewUnpackedHeader.Checksum ==
+ computeChecksum(Ptr, &NewUnpackedHeader));
}
// Nulls out a chunk header. When returning the chunk to the backend, there
@@ -109,114 +124,46 @@ struct ScudoChunk : UnpackedHeader {
// the header invalid. In the extremely rare event where 0 would be a valid
// checksum for the chunk, the state of the chunk is ChunkAvailable anyway.
COMPILER_CHECK(ChunkAvailable == 0);
- void eraseHeader() {
- PackedHeader NullPackedHeader = 0;
- AtomicPackedHeader *AtomicHeader =
- reinterpret_cast<AtomicPackedHeader *>(this);
- atomic_store_relaxed(AtomicHeader, NullPackedHeader);
+ static INLINE void eraseHeader(void *Ptr) {
+ const PackedHeader NullPackedHeader = 0;
+ atomic_store_relaxed(getAtomicHeader(Ptr), NullPackedHeader);
}
// Loads and unpacks the header, verifying the checksum in the process.
- void loadHeader(UnpackedHeader *NewUnpackedHeader) const {
- const AtomicPackedHeader *AtomicHeader =
- reinterpret_cast<const AtomicPackedHeader *>(this);
- PackedHeader NewPackedHeader = atomic_load_relaxed(AtomicHeader);
+ static INLINE
+ void loadHeader(const void *Ptr, UnpackedHeader *NewUnpackedHeader) {
+ PackedHeader NewPackedHeader =
+ atomic_load_relaxed(getConstAtomicHeader(Ptr));
*NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader);
if (UNLIKELY(NewUnpackedHeader->Checksum !=
- computeChecksum(NewUnpackedHeader))) {
- dieWithMessage("ERROR: corrupted chunk header at address %p\n", this);
+ computeChecksum(Ptr, NewUnpackedHeader))) {
+ dieWithMessage("ERROR: corrupted chunk header at address %p\n", Ptr);
}
}
// Packs and stores the header, computing the checksum in the process.
- void storeHeader(UnpackedHeader *NewUnpackedHeader) {
- NewUnpackedHeader->Checksum = computeChecksum(NewUnpackedHeader);
+ static INLINE void storeHeader(void *Ptr, UnpackedHeader *NewUnpackedHeader) {
+ NewUnpackedHeader->Checksum = computeChecksum(Ptr, NewUnpackedHeader);
PackedHeader NewPackedHeader = bit_cast<PackedHeader>(*NewUnpackedHeader);
- AtomicPackedHeader *AtomicHeader =
- reinterpret_cast<AtomicPackedHeader *>(this);
- atomic_store_relaxed(AtomicHeader, NewPackedHeader);
+ atomic_store_relaxed(getAtomicHeader(Ptr), NewPackedHeader);
}
// Packs and stores the header, computing the checksum in the process. We
// compare the current header with the expected provided one to ensure that
// we are not being raced by a corruption occurring in another thread.
- void compareExchangeHeader(UnpackedHeader *NewUnpackedHeader,
- UnpackedHeader *OldUnpackedHeader) {
- NewUnpackedHeader->Checksum = computeChecksum(NewUnpackedHeader);
+ static INLINE void compareExchangeHeader(void *Ptr,
+ UnpackedHeader *NewUnpackedHeader,
+ UnpackedHeader *OldUnpackedHeader) {
+ NewUnpackedHeader->Checksum = computeChecksum(Ptr, NewUnpackedHeader);
PackedHeader NewPackedHeader = bit_cast<PackedHeader>(*NewUnpackedHeader);
PackedHeader OldPackedHeader = bit_cast<PackedHeader>(*OldUnpackedHeader);
- AtomicPackedHeader *AtomicHeader =
- reinterpret_cast<AtomicPackedHeader *>(this);
- if (UNLIKELY(!atomic_compare_exchange_strong(AtomicHeader,
- &OldPackedHeader,
- NewPackedHeader,
- memory_order_relaxed))) {
- dieWithMessage("ERROR: race on chunk header at address %p\n", this);
+ if (UNLIKELY(!atomic_compare_exchange_strong(
+ getAtomicHeader(Ptr), &OldPackedHeader, NewPackedHeader,
+ memory_order_relaxed))) {
+ dieWithMessage("ERROR: race on chunk header at address %p\n", Ptr);
}
}
-};
-
-ScudoChunk *getScudoChunk(uptr UserBeg) {
- return reinterpret_cast<ScudoChunk *>(UserBeg - AlignedChunkHeaderSize);
-}
-
-struct AllocatorOptions {
- u32 QuarantineSizeMb;
- u32 ThreadLocalQuarantineSizeKb;
- bool MayReturnNull;
- s32 ReleaseToOSIntervalMs;
- bool DeallocationTypeMismatch;
- bool DeleteSizeMismatch;
- bool ZeroContents;
-
- void setFrom(const Flags *f, const CommonFlags *cf);
- void copyTo(Flags *f, CommonFlags *cf) const;
-};
-
-void AllocatorOptions::setFrom(const Flags *f, const CommonFlags *cf) {
- MayReturnNull = cf->allocator_may_return_null;
- ReleaseToOSIntervalMs = cf->allocator_release_to_os_interval_ms;
- QuarantineSizeMb = f->QuarantineSizeMb;
- ThreadLocalQuarantineSizeKb = f->ThreadLocalQuarantineSizeKb;
- DeallocationTypeMismatch = f->DeallocationTypeMismatch;
- DeleteSizeMismatch = f->DeleteSizeMismatch;
- ZeroContents = f->ZeroContents;
-}
-
-void AllocatorOptions::copyTo(Flags *f, CommonFlags *cf) const {
- cf->allocator_may_return_null = MayReturnNull;
- cf->allocator_release_to_os_interval_ms = ReleaseToOSIntervalMs;
- f->QuarantineSizeMb = QuarantineSizeMb;
- f->ThreadLocalQuarantineSizeKb = ThreadLocalQuarantineSizeKb;
- f->DeallocationTypeMismatch = DeallocationTypeMismatch;
- f->DeleteSizeMismatch = DeleteSizeMismatch;
- f->ZeroContents = ZeroContents;
-}
-
-static void initScudoInternal(const AllocatorOptions &Options);
-
-static bool ScudoInitIsRunning = false;
-
-void initScudo() {
- SanitizerToolName = "Scudo";
- CHECK(!ScudoInitIsRunning && "Scudo init calls itself!");
- ScudoInitIsRunning = true;
-
- // Check if hardware CRC32 is supported in the binary and by the platform, if
- // so, opt for the CRC32 hardware version of the checksum.
- if (computeHardwareCRC32 && testCPUFeature(CRC32CPUFeature))
- atomic_store_relaxed(&HashAlgorithm, CRC32Hardware);
-
- initFlags();
-
- AllocatorOptions Options;
- Options.setFrom(getFlags(), common_flags());
- initScudoInternal(Options);
-
- // TODO(kostyak): determine if MaybeStartBackgroudThread could be of some use.
-
- ScudoInitIsRunning = false;
-}
+} // namespace Chunk
struct QuarantineCallback {
explicit QuarantineCallback(AllocatorCache *Cache)
@@ -224,52 +171,46 @@ struct QuarantineCallback {
// Chunk recycling function, returns a quarantined chunk to the backend,
// first making sure it hasn't been tampered with.
- void Recycle(ScudoChunk *Chunk) {
+ void Recycle(void *Ptr) {
UnpackedHeader Header;
- Chunk->loadHeader(&Header);
+ Chunk::loadHeader(Ptr, &Header);
if (UNLIKELY(Header.State != ChunkQuarantine)) {
dieWithMessage("ERROR: invalid chunk state when recycling address %p\n",
- Chunk);
+ Ptr);
}
- Chunk->eraseHeader();
- void *Ptr = Chunk->getAllocBeg(&Header);
- if (Header.FromPrimary)
- getBackendAllocator().deallocatePrimary(Cache_, Ptr);
+ Chunk::eraseHeader(Ptr);
+ void *BackendPtr = Chunk::getBackendPtr(Ptr, &Header);
+ if (Header.ClassId)
+ getBackendAllocator().deallocatePrimary(Cache_, BackendPtr,
+ Header.ClassId);
else
- getBackendAllocator().deallocateSecondary(Ptr);
+ getBackendAllocator().deallocateSecondary(BackendPtr);
}
// Internal quarantine allocation and deallocation functions. We first check
// that the batches are indeed serviced by the Primary.
// TODO(kostyak): figure out the best way to protect the batches.
- COMPILER_CHECK(sizeof(QuarantineBatch) < SizeClassMap::kMaxSize);
void *Allocate(uptr Size) {
- return getBackendAllocator().allocatePrimary(Cache_, Size);
+ return getBackendAllocator().allocatePrimary(Cache_, BatchClassId);
}
void Deallocate(void *Ptr) {
- getBackendAllocator().deallocatePrimary(Cache_, Ptr);
+ getBackendAllocator().deallocatePrimary(Cache_, Ptr, BatchClassId);
}
AllocatorCache *Cache_;
+ COMPILER_CHECK(sizeof(QuarantineBatch) < SizeClassMap::kMaxSize);
+ const uptr BatchClassId = SizeClassMap::ClassID(sizeof(QuarantineBatch));
};
-typedef Quarantine<QuarantineCallback, ScudoChunk> ScudoQuarantine;
+typedef Quarantine<QuarantineCallback, void> ScudoQuarantine;
typedef ScudoQuarantine::Cache ScudoQuarantineCache;
COMPILER_CHECK(sizeof(ScudoQuarantineCache) <=
- sizeof(ScudoThreadContext::QuarantineCachePlaceHolder));
+ sizeof(ScudoTSD::QuarantineCachePlaceHolder));
-AllocatorCache *getAllocatorCache(ScudoThreadContext *ThreadContext) {
- return &ThreadContext->Cache;
-}
-
-ScudoQuarantineCache *getQuarantineCache(ScudoThreadContext *ThreadContext) {
- return reinterpret_cast<
- ScudoQuarantineCache *>(ThreadContext->QuarantineCachePlaceHolder);
-}
-
-ScudoPrng *getPrng(ScudoThreadContext *ThreadContext) {
- return &ThreadContext->Prng;
+ScudoQuarantineCache *getQuarantineCache(ScudoTSD *TSD) {
+ return reinterpret_cast<ScudoQuarantineCache *>(
+ TSD->QuarantineCachePlaceHolder);
}
struct ScudoAllocator {
@@ -281,26 +222,22 @@ struct ScudoAllocator {
ScudoBackendAllocator BackendAllocator;
ScudoQuarantine AllocatorQuarantine;
- StaticSpinMutex GlobalPrngMutex;
- ScudoPrng GlobalPrng;
-
- // The fallback caches are used when the thread local caches have been
- // 'detroyed' on thread tear-down. They are protected by a Mutex as they can
- // be accessed by different threads.
- StaticSpinMutex FallbackMutex;
- AllocatorCache FallbackAllocatorCache;
- ScudoQuarantineCache FallbackQuarantineCache;
- ScudoPrng FallbackPrng;
+ u32 QuarantineChunksUpToSize;
bool DeallocationTypeMismatch;
bool ZeroContents;
bool DeleteSizeMismatch;
+ bool CheckRssLimit;
+ uptr HardRssLimitMb;
+ uptr SoftRssLimitMb;
+ atomic_uint8_t RssLimitExceeded;
+ atomic_uint64_t RssLastCheckedAtNS;
+
explicit ScudoAllocator(LinkerInitialized)
- : AllocatorQuarantine(LINKER_INITIALIZED),
- FallbackQuarantineCache(LINKER_INITIALIZED) {}
+ : AllocatorQuarantine(LINKER_INITIALIZED) {}
- void init(const AllocatorOptions &Options) {
+ void performSanityChecks() {
// Verify that the header offset field can hold the maximum offset. In the
// case of the Secondary allocator, it takes care of alignment and the
// offset will always be 0. In the case of the Primary, the worst case
@@ -310,9 +247,9 @@ struct ScudoAllocator {
// result, the maximum offset will be at most the maximum alignment for the
// last size class minus the header size, in multiples of MinAlignment.
UnpackedHeader Header = {};
- uptr MaxPrimaryAlignment =
+ const uptr MaxPrimaryAlignment =
1 << MostSignificantSetBitIndex(SizeClassMap::kMaxSize - MinAlignment);
- uptr MaxOffset =
+ const uptr MaxOffset =
(MaxPrimaryAlignment - AlignedChunkHeaderSize) >> MinAlignmentLog;
Header.Offset = MaxOffset;
if (Header.Offset != MaxOffset) {
@@ -324,36 +261,97 @@ struct ScudoAllocator {
// case scenario happens in the Primary. It will depend on the second to
// last and last class sizes, as well as the dynamic base for the Primary.
// The following is an over-approximation that works for our needs.
- uptr MaxSizeOrUnusedBytes = SizeClassMap::kMaxSize - 1;
+ const uptr MaxSizeOrUnusedBytes = SizeClassMap::kMaxSize - 1;
Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes;
if (Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes) {
dieWithMessage("ERROR: the maximum possible unused bytes doesn't fit in "
"the header\n");
}
- DeallocationTypeMismatch = Options.DeallocationTypeMismatch;
- DeleteSizeMismatch = Options.DeleteSizeMismatch;
- ZeroContents = Options.ZeroContents;
- SetAllocatorMayReturnNull(Options.MayReturnNull);
- BackendAllocator.init(Options.ReleaseToOSIntervalMs);
+ const uptr LargestClassId = SizeClassMap::kLargestClassID;
+ Header.ClassId = LargestClassId;
+ if (Header.ClassId != LargestClassId) {
+ dieWithMessage("ERROR: the largest class ID doesn't fit in the header\n");
+ }
+ }
+
+ void init() {
+ SanitizerToolName = "Scudo";
+ initFlags();
+
+ performSanityChecks();
+
+ // Check if hardware CRC32 is supported in the binary and by the platform,
+ // if so, opt for the CRC32 hardware version of the checksum.
+ if (&computeHardwareCRC32 && hasHardwareCRC32())
+ atomic_store_relaxed(&HashAlgorithm, CRC32Hardware);
+
+ SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+ BackendAllocator.init(common_flags()->allocator_release_to_os_interval_ms);
+ HardRssLimitMb = common_flags()->hard_rss_limit_mb;
+ SoftRssLimitMb = common_flags()->soft_rss_limit_mb;
AllocatorQuarantine.Init(
- static_cast<uptr>(Options.QuarantineSizeMb) << 20,
- static_cast<uptr>(Options.ThreadLocalQuarantineSizeKb) << 10);
- GlobalPrng.init();
- Cookie = GlobalPrng.getU64();
- BackendAllocator.initCache(&FallbackAllocatorCache);
- FallbackPrng.init();
+ static_cast<uptr>(getFlags()->QuarantineSizeKb) << 10,
+ static_cast<uptr>(getFlags()->ThreadLocalQuarantineSizeKb) << 10);
+ QuarantineChunksUpToSize = getFlags()->QuarantineChunksUpToSize;
+ DeallocationTypeMismatch = getFlags()->DeallocationTypeMismatch;
+ DeleteSizeMismatch = getFlags()->DeleteSizeMismatch;
+ ZeroContents = getFlags()->ZeroContents;
+
+ if (UNLIKELY(!GetRandom(reinterpret_cast<void *>(&Cookie), sizeof(Cookie),
+ /*blocking=*/false))) {
+ Cookie = static_cast<u32>((NanoTime() >> 12) ^
+ (reinterpret_cast<uptr>(this) >> 4));
+ }
+
+ CheckRssLimit = HardRssLimitMb || SoftRssLimitMb;
+ if (CheckRssLimit)
+ atomic_store_relaxed(&RssLastCheckedAtNS, MonotonicNanoTime());
}
// Helper function that checks for a valid Scudo chunk. nullptr isn't.
- bool isValidPointer(const void *UserPtr) {
+ bool isValidPointer(const void *Ptr) {
initThreadMaybe();
- if (UNLIKELY(!UserPtr))
+ if (UNLIKELY(!Ptr))
return false;
- uptr UserBeg = reinterpret_cast<uptr>(UserPtr);
- if (!IsAligned(UserBeg, MinAlignment))
+ if (!Chunk::isAligned(Ptr))
return false;
- return getScudoChunk(UserBeg)->isValid();
+ return Chunk::isValid(Ptr);
+ }
+
+ // Opportunistic RSS limit check. This will update the RSS limit status, if
+ // it can, every 100ms, otherwise it will just return the current one.
+ bool isRssLimitExceeded() {
+ u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS);
+ const u64 CurrentCheck = MonotonicNanoTime();
+ if (LIKELY(CurrentCheck < LastCheck + (100ULL * 1000000ULL)))
+ return atomic_load_relaxed(&RssLimitExceeded);
+ if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck,
+ CurrentCheck, memory_order_relaxed))
+ return atomic_load_relaxed(&RssLimitExceeded);
+ // TODO(kostyak): We currently use sanitizer_common's GetRSS which reads the
+ // RSS from /proc/self/statm by default. We might want to
+ // call getrusage directly, even if it's less accurate.
+ const uptr CurrentRssMb = GetRSS() >> 20;
+ if (HardRssLimitMb && HardRssLimitMb < CurrentRssMb) {
+ Report("%s: hard RSS limit exhausted (%zdMb vs %zdMb)\n",
+ SanitizerToolName, HardRssLimitMb, CurrentRssMb);
+ DumpProcessMap();
+ Die();
+ }
+ if (SoftRssLimitMb) {
+ if (atomic_load_relaxed(&RssLimitExceeded)) {
+ if (CurrentRssMb <= SoftRssLimitMb)
+ atomic_store_relaxed(&RssLimitExceeded, false);
+ } else {
+ if (CurrentRssMb > SoftRssLimitMb) {
+ atomic_store_relaxed(&RssLimitExceeded, true);
+ Report("%s: soft RSS limit exhausted (%zdMb vs %zdMb)\n",
+ SanitizerToolName, SoftRssLimitMb, CurrentRssMb);
+ }
+ }
+ }
+ return atomic_load_relaxed(&RssLimitExceeded);
}
// Allocates a chunk.
@@ -375,207 +373,185 @@ struct ScudoAllocator {
if (UNLIKELY(AlignedSize >= MaxAllowedMallocSize))
return FailureHandler::OnBadRequest();
+ if (CheckRssLimit && UNLIKELY(isRssLimitExceeded()))
+ return FailureHandler::OnOOM();
+
// Primary and Secondary backed allocations have a different treatment. We
// deal with alignment requirements of Primary serviced allocations here,
// but the Secondary will take care of its own alignment needs.
- bool FromPrimary = PrimaryAllocator::CanAllocate(AlignedSize, MinAlignment);
-
- void *Ptr;
- u8 Salt;
- uptr AllocSize;
- if (FromPrimary) {
- AllocSize = AlignedSize;
- ScudoThreadContext *ThreadContext = getThreadContextAndLock();
- if (LIKELY(ThreadContext)) {
- Salt = getPrng(ThreadContext)->getU8();
- Ptr = BackendAllocator.allocatePrimary(getAllocatorCache(ThreadContext),
- AllocSize);
- ThreadContext->unlock();
- } else {
- SpinMutexLock l(&FallbackMutex);
- Salt = FallbackPrng.getU8();
- Ptr = BackendAllocator.allocatePrimary(&FallbackAllocatorCache,
- AllocSize);
- }
+ void *BackendPtr;
+ uptr BackendSize;
+ u8 ClassId;
+ if (PrimaryAllocator::CanAllocate(AlignedSize, MinAlignment)) {
+ BackendSize = AlignedSize;
+ ClassId = SizeClassMap::ClassID(BackendSize);
+ ScudoTSD *TSD = getTSDAndLock();
+ BackendPtr = BackendAllocator.allocatePrimary(&TSD->Cache, ClassId);
+ TSD->unlock();
} else {
- {
- SpinMutexLock l(&GlobalPrngMutex);
- Salt = GlobalPrng.getU8();
- }
- AllocSize = NeededSize;
- Ptr = BackendAllocator.allocateSecondary(AllocSize, Alignment);
+ BackendSize = NeededSize;
+ ClassId = 0;
+ BackendPtr = BackendAllocator.allocateSecondary(BackendSize, Alignment);
}
- if (UNLIKELY(!Ptr))
+ if (UNLIKELY(!BackendPtr))
return FailureHandler::OnOOM();
// If requested, we will zero out the entire contents of the returned chunk.
- if ((ForceZeroContents || ZeroContents) && FromPrimary)
- memset(Ptr, 0, BackendAllocator.getActuallyAllocatedSize(
- Ptr, /*FromPrimary=*/true));
+ if ((ForceZeroContents || ZeroContents) && ClassId)
+ memset(BackendPtr, 0,
+ BackendAllocator.getActuallyAllocatedSize(BackendPtr, ClassId));
UnpackedHeader Header = {};
- uptr AllocBeg = reinterpret_cast<uptr>(Ptr);
- uptr UserBeg = AllocBeg + AlignedChunkHeaderSize;
- if (UNLIKELY(!IsAligned(UserBeg, Alignment))) {
+ uptr UserPtr = reinterpret_cast<uptr>(BackendPtr) + AlignedChunkHeaderSize;
+ if (UNLIKELY(!IsAligned(UserPtr, Alignment))) {
// Since the Secondary takes care of alignment, a non-aligned pointer
// means it is from the Primary. It is also the only case where the offset
// field of the header would be non-zero.
- CHECK(FromPrimary);
- UserBeg = RoundUpTo(UserBeg, Alignment);
- uptr Offset = UserBeg - AlignedChunkHeaderSize - AllocBeg;
- Header.Offset = Offset >> MinAlignmentLog;
+ DCHECK(ClassId);
+ const uptr AlignedUserPtr = RoundUpTo(UserPtr, Alignment);
+ Header.Offset = (AlignedUserPtr - UserPtr) >> MinAlignmentLog;
+ UserPtr = AlignedUserPtr;
}
- CHECK_LE(UserBeg + Size, AllocBeg + AllocSize);
+ CHECK_LE(UserPtr + Size, reinterpret_cast<uptr>(BackendPtr) + BackendSize);
Header.State = ChunkAllocated;
Header.AllocType = Type;
- if (FromPrimary) {
- Header.FromPrimary = 1;
+ if (ClassId) {
+ Header.ClassId = ClassId;
Header.SizeOrUnusedBytes = Size;
} else {
// The secondary fits the allocations to a page, so the amount of unused
// bytes is the difference between the end of the user allocation and the
// next page boundary.
- uptr PageSize = GetPageSizeCached();
- uptr TrailingBytes = (UserBeg + Size) & (PageSize - 1);
+ const uptr PageSize = GetPageSizeCached();
+ const uptr TrailingBytes = (UserPtr + Size) & (PageSize - 1);
if (TrailingBytes)
Header.SizeOrUnusedBytes = PageSize - TrailingBytes;
}
- Header.Salt = Salt;
- getScudoChunk(UserBeg)->storeHeader(&Header);
- void *UserPtr = reinterpret_cast<void *>(UserBeg);
- // if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(UserPtr, Size);
- return UserPtr;
+ void *Ptr = reinterpret_cast<void *>(UserPtr);
+ Chunk::storeHeader(Ptr, &Header);
+ // if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(Ptr, Size);
+ return Ptr;
}
- // Place a chunk in the quarantine. In the event of a zero-sized quarantine,
- // we directly deallocate the chunk, otherwise the flow would lead to the
- // chunk being loaded (and checked) twice, and stored (and checksummed) once,
- // with no additional security value.
- void quarantineOrDeallocateChunk(ScudoChunk *Chunk, UnpackedHeader *Header,
+ // Place a chunk in the quarantine or directly deallocate it in the event of
+ // a zero-sized quarantine, or if the size of the chunk is greater than the
+ // quarantine chunk size threshold.
+ void quarantineOrDeallocateChunk(void *Ptr, UnpackedHeader *Header,
uptr Size) {
- bool FromPrimary = Header->FromPrimary;
- bool BypassQuarantine = (AllocatorQuarantine.GetCacheSize() == 0);
+ const bool BypassQuarantine = (AllocatorQuarantine.GetCacheSize() == 0) ||
+ (Size > QuarantineChunksUpToSize);
if (BypassQuarantine) {
- Chunk->eraseHeader();
- void *Ptr = Chunk->getAllocBeg(Header);
- if (FromPrimary) {
- ScudoThreadContext *ThreadContext = getThreadContextAndLock();
- if (LIKELY(ThreadContext)) {
- getBackendAllocator().deallocatePrimary(
- getAllocatorCache(ThreadContext), Ptr);
- ThreadContext->unlock();
- } else {
- SpinMutexLock Lock(&FallbackMutex);
- getBackendAllocator().deallocatePrimary(&FallbackAllocatorCache, Ptr);
- }
+ Chunk::eraseHeader(Ptr);
+ void *BackendPtr = Chunk::getBackendPtr(Ptr, Header);
+ if (Header->ClassId) {
+ ScudoTSD *TSD = getTSDAndLock();
+ getBackendAllocator().deallocatePrimary(&TSD->Cache, BackendPtr,
+ Header->ClassId);
+ TSD->unlock();
} else {
- getBackendAllocator().deallocateSecondary(Ptr);
+ getBackendAllocator().deallocateSecondary(BackendPtr);
}
} else {
+ // If a small memory amount was allocated with a larger alignment, we want
+ // to take that into account. Otherwise the Quarantine would be filled
+ // with tiny chunks, taking a lot of VA memory. This is an approximation
+ // of the usable size, that allows us to not call
+ // GetActuallyAllocatedSize.
+ uptr EstimatedSize = Size + (Header->Offset << MinAlignmentLog);
UnpackedHeader NewHeader = *Header;
NewHeader.State = ChunkQuarantine;
- Chunk->compareExchangeHeader(&NewHeader, Header);
- ScudoThreadContext *ThreadContext = getThreadContextAndLock();
- if (LIKELY(ThreadContext)) {
- AllocatorQuarantine.Put(getQuarantineCache(ThreadContext),
- QuarantineCallback(
- getAllocatorCache(ThreadContext)),
- Chunk, Size);
- ThreadContext->unlock();
- } else {
- SpinMutexLock l(&FallbackMutex);
- AllocatorQuarantine.Put(&FallbackQuarantineCache,
- QuarantineCallback(&FallbackAllocatorCache),
- Chunk, Size);
- }
+ Chunk::compareExchangeHeader(Ptr, &NewHeader, Header);
+ ScudoTSD *TSD = getTSDAndLock();
+ AllocatorQuarantine.Put(getQuarantineCache(TSD),
+ QuarantineCallback(&TSD->Cache), Ptr,
+ EstimatedSize);
+ TSD->unlock();
}
}
- // Deallocates a Chunk, which means adding it to the delayed free list (or
- // Quarantine).
- void deallocate(void *UserPtr, uptr DeleteSize, AllocType Type) {
- initThreadMaybe();
- // if (&__sanitizer_free_hook) __sanitizer_free_hook(UserPtr);
- if (UNLIKELY(!UserPtr))
+ // Deallocates a Chunk, which means either adding it to the quarantine or
+ // directly returning it to the backend if criteria are met.
+ void deallocate(void *Ptr, uptr DeleteSize, AllocType Type) {
+ // For a deallocation, we only ensure minimal initialization, meaning thread
+ // local data will be left uninitialized for now (when using ELF TLS). The
+ // fallback cache will be used instead. This is a workaround for a situation
+ // where the only heap operation performed in a thread would be a free past
+ // the TLS destructors, ending up in initialized thread specific data never
+ // being destroyed properly. Any other heap operation will do a full init.
+ initThreadMaybe(/*MinimalInit=*/true);
+ // if (&__sanitizer_free_hook) __sanitizer_free_hook(Ptr);
+ if (UNLIKELY(!Ptr))
return;
- uptr UserBeg = reinterpret_cast<uptr>(UserPtr);
- if (UNLIKELY(!IsAligned(UserBeg, MinAlignment))) {
+ if (UNLIKELY(!Chunk::isAligned(Ptr))) {
dieWithMessage("ERROR: attempted to deallocate a chunk not properly "
- "aligned at address %p\n", UserPtr);
+ "aligned at address %p\n", Ptr);
}
- ScudoChunk *Chunk = getScudoChunk(UserBeg);
- UnpackedHeader OldHeader;
- Chunk->loadHeader(&OldHeader);
- if (UNLIKELY(OldHeader.State != ChunkAllocated)) {
+ UnpackedHeader Header;
+ Chunk::loadHeader(Ptr, &Header);
+ if (UNLIKELY(Header.State != ChunkAllocated)) {
dieWithMessage("ERROR: invalid chunk state when deallocating address "
- "%p\n", UserPtr);
+ "%p\n", Ptr);
}
if (DeallocationTypeMismatch) {
// The deallocation type has to match the allocation one.
- if (OldHeader.AllocType != Type) {
+ if (Header.AllocType != Type) {
// With the exception of memalign'd Chunks, that can be still be free'd.
- if (OldHeader.AllocType != FromMemalign || Type != FromMalloc) {
- dieWithMessage("ERROR: allocation type mismatch on address %p\n",
- UserPtr);
+ if (Header.AllocType != FromMemalign || Type != FromMalloc) {
+ dieWithMessage("ERROR: allocation type mismatch when deallocating "
+ "address %p\n", Ptr);
}
}
}
- uptr Size = OldHeader.FromPrimary ? OldHeader.SizeOrUnusedBytes :
- Chunk->getUsableSize(&OldHeader) - OldHeader.SizeOrUnusedBytes;
+ uptr Size = Header.ClassId ? Header.SizeOrUnusedBytes :
+ Chunk::getUsableSize(Ptr, &Header) - Header.SizeOrUnusedBytes;
if (DeleteSizeMismatch) {
if (DeleteSize && DeleteSize != Size) {
dieWithMessage("ERROR: invalid sized delete on chunk at address %p\n",
- UserPtr);
+ Ptr);
}
}
-
- // If a small memory amount was allocated with a larger alignment, we want
- // to take that into account. Otherwise the Quarantine would be filled with
- // tiny chunks, taking a lot of VA memory. This is an approximation of the
- // usable size, that allows us to not call GetActuallyAllocatedSize.
- uptr LiableSize = Size + (OldHeader.Offset << MinAlignment);
- quarantineOrDeallocateChunk(Chunk, &OldHeader, LiableSize);
+ quarantineOrDeallocateChunk(Ptr, &Header, Size);
}
// Reallocates a chunk. We can save on a new allocation if the new requested
// size still fits in the chunk.
void *reallocate(void *OldPtr, uptr NewSize) {
initThreadMaybe();
- uptr UserBeg = reinterpret_cast<uptr>(OldPtr);
- if (UNLIKELY(!IsAligned(UserBeg, MinAlignment))) {
+ if (UNLIKELY(!Chunk::isAligned(OldPtr))) {
dieWithMessage("ERROR: attempted to reallocate a chunk not properly "
"aligned at address %p\n", OldPtr);
}
- ScudoChunk *Chunk = getScudoChunk(UserBeg);
UnpackedHeader OldHeader;
- Chunk->loadHeader(&OldHeader);
+ Chunk::loadHeader(OldPtr, &OldHeader);
if (UNLIKELY(OldHeader.State != ChunkAllocated)) {
dieWithMessage("ERROR: invalid chunk state when reallocating address "
"%p\n", OldPtr);
}
- if (UNLIKELY(OldHeader.AllocType != FromMalloc)) {
- dieWithMessage("ERROR: invalid chunk type when reallocating address %p\n",
- OldPtr);
+ if (DeallocationTypeMismatch) {
+ if (UNLIKELY(OldHeader.AllocType != FromMalloc)) {
+ dieWithMessage("ERROR: allocation type mismatch when reallocating "
+ "address %p\n", OldPtr);
+ }
}
- uptr UsableSize = Chunk->getUsableSize(&OldHeader);
+ const uptr UsableSize = Chunk::getUsableSize(OldPtr, &OldHeader);
// The new size still fits in the current chunk, and the size difference
// is reasonable.
if (NewSize <= UsableSize &&
(UsableSize - NewSize) < (SizeClassMap::kMaxSize / 2)) {
UnpackedHeader NewHeader = OldHeader;
NewHeader.SizeOrUnusedBytes =
- OldHeader.FromPrimary ? NewSize : UsableSize - NewSize;
- Chunk->compareExchangeHeader(&NewHeader, &OldHeader);
+ OldHeader.ClassId ? NewSize : UsableSize - NewSize;
+ Chunk::compareExchangeHeader(OldPtr, &NewHeader, &OldHeader);
return OldPtr;
}
// Otherwise, we have to allocate a new chunk and copy the contents of the
// old one.
void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc);
if (NewPtr) {
- uptr OldSize = OldHeader.FromPrimary ? OldHeader.SizeOrUnusedBytes :
+ uptr OldSize = OldHeader.ClassId ? OldHeader.SizeOrUnusedBytes :
UsableSize - OldHeader.SizeOrUnusedBytes;
- memcpy(NewPtr, OldPtr, Min(NewSize, OldSize));
- quarantineOrDeallocateChunk(Chunk, &OldHeader, UsableSize);
+ memcpy(NewPtr, OldPtr, Min(NewSize, UsableSize));
+ quarantineOrDeallocateChunk(OldPtr, &OldHeader, OldSize);
}
return NewPtr;
}
@@ -585,16 +561,14 @@ struct ScudoAllocator {
initThreadMaybe();
if (UNLIKELY(!Ptr))
return 0;
- uptr UserBeg = reinterpret_cast<uptr>(Ptr);
- ScudoChunk *Chunk = getScudoChunk(UserBeg);
UnpackedHeader Header;
- Chunk->loadHeader(&Header);
+ Chunk::loadHeader(Ptr, &Header);
// Getting the usable size of a chunk only makes sense if it's allocated.
if (UNLIKELY(Header.State != ChunkAllocated)) {
dieWithMessage("ERROR: invalid chunk state when sizing address %p\n",
Ptr);
}
- return Chunk->getUsableSize(&Header);
+ return Chunk::getUsableSize(Ptr, &Header);
}
void *calloc(uptr NMemB, uptr Size) {
@@ -604,11 +578,10 @@ struct ScudoAllocator {
return allocate(NMemB * Size, MinAlignment, FromMalloc, true);
}
- void commitBack(ScudoThreadContext *ThreadContext) {
- AllocatorCache *Cache = getAllocatorCache(ThreadContext);
- AllocatorQuarantine.Drain(getQuarantineCache(ThreadContext),
- QuarantineCallback(Cache));
- BackendAllocator.destroyCache(Cache);
+ void commitBack(ScudoTSD *TSD) {
+ AllocatorQuarantine.Drain(getQuarantineCache(TSD),
+ QuarantineCallback(&TSD->Cache));
+ BackendAllocator.destroyCache(&TSD->Cache);
}
uptr getStats(AllocatorStat StatType) {
@@ -617,6 +590,19 @@ struct ScudoAllocator {
BackendAllocator.getStats(stats);
return stats[StatType];
}
+
+ void *handleBadRequest() {
+ initThreadMaybe();
+ return FailureHandler::OnBadRequest();
+ }
+
+ void setRssLimit(uptr LimitMb, bool HardLimit) {
+ if (HardLimit)
+ HardRssLimitMb = LimitMb;
+ else
+ SoftRssLimitMb = LimitMb;
+ CheckRssLimit = HardRssLimitMb || SoftRssLimitMb;
+ }
};
static ScudoAllocator Instance(LINKER_INITIALIZED);
@@ -625,17 +611,17 @@ static ScudoBackendAllocator &getBackendAllocator() {
return Instance.BackendAllocator;
}
-static void initScudoInternal(const AllocatorOptions &Options) {
- Instance.init(Options);
+void initScudo() {
+ Instance.init();
}
-void ScudoThreadContext::init() {
+void ScudoTSD::init(bool Shared) {
+ UnlockRequired = Shared;
getBackendAllocator().initCache(&Cache);
- Prng.init();
memset(QuarantineCachePlaceHolder, 0, sizeof(QuarantineCachePlaceHolder));
}
-void ScudoThreadContext::commitBack() {
+void ScudoTSD::commitBack() {
Instance.commitBack(this);
}
@@ -672,6 +658,10 @@ void *scudoValloc(uptr Size) {
void *scudoPvalloc(uptr Size) {
uptr PageSize = GetPageSizeCached();
+ if (UNLIKELY(CheckForPvallocOverflow(Size, PageSize))) {
+ errno = ENOMEM;
+ return Instance.handleBadRequest();
+ }
// pvalloc(0) should allocate one page.
Size = Size ? RoundUpTo(Size, PageSize) : PageSize;
return SetErrnoOnNull(Instance.allocate(Size, PageSize, FromMemalign));
@@ -679,28 +669,28 @@ void *scudoPvalloc(uptr Size) {
void *scudoMemalign(uptr Alignment, uptr Size) {
if (UNLIKELY(!IsPowerOfTwo(Alignment))) {
- errno = errno_EINVAL;
- return ScudoAllocator::FailureHandler::OnBadRequest();
+ errno = EINVAL;
+ return Instance.handleBadRequest();
}
return SetErrnoOnNull(Instance.allocate(Size, Alignment, FromMemalign));
}
int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size) {
if (UNLIKELY(!CheckPosixMemalignAlignment(Alignment))) {
- ScudoAllocator::FailureHandler::OnBadRequest();
- return errno_EINVAL;
+ Instance.handleBadRequest();
+ return EINVAL;
}
void *Ptr = Instance.allocate(Size, Alignment, FromMemalign);
if (UNLIKELY(!Ptr))
- return errno_ENOMEM;
+ return ENOMEM;
*MemPtr = Ptr;
return 0;
}
void *scudoAlignedAlloc(uptr Alignment, uptr Size) {
if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(Alignment, Size))) {
- errno = errno_EINVAL;
- return ScudoAllocator::FailureHandler::OnBadRequest();
+ errno = EINVAL;
+ return Instance.handleBadRequest();
}
return SetErrnoOnNull(Instance.allocate(Size, Alignment, FromMalloc));
}
@@ -742,3 +732,13 @@ int __sanitizer_get_ownership(const void *Ptr) {
uptr __sanitizer_get_allocated_size(const void *Ptr) {
return Instance.getUsableSize(Ptr);
}
+
+// Interface functions
+
+extern "C" {
+void __scudo_set_rss_limit(unsigned long LimitMb, int HardLimit) { // NOLINT
+ if (!SCUDO_CAN_USE_PUBLIC_INTERFACE)
+ return;
+ Instance.setRssLimit(LimitMb, !!HardLimit);
+}
+} // extern "C"
diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator.h b/contrib/compiler-rt/lib/scudo/scudo_allocator.h
index 29d85995a3ee..a561247def9c 100644
--- a/contrib/compiler-rt/lib/scudo/scudo_allocator.h
+++ b/contrib/compiler-rt/lib/scudo/scudo_allocator.h
@@ -14,21 +14,15 @@
#ifndef SCUDO_ALLOCATOR_H_
#define SCUDO_ALLOCATOR_H_
-#include "scudo_flags.h"
-
-#include "sanitizer_common/sanitizer_allocator.h"
-
-#if !SANITIZER_LINUX
-# error "The Scudo hardened allocator is currently only supported on Linux."
-#endif
+#include "scudo_platform.h"
namespace __scudo {
enum AllocType : u8 {
- FromMalloc = 0, // Memory block came from malloc, realloc, calloc, etc.
- FromNew = 1, // Memory block came from operator new.
- FromNewArray = 2, // Memory block came from operator new [].
- FromMemalign = 3, // Memory block came from memalign, posix_memalign, etc.
+ FromMalloc = 0, // Memory block came from malloc, realloc, calloc, etc.
+ FromNew = 1, // Memory block came from operator new.
+ FromNewArray = 2, // Memory block came from operator new [].
+ FromMemalign = 3, // Memory block came from memalign, posix_memalign, etc.
};
enum ChunkState : u8 {
@@ -45,16 +39,15 @@ enum ChunkState : u8 {
typedef u64 PackedHeader;
struct UnpackedHeader {
u64 Checksum : 16;
- u64 SizeOrUnusedBytes : 19; // Size for Primary backed allocations, amount of
- // unused bytes in the chunk for Secondary ones.
- u64 FromPrimary : 1;
- u64 State : 2; // available, allocated, or quarantined
- u64 AllocType : 2; // malloc, new, new[], or memalign
- u64 Offset : 16; // Offset from the beginning of the backend
- // allocation to the beginning of the chunk
- // itself, in multiples of MinAlignment. See
- // comment about its maximum value and in init().
- u64 Salt : 8;
+ u64 ClassId : 8;
+ u64 SizeOrUnusedBytes : 20; // Size for Primary backed allocations, amount of
+ // unused bytes in the chunk for Secondary ones.
+ u64 State : 2; // available, allocated, or quarantined
+ u64 AllocType : 2; // malloc, new, new[], or memalign
+ u64 Offset : 16; // Offset from the beginning of the backend
+ // allocation to the beginning of the chunk
+ // itself, in multiples of MinAlignment. See
+ // comment about its maximum value and in init().
};
typedef atomic_uint64_t AtomicPackedHeader;
@@ -72,14 +65,6 @@ const uptr AlignedChunkHeaderSize =
#if SANITIZER_CAN_USE_ALLOCATOR64
const uptr AllocatorSpace = ~0ULL;
-# if defined(__aarch64__) && SANITIZER_ANDROID
-const uptr AllocatorSize = 0x4000000000ULL; // 256G.
-# elif defined(__aarch64__)
-const uptr AllocatorSize = 0x10000000000ULL; // 1T.
-# else
-const uptr AllocatorSize = 0x40000000000ULL; // 4T.
-# endif
-typedef DefaultSizeClassMap SizeClassMap;
struct AP64 {
static const uptr kSpaceBeg = AllocatorSpace;
static const uptr kSpaceSize = AllocatorSize;
@@ -91,17 +76,12 @@ struct AP64 {
};
typedef SizeClassAllocator64<AP64> PrimaryAllocator;
#else
-// Currently, the 32-bit Sanitizer allocator has not yet benefited from all the
-// security improvements brought to the 64-bit one. This makes the 32-bit
-// version of Scudo slightly less toughened.
-static const uptr RegionSizeLog = 20;
static const uptr NumRegions = SANITIZER_MMAP_RANGE_SIZE >> RegionSizeLog;
# if SANITIZER_WORDSIZE == 32
typedef FlatByteMap<NumRegions> ByteMap;
# elif SANITIZER_WORDSIZE == 64
typedef TwoLevelByteMap<(NumRegions >> 12), 1 << 12> ByteMap;
# endif // SANITIZER_WORDSIZE
-typedef DefaultSizeClassMap SizeClassMap;
struct AP32 {
static const uptr kSpaceBeg = 0;
static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
@@ -111,7 +91,8 @@ struct AP32 {
typedef __scudo::ByteMap ByteMap;
typedef NoOpMapUnmapCallback MapUnmapCallback;
static const uptr kFlags =
- SizeClassAllocator32FlagMasks::kRandomShuffleChunks;
+ SizeClassAllocator32FlagMasks::kRandomShuffleChunks |
+ SizeClassAllocator32FlagMasks::kUseSeparateSizeClassForBatch;
};
typedef SizeClassAllocator32<AP32> PrimaryAllocator;
#endif // SANITIZER_CAN_USE_ALLOCATOR64
diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator_combined.h b/contrib/compiler-rt/lib/scudo/scudo_allocator_combined.h
index 7599c12abb6d..25e273114c23 100644
--- a/contrib/compiler-rt/lib/scudo/scudo_allocator_combined.h
+++ b/contrib/compiler-rt/lib/scudo/scudo_allocator_combined.h
@@ -31,8 +31,8 @@ class ScudoCombinedAllocator {
// Primary allocations are always MinAlignment aligned, and as such do not
// require an Alignment parameter.
- void *allocatePrimary(AllocatorCache *Cache, uptr Size) {
- return Cache->Allocate(&Primary, Primary.ClassID(Size));
+ void *allocatePrimary(AllocatorCache *Cache, uptr ClassId) {
+ return Cache->Allocate(&Primary, ClassId);
}
// Secondary allocations do not require a Cache, but do require an Alignment
@@ -41,17 +41,17 @@ class ScudoCombinedAllocator {
return Secondary.Allocate(&Stats, Size, Alignment);
}
- void deallocatePrimary(AllocatorCache *Cache, void *Ptr) {
- Cache->Deallocate(&Primary, Primary.GetSizeClass(Ptr), Ptr);
+ void deallocatePrimary(AllocatorCache *Cache, void *Ptr, uptr ClassId) {
+ Cache->Deallocate(&Primary, ClassId, Ptr);
}
void deallocateSecondary(void *Ptr) {
Secondary.Deallocate(&Stats, Ptr);
}
- uptr getActuallyAllocatedSize(void *Ptr, bool FromPrimary) {
- if (FromPrimary)
- return PrimaryAllocator::ClassIdToSize(Primary.GetSizeClass(Ptr));
+ uptr getActuallyAllocatedSize(void *Ptr, uptr ClassId) {
+ if (ClassId)
+ return PrimaryAllocator::ClassIdToSize(ClassId);
return Secondary.GetActuallyAllocatedSize(Ptr);
}
diff --git a/contrib/compiler-rt/lib/scudo/scudo_allocator_secondary.h b/contrib/compiler-rt/lib/scudo/scudo_allocator_secondary.h
index dbfb22565f9c..f2002ed986c3 100644
--- a/contrib/compiler-rt/lib/scudo/scudo_allocator_secondary.h
+++ b/contrib/compiler-rt/lib/scudo/scudo_allocator_secondary.h
@@ -23,23 +23,24 @@
class ScudoLargeMmapAllocator {
public:
-
void Init() {
- PageSize = GetPageSizeCached();
+ PageSizeCached = GetPageSizeCached();
}
void *Allocate(AllocatorStats *Stats, uptr Size, uptr Alignment) {
- uptr UserSize = Size - AlignedChunkHeaderSize;
+ const uptr UserSize = Size - AlignedChunkHeaderSize;
// The Scudo frontend prevents us from allocating more than
// MaxAllowedMallocSize, so integer overflow checks would be superfluous.
- uptr MapSize = Size + SecondaryHeaderSize;
+ uptr MapSize = Size + AlignedReservedAddressRangeSize;
if (Alignment > MinAlignment)
MapSize += Alignment;
+ const uptr PageSize = PageSizeCached;
MapSize = RoundUpTo(MapSize, PageSize);
// Account for 2 guard pages, one before and one after the chunk.
MapSize += 2 * PageSize;
- uptr MapBeg = reinterpret_cast<uptr>(MmapNoAccess(MapSize));
+ ReservedAddressRange AddressRange;
+ uptr MapBeg = AddressRange.Init(MapSize);
if (MapBeg == ~static_cast<uptr>(0))
return ReturnNullOrDieOnFailure::OnOOM();
// A page-aligned pointer is assumed after that, so check it now.
@@ -62,27 +63,27 @@ class ScudoLargeMmapAllocator {
PageSize;
CHECK_GE(NewMapBeg, MapBeg);
if (NewMapBeg != MapBeg) {
- UnmapOrDie(reinterpret_cast<void *>(MapBeg), NewMapBeg - MapBeg);
+ AddressRange.Unmap(MapBeg, NewMapBeg - MapBeg);
MapBeg = NewMapBeg;
}
UserEnd = UserBeg + UserSize;
}
uptr NewMapEnd = RoundUpTo(UserEnd, PageSize) + PageSize;
if (NewMapEnd != MapEnd) {
- UnmapOrDie(reinterpret_cast<void *>(NewMapEnd), MapEnd - NewMapEnd);
+ AddressRange.Unmap(NewMapEnd, MapEnd - NewMapEnd);
MapEnd = NewMapEnd;
}
MapSize = MapEnd - MapBeg;
}
CHECK_LE(UserEnd, MapEnd - PageSize);
- // Actually mmap the memory, preserving the guard pages on either side.
- CHECK_EQ(MapBeg + PageSize, reinterpret_cast<uptr>(
- MmapFixedOrDie(MapBeg + PageSize, MapSize - 2 * PageSize)));
- uptr Ptr = UserBeg - AlignedChunkHeaderSize;
- SecondaryHeader *Header = getHeader(Ptr);
- Header->MapBeg = MapBeg;
- Header->MapSize = MapSize;
+ // Actually mmap the memory, preserving the guard pages on either side
+ CHECK_EQ(MapBeg + PageSize,
+ AddressRange.Map(MapBeg + PageSize, MapSize - 2 * PageSize));
+ const uptr Ptr = UserBeg - AlignedChunkHeaderSize;
+ ReservedAddressRange *StoredRange = getReservedAddressRange(Ptr);
+ *StoredRange = AddressRange;
+
// The primary adds the whole class size to the stats when allocating a
// chunk, so we will do something similar here. But we will not account for
// the guard pages.
@@ -96,42 +97,43 @@ class ScudoLargeMmapAllocator {
}
void Deallocate(AllocatorStats *Stats, void *Ptr) {
- SecondaryHeader *Header = getHeader(Ptr);
+ // Since we're unmapping the entirety of where the ReservedAddressRange
+ // actually is, copy onto the stack.
+ const uptr PageSize = PageSizeCached;
+ ReservedAddressRange AddressRange = *getReservedAddressRange(Ptr);
{
SpinMutexLock l(&StatsMutex);
- Stats->Sub(AllocatorStatAllocated, Header->MapSize - 2 * PageSize);
- Stats->Sub(AllocatorStatMapped, Header->MapSize - 2 * PageSize);
+ Stats->Sub(AllocatorStatAllocated, AddressRange.size() - 2 * PageSize);
+ Stats->Sub(AllocatorStatMapped, AddressRange.size() - 2 * PageSize);
}
- UnmapOrDie(reinterpret_cast<void *>(Header->MapBeg), Header->MapSize);
+ AddressRange.Unmap(reinterpret_cast<uptr>(AddressRange.base()),
+ AddressRange.size());
}
uptr GetActuallyAllocatedSize(void *Ptr) {
- SecondaryHeader *Header = getHeader(Ptr);
- // Deduct PageSize as MapSize includes the trailing guard page.
- uptr MapEnd = Header->MapBeg + Header->MapSize - PageSize;
+ ReservedAddressRange *StoredRange = getReservedAddressRange(Ptr);
+ // Deduct PageSize as ReservedAddressRange size includes the trailing guard
+ // page.
+ uptr MapEnd = reinterpret_cast<uptr>(StoredRange->base()) +
+ StoredRange->size() - PageSizeCached;
return MapEnd - reinterpret_cast<uptr>(Ptr);
}
private:
- // A Secondary allocated chunk header contains the base of the mapping and
- // its size, which comprises the guard pages.
- struct SecondaryHeader {
- uptr MapBeg;
- uptr MapSize;
- };
- // Check that sizeof(SecondaryHeader) is a multiple of MinAlignment.
- COMPILER_CHECK((sizeof(SecondaryHeader) & (MinAlignment - 1)) == 0);
-
- SecondaryHeader *getHeader(uptr Ptr) {
- return reinterpret_cast<SecondaryHeader*>(Ptr - sizeof(SecondaryHeader));
+ ReservedAddressRange *getReservedAddressRange(uptr Ptr) {
+ return reinterpret_cast<ReservedAddressRange*>(
+ Ptr - sizeof(ReservedAddressRange));
}
- SecondaryHeader *getHeader(const void *Ptr) {
- return getHeader(reinterpret_cast<uptr>(Ptr));
+ ReservedAddressRange *getReservedAddressRange(const void *Ptr) {
+ return getReservedAddressRange(reinterpret_cast<uptr>(Ptr));
}
- const uptr SecondaryHeaderSize = sizeof(SecondaryHeader);
- const uptr HeadersSize = SecondaryHeaderSize + AlignedChunkHeaderSize;
- uptr PageSize;
+ static constexpr uptr AlignedReservedAddressRangeSize =
+ (sizeof(ReservedAddressRange) + MinAlignment - 1) & ~(MinAlignment - 1);
+ static constexpr uptr HeadersSize =
+ AlignedReservedAddressRangeSize + AlignedChunkHeaderSize;
+
+ uptr PageSizeCached;
SpinMutex StatsMutex;
};
diff --git a/contrib/compiler-rt/lib/scudo/scudo_crc32.h b/contrib/compiler-rt/lib/scudo/scudo_crc32.h
index 5ffcc62658cc..e89e430f4085 100644
--- a/contrib/compiler-rt/lib/scudo/scudo_crc32.h
+++ b/contrib/compiler-rt/lib/scudo/scudo_crc32.h
@@ -40,7 +40,7 @@ enum : u8 {
CRC32Hardware = 1,
};
-const static u32 CRC32Table[] = {
+static const u32 CRC32Table[] = {
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f,
0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2,
diff --git a/contrib/compiler-rt/lib/scudo/scudo_flags.cpp b/contrib/compiler-rt/lib/scudo/scudo_flags.cpp
index 90f0cbf4bb86..2aff3ef1e8fa 100644
--- a/contrib/compiler-rt/lib/scudo/scudo_flags.cpp
+++ b/contrib/compiler-rt/lib/scudo/scudo_flags.cpp
@@ -17,12 +17,11 @@
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_flag_parser.h"
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-const char* __scudo_default_options();
+SANITIZER_INTERFACE_WEAK_DEF(const char*, __scudo_default_options, void);
namespace __scudo {
-Flags ScudoFlags; // Use via getFlags().
+static Flags ScudoFlags; // Use via getFlags().
void Flags::setDefaults() {
#define SCUDO_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
@@ -37,7 +36,7 @@ static void RegisterScudoFlags(FlagParser *parser, Flags *f) {
#undef SCUDO_FLAG
}
-static const char *callGetScudoDefaultOptions() {
+static const char *getScudoDefaultOptions() {
return (&__scudo_default_options) ? __scudo_default_options() : "";
}
@@ -57,8 +56,7 @@ void initFlags() {
RegisterCommonFlags(&ScudoParser);
// Override from user-specified string.
- const char *ScudoDefaultOptions = callGetScudoDefaultOptions();
- ScudoParser.ParseString(ScudoDefaultOptions);
+ ScudoParser.ParseString(getScudoDefaultOptions());
// Override from environment.
ScudoParser.ParseString(GetEnv("SCUDO_OPTIONS"));
@@ -67,27 +65,52 @@ void initFlags() {
// Sanity checks and default settings for the Quarantine parameters.
- if (f->QuarantineSizeMb < 0) {
- const int DefaultQuarantineSizeMb = FIRST_32_SECOND_64(4, 16);
- f->QuarantineSizeMb = DefaultQuarantineSizeMb;
+ if (f->QuarantineSizeMb >= 0) {
+ // Backward compatible logic if QuarantineSizeMb is set.
+ if (f->QuarantineSizeKb >= 0) {
+ dieWithMessage("ERROR: please use either QuarantineSizeMb (deprecated) "
+ "or QuarantineSizeKb, but not both\n");
+ }
+ if (f->QuarantineChunksUpToSize >= 0) {
+ dieWithMessage("ERROR: QuarantineChunksUpToSize cannot be used in "
+ " conjunction with the deprecated QuarantineSizeMb option\n");
+ }
+ // If everything is in order, update QuarantineSizeKb accordingly.
+ f->QuarantineSizeKb = f->QuarantineSizeMb * 1024;
+ } else {
+ // Otherwise proceed with the new options.
+ if (f->QuarantineSizeKb < 0) {
+ const int DefaultQuarantineSizeKb = FIRST_32_SECOND_64(64, 256);
+ f->QuarantineSizeKb = DefaultQuarantineSizeKb;
+ }
+ if (f->QuarantineChunksUpToSize < 0) {
+ const int DefaultQuarantineChunksUpToSize = FIRST_32_SECOND_64(512, 2048);
+ f->QuarantineChunksUpToSize = DefaultQuarantineChunksUpToSize;
+ }
}
- // We enforce an upper limit for the quarantine size of 4Gb.
- if (f->QuarantineSizeMb > (4 * 1024)) {
+
+ // We enforce an upper limit for the chunk quarantine threshold of 4Mb.
+ if (f->QuarantineChunksUpToSize > (4 * 1024 * 1024)) {
+ dieWithMessage("ERROR: the chunk quarantine threshold is too large\n");
+ }
+
+ // We enforce an upper limit for the quarantine size of 32Mb.
+ if (f->QuarantineSizeKb > (32 * 1024)) {
dieWithMessage("ERROR: the quarantine size is too large\n");
}
+
if (f->ThreadLocalQuarantineSizeKb < 0) {
- const int DefaultThreadLocalQuarantineSizeKb =
- FIRST_32_SECOND_64(64, 256);
+ const int DefaultThreadLocalQuarantineSizeKb = FIRST_32_SECOND_64(16, 64);
f->ThreadLocalQuarantineSizeKb = DefaultThreadLocalQuarantineSizeKb;
}
- // And an upper limit of 128Mb for the thread quarantine cache.
- if (f->ThreadLocalQuarantineSizeKb > (128 * 1024)) {
+ // And an upper limit of 8Mb for the thread quarantine cache.
+ if (f->ThreadLocalQuarantineSizeKb > (8 * 1024)) {
dieWithMessage("ERROR: the per thread quarantine cache size is too "
- "large\n");
+ "large\n");
}
- if (f->ThreadLocalQuarantineSizeKb == 0 && f->QuarantineSizeMb > 0) {
+ if (f->ThreadLocalQuarantineSizeKb == 0 && f->QuarantineSizeKb > 0) {
dieWithMessage("ERROR: ThreadLocalQuarantineSizeKb can be set to 0 only "
- "when QuarantineSizeMb is set to 0\n");
+ "when QuarantineSizeKb is set to 0\n");
}
}
diff --git a/contrib/compiler-rt/lib/scudo/scudo_flags.inc b/contrib/compiler-rt/lib/scudo/scudo_flags.inc
index 45f9ea846e1a..f180478fdac3 100644
--- a/contrib/compiler-rt/lib/scudo/scudo_flags.inc
+++ b/contrib/compiler-rt/lib/scudo/scudo_flags.inc
@@ -15,17 +15,27 @@
# error "Define SCUDO_FLAG prior to including this file!"
#endif
-// Default value is set in scudo_flags.cpp based on architecture.
SCUDO_FLAG(int, QuarantineSizeMb, -1,
- "Size (in Mb) of quarantine used to delay the actual deallocation "
- "of chunks. Lower value may reduce memory usage but decrease the "
- "effectiveness of the mitigation.")
+ "Deprecated. Please use QuarantineSizeKb.")
+
+// Default value is set in scudo_flags.cpp based on architecture.
+SCUDO_FLAG(int, QuarantineSizeKb, -1,
+ "Size in KB of quarantine used to delay the actual deallocation of "
+ "chunks. Lower value may reduce memory usage but decrease the "
+ "effectiveness of the mitigation. Defaults to 64KB (32-bit) or "
+ "256KB (64-bit)")
// Default value is set in scudo_flags.cpp based on architecture.
SCUDO_FLAG(int, ThreadLocalQuarantineSizeKb, -1,
- "Size (in Kb) of per-thread cache used to offload the global "
+ "Size in KB of per-thread cache used to offload the global "
"quarantine. Lower value may reduce memory usage but might increase "
- "the contention on the global quarantine.")
+ "the contention on the global quarantine. Defaults to 16KB (32-bit) "
+ "or 64KB (64-bit)")
+
+// Default value is set in scudo_flags.cpp based on architecture.
+SCUDO_FLAG(int, QuarantineChunksUpToSize, -1,
+ "Size in bytes up to which chunks will be quarantined (if lower than"
+ "or equal to). Defaults to 256 (32-bit) or 2048 (64-bit)")
SCUDO_FLAG(bool, DeallocationTypeMismatch, true,
"Report errors on malloc/delete, new/free, new/delete[], etc.")
diff --git a/contrib/compiler-rt/lib/scudo/scudo_interface_internal.h b/contrib/compiler-rt/lib/scudo/scudo_interface_internal.h
new file mode 100644
index 000000000000..3f39e0c4ee0b
--- /dev/null
+++ b/contrib/compiler-rt/lib/scudo/scudo_interface_internal.h
@@ -0,0 +1,22 @@
+//===-- scudo_interface_internal.h ------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Private Scudo interface header.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_INTERFACE_INTERNAL_H_
+#define SCUDO_INTERFACE_INTERNAL_H_
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __scudo_set_rss_limit(unsigned long LimitMb, int HardLimit); // NOLINT
+} // extern "C"
+
+#endif // SCUDO_INTERFACE_INTERNAL_H_
diff --git a/contrib/compiler-rt/lib/scudo/scudo_new_delete.cpp b/contrib/compiler-rt/lib/scudo/scudo_new_delete.cpp
index cdefb127b965..c5a1abbed82b 100644
--- a/contrib/compiler-rt/lib/scudo/scudo_new_delete.cpp
+++ b/contrib/compiler-rt/lib/scudo/scudo_new_delete.cpp
@@ -15,7 +15,7 @@
#include "interception/interception.h"
-#include <cstddef>
+#include <stddef.h>
using namespace __scudo;
diff --git a/contrib/compiler-rt/lib/scudo/scudo_platform.h b/contrib/compiler-rt/lib/scudo/scudo_platform.h
new file mode 100644
index 000000000000..e1c9c32e9a62
--- /dev/null
+++ b/contrib/compiler-rt/lib/scudo/scudo_platform.h
@@ -0,0 +1,80 @@
+//===-- scudo_platform.h ----------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Scudo platform specific definitions.
+/// TODO(kostyak): add tests for the compile time defines.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_PLATFORM_H_
+#define SCUDO_PLATFORM_H_
+
+#include "sanitizer_common/sanitizer_allocator.h"
+
+#if !SANITIZER_LINUX && !SANITIZER_FUCHSIA
+# error "The Scudo hardened allocator is not supported on this platform."
+#endif
+
+#define SCUDO_TSD_EXCLUSIVE_SUPPORTED (!SANITIZER_ANDROID && !SANITIZER_FUCHSIA)
+
+#ifndef SCUDO_TSD_EXCLUSIVE
+// SCUDO_TSD_EXCLUSIVE wasn't defined, use a default TSD model for the platform.
+# if SANITIZER_ANDROID || SANITIZER_FUCHSIA
+// Android and Fuchsia use a pool of TSDs shared between threads.
+# define SCUDO_TSD_EXCLUSIVE 0
+# elif SANITIZER_LINUX && !SANITIZER_ANDROID
+// Non-Android Linux use an exclusive TSD per thread.
+# define SCUDO_TSD_EXCLUSIVE 1
+# else
+# error "No default TSD model defined for this platform."
+# endif // SANITIZER_ANDROID || SANITIZER_FUCHSIA
+#endif // SCUDO_TSD_EXCLUSIVE
+
+// If the exclusive TSD model is chosen, make sure the platform supports it.
+#if SCUDO_TSD_EXCLUSIVE && !SCUDO_TSD_EXCLUSIVE_SUPPORTED
+# error "The exclusive TSD model is not supported on this platform."
+#endif
+
+// Maximum number of TSDs that can be created for the Shared model.
+#ifndef SCUDO_SHARED_TSD_POOL_SIZE
+# define SCUDO_SHARED_TSD_POOL_SIZE 32U
+#endif // SCUDO_SHARED_TSD_POOL_SIZE
+
+// The following allows the public interface functions to be disabled.
+#ifndef SCUDO_CAN_USE_PUBLIC_INTERFACE
+# define SCUDO_CAN_USE_PUBLIC_INTERFACE 1
+#endif
+
+namespace __scudo {
+
+#if SANITIZER_CAN_USE_ALLOCATOR64
+# if defined(__aarch64__) && SANITIZER_ANDROID
+const uptr AllocatorSize = 0x4000000000ULL; // 256G.
+# elif defined(__aarch64__)
+const uptr AllocatorSize = 0x10000000000ULL; // 1T.
+# else
+const uptr AllocatorSize = 0x40000000000ULL; // 4T.
+# endif
+#else
+const uptr RegionSizeLog = SANITIZER_ANDROID ? 19 : 20;
+#endif // SANITIZER_CAN_USE_ALLOCATOR64
+
+#if !defined(SCUDO_SIZE_CLASS_MAP)
+# define SCUDO_SIZE_CLASS_MAP Default
+#endif
+
+#define SIZE_CLASS_MAP_TYPE SIZE_CLASS_MAP_TYPE_(SCUDO_SIZE_CLASS_MAP)
+#define SIZE_CLASS_MAP_TYPE_(T) SIZE_CLASS_MAP_TYPE__(T)
+#define SIZE_CLASS_MAP_TYPE__(T) T##SizeClassMap
+
+typedef SIZE_CLASS_MAP_TYPE SizeClassMap;
+
+} // namespace __scudo
+
+#endif // SCUDO_PLATFORM_H_
diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls.h b/contrib/compiler-rt/lib/scudo/scudo_tls.h
deleted file mode 100644
index 20c49204cf13..000000000000
--- a/contrib/compiler-rt/lib/scudo/scudo_tls.h
+++ /dev/null
@@ -1,47 +0,0 @@
-//===-- scudo_tls.h ---------------------------------------------*- C++ -*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-///
-/// Scudo thread local structure definition.
-/// Implementation will differ based on the thread local storage primitives
-/// offered by the underlying platform.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef SCUDO_TLS_H_
-#define SCUDO_TLS_H_
-
-#include "scudo_allocator.h"
-#include "scudo_utils.h"
-
-#include "sanitizer_common/sanitizer_linux.h"
-#include "sanitizer_common/sanitizer_platform.h"
-
-namespace __scudo {
-
-// Platform specific base thread context definitions.
-#include "scudo_tls_context_android.inc"
-#include "scudo_tls_context_linux.inc"
-
-struct ALIGNED(64) ScudoThreadContext : public ScudoThreadContextPlatform {
- AllocatorCache Cache;
- ScudoPrng Prng;
- uptr QuarantineCachePlaceHolder[4];
- void init();
- void commitBack();
-};
-
-void initThread();
-
-// Platform specific dastpath functions definitions.
-#include "scudo_tls_android.inc"
-#include "scudo_tls_linux.inc"
-
-} // namespace __scudo
-
-#endif // SCUDO_TLS_H_
diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_android.cpp b/contrib/compiler-rt/lib/scudo/scudo_tls_android.cpp
deleted file mode 100644
index ec74e37c8dbc..000000000000
--- a/contrib/compiler-rt/lib/scudo/scudo_tls_android.cpp
+++ /dev/null
@@ -1,95 +0,0 @@
-//===-- scudo_tls_android.cpp -----------------------------------*- C++ -*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-///
-/// Scudo thread local structure implementation for Android.
-///
-//===----------------------------------------------------------------------===//
-
-#include "sanitizer_common/sanitizer_platform.h"
-
-#if SANITIZER_LINUX && SANITIZER_ANDROID
-
-#include "scudo_tls.h"
-
-#include <pthread.h>
-
-namespace __scudo {
-
-static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT;
-static pthread_key_t PThreadKey;
-
-static atomic_uint32_t ThreadContextCurrentIndex;
-static ScudoThreadContext *ThreadContexts;
-static uptr NumberOfContexts;
-
-// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used as they allocate memory.
-static uptr getNumberOfCPUs() {
- cpu_set_t CPUs;
- CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0);
- return CPU_COUNT(&CPUs);
-}
-
-static void initOnce() {
- // Hack: TLS_SLOT_TSAN was introduced in N. To be able to use it on M for
- // testing, we create an unused key. Since the key_data array follows the tls
- // array, it basically gives us the extra entry we need.
- // TODO(kostyak): remove and restrict to N and above.
- CHECK_EQ(pthread_key_create(&PThreadKey, NULL), 0);
- initScudo();
- NumberOfContexts = getNumberOfCPUs();
- ThreadContexts = reinterpret_cast<ScudoThreadContext *>(
- MmapOrDie(sizeof(ScudoThreadContext) * NumberOfContexts, __func__));
- for (uptr i = 0; i < NumberOfContexts; i++)
- ThreadContexts[i].init();
-}
-
-void initThread() {
- pthread_once(&GlobalInitialized, initOnce);
- // Initial context assignment is done in a plain round-robin fashion.
- u32 Index = atomic_fetch_add(&ThreadContextCurrentIndex, 1,
- memory_order_relaxed);
- ScudoThreadContext *ThreadContext =
- &ThreadContexts[Index % NumberOfContexts];
- *get_android_tls_ptr() = reinterpret_cast<uptr>(ThreadContext);
-}
-
-ScudoThreadContext *getThreadContextAndLockSlow() {
- ScudoThreadContext *ThreadContext;
- // Go through all the contexts and find the first unlocked one.
- for (u32 i = 0; i < NumberOfContexts; i++) {
- ThreadContext = &ThreadContexts[i];
- if (ThreadContext->tryLock()) {
- *get_android_tls_ptr() = reinterpret_cast<uptr>(ThreadContext);
- return ThreadContext;
- }
- }
- // No luck, find the one with the lowest precedence, and slow lock it.
- u64 Precedence = UINT64_MAX;
- for (u32 i = 0; i < NumberOfContexts; i++) {
- u64 SlowLockPrecedence = ThreadContexts[i].getSlowLockPrecedence();
- if (SlowLockPrecedence && SlowLockPrecedence < Precedence) {
- ThreadContext = &ThreadContexts[i];
- Precedence = SlowLockPrecedence;
- }
- }
- if (LIKELY(Precedence != UINT64_MAX)) {
- ThreadContext->lock();
- *get_android_tls_ptr() = reinterpret_cast<uptr>(ThreadContext);
- return ThreadContext;
- }
- // Last resort (can this happen?), stick with the current one.
- ThreadContext =
- reinterpret_cast<ScudoThreadContext *>(*get_android_tls_ptr());
- ThreadContext->lock();
- return ThreadContext;
-}
-
-} // namespace __scudo
-
-#endif // SANITIZER_LINUX && SANITIZER_ANDROID
diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_android.inc b/contrib/compiler-rt/lib/scudo/scudo_tls_android.inc
deleted file mode 100644
index 8ecad7a30a6c..000000000000
--- a/contrib/compiler-rt/lib/scudo/scudo_tls_android.inc
+++ /dev/null
@@ -1,44 +0,0 @@
-//===-- scudo_tls_android.inc -----------------------------------*- C++ -*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-///
-/// Scudo thread local structure fastpath functions implementation for Android.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef SCUDO_TLS_ANDROID_H_
-#define SCUDO_TLS_ANDROID_H_
-
-#ifndef SCUDO_TLS_H_
-# error "This file must be included inside scudo_tls.h."
-#endif // SCUDO_TLS_H_
-
-#if SANITIZER_LINUX && SANITIZER_ANDROID
-
-ALWAYS_INLINE void initThreadMaybe() {
- if (LIKELY(*get_android_tls_ptr()))
- return;
- initThread();
-}
-
-ScudoThreadContext *getThreadContextAndLockSlow();
-
-ALWAYS_INLINE ScudoThreadContext *getThreadContextAndLock() {
- ScudoThreadContext *ThreadContext =
- reinterpret_cast<ScudoThreadContext *>(*get_android_tls_ptr());
- CHECK(ThreadContext);
- // Try to lock the currently associated context.
- if (ThreadContext->tryLock())
- return ThreadContext;
- // If it failed, go the slow path.
- return getThreadContextAndLockSlow();
-}
-
-#endif // SANITIZER_LINUX && SANITIZER_ANDROID
-
-#endif // SCUDO_TLS_ANDROID_H_
diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_context_android.inc b/contrib/compiler-rt/lib/scudo/scudo_tls_context_android.inc
deleted file mode 100644
index f1951319d487..000000000000
--- a/contrib/compiler-rt/lib/scudo/scudo_tls_context_android.inc
+++ /dev/null
@@ -1,54 +0,0 @@
-//===-- scudo_tls_context_android.inc ---------------------------*- C++ -*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-///
-/// Android specific base thread context definition.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef SCUDO_TLS_CONTEXT_ANDROID_INC_
-#define SCUDO_TLS_CONTEXT_ANDROID_INC_
-
-#ifndef SCUDO_TLS_H_
-# error "This file must be included inside scudo_tls.h."
-#endif // SCUDO_TLS_H_
-
-#if SANITIZER_LINUX && SANITIZER_ANDROID
-
-struct ScudoThreadContextPlatform {
- INLINE bool tryLock() {
- if (Mutex.TryLock()) {
- atomic_store_relaxed(&SlowLockPrecedence, 0);
- return true;
- }
- if (atomic_load_relaxed(&SlowLockPrecedence) == 0)
- atomic_store_relaxed(&SlowLockPrecedence, NanoTime());
- return false;
- }
-
- INLINE void lock() {
- Mutex.Lock();
- atomic_store_relaxed(&SlowLockPrecedence, 0);
- }
-
- INLINE void unlock() {
- Mutex.Unlock();
- }
-
- INLINE u64 getSlowLockPrecedence() {
- return atomic_load_relaxed(&SlowLockPrecedence);
- }
-
- private:
- StaticSpinMutex Mutex;
- atomic_uint64_t SlowLockPrecedence;
-};
-
-#endif // SANITIZER_LINUX && SANITIZER_ANDROID
-
-#endif // SCUDO_TLS_CONTEXT_ANDROID_INC_
diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_context_linux.inc b/contrib/compiler-rt/lib/scudo/scudo_tls_context_linux.inc
deleted file mode 100644
index 8d292bdbc932..000000000000
--- a/contrib/compiler-rt/lib/scudo/scudo_tls_context_linux.inc
+++ /dev/null
@@ -1,29 +0,0 @@
-//===-- scudo_tls_context_linux.inc -----------------------------*- C++ -*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-///
-/// Linux specific base thread context definition.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef SCUDO_TLS_CONTEXT_LINUX_INC_
-#define SCUDO_TLS_CONTEXT_LINUX_INC_
-
-#ifndef SCUDO_TLS_H_
-# error "This file must be included inside scudo_tls.h."
-#endif // SCUDO_TLS_H_
-
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
-
-struct ScudoThreadContextPlatform {
- ALWAYS_INLINE void unlock() {}
-};
-
-#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
-
-#endif // SCUDO_TLS_CONTEXT_LINUX_INC_
diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_linux.inc b/contrib/compiler-rt/lib/scudo/scudo_tls_linux.inc
deleted file mode 100644
index 242ee3329ea8..000000000000
--- a/contrib/compiler-rt/lib/scudo/scudo_tls_linux.inc
+++ /dev/null
@@ -1,48 +0,0 @@
-//===-- scudo_tls_linux.inc -------------------------------------*- C++ -*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-///
-/// Scudo thread local structure fastpath functions implementation for platforms
-/// supporting thread_local.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef SCUDO_TLS_LINUX_H_
-#define SCUDO_TLS_LINUX_H_
-
-#ifndef SCUDO_TLS_H_
-# error "This file must be included inside scudo_tls.h."
-#endif // SCUDO_TLS_H_
-
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
-
-enum ThreadState : u8 {
- ThreadNotInitialized = 0,
- ThreadInitialized,
- ThreadTornDown,
-};
-__attribute__((tls_model("initial-exec")))
-extern THREADLOCAL ThreadState ScudoThreadState;
-__attribute__((tls_model("initial-exec")))
-extern THREADLOCAL ScudoThreadContext ThreadLocalContext;
-
-ALWAYS_INLINE void initThreadMaybe() {
- if (LIKELY(ScudoThreadState != ThreadNotInitialized))
- return;
- initThread();
-}
-
-ALWAYS_INLINE ScudoThreadContext *getThreadContextAndLock() {
- if (UNLIKELY(ScudoThreadState == ThreadTornDown))
- return nullptr;
- return &ThreadLocalContext;
-}
-
-#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
-
-#endif // SCUDO_TLS_LINUX_H_
diff --git a/contrib/compiler-rt/lib/scudo/scudo_tsd.h b/contrib/compiler-rt/lib/scudo/scudo_tsd.h
new file mode 100644
index 000000000000..80464b5ea1e4
--- /dev/null
+++ b/contrib/compiler-rt/lib/scudo/scudo_tsd.h
@@ -0,0 +1,72 @@
+//===-- scudo_tsd.h ---------------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Scudo thread specific data definition.
+/// Implementation will differ based on the thread local storage primitives
+/// offered by the underlying platform.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_TSD_H_
+#define SCUDO_TSD_H_
+
+#include "scudo_allocator.h"
+#include "scudo_utils.h"
+
+#include <pthread.h>
+
+namespace __scudo {
+
+struct ALIGNED(64) ScudoTSD {
+ AllocatorCache Cache;
+ uptr QuarantineCachePlaceHolder[4];
+
+ void init(bool Shared);
+ void commitBack();
+
+ INLINE bool tryLock() {
+ if (Mutex.TryLock()) {
+ atomic_store_relaxed(&Precedence, 0);
+ return true;
+ }
+ if (atomic_load_relaxed(&Precedence) == 0)
+ atomic_store_relaxed(&Precedence, MonotonicNanoTime());
+ return false;
+ }
+
+ INLINE void lock() {
+ Mutex.Lock();
+ atomic_store_relaxed(&Precedence, 0);
+ }
+
+ INLINE void unlock() {
+ if (!UnlockRequired)
+ return;
+ Mutex.Unlock();
+ }
+
+ INLINE u64 getPrecedence() {
+ return atomic_load_relaxed(&Precedence);
+ }
+
+ private:
+ bool UnlockRequired;
+ StaticSpinMutex Mutex;
+ atomic_uint64_t Precedence;
+};
+
+void initThread(bool MinimalInit);
+
+// TSD model specific fastpath functions definitions.
+#include "scudo_tsd_exclusive.inc"
+#include "scudo_tsd_shared.inc"
+
+} // namespace __scudo
+
+#endif // SCUDO_TSD_H_
diff --git a/contrib/compiler-rt/lib/scudo/scudo_tls_linux.cpp b/contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp
index 1e38233f339c..1084dfac91e1 100644
--- a/contrib/compiler-rt/lib/scudo/scudo_tls_linux.cpp
+++ b/contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp
@@ -1,4 +1,4 @@
-//===-- scudo_tls_linux.cpp -------------------------------------*- C++ -*-===//
+//===-- scudo_tsd_exclusive.cpp ---------------------------------*- C++ -*-===//
//
// The LLVM Compiler Infrastructure
//
@@ -7,18 +7,13 @@
//
//===----------------------------------------------------------------------===//
///
-/// Scudo thread local structure implementation for platforms supporting
-/// thread_local.
+/// Scudo exclusive TSD implementation.
///
//===----------------------------------------------------------------------===//
-#include "sanitizer_common/sanitizer_platform.h"
+#include "scudo_tsd.h"
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
-
-#include "scudo_tls.h"
-
-#include <pthread.h>
+#if SCUDO_TSD_EXCLUSIVE
namespace __scudo {
@@ -28,7 +23,11 @@ static pthread_key_t PThreadKey;
__attribute__((tls_model("initial-exec")))
THREADLOCAL ThreadState ScudoThreadState = ThreadNotInitialized;
__attribute__((tls_model("initial-exec")))
-THREADLOCAL ScudoThreadContext ThreadLocalContext;
+THREADLOCAL ScudoTSD TSD;
+
+// Fallback TSD for when the thread isn't initialized yet or is torn down. It
+// can be shared between multiple threads and as such must be locked.
+ScudoTSD FallbackTSD;
static void teardownThread(void *Ptr) {
uptr I = reinterpret_cast<uptr>(Ptr);
@@ -43,7 +42,7 @@ static void teardownThread(void *Ptr) {
reinterpret_cast<void *>(I - 1)) == 0))
return;
}
- ThreadLocalContext.commitBack();
+ TSD.commitBack();
ScudoThreadState = ThreadTornDown;
}
@@ -51,16 +50,19 @@ static void teardownThread(void *Ptr) {
static void initOnce() {
CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread), 0);
initScudo();
+ FallbackTSD.init(/*Shared=*/true);
}
-void initThread() {
+void initThread(bool MinimalInit) {
CHECK_EQ(pthread_once(&GlobalInitialized, initOnce), 0);
+ if (UNLIKELY(MinimalInit))
+ return;
CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast<void *>(
GetPthreadDestructorIterations())), 0);
- ThreadLocalContext.init();
+ TSD.init(/*Shared=*/false);
ScudoThreadState = ThreadInitialized;
}
} // namespace __scudo
-#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
+#endif // SCUDO_TSD_EXCLUSIVE
diff --git a/contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc b/contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc
new file mode 100644
index 000000000000..567b6a1edd12
--- /dev/null
+++ b/contrib/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc
@@ -0,0 +1,46 @@
+//===-- scudo_tsd_exclusive.inc ---------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Scudo exclusive TSD fastpath functions implementation.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_TSD_H_
+# error "This file must be included inside scudo_tsd.h."
+#endif // SCUDO_TSD_H_
+
+#if SCUDO_TSD_EXCLUSIVE
+
+enum ThreadState : u8 {
+ ThreadNotInitialized = 0,
+ ThreadInitialized,
+ ThreadTornDown,
+};
+__attribute__((tls_model("initial-exec")))
+extern THREADLOCAL ThreadState ScudoThreadState;
+__attribute__((tls_model("initial-exec")))
+extern THREADLOCAL ScudoTSD TSD;
+
+extern ScudoTSD FallbackTSD;
+
+ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
+ if (LIKELY(ScudoThreadState != ThreadNotInitialized))
+ return;
+ initThread(MinimalInit);
+}
+
+ALWAYS_INLINE ScudoTSD *getTSDAndLock() {
+ if (UNLIKELY(ScudoThreadState != ThreadInitialized)) {
+ FallbackTSD.lock();
+ return &FallbackTSD;
+ }
+ return &TSD;
+}
+
+#endif // SCUDO_TSD_EXCLUSIVE
diff --git a/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.cpp b/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.cpp
new file mode 100644
index 000000000000..3e13e5d3a109
--- /dev/null
+++ b/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.cpp
@@ -0,0 +1,87 @@
+//===-- scudo_tsd_shared.cpp ------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Scudo shared TSD implementation.
+///
+//===----------------------------------------------------------------------===//
+
+#include "scudo_tsd.h"
+
+#if !SCUDO_TSD_EXCLUSIVE
+
+namespace __scudo {
+
+static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT;
+pthread_key_t PThreadKey;
+
+static atomic_uint32_t CurrentIndex;
+static ScudoTSD *TSDs;
+static u32 NumberOfTSDs;
+
+static void initOnce() {
+ CHECK_EQ(pthread_key_create(&PThreadKey, NULL), 0);
+ initScudo();
+ NumberOfTSDs = Min(Max(1U, GetNumberOfCPUsCached()),
+ static_cast<u32>(SCUDO_SHARED_TSD_POOL_SIZE));
+ TSDs = reinterpret_cast<ScudoTSD *>(
+ MmapOrDie(sizeof(ScudoTSD) * NumberOfTSDs, "ScudoTSDs"));
+ for (u32 i = 0; i < NumberOfTSDs; i++)
+ TSDs[i].init(/*Shared=*/true);
+}
+
+ALWAYS_INLINE void setCurrentTSD(ScudoTSD *TSD) {
+#if SANITIZER_ANDROID
+ *get_android_tls_ptr() = reinterpret_cast<uptr>(TSD);
+#else
+ CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast<void *>(TSD)), 0);
+#endif // SANITIZER_ANDROID
+}
+
+void initThread(bool MinimalInit) {
+ pthread_once(&GlobalInitialized, initOnce);
+ // Initial context assignment is done in a plain round-robin fashion.
+ u32 Index = atomic_fetch_add(&CurrentIndex, 1, memory_order_relaxed);
+ setCurrentTSD(&TSDs[Index % NumberOfTSDs]);
+}
+
+ScudoTSD *getTSDAndLockSlow() {
+ ScudoTSD *TSD;
+ if (NumberOfTSDs > 1) {
+ // Go through all the contexts and find the first unlocked one.
+ for (u32 i = 0; i < NumberOfTSDs; i++) {
+ TSD = &TSDs[i];
+ if (TSD->tryLock()) {
+ setCurrentTSD(TSD);
+ return TSD;
+ }
+ }
+ // No luck, find the one with the lowest Precedence, and slow lock it.
+ u64 LowestPrecedence = UINT64_MAX;
+ for (u32 i = 0; i < NumberOfTSDs; i++) {
+ u64 Precedence = TSDs[i].getPrecedence();
+ if (Precedence && Precedence < LowestPrecedence) {
+ TSD = &TSDs[i];
+ LowestPrecedence = Precedence;
+ }
+ }
+ if (LIKELY(LowestPrecedence != UINT64_MAX)) {
+ TSD->lock();
+ setCurrentTSD(TSD);
+ return TSD;
+ }
+ }
+ // Last resort, stick with the current one.
+ TSD = getCurrentTSD();
+ TSD->lock();
+ return TSD;
+}
+
+} // namespace __scudo
+
+#endif // !SCUDO_TSD_EXCLUSIVE
diff --git a/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.inc b/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.inc
new file mode 100644
index 000000000000..79fcd651ed2d
--- /dev/null
+++ b/contrib/compiler-rt/lib/scudo/scudo_tsd_shared.inc
@@ -0,0 +1,48 @@
+//===-- scudo_tsd_shared.inc ------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Scudo shared TSD fastpath functions implementation.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_TSD_H_
+# error "This file must be included inside scudo_tsd.h."
+#endif // SCUDO_TSD_H_
+
+#if !SCUDO_TSD_EXCLUSIVE
+
+extern pthread_key_t PThreadKey;
+
+ALWAYS_INLINE ScudoTSD* getCurrentTSD() {
+#if SANITIZER_ANDROID
+ return reinterpret_cast<ScudoTSD *>(*get_android_tls_ptr());
+#else
+ return reinterpret_cast<ScudoTSD *>(pthread_getspecific(PThreadKey));
+#endif // SANITIZER_ANDROID
+}
+
+ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
+ if (LIKELY(getCurrentTSD()))
+ return;
+ initThread(MinimalInit);
+}
+
+ScudoTSD *getTSDAndLockSlow();
+
+ALWAYS_INLINE ScudoTSD *getTSDAndLock() {
+ ScudoTSD *TSD = getCurrentTSD();
+ CHECK(TSD && "No TSD associated with the current thread!");
+ // Try to lock the currently associated context.
+ if (TSD->tryLock())
+ return TSD;
+ // If it failed, go the slow path.
+ return getTSDAndLockSlow();
+}
+
+#endif // !SCUDO_TSD_EXCLUSIVE
diff --git a/contrib/compiler-rt/lib/scudo/scudo_utils.cpp b/contrib/compiler-rt/lib/scudo/scudo_utils.cpp
index f7903ff34c73..2f936bf9e780 100644
--- a/contrib/compiler-rt/lib/scudo/scudo_utils.cpp
+++ b/contrib/compiler-rt/lib/scudo/scudo_utils.cpp
@@ -13,17 +13,18 @@
#include "scudo_utils.h"
-#include <errno.h>
-#include <fcntl.h>
-#include <stdarg.h>
-#include <unistd.h>
#if defined(__x86_64__) || defined(__i386__)
# include <cpuid.h>
-#endif
-#if defined(__arm__) || defined(__aarch64__)
-# include <sys/auxv.h>
+#elif defined(__arm__) || defined(__aarch64__)
+# include "sanitizer_common/sanitizer_getauxval.h"
+# if SANITIZER_POSIX
+# include "sanitizer_common/sanitizer_posix.h"
+# include <fcntl.h>
+# endif
#endif
+#include <stdarg.h>
+
// TODO(kostyak): remove __sanitizer *Printf uses in favor for our own less
// complicated string formatting code. The following is a
// temporary workaround to be able to use __sanitizer::VSNPrintf.
@@ -36,13 +37,12 @@ extern int VSNPrintf(char *buff, int buff_length, const char *format,
namespace __scudo {
-FORMAT(1, 2)
-void NORETURN dieWithMessage(const char *Format, ...) {
+FORMAT(1, 2) void NORETURN dieWithMessage(const char *Format, ...) {
// Our messages are tiny, 256 characters is more than enough.
char Message[256];
va_list Args;
va_start(Args, Format);
- __sanitizer::VSNPrintf(Message, sizeof(Message), Format, Args);
+ VSNPrintf(Message, sizeof(Message), Format, Args);
va_end(Args);
RawWrite(Message);
Die();
@@ -51,76 +51,68 @@ void NORETURN dieWithMessage(const char *Format, ...) {
#if defined(__x86_64__) || defined(__i386__)
// i386 and x86_64 specific code to detect CRC32 hardware support via CPUID.
// CRC32 requires the SSE 4.2 instruction set.
-typedef struct {
- u32 Eax;
- u32 Ebx;
- u32 Ecx;
- u32 Edx;
-} CPUIDRegs;
-
-static void getCPUID(CPUIDRegs *Regs, u32 Level)
-{
- __get_cpuid(Level, &Regs->Eax, &Regs->Ebx, &Regs->Ecx, &Regs->Edx);
+# ifndef bit_SSE4_2
+# define bit_SSE4_2 bit_SSE42 // clang and gcc have different defines.
+# endif
+bool hasHardwareCRC32() {
+ u32 Eax, Ebx, Ecx, Edx;
+ __get_cpuid(0, &Eax, &Ebx, &Ecx, &Edx);
+ const bool IsIntel = (Ebx == signature_INTEL_ebx) &&
+ (Edx == signature_INTEL_edx) &&
+ (Ecx == signature_INTEL_ecx);
+ const bool IsAMD = (Ebx == signature_AMD_ebx) &&
+ (Edx == signature_AMD_edx) &&
+ (Ecx == signature_AMD_ecx);
+ if (!IsIntel && !IsAMD)
+ return false;
+ __get_cpuid(1, &Eax, &Ebx, &Ecx, &Edx);
+ return !!(Ecx & bit_SSE4_2);
}
-
-CPUIDRegs getCPUFeatures() {
- CPUIDRegs VendorRegs = {};
- getCPUID(&VendorRegs, 0);
- bool IsIntel =
- (VendorRegs.Ebx == signature_INTEL_ebx) &&
- (VendorRegs.Edx == signature_INTEL_edx) &&
- (VendorRegs.Ecx == signature_INTEL_ecx);
- bool IsAMD =
- (VendorRegs.Ebx == signature_AMD_ebx) &&
- (VendorRegs.Edx == signature_AMD_edx) &&
- (VendorRegs.Ecx == signature_AMD_ecx);
- // Default to an empty feature set if not on a supported CPU.
- CPUIDRegs FeaturesRegs = {};
- if (IsIntel || IsAMD) {
- getCPUID(&FeaturesRegs, 1);
- }
- return FeaturesRegs;
-}
-
-#ifndef bit_SSE4_2
-# define bit_SSE4_2 bit_SSE42 // clang and gcc have different defines.
-#endif
-
-bool testCPUFeature(CPUFeature Feature)
-{
- CPUIDRegs FeaturesRegs = getCPUFeatures();
-
- switch (Feature) {
- case CRC32CPUFeature: // CRC32 is provided by SSE 4.2.
- return !!(FeaturesRegs.Ecx & bit_SSE4_2);
- default:
+#elif defined(__arm__) || defined(__aarch64__)
+// For ARM and AArch64, hardware CRC32 support is indicated in the AT_HWCAP
+// auxiliary vector.
+# ifndef AT_HWCAP
+# define AT_HWCAP 16
+# endif
+# ifndef HWCAP_CRC32
+# define HWCAP_CRC32 (1 << 7) // HWCAP_CRC32 is missing on older platforms.
+# endif
+# if SANITIZER_POSIX
+bool hasHardwareCRC32ARMPosix() {
+ uptr F = internal_open("/proc/self/auxv", O_RDONLY);
+ if (internal_iserror(F))
+ return false;
+ struct { uptr Tag; uptr Value; } Entry = { 0, 0 };
+ for (;;) {
+ uptr N = internal_read(F, &Entry, sizeof(Entry));
+ if (internal_iserror(N) || N != sizeof(Entry) ||
+ (Entry.Tag == 0 && Entry.Value == 0) || Entry.Tag == AT_HWCAP)
break;
}
- return false;
+ internal_close(F);
+ return (Entry.Tag == AT_HWCAP && (Entry.Value & HWCAP_CRC32) != 0);
}
-#elif defined(__arm__) || defined(__aarch64__)
-// For ARM and AArch64, hardware CRC32 support is indicated in the
-// AT_HWVAL auxiliary vector.
-
-#ifndef HWCAP_CRC32
-# define HWCAP_CRC32 (1<<7) // HWCAP_CRC32 is missing on older platforms.
-#endif
+# else
+bool hasHardwareCRC32ARMPosix() { return false; }
+# endif // SANITIZER_POSIX
-bool testCPUFeature(CPUFeature Feature) {
- uptr HWCap = getauxval(AT_HWCAP);
+// Bionic doesn't initialize its globals early enough. This causes issues when
+// trying to access them from a preinit_array (b/25751302) or from another
+// constructor called before the libc one (b/68046352). __progname is
+// initialized after the other globals, so we can check its value to know if
+// calling getauxval is safe.
+extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname;
+INLINE bool areBionicGlobalsInitialized() {
+ return !SANITIZER_ANDROID || (&__progname && __progname);
+}
- switch (Feature) {
- case CRC32CPUFeature:
- return !!(HWCap & HWCAP_CRC32);
- default:
- break;
- }
- return false;
+bool hasHardwareCRC32() {
+ if (&getauxval && areBionicGlobalsInitialized())
+ return !!(getauxval(AT_HWCAP) & HWCAP_CRC32);
+ return hasHardwareCRC32ARMPosix();
}
#else
-bool testCPUFeature(CPUFeature Feature) {
- return false;
-}
+bool hasHardwareCRC32() { return false; }
#endif // defined(__x86_64__) || defined(__i386__)
} // namespace __scudo
diff --git a/contrib/compiler-rt/lib/scudo/scudo_utils.h b/contrib/compiler-rt/lib/scudo/scudo_utils.h
index 6c6c9d893404..43448e0831e8 100644
--- a/contrib/compiler-rt/lib/scudo/scudo_utils.h
+++ b/contrib/compiler-rt/lib/scudo/scudo_utils.h
@@ -14,14 +14,14 @@
#ifndef SCUDO_UTILS_H_
#define SCUDO_UTILS_H_
-#include <string.h>
-
#include "sanitizer_common/sanitizer_common.h"
+#include <string.h>
+
namespace __scudo {
template <class Dest, class Source>
-inline Dest bit_cast(const Source& source) {
+INLINE Dest bit_cast(const Source& source) {
static_assert(sizeof(Dest) == sizeof(Source), "Sizes are not equal!");
Dest dest;
memcpy(&dest, &source, sizeof(dest));
@@ -30,63 +30,7 @@ inline Dest bit_cast(const Source& source) {
void NORETURN dieWithMessage(const char *Format, ...);
-enum CPUFeature {
- CRC32CPUFeature = 0,
- MaxCPUFeature,
-};
-bool testCPUFeature(CPUFeature feature);
-
-INLINE u64 rotl(const u64 X, int K) {
- return (X << K) | (X >> (64 - K));
-}
-
-// XoRoShiRo128+ PRNG (http://xoroshiro.di.unimi.it/).
-struct XoRoShiRo128Plus {
- public:
- void init() {
- if (UNLIKELY(!GetRandom(reinterpret_cast<void *>(State), sizeof(State)))) {
- // Early processes (eg: init) do not have /dev/urandom yet, but we still
- // have to provide them with some degree of entropy. Not having a secure
- // seed is not as problematic for them, as they are less likely to be
- // the target of heap based vulnerabilities exploitation attempts.
- State[0] = NanoTime();
- State[1] = 0;
- }
- fillCache();
- }
- u8 getU8() {
- if (UNLIKELY(isCacheEmpty()))
- fillCache();
- const u8 Result = static_cast<u8>(CachedBytes & 0xff);
- CachedBytes >>= 8;
- CachedBytesAvailable--;
- return Result;
- }
- u64 getU64() { return next(); }
-
- private:
- u8 CachedBytesAvailable;
- u64 CachedBytes;
- u64 State[2];
- u64 next() {
- const u64 S0 = State[0];
- u64 S1 = State[1];
- const u64 Result = S0 + S1;
- S1 ^= S0;
- State[0] = rotl(S0, 55) ^ S1 ^ (S1 << 14);
- State[1] = rotl(S1, 36);
- return Result;
- }
- bool isCacheEmpty() {
- return CachedBytesAvailable == 0;
- }
- void fillCache() {
- CachedBytes = next();
- CachedBytesAvailable = sizeof(CachedBytes);
- }
-};
-
-typedef XoRoShiRo128Plus ScudoPrng;
+bool hasHardwareCRC32();
} // namespace __scudo
diff --git a/contrib/compiler-rt/lib/stats/stats.cc b/contrib/compiler-rt/lib/stats/stats.cc
index df9845a38de9..6a6eb3a3cdbc 100644
--- a/contrib/compiler-rt/lib/stats/stats.cc
+++ b/contrib/compiler-rt/lib/stats/stats.cc
@@ -13,6 +13,7 @@
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_file.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
#if SANITIZER_POSIX
#include "sanitizer_common/sanitizer_posix.h"
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_fd.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_fd.cc
index d84df4a6413c..f13a7432ec95 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_fd.cc
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_fd.cc
@@ -48,8 +48,8 @@ static bool bogusfd(int fd) {
}
static FdSync *allocsync(ThreadState *thr, uptr pc) {
- FdSync *s = (FdSync*)user_alloc(thr, pc, sizeof(FdSync), kDefaultAlignment,
- false);
+ FdSync *s = (FdSync*)user_alloc_internal(thr, pc, sizeof(FdSync),
+ kDefaultAlignment, false);
atomic_store(&s->rc, 1, memory_order_relaxed);
return s;
}
@@ -79,7 +79,7 @@ static FdDesc *fddesc(ThreadState *thr, uptr pc, int fd) {
if (l1 == 0) {
uptr size = kTableSizeL2 * sizeof(FdDesc);
// We need this to reside in user memory to properly catch races on it.
- void *p = user_alloc(thr, pc, size, kDefaultAlignment, false);
+ void *p = user_alloc_internal(thr, pc, size, kDefaultAlignment, false);
internal_memset(p, 0, size);
MemoryResetRange(thr, (uptr)&fddesc, (uptr)p, size);
if (atomic_compare_exchange_strong(pl1, &l1, (uptr)p, memory_order_acq_rel))
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc
index 001123f4941e..d92976ad6e9e 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc
@@ -17,6 +17,7 @@
#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
#include "sanitizer_common/sanitizer_platform_limits_posix.h"
#include "sanitizer_common/sanitizer_placement_new.h"
#include "sanitizer_common/sanitizer_posix.h"
@@ -39,6 +40,21 @@ using namespace __tsan; // NOLINT
#define stderr __stderrp
#endif
+#if SANITIZER_NETBSD
+#define dirfd(dirp) (*(int *)(dirp))
+#define fileno_unlocked fileno
+
+#if _LP64
+#define __sF_size 152
+#else
+#define __sF_size 88
+#endif
+
+#define stdout ((char*)&__sF + (__sF_size * 1))
+#define stderr ((char*)&__sF + (__sF_size * 2))
+
+#endif
+
#if SANITIZER_ANDROID
#define mallopt(a, b)
#endif
@@ -49,11 +65,6 @@ const int kSigCount = 129;
const int kSigCount = 65;
#endif
-struct my_siginfo_t {
- // The size is determined by looking at sizeof of real siginfo_t on linux.
- u64 opaque[128 / sizeof(u64)];
-};
-
#ifdef __mips__
struct ucontext_t {
u64 opaque[768 / sizeof(u64) + 1];
@@ -84,19 +95,25 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
extern "C" void *pthread_self();
extern "C" void _exit(int status);
extern "C" int fileno_unlocked(void *stream);
+#if !SANITIZER_NETBSD
extern "C" int dirfd(void *dirp);
-#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID
+#endif
+#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_NETBSD
extern "C" int mallopt(int param, int value);
#endif
+#if SANITIZER_NETBSD
+extern __sanitizer_FILE __sF[];
+#else
extern __sanitizer_FILE *stdout, *stderr;
-#if !SANITIZER_FREEBSD && !SANITIZER_MAC
+#endif
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
const int PTHREAD_MUTEX_RECURSIVE = 1;
const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
#else
const int PTHREAD_MUTEX_RECURSIVE = 2;
const int PTHREAD_MUTEX_RECURSIVE_NP = 2;
#endif
-#if !SANITIZER_FREEBSD && !SANITIZER_MAC
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
const int EPOLL_CTL_ADD = 1;
#endif
const int SIGILL = 4;
@@ -105,7 +122,7 @@ const int SIGFPE = 8;
const int SIGSEGV = 11;
const int SIGPIPE = 13;
const int SIGTERM = 15;
-#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC
+#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
const int SIGBUS = 10;
const int SIGSYS = 12;
#else
@@ -113,7 +130,9 @@ const int SIGBUS = 7;
const int SIGSYS = 31;
#endif
void *const MAP_FAILED = (void*)-1;
-#if !SANITIZER_MAC
+#if SANITIZER_NETBSD
+const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567;
+#elif !SANITIZER_MAC
const int PTHREAD_BARRIER_SERIAL_THREAD = -1;
#endif
const int MAP_FIXED = 0x10;
@@ -125,48 +144,7 @@ typedef long long_t; // NOLINT
# define F_TLOCK 2 /* Test and lock a region for exclusive use. */
# define F_TEST 3 /* Test a region for other processes locks. */
-typedef void (*sighandler_t)(int sig);
-typedef void (*sigactionhandler_t)(int sig, my_siginfo_t *siginfo, void *uctx);
-
-#if SANITIZER_ANDROID
-struct sigaction_t {
- u32 sa_flags;
- union {
- sighandler_t sa_handler;
- sigactionhandler_t sa_sigaction;
- };
- __sanitizer_sigset_t sa_mask;
- void (*sa_restorer)();
-};
-#else
-struct sigaction_t {
-#ifdef __mips__
- u32 sa_flags;
-#endif
- union {
- sighandler_t sa_handler;
- sigactionhandler_t sa_sigaction;
- };
-#if SANITIZER_FREEBSD
- int sa_flags;
- __sanitizer_sigset_t sa_mask;
-#elif SANITIZER_MAC
- __sanitizer_sigset_t sa_mask;
- int sa_flags;
-#else
- __sanitizer_sigset_t sa_mask;
-#ifndef __mips__
- int sa_flags;
-#endif
- void (*sa_restorer)();
-#endif
-};
-#endif
-
-const sighandler_t SIG_DFL = (sighandler_t)0;
-const sighandler_t SIG_IGN = (sighandler_t)1;
-const sighandler_t SIG_ERR = (sighandler_t)-1;
-#if SANITIZER_FREEBSD || SANITIZER_MAC
+#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
const int SA_SIGINFO = 0x40;
const int SIG_SETMASK = 3;
#elif defined(__mips__)
@@ -180,13 +158,11 @@ const int SIG_SETMASK = 2;
#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \
(!cur_thread()->is_inited)
-static sigaction_t sigactions[kSigCount];
-
namespace __tsan {
struct SignalDesc {
bool armed;
bool sigaction;
- my_siginfo_t siginfo;
+ __sanitizer_siginfo siginfo;
ucontext_t ctx;
};
@@ -200,11 +176,41 @@ struct ThreadSignalContext {
__sanitizer_sigset_t oldset;
};
-// The object is 64-byte aligned, because we want hot data to be located in
-// a single cache line if possible (it's accessed in every interceptor).
-static ALIGNED(64) char libignore_placeholder[sizeof(LibIgnore)];
+// The sole reason tsan wraps atexit callbacks is to establish synchronization
+// between callback setup and callback execution.
+struct AtExitCtx {
+ void (*f)();
+ void *arg;
+};
+
+// InterceptorContext holds all global data required for interceptors.
+// It's explicitly constructed in InitializeInterceptors with placement new
+// and is never destroyed. This allows usage of members with non-trivial
+// constructors and destructors.
+struct InterceptorContext {
+ // The object is 64-byte aligned, because we want hot data to be located
+ // in a single cache line if possible (it's accessed in every interceptor).
+ ALIGNED(64) LibIgnore libignore;
+ __sanitizer_sigaction sigactions[kSigCount];
+#if !SANITIZER_MAC && !SANITIZER_NETBSD
+ unsigned finalize_key;
+#endif
+
+ BlockingMutex atexit_mu;
+ Vector<struct AtExitCtx *> AtExitStack;
+
+ InterceptorContext()
+ : libignore(LINKER_INITIALIZED), AtExitStack() {
+ }
+};
+
+static ALIGNED(64) char interceptor_placeholder[sizeof(InterceptorContext)];
+InterceptorContext *interceptor_ctx() {
+ return reinterpret_cast<InterceptorContext*>(&interceptor_placeholder[0]);
+}
+
LibIgnore *libignore() {
- return reinterpret_cast<LibIgnore*>(&libignore_placeholder[0]);
+ return &interceptor_ctx()->libignore;
}
void InitializeLibIgnore() {
@@ -232,10 +238,6 @@ static ThreadSignalContext *SigCtx(ThreadState *thr) {
return ctx;
}
-#if !SANITIZER_MAC
-static unsigned g_thread_finalize_key;
-#endif
-
ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname,
uptr pc)
: thr_(thr), pc_(pc), in_ignored_lib_(false), ignoring_(false) {
@@ -284,8 +286,18 @@ void ScopedInterceptor::DisableIgnores() {
#define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
#if SANITIZER_FREEBSD
# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
+#elif SANITIZER_NETBSD
+# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \
+ INTERCEPT_FUNCTION(__libc_##func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \
+ INTERCEPT_FUNCTION(__libc_thr_##func)
#else
# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
+# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
#endif
#define READ_STRING_OF_LEN(thr, pc, s, len, n) \
@@ -346,17 +358,30 @@ TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) {
return res;
}
-// The sole reason tsan wraps atexit callbacks is to establish synchronization
-// between callback setup and callback execution.
-struct AtExitCtx {
- void (*f)();
- void *arg;
-};
+TSAN_INTERCEPTOR(int, pause, int fake) {
+ SCOPED_TSAN_INTERCEPTOR(pause, fake);
+ return BLOCK_REAL(pause)(fake);
+}
-static void at_exit_wrapper(void *arg) {
- ThreadState *thr = cur_thread();
- uptr pc = 0;
- Acquire(thr, pc, (uptr)arg);
+static void at_exit_wrapper() {
+ AtExitCtx *ctx;
+ {
+ // Ensure thread-safety.
+ BlockingMutexLock l(&interceptor_ctx()->atexit_mu);
+
+ // Pop AtExitCtx from the top of the stack of callback functions
+ uptr element = interceptor_ctx()->AtExitStack.Size() - 1;
+ ctx = interceptor_ctx()->AtExitStack[element];
+ interceptor_ctx()->AtExitStack.PopBack();
+ }
+
+ Acquire(cur_thread(), (uptr)0, (uptr)ctx);
+ ((void(*)())ctx->f)();
+ InternalFree(ctx);
+}
+
+static void cxa_at_exit_wrapper(void *arg) {
+ Acquire(cur_thread(), 0, (uptr)arg);
AtExitCtx *ctx = (AtExitCtx*)arg;
((void(*)(void *arg))ctx->f)(ctx->arg);
InternalFree(ctx);
@@ -367,7 +392,7 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(),
#if !SANITIZER_ANDROID
TSAN_INTERCEPTOR(int, atexit, void (*f)()) {
- if (cur_thread()->in_symbolizer)
+ if (UNLIKELY(cur_thread()->in_symbolizer))
return 0;
// We want to setup the atexit callback even if we are in ignored lib
// or after fork.
@@ -377,7 +402,7 @@ TSAN_INTERCEPTOR(int, atexit, void (*f)()) {
#endif
TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) {
- if (cur_thread()->in_symbolizer)
+ if (UNLIKELY(cur_thread()->in_symbolizer))
return 0;
SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso);
return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso);
@@ -392,12 +417,27 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(),
// Memory allocation in __cxa_atexit will race with free during exit,
// because we do not see synchronization around atexit callback list.
ThreadIgnoreBegin(thr, pc);
- int res = REAL(__cxa_atexit)(at_exit_wrapper, ctx, dso);
+ int res;
+ if (!dso) {
+ // NetBSD does not preserve the 2nd argument if dso is equal to 0
+ // Store ctx in a local stack-like structure
+
+ // Ensure thread-safety.
+ BlockingMutexLock l(&interceptor_ctx()->atexit_mu);
+
+ res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_wrapper, 0, 0);
+ // Push AtExitCtx on the top of the stack of callback functions
+ if (!res) {
+ interceptor_ctx()->AtExitStack.PushBack(ctx);
+ }
+ } else {
+ res = REAL(__cxa_atexit)(cxa_at_exit_wrapper, ctx, dso);
+ }
ThreadIgnoreEnd(thr, pc);
return res;
}
-#if !SANITIZER_MAC
+#if !SANITIZER_MAC && !SANITIZER_NETBSD
static void on_exit_wrapper(int status, void *arg) {
ThreadState *thr = cur_thread();
uptr pc = 0;
@@ -408,7 +448,7 @@ static void on_exit_wrapper(int status, void *arg) {
}
TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) {
- if (cur_thread()->in_symbolizer)
+ if (UNLIKELY(cur_thread()->in_symbolizer))
return 0;
SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg);
AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx));
@@ -422,6 +462,9 @@ TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) {
ThreadIgnoreEnd(thr, pc);
return res;
}
+#define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit)
+#else
+#define TSAN_MAYBE_INTERCEPT_ON_EXIT
#endif
// Cleanup old bufs.
@@ -461,13 +504,15 @@ static void LongJmp(ThreadState *thr, uptr *env) {
uptr mangled_sp = env[0];
#elif SANITIZER_FREEBSD
uptr mangled_sp = env[2];
+#elif SANITIZER_NETBSD
+ uptr mangled_sp = env[6];
#elif SANITIZER_MAC
# ifdef __aarch64__
uptr mangled_sp = env[13];
# else
uptr mangled_sp = env[2];
# endif
-#elif defined(SANITIZER_LINUX)
+#elif SANITIZER_LINUX
# ifdef __aarch64__
uptr mangled_sp = env[13];
# elif defined(__mips64)
@@ -510,10 +555,29 @@ TSAN_INTERCEPTOR(int, setjmp, void *env);
TSAN_INTERCEPTOR(int, _setjmp, void *env);
TSAN_INTERCEPTOR(int, sigsetjmp, void *env);
#else // SANITIZER_MAC
+
+#if SANITIZER_NETBSD
+#define setjmp_symname __setjmp14
+#define sigsetjmp_symname __sigsetjmp14
+#else
+#define setjmp_symname setjmp
+#define sigsetjmp_symname sigsetjmp
+#endif
+
+#define TSAN_INTERCEPTOR_SETJMP_(x) __interceptor_ ## x
+#define TSAN_INTERCEPTOR_SETJMP__(x) TSAN_INTERCEPTOR_SETJMP_(x)
+#define TSAN_INTERCEPTOR_SETJMP TSAN_INTERCEPTOR_SETJMP__(setjmp_symname)
+#define TSAN_INTERCEPTOR_SIGSETJMP TSAN_INTERCEPTOR_SETJMP__(sigsetjmp_symname)
+
+#define TSAN_STRING_SETJMP_(x) # x
+#define TSAN_STRING_SETJMP__(x) TSAN_STRING_SETJMP_(x)
+#define TSAN_STRING_SETJMP TSAN_STRING_SETJMP__(setjmp_symname)
+#define TSAN_STRING_SIGSETJMP TSAN_STRING_SETJMP__(sigsetjmp_symname)
+
// Not called. Merely to satisfy TSAN_INTERCEPT().
extern "C" SANITIZER_INTERFACE_ATTRIBUTE
-int __interceptor_setjmp(void *env);
-extern "C" int __interceptor_setjmp(void *env) {
+int TSAN_INTERCEPTOR_SETJMP(void *env);
+extern "C" int TSAN_INTERCEPTOR_SETJMP(void *env) {
CHECK(0);
return 0;
}
@@ -527,51 +591,75 @@ extern "C" int __interceptor__setjmp(void *env) {
}
extern "C" SANITIZER_INTERFACE_ATTRIBUTE
-int __interceptor_sigsetjmp(void *env);
-extern "C" int __interceptor_sigsetjmp(void *env) {
+int TSAN_INTERCEPTOR_SIGSETJMP(void *env);
+extern "C" int TSAN_INTERCEPTOR_SIGSETJMP(void *env) {
CHECK(0);
return 0;
}
+#if !SANITIEZER_NETBSD
extern "C" SANITIZER_INTERFACE_ATTRIBUTE
int __interceptor___sigsetjmp(void *env);
extern "C" int __interceptor___sigsetjmp(void *env) {
CHECK(0);
return 0;
}
+#endif
-extern "C" int setjmp(void *env);
+extern "C" int setjmp_symname(void *env);
extern "C" int _setjmp(void *env);
-extern "C" int sigsetjmp(void *env);
+extern "C" int sigsetjmp_symname(void *env);
+#if !SANITIEZER_NETBSD
extern "C" int __sigsetjmp(void *env);
-DEFINE_REAL(int, setjmp, void *env)
+#endif
+DEFINE_REAL(int, setjmp_symname, void *env)
DEFINE_REAL(int, _setjmp, void *env)
-DEFINE_REAL(int, sigsetjmp, void *env)
+DEFINE_REAL(int, sigsetjmp_symname, void *env)
+#if !SANITIEZER_NETBSD
DEFINE_REAL(int, __sigsetjmp, void *env)
+#endif
#endif // SANITIZER_MAC
-TSAN_INTERCEPTOR(void, longjmp, uptr *env, int val) {
+#if SANITIZER_NETBSD
+#define longjmp_symname __longjmp14
+#define siglongjmp_symname __siglongjmp14
+#else
+#define longjmp_symname longjmp
+#define siglongjmp_symname siglongjmp
+#endif
+
+TSAN_INTERCEPTOR(void, longjmp_symname, uptr *env, int val) {
// Note: if we call REAL(longjmp) in the context of ScopedInterceptor,
// bad things will happen. We will jump over ScopedInterceptor dtor and can
// leave thr->in_ignored_lib set.
{
- SCOPED_INTERCEPTOR_RAW(longjmp, env, val);
+ SCOPED_INTERCEPTOR_RAW(longjmp_symname, env, val);
}
LongJmp(cur_thread(), env);
- REAL(longjmp)(env, val);
+ REAL(longjmp_symname)(env, val);
}
-TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) {
+TSAN_INTERCEPTOR(void, siglongjmp_symname, uptr *env, int val) {
{
- SCOPED_INTERCEPTOR_RAW(siglongjmp, env, val);
+ SCOPED_INTERCEPTOR_RAW(siglongjmp_symname, env, val);
}
LongJmp(cur_thread(), env);
- REAL(siglongjmp)(env, val);
+ REAL(siglongjmp_symname)(env, val);
}
+#if SANITIZER_NETBSD
+TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) {
+ {
+ SCOPED_INTERCEPTOR_RAW(_longjmp, env, val);
+ }
+ LongJmp(cur_thread(), env);
+ REAL(_longjmp)(env, val);
+}
+#endif
+
#if !SANITIZER_MAC
TSAN_INTERCEPTOR(void*, malloc, uptr size) {
- if (cur_thread()->in_symbolizer)
+ if (UNLIKELY(cur_thread()->in_symbolizer))
return InternalAlloc(size);
void *p = 0;
{
@@ -584,11 +672,11 @@ TSAN_INTERCEPTOR(void*, malloc, uptr size) {
TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz);
- return user_alloc(thr, pc, sz, align);
+ return user_memalign(thr, pc, align, sz);
}
TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
- if (cur_thread()->in_symbolizer)
+ if (UNLIKELY(cur_thread()->in_symbolizer))
return InternalCalloc(size, n);
void *p = 0;
{
@@ -600,7 +688,7 @@ TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
}
TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) {
- if (cur_thread()->in_symbolizer)
+ if (UNLIKELY(cur_thread()->in_symbolizer))
return InternalRealloc(p, size);
if (p)
invoke_free_hook(p);
@@ -615,7 +703,7 @@ TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) {
TSAN_INTERCEPTOR(void, free, void *p) {
if (p == 0)
return;
- if (cur_thread()->in_symbolizer)
+ if (UNLIKELY(cur_thread()->in_symbolizer))
return InternalFree(p);
invoke_free_hook(p);
SCOPED_INTERCEPTOR_RAW(free, p);
@@ -625,7 +713,7 @@ TSAN_INTERCEPTOR(void, free, void *p) {
TSAN_INTERCEPTOR(void, cfree, void *p) {
if (p == 0)
return;
- if (cur_thread()->in_symbolizer)
+ if (UNLIKELY(cur_thread()->in_symbolizer))
return InternalFree(p);
invoke_free_hook(p);
SCOPED_INTERCEPTOR_RAW(cfree, p);
@@ -730,7 +818,7 @@ TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) {
#if SANITIZER_LINUX
TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
SCOPED_INTERCEPTOR_RAW(memalign, align, sz);
- return user_alloc(thr, pc, sz, align);
+ return user_memalign(thr, pc, align, sz);
}
#define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign)
#else
@@ -739,21 +827,29 @@ TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
#if !SANITIZER_MAC
TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) {
- SCOPED_INTERCEPTOR_RAW(memalign, align, sz);
- return user_alloc(thr, pc, sz, align);
+ if (UNLIKELY(cur_thread()->in_symbolizer))
+ return InternalAlloc(sz, nullptr, align);
+ SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz);
+ return user_aligned_alloc(thr, pc, align, sz);
}
TSAN_INTERCEPTOR(void*, valloc, uptr sz) {
+ if (UNLIKELY(cur_thread()->in_symbolizer))
+ return InternalAlloc(sz, nullptr, GetPageSizeCached());
SCOPED_INTERCEPTOR_RAW(valloc, sz);
- return user_alloc(thr, pc, sz, GetPageSizeCached());
+ return user_valloc(thr, pc, sz);
}
#endif
#if SANITIZER_LINUX
TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
+ if (UNLIKELY(cur_thread()->in_symbolizer)) {
+ uptr PageSize = GetPageSizeCached();
+ sz = sz ? RoundUpTo(sz, PageSize) : PageSize;
+ return InternalAlloc(sz, nullptr, PageSize);
+ }
SCOPED_INTERCEPTOR_RAW(pvalloc, sz);
- sz = RoundUp(sz, GetPageSizeCached());
- return user_alloc(thr, pc, sz, GetPageSizeCached());
+ return user_pvalloc(thr, pc, sz);
}
#define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc)
#else
@@ -762,9 +858,15 @@ TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
#if !SANITIZER_MAC
TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
+ if (UNLIKELY(cur_thread()->in_symbolizer)) {
+ void *p = InternalAlloc(sz, nullptr, align);
+ if (!p)
+ return errno_ENOMEM;
+ *memptr = p;
+ return 0;
+ }
SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz);
- *memptr = user_alloc(thr, pc, sz, align);
- return 0;
+ return user_posix_memalign(thr, pc, memptr, align, sz);
}
#endif
@@ -830,11 +932,12 @@ void DestroyThreadState() {
}
} // namespace __tsan
-#if !SANITIZER_MAC
+#if !SANITIZER_MAC && !SANITIZER_NETBSD
static void thread_finalize(void *v) {
uptr iter = (uptr)v;
if (iter > 1) {
- if (pthread_setspecific(g_thread_finalize_key, (void*)(iter - 1))) {
+ if (pthread_setspecific(interceptor_ctx()->finalize_key,
+ (void*)(iter - 1))) {
Printf("ThreadSanitizer: failed to set thread key\n");
Die();
}
@@ -860,9 +963,9 @@ extern "C" void *__tsan_thread_start_func(void *arg) {
ThreadState *thr = cur_thread();
// Thread-local state is not initialized yet.
ScopedIgnoreInterceptors ignore;
-#if !SANITIZER_MAC
+#if !SANITIZER_MAC && !SANITIZER_NETBSD
ThreadIgnoreBegin(thr, 0);
- if (pthread_setspecific(g_thread_finalize_key,
+ if (pthread_setspecific(interceptor_ctx()->finalize_key,
(void *)GetPthreadDestructorIterations())) {
Printf("ThreadSanitizer: failed to set thread key\n");
Die();
@@ -887,6 +990,9 @@ extern "C" void *__tsan_thread_start_func(void *arg) {
TSAN_INTERCEPTOR(int, pthread_create,
void *th, void *attr, void *(*callback)(void*), void * param) {
SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param);
+
+ MaybeSpawnBackgroundThread();
+
if (ctx->after_multithreaded_fork) {
if (flags()->die_after_fork) {
Report("ThreadSanitizer: starting new threads after multi-threaded "
@@ -1312,10 +1418,15 @@ TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) {
if (o == 0 || f == 0)
return errno_EINVAL;
atomic_uint32_t *a;
- if (!SANITIZER_MAC)
- a = static_cast<atomic_uint32_t*>(o);
- else // On OS X, pthread_once_t has a header with a long-sized signature.
+
+ if (SANITIZER_MAC)
a = static_cast<atomic_uint32_t*>((void *)((char *)o + sizeof(long_t)));
+ else if (SANITIZER_NETBSD)
+ a = static_cast<atomic_uint32_t*>
+ ((void *)((char *)o + __sanitizer::pthread_mutex_t_sz));
+ else
+ a = static_cast<atomic_uint32_t*>(o);
+
u32 v = atomic_load(a, memory_order_acquire);
if (v == 0 && atomic_compare_exchange_strong(a, &v, 1,
memory_order_relaxed)) {
@@ -1347,7 +1458,7 @@ TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
#endif
TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {
-#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID
+#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD
SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf);
if (fd > 0)
FdAccess(thr, pc, fd);
@@ -1762,7 +1873,9 @@ TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set,
namespace __tsan {
static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
- bool sigact, int sig, my_siginfo_t *info, void *uctx) {
+ bool sigact, int sig,
+ __sanitizer_siginfo *info, void *uctx) {
+ __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions;
if (acquire)
Acquire(thr, 0, (uptr)&sigactions[sig]);
// Signals are generally asynchronous, so if we receive a signals when
@@ -1784,14 +1897,13 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
// This code races with sigaction. Be careful to not read sa_sigaction twice.
// Also need to remember pc for reporting before the call,
// because the handler can reset it.
- volatile uptr pc = sigact ?
- (uptr)sigactions[sig].sa_sigaction :
- (uptr)sigactions[sig].sa_handler;
- if (pc != (uptr)SIG_DFL && pc != (uptr)SIG_IGN) {
+ volatile uptr pc =
+ sigact ? (uptr)sigactions[sig].sigaction : (uptr)sigactions[sig].handler;
+ if (pc != sig_dfl && pc != sig_ign) {
if (sigact)
- ((sigactionhandler_t)pc)(sig, info, uctx);
+ ((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx);
else
- ((sighandler_t)pc)(sig);
+ ((__sanitizer_sighandler_ptr)pc)(sig);
}
if (!ctx->after_multithreaded_fork) {
thr->ignore_reads_and_writes = ignore_reads_and_writes;
@@ -1855,7 +1967,8 @@ static bool is_sync_signal(ThreadSignalContext *sctx, int sig) {
}
void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,
- my_siginfo_t *info, void *ctx) {
+ __sanitizer_siginfo *info,
+ void *ctx) {
ThreadState *thr = cur_thread();
ThreadSignalContext *sctx = SigCtx(thr);
if (sig < 0 || sig >= kSigCount) {
@@ -1905,58 +2018,10 @@ static void rtl_sighandler(int sig) {
rtl_generic_sighandler(false, sig, 0, 0);
}
-static void rtl_sigaction(int sig, my_siginfo_t *info, void *ctx) {
+static void rtl_sigaction(int sig, __sanitizer_siginfo *info, void *ctx) {
rtl_generic_sighandler(true, sig, info, ctx);
}
-TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) {
- // Note: if we call REAL(sigaction) directly for any reason without proxying
- // the signal handler through rtl_sigaction, very bad things will happen.
- // The handler will run synchronously and corrupt tsan per-thread state.
- SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old);
- if (old)
- internal_memcpy(old, &sigactions[sig], sizeof(*old));
- if (act == 0)
- return 0;
- // Copy act into sigactions[sig].
- // Can't use struct copy, because compiler can emit call to memcpy.
- // Can't use internal_memcpy, because it copies byte-by-byte,
- // and signal handler reads the sa_handler concurrently. It it can read
- // some bytes from old value and some bytes from new value.
- // Use volatile to prevent insertion of memcpy.
- sigactions[sig].sa_handler = *(volatile sighandler_t*)&act->sa_handler;
- sigactions[sig].sa_flags = *(volatile int*)&act->sa_flags;
- internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask,
- sizeof(sigactions[sig].sa_mask));
-#if !SANITIZER_FREEBSD && !SANITIZER_MAC
- sigactions[sig].sa_restorer = act->sa_restorer;
-#endif
- sigaction_t newact;
- internal_memcpy(&newact, act, sizeof(newact));
- internal_sigfillset(&newact.sa_mask);
- if (act->sa_handler != SIG_IGN && act->sa_handler != SIG_DFL) {
- if (newact.sa_flags & SA_SIGINFO)
- newact.sa_sigaction = rtl_sigaction;
- else
- newact.sa_handler = rtl_sighandler;
- }
- ReleaseStore(thr, pc, (uptr)&sigactions[sig]);
- int res = REAL(sigaction)(sig, &newact, 0);
- return res;
-}
-
-TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) {
- sigaction_t act;
- act.sa_handler = h;
- internal_memset(&act.sa_mask, -1, sizeof(act.sa_mask));
- act.sa_flags = 0;
- sigaction_t old;
- int res = sigaction(sig, &act, &old);
- if (res)
- return SIG_ERR;
- return old.sa_handler;
-}
-
TSAN_INTERCEPTOR(int, raise, int sig) {
SCOPED_TSAN_INTERCEPTOR(raise, sig);
ThreadSignalContext *sctx = SigCtx(thr);
@@ -2020,7 +2085,7 @@ TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service,
}
TSAN_INTERCEPTOR(int, fork, int fake) {
- if (cur_thread()->in_symbolizer)
+ if (UNLIKELY(cur_thread()->in_symbolizer))
return REAL(fork)(fake);
SCOPED_INTERCEPTOR_RAW(fork, fake);
ForkBefore(thr, pc);
@@ -2270,6 +2335,77 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
#include "sanitizer_common/sanitizer_common_interceptors.inc"
+static int sigaction_impl(int sig, const __sanitizer_sigaction *act,
+ __sanitizer_sigaction *old);
+static __sanitizer_sighandler_ptr signal_impl(int sig,
+ __sanitizer_sighandler_ptr h);
+
+#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \
+ { return sigaction_impl(signo, act, oldact); }
+
+#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \
+ { return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); }
+
+#include "sanitizer_common/sanitizer_signal_interceptors.inc"
+
+int sigaction_impl(int sig, const __sanitizer_sigaction *act,
+ __sanitizer_sigaction *old) {
+ // Note: if we call REAL(sigaction) directly for any reason without proxying
+ // the signal handler through rtl_sigaction, very bad things will happen.
+ // The handler will run synchronously and corrupt tsan per-thread state.
+ SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old);
+ __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions;
+ __sanitizer_sigaction old_stored;
+ if (old) internal_memcpy(&old_stored, &sigactions[sig], sizeof(old_stored));
+ __sanitizer_sigaction newact;
+ if (act) {
+ // Copy act into sigactions[sig].
+ // Can't use struct copy, because compiler can emit call to memcpy.
+ // Can't use internal_memcpy, because it copies byte-by-byte,
+ // and signal handler reads the handler concurrently. It it can read
+ // some bytes from old value and some bytes from new value.
+ // Use volatile to prevent insertion of memcpy.
+ sigactions[sig].handler =
+ *(volatile __sanitizer_sighandler_ptr const *)&act->handler;
+ sigactions[sig].sa_flags = *(volatile int const *)&act->sa_flags;
+ internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask,
+ sizeof(sigactions[sig].sa_mask));
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
+ sigactions[sig].sa_restorer = act->sa_restorer;
+#endif
+ internal_memcpy(&newact, act, sizeof(newact));
+ internal_sigfillset(&newact.sa_mask);
+ if ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl) {
+ if (newact.sa_flags & SA_SIGINFO)
+ newact.sigaction = rtl_sigaction;
+ else
+ newact.handler = rtl_sighandler;
+ }
+ ReleaseStore(thr, pc, (uptr)&sigactions[sig]);
+ act = &newact;
+ }
+ int res = REAL(sigaction)(sig, act, old);
+ if (res == 0 && old) {
+ uptr cb = (uptr)old->sigaction;
+ if (cb == (uptr)rtl_sigaction || cb == (uptr)rtl_sighandler) {
+ internal_memcpy(old, &old_stored, sizeof(*old));
+ }
+ }
+ return res;
+}
+
+static __sanitizer_sighandler_ptr signal_impl(int sig,
+ __sanitizer_sighandler_ptr h) {
+ __sanitizer_sigaction act;
+ act.handler = h;
+ internal_memset(&act.sa_mask, -1, sizeof(act.sa_mask));
+ act.sa_flags = 0;
+ __sanitizer_sigaction old;
+ int res = sigaction_symname(sig, &act, &old);
+ if (res) return (__sanitizer_sighandler_ptr)sig_err;
+ return old.handler;
+}
+
#define TSAN_SYSCALL() \
ThreadState *thr = cur_thread(); \
if (thr->ignore_interceptors) \
@@ -2290,7 +2426,7 @@ struct ScopedSyscall {
}
};
-#if !SANITIZER_FREEBSD && !SANITIZER_MAC
+#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {
TSAN_SYSCALL();
MemoryAccessRange(thr, pc, p, s, write);
@@ -2409,6 +2545,34 @@ TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) {
}
#endif
+#if SANITIZER_NETBSD
+TSAN_INTERCEPTOR(void, _lwp_exit) {
+ SCOPED_TSAN_INTERCEPTOR(_lwp_exit);
+ DestroyThreadState();
+ REAL(_lwp_exit)();
+}
+#define TSAN_MAYBE_INTERCEPT__LWP_EXIT TSAN_INTERCEPT(_lwp_exit)
+#else
+#define TSAN_MAYBE_INTERCEPT__LWP_EXIT
+#endif
+
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init, void *c, void *a);
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal, void *c);
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast, void *c);
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_wait, void *c, void *m);
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy, void *c);
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init, void *m, void *a);
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy, void *m);
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock, void *m);
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init, void *m, void *a);
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy, void *m);
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock, void *m);
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_tryrdlock, void *m);
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_wrlock, void *m);
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_trywrlock, void *m);
+TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_unlock, void *m);
+TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(int, once, void *o, void (*f)());
+
namespace __tsan {
static void finalize(void *arg) {
@@ -2440,20 +2604,30 @@ void InitializeInterceptors() {
mallopt(-3, 32*1024); // M_MMAP_THRESHOLD
#endif
+ new(interceptor_ctx()) InterceptorContext();
+
InitializeCommonInterceptors();
+ InitializeSignalInterceptors();
#if !SANITIZER_MAC
// We can not use TSAN_INTERCEPT to get setjmp addr,
// because it does &setjmp and setjmp is not present in some versions of libc.
using __interception::GetRealFunctionAddress;
- GetRealFunctionAddress("setjmp", (uptr*)&REAL(setjmp), 0, 0);
+ GetRealFunctionAddress(TSAN_STRING_SETJMP,
+ (uptr*)&REAL(setjmp_symname), 0, 0);
GetRealFunctionAddress("_setjmp", (uptr*)&REAL(_setjmp), 0, 0);
- GetRealFunctionAddress("sigsetjmp", (uptr*)&REAL(sigsetjmp), 0, 0);
+ GetRealFunctionAddress(TSAN_STRING_SIGSETJMP,
+ (uptr*)&REAL(sigsetjmp_symname), 0, 0);
+#if !SANITIZER_NETBSD
GetRealFunctionAddress("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0);
#endif
+#endif
- TSAN_INTERCEPT(longjmp);
- TSAN_INTERCEPT(siglongjmp);
+ TSAN_INTERCEPT(longjmp_symname);
+ TSAN_INTERCEPT(siglongjmp_symname);
+#if SANITIZER_NETBSD
+ TSAN_INTERCEPT(_longjmp);
+#endif
TSAN_INTERCEPT(malloc);
TSAN_INTERCEPT(__libc_memalign);
@@ -2548,8 +2722,6 @@ void InitializeInterceptors() {
TSAN_INTERCEPT(rmdir);
TSAN_INTERCEPT(closedir);
- TSAN_INTERCEPT(sigaction);
- TSAN_INTERCEPT(signal);
TSAN_INTERCEPT(sigsuspend);
TSAN_INTERCEPT(sigblock);
TSAN_INTERCEPT(sigsetmask);
@@ -2560,6 +2732,7 @@ void InitializeInterceptors() {
TSAN_INTERCEPT(sleep);
TSAN_INTERCEPT(usleep);
TSAN_INTERCEPT(nanosleep);
+ TSAN_INTERCEPT(pause);
TSAN_INTERCEPT(gettimeofday);
TSAN_INTERCEPT(getaddrinfo);
@@ -2568,7 +2741,7 @@ void InitializeInterceptors() {
#if !SANITIZER_ANDROID
TSAN_INTERCEPT(dl_iterate_phdr);
#endif
- TSAN_INTERCEPT(on_exit);
+ TSAN_MAYBE_INTERCEPT_ON_EXIT;
TSAN_INTERCEPT(__cxa_atexit);
TSAN_INTERCEPT(_exit);
@@ -2576,6 +2749,8 @@ void InitializeInterceptors() {
TSAN_INTERCEPT(__tls_get_addr);
#endif
+ TSAN_MAYBE_INTERCEPT__LWP_EXIT;
+
#if !SANITIZER_MAC && !SANITIZER_ANDROID
// Need to setup it, because interceptors check that the function is resolved.
// But atexit is emitted directly into the module, so can't be resolved.
@@ -2587,13 +2762,30 @@ void InitializeInterceptors() {
Die();
}
-#if !SANITIZER_MAC
- if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) {
+#if !SANITIZER_MAC && !SANITIZER_NETBSD
+ if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) {
Printf("ThreadSanitizer: failed to create thread key\n");
Die();
}
#endif
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_wait);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_tryrdlock);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_wrlock);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_trywrlock);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_unlock);
+ TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(once);
+
FdInit();
}
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h
index de47466501da..959a39465e33 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h
@@ -49,4 +49,16 @@ LibIgnore *libignore();
#define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)
+#if SANITIZER_NETBSD
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) \
+ TSAN_INTERCEPTOR(ret, __libc_##func, __VA_ARGS__) \
+ ALIAS(WRAPPER_NAME(pthread_##func));
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) \
+ TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \
+ ALIAS(WRAPPER_NAME(pthread_##func));
+#else
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...)
+# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...)
+#endif
+
#endif // TSAN_INTERCEPTORS_H
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc
index 4f1079467331..3b3a4a2815c3 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cc
@@ -100,7 +100,7 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \
SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
return tsan_atomic_f##_compare_exchange_strong( \
- (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
+ (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
kMacOrderNonBarrier, kMacOrderNonBarrier); \
} \
\
@@ -108,7 +108,7 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
t volatile *ptr) { \
SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
return tsan_atomic_f##_compare_exchange_strong( \
- (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
+ (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
kMacOrderBarrier, kMacOrderNonBarrier); \
}
@@ -122,14 +122,14 @@ OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
int64_t)
-#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \
- TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
- SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \
- char *byte_ptr = ((char *)ptr) + (n >> 3); \
- char bit = 0x80u >> (n & 7); \
- char mask = clear ? ~bit : bit; \
- char orig_byte = op((a8 *)byte_ptr, mask, mo); \
- return orig_byte & bit; \
+#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \
+ TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
+ SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \
+ volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \
+ char bit = 0x80u >> (n & 7); \
+ char mask = clear ? ~bit : bit; \
+ char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \
+ return orig_byte & bit; \
}
#define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc
index f68a0468de53..a7922a42e42c 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc
@@ -14,6 +14,7 @@
#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_placement_new.h"
#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_vector.h"
#include "tsan_interface_ann.h"
#include "tsan_mutex.h"
#include "tsan_report.h"
@@ -21,7 +22,6 @@
#include "tsan_mman.h"
#include "tsan_flags.h"
#include "tsan_platform.h"
-#include "tsan_vector.h"
#define CALLERPC ((uptr)__builtin_return_address(0))
@@ -185,10 +185,10 @@ void PrintMatchedBenignRaces() {
int unique_count = 0;
int hit_count = 0;
int add_count = 0;
- Vector<ExpectRace> hit_matched(MBlockScopedBuf);
+ Vector<ExpectRace> hit_matched;
CollectMatchedBenignRaces(&hit_matched, &unique_count, &hit_count,
&ExpectRace::hitcount);
- Vector<ExpectRace> add_matched(MBlockScopedBuf);
+ Vector<ExpectRace> add_matched;
CollectMatchedBenignRaces(&add_matched, &unique_count, &add_count,
&ExpectRace::addcount);
if (hit_matched.Size()) {
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc
index 8c759a3be4e1..eb22e4baa710 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_libdispatch_mac.cc
@@ -86,7 +86,8 @@ static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc,
void *orig_context,
dispatch_function_t orig_work) {
tsan_block_context_t *new_context =
- (tsan_block_context_t *)user_alloc(thr, pc, sizeof(tsan_block_context_t));
+ (tsan_block_context_t *)user_alloc_internal(thr, pc,
+ sizeof(tsan_block_context_t));
new_context->queue = queue;
new_context->orig_context = orig_context;
new_context->orig_work = orig_work;
@@ -176,7 +177,8 @@ static void invoke_and_release_block(void *param) {
}
#define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \
- TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
+ TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \
+ DISPATCH_NOESCAPE dispatch_block_t block) { \
SCOPED_TSAN_INTERCEPTOR(name, q, block); \
SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
dispatch_block_t heap_block = Block_copy(block); \
@@ -266,7 +268,7 @@ TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
// need to undefine the macro.
#undef dispatch_once
TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
- dispatch_block_t block) {
+ DISPATCH_NOESCAPE dispatch_block_t block) {
SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
u32 v = atomic_load(a, memory_order_acquire);
@@ -476,7 +478,8 @@ TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
}
TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
- dispatch_queue_t queue, void (^block)(size_t)) {
+ dispatch_queue_t queue,
+ DISPATCH_NOESCAPE void (^block)(size_t)) {
SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
void *parent_to_child_sync = nullptr;
@@ -519,9 +522,9 @@ TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,
return REAL(dispatch_data_create)(buffer, size, q, destructor);
if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)
- destructor = ^(void) { WRAP(free)((void *)buffer); };
+ destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); };
else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)
- destructor = ^(void) { WRAP(munmap)((void *)buffer, size); };
+ destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); };
SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
dispatch_block_t heap_block = Block_copy(destructor);
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc
index 8d31ccbcaaca..455c95df6c50 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cc
@@ -26,7 +26,7 @@ using namespace __tsan;
#define COMMON_MALLOC_FORCE_UNLOCK()
#define COMMON_MALLOC_MEMALIGN(alignment, size) \
void *p = \
- user_alloc(cur_thread(), StackTrace::GetCurrentPc(), size, alignment)
+ user_memalign(cur_thread(), StackTrace::GetCurrentPc(), alignment, size)
#define COMMON_MALLOC_MALLOC(size) \
if (cur_thread()->in_symbolizer) return InternalAlloc(size); \
SCOPED_INTERCEPTOR_RAW(malloc, size); \
@@ -43,7 +43,7 @@ using namespace __tsan;
if (cur_thread()->in_symbolizer) \
return InternalAlloc(size, nullptr, GetPageSizeCached()); \
SCOPED_INTERCEPTOR_RAW(valloc, size); \
- void *p = user_alloc(thr, pc, size, GetPageSizeCached())
+ void *p = user_valloc(thr, pc, size)
#define COMMON_MALLOC_FREE(ptr) \
if (cur_thread()->in_symbolizer) return InternalFree(ptr); \
SCOPED_INTERCEPTOR_RAW(free, ptr); \
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc
index f79dccddba9f..19680238bf76 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.cc
@@ -13,6 +13,7 @@
#include "sanitizer_common/sanitizer_allocator_checks.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_placement_new.h"
#include "tsan_mman.h"
#include "tsan_rtl.h"
@@ -149,11 +150,12 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
OutputReport(thr, rep);
}
-void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) {
+void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, uptr align,
+ bool signal) {
if ((sz >= (1ull << 40)) || (align >= (1ull << 40)))
return Allocator::FailureHandler::OnBadRequest();
void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align);
- if (p == 0)
+ if (UNLIKELY(p == 0))
return 0;
if (ctx && ctx->initialized)
OnUserAlloc(thr, pc, (uptr)p, sz, true);
@@ -162,15 +164,6 @@ void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) {
return p;
}
-void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) {
- if (CheckForCallocOverflow(size, n))
- return Allocator::FailureHandler::OnBadRequest();
- void *p = user_alloc(thr, pc, n * size);
- if (p)
- internal_memset(p, 0, n * size);
- return p;
-}
-
void user_free(ThreadState *thr, uptr pc, void *p, bool signal) {
ScopedGlobalProcessor sgp;
if (ctx && ctx->initialized)
@@ -180,6 +173,19 @@ void user_free(ThreadState *thr, uptr pc, void *p, bool signal) {
SignalUnsafeCall(thr, pc);
}
+void *user_alloc(ThreadState *thr, uptr pc, uptr sz) {
+ return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, kDefaultAlignment));
+}
+
+void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) {
+ if (UNLIKELY(CheckForCallocOverflow(size, n)))
+ return SetErrnoOnNull(Allocator::FailureHandler::OnBadRequest());
+ void *p = user_alloc_internal(thr, pc, n * size);
+ if (p)
+ internal_memset(p, 0, n * size);
+ return SetErrnoOnNull(p);
+}
+
void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {
DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p);
ctx->metamap.AllocBlock(thr, pc, p, sz);
@@ -200,15 +206,64 @@ void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) {
void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) {
// FIXME: Handle "shrinking" more efficiently,
// it seems that some software actually does this.
- void *p2 = user_alloc(thr, pc, sz);
- if (p2 == 0)
- return 0;
- if (p) {
- uptr oldsz = user_alloc_usable_size(p);
- internal_memcpy(p2, p, min(oldsz, sz));
+ if (!p)
+ return SetErrnoOnNull(user_alloc_internal(thr, pc, sz));
+ if (!sz) {
+ user_free(thr, pc, p);
+ return nullptr;
+ }
+ void *new_p = user_alloc_internal(thr, pc, sz);
+ if (new_p) {
+ uptr old_sz = user_alloc_usable_size(p);
+ internal_memcpy(new_p, p, min(old_sz, sz));
user_free(thr, pc, p);
}
- return p2;
+ return SetErrnoOnNull(new_p);
+}
+
+void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz) {
+ if (UNLIKELY(!IsPowerOfTwo(align))) {
+ errno = errno_EINVAL;
+ return Allocator::FailureHandler::OnBadRequest();
+ }
+ return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align));
+}
+
+int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align,
+ uptr sz) {
+ if (UNLIKELY(!CheckPosixMemalignAlignment(align))) {
+ Allocator::FailureHandler::OnBadRequest();
+ return errno_EINVAL;
+ }
+ void *ptr = user_alloc_internal(thr, pc, sz, align);
+ if (UNLIKELY(!ptr))
+ return errno_ENOMEM;
+ CHECK(IsAligned((uptr)ptr, align));
+ *memptr = ptr;
+ return 0;
+}
+
+void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz) {
+ if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(align, sz))) {
+ errno = errno_EINVAL;
+ return Allocator::FailureHandler::OnBadRequest();
+ }
+ return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, align));
+}
+
+void *user_valloc(ThreadState *thr, uptr pc, uptr sz) {
+ return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, GetPageSizeCached()));
+}
+
+void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz) {
+ uptr PageSize = GetPageSizeCached();
+ if (UNLIKELY(CheckForPvallocOverflow(sz, PageSize))) {
+ errno = errno_ENOMEM;
+ return Allocator::FailureHandler::OnBadRequest();
+ }
+ // pvalloc(0) should allocate one page.
+ sz = sz ? RoundUpTo(sz, PageSize) : PageSize;
+ return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, PageSize));
}
uptr user_alloc_usable_size(const void *p) {
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h
index 8cdeeb35aa6b..6042c5c5d96d 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_mman.h
@@ -27,13 +27,20 @@ void AllocatorProcFinish(Processor *proc);
void AllocatorPrintStats();
// For user allocations.
-void *user_alloc(ThreadState *thr, uptr pc, uptr sz,
- uptr align = kDefaultAlignment, bool signal = true);
-void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n);
+void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz,
+ uptr align = kDefaultAlignment, bool signal = true);
// Does not accept NULL.
void user_free(ThreadState *thr, uptr pc, void *p, bool signal = true);
+// Interceptor implementations.
+void *user_alloc(ThreadState *thr, uptr pc, uptr sz);
+void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n);
void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz);
-void *user_alloc_aligned(ThreadState *thr, uptr pc, uptr sz, uptr align);
+void *user_memalign(ThreadState *thr, uptr pc, uptr align, uptr sz);
+int user_posix_memalign(ThreadState *thr, uptr pc, void **memptr, uptr align,
+ uptr sz);
+void *user_aligned_alloc(ThreadState *thr, uptr pc, uptr align, uptr sz);
+void *user_valloc(ThreadState *thr, uptr pc, uptr sz);
+void *user_pvalloc(ThreadState *thr, uptr pc, uptr sz);
uptr user_alloc_usable_size(const void *p);
// Invoking malloc/free hooks that may be installed by the user.
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h
index bea1daba3952..bfac70df12d9 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform.h
@@ -42,6 +42,19 @@ C/C++ on linux/x86_64 and freebsd/x86_64
7b00 0000 0000 - 7c00 0000 0000: heap
7c00 0000 0000 - 7e80 0000 0000: -
7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
+
+C/C++ on netbsd/amd64 can reuse the same mapping:
+ * The address space starts from 0x1000 (option with 0x0) and ends with
+ 0x7f7ffffff000.
+ * LoAppMem-kHeapMemEnd can be reused as it is.
+ * No VDSO support.
+ * No MidAppMem region.
+ * No additional HeapMem region.
+ * HiAppMem contains the stack, loader, shared libraries and heap.
+ * Stack on NetBSD/amd64 has prereserved 128MB.
+ * Heap grows downwards (top-down).
+ * ASLR must be disabled per-process or globally.
+
*/
struct Mapping {
static const uptr kMetaShadowBeg = 0x300000000000ull;
@@ -111,7 +124,8 @@ C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
0c00 0000 00 - 0d00 0000 00: - (4 GB)
0d00 0000 00 - 0e00 0000 00: metainfo (4 GB)
0e00 0000 00 - 0f00 0000 00: - (4 GB)
-0f00 0000 00 - 1000 0000 00: traces (4 GB)
+0f00 0000 00 - 0fc0 0000 00: traces (3 GB)
+0fc0 0000 00 - 1000 0000 00: -
*/
struct Mapping {
static const uptr kLoAppMemBeg = 0x0100000000ull;
@@ -123,9 +137,9 @@ struct Mapping {
static const uptr kMetaShadowBeg = 0x0d00000000ull;
static const uptr kMetaShadowEnd = 0x0e00000000ull;
static const uptr kTraceMemBeg = 0x0f00000000ull;
- static const uptr kTraceMemEnd = 0x1000000000ull;
- static const uptr kHiAppMemBeg = 0x1000000000ull;
- static const uptr kHiAppMemEnd = 0x1000000000ull;
+ static const uptr kTraceMemEnd = 0x0fc0000000ull;
+ static const uptr kHiAppMemBeg = 0x0fc0000000ull;
+ static const uptr kHiAppMemEnd = 0x0fc0000000ull;
static const uptr kAppMemMsk = 0x0ull;
static const uptr kAppMemXor = 0x0ull;
static const uptr kVdsoBeg = 0x7000000000000000ull;
@@ -303,6 +317,38 @@ struct Mapping46 {
static const uptr kVdsoBeg = 0x7800000000000000ull;
};
+/*
+C/C++ on linux/powerpc64 (47-bit VMA)
+0000 0000 1000 - 0100 0000 0000: main binary
+0100 0000 0000 - 0200 0000 0000: -
+0100 0000 0000 - 1000 0000 0000: shadow
+1000 0000 0000 - 1000 0000 0000: -
+1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
+2000 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 2200 0000 0000: traces
+2200 0000 0000 - 7d00 0000 0000: -
+7d00 0000 0000 - 7e00 0000 0000: heap
+7e00 0000 0000 - 7e80 0000 0000: -
+7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
+*/
+struct Mapping47 {
+ static const uptr kMetaShadowBeg = 0x100000000000ull;
+ static const uptr kMetaShadowEnd = 0x200000000000ull;
+ static const uptr kTraceMemBeg = 0x200000000000ull;
+ static const uptr kTraceMemEnd = 0x220000000000ull;
+ static const uptr kShadowBeg = 0x010000000000ull;
+ static const uptr kShadowEnd = 0x100000000000ull;
+ static const uptr kHeapMemBeg = 0x7d0000000000ull;
+ static const uptr kHeapMemEnd = 0x7e0000000000ull;
+ static const uptr kLoAppMemBeg = 0x000000001000ull;
+ static const uptr kLoAppMemEnd = 0x010000000000ull;
+ static const uptr kHiAppMemBeg = 0x7e8000000000ull;
+ static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits
+ static const uptr kAppMemMsk = 0x7c0000000000ull;
+ static const uptr kAppMemXor = 0x020000000000ull;
+ static const uptr kVdsoBeg = 0x7800000000000000ull;
+};
+
// Indicates the runtime will define the memory regions at runtime.
#define TSAN_RUNTIME_VMA 1
#endif
@@ -429,11 +475,13 @@ uptr MappingArchImpl(void) {
DCHECK(0);
return 0;
#elif defined(__powerpc64__)
- if (vmaSize == 44)
- return MappingImpl<Mapping44, Type>();
- else
- return MappingImpl<Mapping46, Type>();
+ switch (vmaSize) {
+ case 44: return MappingImpl<Mapping44, Type>();
+ case 46: return MappingImpl<Mapping46, Type>();
+ case 47: return MappingImpl<Mapping47, Type>();
+ }
DCHECK(0);
+ return 0;
#else
return MappingImpl<Mapping, Type>();
#endif
@@ -582,11 +630,13 @@ bool IsAppMem(uptr mem) {
DCHECK(0);
return false;
#elif defined(__powerpc64__)
- if (vmaSize == 44)
- return IsAppMemImpl<Mapping44>(mem);
- else
- return IsAppMemImpl<Mapping46>(mem);
+ switch (vmaSize) {
+ case 44: return IsAppMemImpl<Mapping44>(mem);
+ case 46: return IsAppMemImpl<Mapping46>(mem);
+ case 47: return IsAppMemImpl<Mapping47>(mem);
+ }
DCHECK(0);
+ return false;
#else
return IsAppMemImpl<Mapping>(mem);
#endif
@@ -609,11 +659,13 @@ bool IsShadowMem(uptr mem) {
DCHECK(0);
return false;
#elif defined(__powerpc64__)
- if (vmaSize == 44)
- return IsShadowMemImpl<Mapping44>(mem);
- else
- return IsShadowMemImpl<Mapping46>(mem);
+ switch (vmaSize) {
+ case 44: return IsShadowMemImpl<Mapping44>(mem);
+ case 46: return IsShadowMemImpl<Mapping46>(mem);
+ case 47: return IsShadowMemImpl<Mapping47>(mem);
+ }
DCHECK(0);
+ return false;
#else
return IsShadowMemImpl<Mapping>(mem);
#endif
@@ -636,11 +688,13 @@ bool IsMetaMem(uptr mem) {
DCHECK(0);
return false;
#elif defined(__powerpc64__)
- if (vmaSize == 44)
- return IsMetaMemImpl<Mapping44>(mem);
- else
- return IsMetaMemImpl<Mapping46>(mem);
+ switch (vmaSize) {
+ case 44: return IsMetaMemImpl<Mapping44>(mem);
+ case 46: return IsMetaMemImpl<Mapping46>(mem);
+ case 47: return IsMetaMemImpl<Mapping47>(mem);
+ }
DCHECK(0);
+ return false;
#else
return IsMetaMemImpl<Mapping>(mem);
#endif
@@ -673,11 +727,13 @@ uptr MemToShadow(uptr x) {
DCHECK(0);
return 0;
#elif defined(__powerpc64__)
- if (vmaSize == 44)
- return MemToShadowImpl<Mapping44>(x);
- else
- return MemToShadowImpl<Mapping46>(x);
+ switch (vmaSize) {
+ case 44: return MemToShadowImpl<Mapping44>(x);
+ case 46: return MemToShadowImpl<Mapping46>(x);
+ case 47: return MemToShadowImpl<Mapping47>(x);
+ }
DCHECK(0);
+ return 0;
#else
return MemToShadowImpl<Mapping>(x);
#endif
@@ -712,11 +768,13 @@ u32 *MemToMeta(uptr x) {
DCHECK(0);
return 0;
#elif defined(__powerpc64__)
- if (vmaSize == 44)
- return MemToMetaImpl<Mapping44>(x);
- else
- return MemToMetaImpl<Mapping46>(x);
+ switch (vmaSize) {
+ case 44: return MemToMetaImpl<Mapping44>(x);
+ case 46: return MemToMetaImpl<Mapping46>(x);
+ case 47: return MemToMetaImpl<Mapping47>(x);
+ }
DCHECK(0);
+ return 0;
#else
return MemToMetaImpl<Mapping>(x);
#endif
@@ -764,11 +822,13 @@ uptr ShadowToMem(uptr s) {
DCHECK(0);
return 0;
#elif defined(__powerpc64__)
- if (vmaSize == 44)
- return ShadowToMemImpl<Mapping44>(s);
- else
- return ShadowToMemImpl<Mapping46>(s);
+ switch (vmaSize) {
+ case 44: return ShadowToMemImpl<Mapping44>(s);
+ case 46: return ShadowToMemImpl<Mapping46>(s);
+ case 47: return ShadowToMemImpl<Mapping47>(s);
+ }
DCHECK(0);
+ return 0;
#else
return ShadowToMemImpl<Mapping>(s);
#endif
@@ -799,11 +859,13 @@ uptr GetThreadTrace(int tid) {
DCHECK(0);
return 0;
#elif defined(__powerpc64__)
- if (vmaSize == 44)
- return GetThreadTraceImpl<Mapping44>(tid);
- else
- return GetThreadTraceImpl<Mapping46>(tid);
+ switch (vmaSize) {
+ case 44: return GetThreadTraceImpl<Mapping44>(tid);
+ case 46: return GetThreadTraceImpl<Mapping46>(tid);
+ case 47: return GetThreadTraceImpl<Mapping47>(tid);
+ }
DCHECK(0);
+ return 0;
#else
return GetThreadTraceImpl<Mapping>(tid);
#endif
@@ -829,11 +891,13 @@ uptr GetThreadTraceHeader(int tid) {
DCHECK(0);
return 0;
#elif defined(__powerpc64__)
- if (vmaSize == 44)
- return GetThreadTraceHeaderImpl<Mapping44>(tid);
- else
- return GetThreadTraceHeaderImpl<Mapping46>(tid);
+ switch (vmaSize) {
+ case 44: return GetThreadTraceHeaderImpl<Mapping44>(tid);
+ case 46: return GetThreadTraceHeaderImpl<Mapping46>(tid);
+ case 47: return GetThreadTraceHeaderImpl<Mapping47>(tid);
+ }
DCHECK(0);
+ return 0;
#else
return GetThreadTraceHeaderImpl<Mapping>(tid);
#endif
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc
index ead1e5704989..e14d5f575a2e 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cc
@@ -14,11 +14,12 @@
#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_LINUX || SANITIZER_FREEBSD
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
#include "sanitizer_common/sanitizer_platform_limits_posix.h"
#include "sanitizer_common/sanitizer_posix.h"
#include "sanitizer_common/sanitizer_procmaps.h"
@@ -216,9 +217,9 @@ void InitializePlatformEarly() {
Die();
}
#elif defined(__powerpc64__)
- if (vmaSize != 44 && vmaSize != 46) {
+ if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) {
Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
- Printf("FATAL: Found %d - Supported 44 and 46\n", vmaSize);
+ Printf("FATAL: Found %d - Supported 44, 46, and 47\n", vmaSize);
Die();
}
#endif
@@ -401,4 +402,4 @@ void cur_thread_finalize() {
} // namespace __tsan
-#endif // SANITIZER_LINUX || SANITIZER_FREEBSD
+#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc
index 73a656ffca5e..f8d7324b763c 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cc
@@ -167,8 +167,8 @@ void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
#else // !SANITIZER_GO
"app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
#endif
- "stacks: %ld unique IDs, %ld kB allocated\n"
- "threads: %ld total, %ld live\n"
+ "stacks: %zd unique IDs, %zd kB allocated\n"
+ "threads: %zd total, %zd live\n"
"------------------------------\n",
ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024,
MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024,
@@ -231,7 +231,7 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread,
void InitializePlatformEarly() {
#if defined(__aarch64__)
- uptr max_vm = GetMaxVirtualAddress() + 1;
+ uptr max_vm = GetMaxUserVirtualAddress() + 1;
if (max_vm != Mapping::kHiAppMemEnd) {
Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
max_vm, Mapping::kHiAppMemEnd);
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc
index 32cc3325fe40..af47076968d3 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.cc
@@ -13,6 +13,7 @@
#include "tsan_report.h"
#include "tsan_platform.h"
#include "tsan_rtl.h"
+#include "sanitizer_common/sanitizer_file.h"
#include "sanitizer_common/sanitizer_placement_new.h"
#include "sanitizer_common/sanitizer_report_decorator.h"
#include "sanitizer_common/sanitizer_stacktrace_printer.h"
@@ -38,34 +39,27 @@ ReportLocation *ReportLocation::New(ReportLocationType type) {
class Decorator: public __sanitizer::SanitizerCommonDecorator {
public:
Decorator() : SanitizerCommonDecorator() { }
- const char *Warning() { return Red(); }
- const char *EndWarning() { return Default(); }
const char *Access() { return Blue(); }
- const char *EndAccess() { return Default(); }
const char *ThreadDescription() { return Cyan(); }
- const char *EndThreadDescription() { return Default(); }
const char *Location() { return Green(); }
- const char *EndLocation() { return Default(); }
const char *Sleep() { return Yellow(); }
- const char *EndSleep() { return Default(); }
const char *Mutex() { return Magenta(); }
- const char *EndMutex() { return Default(); }
};
ReportDesc::ReportDesc()
: tag(kExternalTagNone)
- , stacks(MBlockReportStack)
- , mops(MBlockReportMop)
- , locs(MBlockReportLoc)
- , mutexes(MBlockReportMutex)
- , threads(MBlockReportThread)
- , unique_tids(MBlockReportThread)
+ , stacks()
+ , mops()
+ , locs()
+ , mutexes()
+ , threads()
+ , unique_tids()
, sleep()
, count() {
}
ReportMop::ReportMop()
- : mset(MBlockReportMutex) {
+ : mset() {
}
ReportDesc::~ReportDesc() {
@@ -180,7 +174,7 @@ static void PrintMop(const ReportMop *mop, bool first) {
}
PrintMutexSet(mop->mset);
Printf(":\n");
- Printf("%s", d.EndAccess());
+ Printf("%s", d.Default());
PrintStack(mop->stack);
}
@@ -221,20 +215,20 @@ static void PrintLocation(const ReportLocation *loc) {
loc->fd, thread_name(thrbuf, loc->tid));
print_stack = true;
}
- Printf("%s", d.EndLocation());
+ Printf("%s", d.Default());
if (print_stack)
PrintStack(loc->stack);
}
static void PrintMutexShort(const ReportMutex *rm, const char *after) {
Decorator d;
- Printf("%sM%zd%s%s", d.Mutex(), rm->id, d.EndMutex(), after);
+ Printf("%sM%zd%s%s", d.Mutex(), rm->id, d.Default(), after);
}
static void PrintMutexShortWithAddress(const ReportMutex *rm,
const char *after) {
Decorator d;
- Printf("%sM%zd (%p)%s%s", d.Mutex(), rm->id, rm->addr, d.EndMutex(), after);
+ Printf("%sM%zd (%p)%s%s", d.Mutex(), rm->id, rm->addr, d.Default(), after);
}
static void PrintMutex(const ReportMutex *rm) {
@@ -242,11 +236,11 @@ static void PrintMutex(const ReportMutex *rm) {
if (rm->destroyed) {
Printf("%s", d.Mutex());
Printf(" Mutex M%llu is already destroyed.\n\n", rm->id);
- Printf("%s", d.EndMutex());
+ Printf("%s", d.Default());
} else {
Printf("%s", d.Mutex());
Printf(" Mutex M%llu (%p) created at:\n", rm->id, rm->addr);
- Printf("%s", d.EndMutex());
+ Printf("%s", d.Default());
PrintStack(rm->stack);
}
}
@@ -264,7 +258,7 @@ static void PrintThread(const ReportThread *rt) {
if (rt->workerthread) {
Printf(" (tid=%zu, %s) is a GCD worker thread\n", rt->os_id, thread_status);
Printf("\n");
- Printf("%s", d.EndThreadDescription());
+ Printf("%s", d.Default());
return;
}
Printf(" (tid=%zu, %s) created by %s", rt->os_id, thread_status,
@@ -272,7 +266,7 @@ static void PrintThread(const ReportThread *rt) {
if (rt->stack)
Printf(" at:");
Printf("\n");
- Printf("%s", d.EndThreadDescription());
+ Printf("%s", d.Default());
PrintStack(rt->stack);
}
@@ -280,7 +274,7 @@ static void PrintSleep(const ReportStack *s) {
Decorator d;
Printf("%s", d.Sleep());
Printf(" As if synchronized via sleep:\n");
- Printf("%s", d.EndSleep());
+ Printf("%s", d.Default());
PrintStack(s);
}
@@ -324,7 +318,7 @@ void PrintReport(const ReportDesc *rep) {
Printf("%s", d.Warning());
Printf("WARNING: ThreadSanitizer: %s (pid=%d)\n", rep_typ_str,
(int)internal_getpid());
- Printf("%s", d.EndWarning());
+ Printf("%s", d.Default());
if (rep->typ == ReportTypeDeadlock) {
char thrbuf[kThreadBufSize];
@@ -342,7 +336,7 @@ void PrintReport(const ReportDesc *rep) {
PrintMutexShort(rep->mutexes[i], " in ");
Printf("%s", d.ThreadDescription());
Printf("%s:\n", thread_name(thrbuf, rep->unique_tids[i]));
- Printf("%s", d.EndThreadDescription());
+ Printf("%s", d.Default());
if (flags()->second_deadlock_stack) {
PrintStack(rep->stacks[2*i]);
Printf(" Mutex ");
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.h
index bc1582f90fe4..cdc999c6af1d 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_report.h
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_report.h
@@ -14,8 +14,8 @@
#define TSAN_REPORT_H
#include "sanitizer_common/sanitizer_symbolizer.h"
+#include "sanitizer_common/sanitizer_vector.h"
#include "tsan_defs.h"
-#include "tsan_vector.h"
namespace __tsan {
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc
index a01525302b02..17b820977c26 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.cc
@@ -14,6 +14,7 @@
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_file.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_stackdepot.h"
#include "sanitizer_common/sanitizer_placement_new.h"
@@ -101,8 +102,8 @@ Context::Context()
, thread_registry(new(thread_registry_placeholder) ThreadRegistry(
CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse))
, racy_mtx(MutexTypeRacy, StatMtxRacy)
- , racy_stacks(MBlockRacyStacks)
- , racy_addresses(MBlockRacyAddresses)
+ , racy_stacks()
+ , racy_addresses()
, fired_suppressions_mtx(MutexTypeFired, StatMtxFired)
, fired_suppressions(8)
, clock_alloc("clock allocator") {
@@ -120,7 +121,7 @@ ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch,
// , ignore_interceptors()
, clock(tid, reuse_count)
#if !SANITIZER_GO
- , jmp_bufs(MBlockJmpBuf)
+ , jmp_bufs()
#endif
, tid(tid)
, unique_id(unique_id)
@@ -213,7 +214,7 @@ static void BackgroundThread(void *arg) {
memory_order_relaxed);
if (last != 0 && last + flags()->flush_symbolizer_ms * kMs2Ns < now) {
Lock l(&ctx->report_mtx);
- SpinMutexLock l2(&CommonSanitizerReportMutex);
+ ScopedErrorReportLock l2;
SymbolizeFlush();
atomic_store(&ctx->last_symbolize_time_ns, 0, memory_order_relaxed);
}
@@ -323,6 +324,21 @@ static void CheckShadowMapping() {
}
}
+#if !SANITIZER_GO
+static void OnStackUnwind(const SignalContext &sig, const void *,
+ BufferedStackTrace *stack) {
+ uptr top = 0;
+ uptr bottom = 0;
+ bool fast = common_flags()->fast_unwind_on_fatal;
+ if (fast) GetThreadStackTopAndBottom(false, &top, &bottom);
+ stack->Unwind(kStackTraceMax, sig.pc, sig.bp, sig.context, top, bottom, fast);
+}
+
+static void TsanOnDeadlySignal(int signo, void *siginfo, void *context) {
+ HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr);
+}
+#endif
+
void Initialize(ThreadState *thr) {
// Thread safe because done before all threads exist.
static bool is_initialized = false;
@@ -360,6 +376,7 @@ void Initialize(ThreadState *thr) {
#if !SANITIZER_GO
InitializeShadowMemory();
InitializeAllocatorLate();
+ InstallDeadlySignalHandlers(TsanOnDeadlySignal);
#endif
// Setup correct file descriptor for error reports.
__sanitizer_set_report_path(common_flags()->log_path);
@@ -367,13 +384,6 @@ void Initialize(ThreadState *thr) {
#if !SANITIZER_GO
InitializeLibIgnore();
Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
- // On MIPS, TSan initialization is run before
- // __pthread_initialize_minimal_internal() is finished, so we can not spawn
- // new threads.
-#ifndef __mips__
- StartBackgroundThread();
- SetSandboxingCallback(StopBackgroundThread);
-#endif
#endif
VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n",
@@ -402,6 +412,21 @@ void Initialize(ThreadState *thr) {
OnInitialize();
}
+void MaybeSpawnBackgroundThread() {
+ // On MIPS, TSan initialization is run before
+ // __pthread_initialize_minimal_internal() is finished, so we can not spawn
+ // new threads.
+#if !SANITIZER_GO && !defined(__mips__)
+ static atomic_uint32_t bg_thread = {};
+ if (atomic_load(&bg_thread, memory_order_relaxed) == 0 &&
+ atomic_exchange(&bg_thread, 1, memory_order_relaxed) == 0) {
+ StartBackgroundThread();
+ SetSandboxingCallback(StopBackgroundThread);
+ }
+#endif
+}
+
+
int Finalize(ThreadState *thr) {
bool failed = false;
@@ -412,8 +437,7 @@ int Finalize(ThreadState *thr) {
// Wait for pending reports.
ctx->report_mtx.Lock();
- CommonSanitizerReportMutex.Lock();
- CommonSanitizerReportMutex.Unlock();
+ { ScopedErrorReportLock l; }
ctx->report_mtx.Unlock();
#if !SANITIZER_GO
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h
index 2cf2e168454d..7d44ccae9893 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl.h
@@ -34,12 +34,13 @@
#include "sanitizer_common/sanitizer_libignore.h"
#include "sanitizer_common/sanitizer_suppressions.h"
#include "sanitizer_common/sanitizer_thread_registry.h"
+#include "sanitizer_common/sanitizer_vector.h"
#include "tsan_clock.h"
#include "tsan_defs.h"
#include "tsan_flags.h"
+#include "tsan_mman.h"
#include "tsan_sync.h"
#include "tsan_trace.h"
-#include "tsan_vector.h"
#include "tsan_report.h"
#include "tsan_platform.h"
#include "tsan_mutexset.h"
@@ -574,11 +575,8 @@ const char *GetObjectTypeFromTag(uptr tag);
const char *GetReportHeaderFromTag(uptr tag);
uptr TagFromShadowStackFrame(uptr pc);
-class ScopedReport {
+class ScopedReportBase {
public:
- explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone);
- ~ScopedReport();
-
void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, StackTrace stack,
const MutexSet *mset);
void AddStack(StackTrace stack, bool suppressable = false);
@@ -593,6 +591,10 @@ class ScopedReport {
const ReportDesc *GetReport() const;
+ protected:
+ ScopedReportBase(ReportType typ, uptr tag);
+ ~ScopedReportBase();
+
private:
ReportDesc *rep_;
// Symbolizer makes lots of intercepted calls. If we try to process them,
@@ -601,8 +603,17 @@ class ScopedReport {
void AddDeadMutex(u64 id);
- ScopedReport(const ScopedReport&);
- void operator = (const ScopedReport&);
+ ScopedReportBase(const ScopedReportBase &) = delete;
+ void operator=(const ScopedReportBase &) = delete;
+};
+
+class ScopedReport : public ScopedReportBase {
+ public:
+ explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone);
+ ~ScopedReport();
+
+ private:
+ ScopedErrorReportLock lock_;
};
ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack);
@@ -690,6 +701,7 @@ void PrintCurrentStack(ThreadState *thr, uptr pc);
void PrintCurrentStackSlow(uptr pc); // uses libunwind
void Initialize(ThreadState *thr);
+void MaybeSpawnBackgroundThread();
int Finalize(ThreadState *thr);
void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
@@ -825,7 +837,7 @@ void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs,
return;
DCHECK_GE((int)typ, 0);
DCHECK_LE((int)typ, 7);
- DCHECK_EQ(GetLsb(addr, 61), addr);
+ DCHECK_EQ(GetLsb(addr, kEventPCBits), addr);
StatInc(thr, StatEvents);
u64 pos = fs.GetTracePos();
if (UNLIKELY((pos % kTracePartSize) == 0)) {
@@ -837,7 +849,7 @@ void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs,
}
Event *trace = (Event*)GetThreadTrace(fs.tid());
Event *evp = &trace[pos];
- Event ev = (u64)addr | ((u64)typ << 61);
+ Event ev = (u64)addr | ((u64)typ << kEventPCBits);
*evp = ev;
}
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S
index f8c6453a5da4..f1e4dbb50fbb 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S
@@ -6,7 +6,7 @@
#if !defined(__APPLE__)
.section .bss
.type __tsan_pointer_chk_guard, %object
-ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__tsan_pointer_chk_guard))
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__tsan_pointer_chk_guard))
__tsan_pointer_chk_guard:
.zero 8
#endif
@@ -51,7 +51,7 @@ _sigsetjmp$non_lazy_ptr:
// original ones.
ASM_HIDDEN(_Z18InitializeGuardPtrv)
.global _Z18InitializeGuardPtrv
-ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv))
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv))
_Z18InitializeGuardPtrv:
CFI_STARTPROC
// Allocates a jmp_buf for the setjmp call.
@@ -88,14 +88,14 @@ _Z18InitializeGuardPtrv:
CFI_DEF_CFA (31, 0)
ret
CFI_ENDPROC
-ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv))
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv))
#endif
ASM_HIDDEN(__tsan_setjmp)
.comm _ZN14__interception11real_setjmpE,8,8
-.globl ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)
-ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp))
-ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp):
+.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
+ASM_SYMBOL_INTERCEPTOR(setjmp):
CFI_STARTPROC
// save env parameters for function call
@@ -125,7 +125,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp):
#endif
// call tsan interceptor
- bl ASM_TSAN_SYMBOL(__tsan_setjmp)
+ bl ASM_SYMBOL(__tsan_setjmp)
// restore env parameter
mov x0, x19
@@ -148,12 +148,12 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp):
br x1
CFI_ENDPROC
-ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp))
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
.comm _ZN14__interception12real__setjmpE,8,8
-.globl ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)
-ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp))
-ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp):
+.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
+ASM_SYMBOL_INTERCEPTOR(_setjmp):
CFI_STARTPROC
// save env parameters for function call
@@ -183,7 +183,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp):
#endif
// call tsan interceptor
- bl ASM_TSAN_SYMBOL(__tsan_setjmp)
+ bl ASM_SYMBOL(__tsan_setjmp)
// Restore jmp_buf parameter
mov x0, x19
@@ -206,12 +206,12 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp):
br x1
CFI_ENDPROC
-ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp))
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
.comm _ZN14__interception14real_sigsetjmpE,8,8
-.globl ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)
-ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp))
-ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp):
+.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
CFI_STARTPROC
// save env parameters for function call
@@ -243,7 +243,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp):
#endif
// call tsan interceptor
- bl ASM_TSAN_SYMBOL(__tsan_setjmp)
+ bl ASM_SYMBOL(__tsan_setjmp)
// restore env parameter
mov w1, w20
@@ -268,13 +268,13 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp):
#endif
br x2
CFI_ENDPROC
-ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp))
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
#if !defined(__APPLE__)
.comm _ZN14__interception16real___sigsetjmpE,8,8
-.globl ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)
-ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
-ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp):
+.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
CFI_STARTPROC
// save env parameters for function call
@@ -303,7 +303,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp):
#endif
// call tsan interceptor
- bl ASM_TSAN_SYMBOL(__tsan_setjmp)
+ bl ASM_SYMBOL(__tsan_setjmp)
mov w1, w20
mov x0, x19
@@ -321,12 +321,12 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp):
ldr x2, [x2, #:got_lo12:_ZN14__interception16real___sigsetjmpE]
ldr x2, [x2]
#else
- adrp x2, ASM_TSAN_SYMBOL(__sigsetjmp)@page
- add x2, x2, ASM_TSAN_SYMBOL(__sigsetjmp)@pageoff
+ adrp x2, ASM_SYMBOL(__sigsetjmp)@page
+ add x2, x2, ASM_SYMBOL(__sigsetjmp)@pageoff
#endif
br x2
CFI_ENDPROC
-ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
#endif
#if defined(__FreeBSD__) || defined(__linux__)
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S
index 98947fd2a1ba..8af61bf0e892 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S
@@ -10,8 +10,8 @@
#endif
ASM_HIDDEN(__tsan_trace_switch)
-.globl ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk)
-ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk):
+.globl ASM_SYMBOL(__tsan_trace_switch_thunk)
+ASM_SYMBOL(__tsan_trace_switch_thunk):
CFI_STARTPROC
# Save scratch registers.
push %rax
@@ -50,7 +50,7 @@ ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk):
shr $4, %rsp # clear 4 lsb, align to 16
shl $4, %rsp
- call ASM_TSAN_SYMBOL(__tsan_trace_switch)
+ call ASM_SYMBOL(__tsan_trace_switch)
# Unalign stack frame back.
mov %rbx, %rsp # restore the original rsp
@@ -90,8 +90,8 @@ ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk):
CFI_ENDPROC
ASM_HIDDEN(__tsan_report_race)
-.globl ASM_TSAN_SYMBOL(__tsan_report_race_thunk)
-ASM_TSAN_SYMBOL(__tsan_report_race_thunk):
+.globl ASM_SYMBOL(__tsan_report_race_thunk)
+ASM_SYMBOL(__tsan_report_race_thunk):
CFI_STARTPROC
# Save scratch registers.
push %rax
@@ -130,7 +130,7 @@ ASM_TSAN_SYMBOL(__tsan_report_race_thunk):
shr $4, %rsp # clear 4 lsb, align to 16
shl $4, %rsp
- call ASM_TSAN_SYMBOL(__tsan_report_race)
+ call ASM_SYMBOL(__tsan_report_race)
# Unalign stack frame back.
mov %rbx, %rsp # restore the original rsp
@@ -170,19 +170,27 @@ ASM_TSAN_SYMBOL(__tsan_report_race_thunk):
CFI_ENDPROC
ASM_HIDDEN(__tsan_setjmp)
-#if !defined(__APPLE__)
+#if defined(__NetBSD__)
+.comm _ZN14__interception15real___setjmp14E,8,8
+#elif !defined(__APPLE__)
.comm _ZN14__interception11real_setjmpE,8,8
#endif
-.globl ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)
-ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp))
-ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp):
+#if defined(__NetBSD__)
+.globl ASM_SYMBOL_INTERCEPTOR(__setjmp14)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
+ASM_SYMBOL_INTERCEPTOR(__setjmp14):
+#else
+.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
+ASM_SYMBOL_INTERCEPTOR(setjmp):
+#endif
CFI_STARTPROC
// save env parameter
push %rdi
CFI_ADJUST_CFA_OFFSET(8)
CFI_REL_OFFSET(%rdi, 0)
// obtain %rsp
-#if defined(__FreeBSD__)
+#if defined(__FreeBSD__) || defined(__NetBSD__)
lea 8(%rsp), %rdi
mov %rdi, %rsi
#elif defined(__APPLE__)
@@ -197,33 +205,40 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp):
# error "Unknown platform"
#endif
// call tsan interceptor
- call ASM_TSAN_SYMBOL(__tsan_setjmp)
+ call ASM_SYMBOL(__tsan_setjmp)
// restore env parameter
pop %rdi
CFI_ADJUST_CFA_OFFSET(-8)
CFI_RESTORE(%rdi)
// tail jump to libc setjmp
movl $0, %eax
-#if !defined(__APPLE__)
+#if defined(__NetBSD__)
+ movq _ZN14__interception15real___setjmp14E@GOTPCREL(%rip), %rdx
+ jmp *(%rdx)
+#elif !defined(__APPLE__)
movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx
jmp *(%rdx)
#else
- jmp ASM_TSAN_SYMBOL(setjmp)
+ jmp ASM_SYMBOL(setjmp)
#endif
CFI_ENDPROC
-ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp))
+#if defined(__NetBSD__)
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
+#else
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
+#endif
.comm _ZN14__interception12real__setjmpE,8,8
-.globl ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)
-ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp))
-ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp):
+.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
+ASM_SYMBOL_INTERCEPTOR(_setjmp):
CFI_STARTPROC
// save env parameter
push %rdi
CFI_ADJUST_CFA_OFFSET(8)
CFI_REL_OFFSET(%rdi, 0)
// obtain %rsp
-#if defined(__FreeBSD__)
+#if defined(__FreeBSD__) || defined(__NetBSD__)
lea 8(%rsp), %rdi
mov %rdi, %rsi
#elif defined(__APPLE__)
@@ -238,7 +253,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp):
# error "Unknown platform"
#endif
// call tsan interceptor
- call ASM_TSAN_SYMBOL(__tsan_setjmp)
+ call ASM_SYMBOL(__tsan_setjmp)
// restore env parameter
pop %rdi
CFI_ADJUST_CFA_OFFSET(-8)
@@ -249,15 +264,22 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp):
movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx
jmp *(%rdx)
#else
- jmp ASM_TSAN_SYMBOL(_setjmp)
+ jmp ASM_SYMBOL(_setjmp)
#endif
CFI_ENDPROC
-ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp))
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
+#if defined(__NetBSD__)
+.comm _ZN14__interception18real___sigsetjmp14E,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
+ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14):
+#else
.comm _ZN14__interception14real_sigsetjmpE,8,8
-.globl ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)
-ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp))
-ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp):
+.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
+#endif
CFI_STARTPROC
// save env parameter
push %rdi
@@ -271,7 +293,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp):
sub $8, %rsp
CFI_ADJUST_CFA_OFFSET(8)
// obtain %rsp
-#if defined(__FreeBSD__)
+#if defined(__FreeBSD__) || defined(__NetBSD__)
lea 24(%rsp), %rdi
mov %rdi, %rsi
#elif defined(__APPLE__)
@@ -286,7 +308,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp):
# error "Unknown platform"
#endif
// call tsan interceptor
- call ASM_TSAN_SYMBOL(__tsan_setjmp)
+ call ASM_SYMBOL(__tsan_setjmp)
// unalign stack frame
add $8, %rsp
CFI_ADJUST_CFA_OFFSET(-8)
@@ -300,20 +322,27 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp):
CFI_RESTORE(%rdi)
// tail jump to libc sigsetjmp
movl $0, %eax
-#if !defined(__APPLE__)
+#if defined(__NetBSD__)
+ movq _ZN14__interception18real___sigsetjmp14E@GOTPCREL(%rip), %rdx
+ jmp *(%rdx)
+#elif !defined(__APPLE__)
movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx
jmp *(%rdx)
#else
- jmp ASM_TSAN_SYMBOL(sigsetjmp)
+ jmp ASM_SYMBOL(sigsetjmp)
#endif
CFI_ENDPROC
-ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp))
+#if defined(__NetBSD__)
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
+#else
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
+#endif
-#if !defined(__APPLE__)
+#if !defined(__APPLE__) && !defined(__NetBSD__)
.comm _ZN14__interception16real___sigsetjmpE,8,8
-.globl ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)
-ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
-ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp):
+.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
CFI_STARTPROC
// save env parameter
push %rdi
@@ -337,7 +366,7 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp):
rol $0x11, %rsi
#endif
// call tsan interceptor
- call ASM_TSAN_SYMBOL(__tsan_setjmp)
+ call ASM_SYMBOL(__tsan_setjmp)
// unalign stack frame
add $8, %rsp
CFI_ADJUST_CFA_OFFSET(-8)
@@ -354,11 +383,12 @@ ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp):
movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx
jmp *(%rdx)
CFI_ENDPROC
-ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
-#endif // !defined(__APPLE__)
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
+#endif // !defined(__APPLE__) && !defined(__NetBSD__)
#if defined(__FreeBSD__) || defined(__linux__)
/* We do not need executable stack. */
+/* This note is not needed on NetBSD. */
.section .note.GNU-stack,"",@progbits
#endif
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc
index 2f85811620f1..152b965ad535 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cc
@@ -84,7 +84,9 @@ void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
if (s == 0)
return;
- if ((flagz & MutexFlagLinkerInit) || s->IsFlagSet(MutexFlagLinkerInit)) {
+ if ((flagz & MutexFlagLinkerInit)
+ || s->IsFlagSet(MutexFlagLinkerInit)
+ || ((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) {
// Destroy is no-op for linker-initialized mutexes.
s->mtx.Unlock();
return;
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc
index 85a982941ed7..cc582ab50190 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cc
@@ -143,30 +143,28 @@ static ReportStack *SymbolizeStack(StackTrace trace) {
return stack;
}
-ScopedReport::ScopedReport(ReportType typ, uptr tag) {
+ScopedReportBase::ScopedReportBase(ReportType typ, uptr tag) {
ctx->thread_registry->CheckLocked();
void *mem = internal_alloc(MBlockReport, sizeof(ReportDesc));
rep_ = new(mem) ReportDesc;
rep_->typ = typ;
rep_->tag = tag;
ctx->report_mtx.Lock();
- CommonSanitizerReportMutex.Lock();
}
-ScopedReport::~ScopedReport() {
- CommonSanitizerReportMutex.Unlock();
+ScopedReportBase::~ScopedReportBase() {
ctx->report_mtx.Unlock();
DestroyAndFree(rep_);
}
-void ScopedReport::AddStack(StackTrace stack, bool suppressable) {
+void ScopedReportBase::AddStack(StackTrace stack, bool suppressable) {
ReportStack **rs = rep_->stacks.PushBack();
*rs = SymbolizeStack(stack);
(*rs)->suppressable = suppressable;
}
-void ScopedReport::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s,
- StackTrace stack, const MutexSet *mset) {
+void ScopedReportBase::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s,
+ StackTrace stack, const MutexSet *mset) {
void *mem = internal_alloc(MBlockReportMop, sizeof(ReportMop));
ReportMop *mop = new(mem) ReportMop;
rep_->mops.PushBack(mop);
@@ -187,11 +185,11 @@ void ScopedReport::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s,
}
}
-void ScopedReport::AddUniqueTid(int unique_tid) {
+void ScopedReportBase::AddUniqueTid(int unique_tid) {
rep_->unique_tids.PushBack(unique_tid);
}
-void ScopedReport::AddThread(const ThreadContext *tctx, bool suppressable) {
+void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) {
for (uptr i = 0; i < rep_->threads.Size(); i++) {
if ((u32)rep_->threads[i]->id == tctx->tid)
return;
@@ -255,14 +253,14 @@ ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) {
}
#endif
-void ScopedReport::AddThread(int unique_tid, bool suppressable) {
+void ScopedReportBase::AddThread(int unique_tid, bool suppressable) {
#if !SANITIZER_GO
if (const ThreadContext *tctx = FindThreadByUidLocked(unique_tid))
AddThread(tctx, suppressable);
#endif
}
-void ScopedReport::AddMutex(const SyncVar *s) {
+void ScopedReportBase::AddMutex(const SyncVar *s) {
for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
if (rep_->mutexes[i]->id == s->uid)
return;
@@ -276,7 +274,7 @@ void ScopedReport::AddMutex(const SyncVar *s) {
rm->stack = SymbolizeStackId(s->creation_stack_id);
}
-u64 ScopedReport::AddMutex(u64 id) {
+u64 ScopedReportBase::AddMutex(u64 id) {
u64 uid = 0;
u64 mid = id;
uptr addr = SyncVar::SplitId(id, &uid);
@@ -295,7 +293,7 @@ u64 ScopedReport::AddMutex(u64 id) {
return mid;
}
-void ScopedReport::AddDeadMutex(u64 id) {
+void ScopedReportBase::AddDeadMutex(u64 id) {
for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
if (rep_->mutexes[i]->id == id)
return;
@@ -309,7 +307,7 @@ void ScopedReport::AddDeadMutex(u64 id) {
rm->stack = 0;
}
-void ScopedReport::AddLocation(uptr addr, uptr size) {
+void ScopedReportBase::AddLocation(uptr addr, uptr size) {
if (addr == 0)
return;
#if !SANITIZER_GO
@@ -364,18 +362,19 @@ void ScopedReport::AddLocation(uptr addr, uptr size) {
}
#if !SANITIZER_GO
-void ScopedReport::AddSleep(u32 stack_id) {
+void ScopedReportBase::AddSleep(u32 stack_id) {
rep_->sleep = SymbolizeStackId(stack_id);
}
#endif
-void ScopedReport::SetCount(int count) {
- rep_->count = count;
-}
+void ScopedReportBase::SetCount(int count) { rep_->count = count; }
-const ReportDesc *ScopedReport::GetReport() const {
- return rep_;
-}
+const ReportDesc *ScopedReportBase::GetReport() const { return rep_; }
+
+ScopedReport::ScopedReport(ReportType typ, uptr tag)
+ : ScopedReportBase(typ, tag) {}
+
+ScopedReport::~ScopedReport() {}
void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
MutexSet *mset, uptr *tag) {
@@ -394,7 +393,7 @@ void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
const u64 ebegin = RoundDown(eend, kTracePartSize);
DPrintf("#%d: RestoreStack epoch=%zu ebegin=%zu eend=%zu partidx=%d\n",
tid, (uptr)epoch, (uptr)ebegin, (uptr)eend, partidx);
- Vector<uptr> stack(MBlockReportStack);
+ Vector<uptr> stack;
stack.Resize(hdr->stack0.size + 64);
for (uptr i = 0; i < hdr->stack0.size; i++) {
stack[i] = hdr->stack0.trace[i];
@@ -406,8 +405,8 @@ void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
Event *events = (Event*)GetThreadTrace(tid);
for (uptr i = ebegin; i <= eend; i++) {
Event ev = events[i];
- EventType typ = (EventType)(ev >> 61);
- uptr pc = (uptr)(ev & ((1ull << 61) - 1));
+ EventType typ = (EventType)(ev >> kEventPCBits);
+ uptr pc = (uptr)(ev & ((1ull << kEventPCBits) - 1));
DPrintf2(" %zu typ=%d pc=%zx\n", i, typ, pc);
if (typ == EventTypeMop) {
stack[pos] = pc;
@@ -634,13 +633,33 @@ void ReportRace(ThreadState *thr) {
const uptr kMop = 2;
VarSizeStackTrace traces[kMop];
uptr tags[kMop] = {kExternalTagNone};
- const uptr toppc = TraceTopPC(thr);
+ uptr toppc = TraceTopPC(thr);
+ if (toppc >> kEventPCBits) {
+ // This is a work-around for a known issue.
+ // The scenario where this happens is rather elaborate and requires
+ // an instrumented __sanitizer_report_error_summary callback and
+ // a __tsan_symbolize_external callback and a race during a range memory
+ // access larger than 8 bytes. MemoryAccessRange adds the current PC to
+ // the trace and starts processing memory accesses. A first memory access
+ // triggers a race, we report it and call the instrumented
+ // __sanitizer_report_error_summary, which adds more stuff to the trace
+ // since it is intrumented. Then a second memory access in MemoryAccessRange
+ // also triggers a race and we get here and call TraceTopPC to get the
+ // current PC, however now it contains some unrelated events from the
+ // callback. Most likely, TraceTopPC will now return a EventTypeFuncExit
+ // event. Later we subtract -1 from it (in GetPreviousInstructionPc)
+ // and the resulting PC has kExternalPCBit set, so we pass it to
+ // __tsan_symbolize_external. __tsan_symbolize_external is within its rights
+ // to crash since the PC is completely bogus.
+ // test/tsan/double_race.cc contains a test case for this.
+ toppc = 0;
+ }
ObtainCurrentStack(thr, toppc, &traces[0], &tags[0]);
if (IsFiredSuppression(ctx, typ, traces[0]))
return;
// MutexSet is too large to live on stack.
- Vector<u64> mset_buffer(MBlockScopedBuf);
+ Vector<u64> mset_buffer;
mset_buffer.Resize(sizeof(MutexSet) / sizeof(u64) + 1);
MutexSet *mset2 = new(&mset_buffer[0]) MutexSet();
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc
index 83fab082afe3..e4d65b9a909b 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc
@@ -211,7 +211,7 @@ void ThreadFinalize(ThreadState *thr) {
if (!flags()->report_thread_leaks)
return;
ThreadRegistryLock l(ctx->thread_registry);
- Vector<ThreadLeak> leaks(MBlockScopedBuf);
+ Vector<ThreadLeak> leaks;
ctx->thread_registry->RunCallbackForEachThreadLocked(
MaybeReportThreadLeak, &leaks);
for (uptr i = 0; i < leaks.Size(); i++) {
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.h
index b83c09ff784e..9039970bcf86 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.h
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_sync.h
@@ -34,6 +34,7 @@ enum MutexFlags {
MutexFlagTryLockFailed = 1 << 5, // __tsan_mutex_try_lock_failed
MutexFlagRecursiveLock = 1 << 6, // __tsan_mutex_recursive_lock
MutexFlagRecursiveUnlock = 1 << 7, // __tsan_mutex_recursive_unlock
+ MutexFlagNotStatic = 1 << 8, // __tsan_mutex_not_static
// The following flags are runtime private.
// Mutex API misuse was detected, so don't report any more.
@@ -43,7 +44,8 @@ enum MutexFlags {
// Must list all mutex creation flags.
MutexCreationFlagMask = MutexFlagLinkerInit |
MutexFlagWriteReentrant |
- MutexFlagReadReentrant,
+ MutexFlagReadReentrant |
+ MutexFlagNotStatic,
};
struct SyncVar {
diff --git a/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h b/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h
index 96a18ac4101a..9aef375cbfe6 100644
--- a/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h
+++ b/contrib/compiler-rt/lib/tsan/rtl/tsan_trace.h
@@ -41,6 +41,8 @@ enum EventType {
// u64 addr : 61; // Associated pc.
typedef u64 Event;
+const uptr kEventPCBits = 61;
+
struct TraceHeader {
#if !SANITIZER_GO
BufferedStackTrace stack0; // Start stack for the trace.
diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc b/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc
index 0a87e6e8e3a4..73c69363215a 100644
--- a/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc
+++ b/contrib/compiler-rt/lib/ubsan/ubsan_checks.inc
@@ -29,6 +29,7 @@ UBSAN_CHECK(UnsignedIntegerOverflow, "unsigned-integer-overflow",
UBSAN_CHECK(IntegerDivideByZero, "integer-divide-by-zero",
"integer-divide-by-zero")
UBSAN_CHECK(FloatDivideByZero, "float-divide-by-zero", "float-divide-by-zero")
+UBSAN_CHECK(InvalidBuiltin, "invalid-builtin-use", "invalid-builtin-use")
UBSAN_CHECK(InvalidShiftBase, "invalid-shift-base", "shift-base")
UBSAN_CHECK(InvalidShiftExponent, "invalid-shift-exponent", "shift-exponent")
UBSAN_CHECK(OutOfBoundsIndex, "out-of-bounds-index", "bounds")
diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc b/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc
index 742802b8f341..d5b68407bb37 100644
--- a/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc
+++ b/contrib/compiler-rt/lib/ubsan/ubsan_diag.cc
@@ -26,21 +26,24 @@
using namespace __ubsan;
+void __ubsan::GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc,
+ uptr bp, void *context, bool fast) {
+ uptr top = 0;
+ uptr bottom = 0;
+ if (fast)
+ GetThreadStackTopAndBottom(false, &top, &bottom);
+ stack->Unwind(max_depth, pc, bp, context, top, bottom, fast);
+}
+
static void MaybePrintStackTrace(uptr pc, uptr bp) {
// We assume that flags are already parsed, as UBSan runtime
// will definitely be called when we print the first diagnostics message.
if (!flags()->print_stacktrace)
return;
- uptr top = 0;
- uptr bottom = 0;
- bool request_fast_unwind = common_flags()->fast_unwind_on_fatal;
- if (request_fast_unwind)
- __sanitizer::GetThreadStackTopAndBottom(false, &top, &bottom);
-
BufferedStackTrace stack;
- stack.Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom,
- request_fast_unwind);
+ GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr,
+ common_flags()->fast_unwind_on_fatal);
stack.Print();
}
@@ -97,9 +100,7 @@ class Decorator : public SanitizerCommonDecorator {
public:
Decorator() : SanitizerCommonDecorator() {}
const char *Highlight() const { return Green(); }
- const char *EndHighlight() const { return Default(); }
const char *Note() const { return Black(); }
- const char *EndNote() const { return Default(); }
};
}
@@ -295,7 +296,7 @@ static void PrintMemorySnippet(const Decorator &Decor, MemoryLocation Loc,
Buffer.append("%c", P == Loc ? '^' : Byte);
Buffer.append("%c", Byte);
}
- Buffer.append("%s\n", Decor.EndHighlight());
+ Buffer.append("%s\n", Decor.Default());
// Go over the line again, and print names for the ranges.
InRange = 0;
@@ -335,7 +336,7 @@ static void PrintMemorySnippet(const Decorator &Decor, MemoryLocation Loc,
Diag::~Diag() {
// All diagnostics should be printed under report mutex.
- CommonSanitizerReportMutex.CheckLocked();
+ ScopedReport::CheckLocked();
Decorator Decor;
InternalScopedString Buffer(1024);
@@ -345,12 +346,12 @@ Diag::~Diag() {
switch (Level) {
case DL_Error:
- Buffer.append("%s runtime error: %s%s", Decor.Warning(), Decor.EndWarning(),
+ Buffer.append("%s runtime error: %s%s", Decor.Warning(), Decor.Default(),
Decor.Bold());
break;
case DL_Note:
- Buffer.append("%s note: %s", Decor.Note(), Decor.EndNote());
+ Buffer.append("%s note: %s", Decor.Note(), Decor.Default());
break;
}
@@ -363,17 +364,15 @@ Diag::~Diag() {
PrintMemorySnippet(Decor, Loc.getMemoryLocation(), Ranges, NumRanges, Args);
}
+ScopedReport::Initializer::Initializer() { InitAsStandaloneIfNecessary(); }
+
ScopedReport::ScopedReport(ReportOptions Opts, Location SummaryLoc,
ErrorType Type)
- : Opts(Opts), SummaryLoc(SummaryLoc), Type(Type) {
- InitAsStandaloneIfNecessary();
- CommonSanitizerReportMutex.Lock();
-}
+ : Opts(Opts), SummaryLoc(SummaryLoc), Type(Type) {}
ScopedReport::~ScopedReport() {
MaybePrintStackTrace(Opts.pc, Opts.bp);
MaybeReportErrorSummary(SummaryLoc, Type);
- CommonSanitizerReportMutex.Unlock();
if (flags()->halt_on_error)
Die();
}
diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_diag.h b/contrib/compiler-rt/lib/ubsan/ubsan_diag.h
index 3edb67a03c1f..7370b1b62421 100644
--- a/contrib/compiler-rt/lib/ubsan/ubsan_diag.h
+++ b/contrib/compiler-rt/lib/ubsan/ubsan_diag.h
@@ -231,10 +231,19 @@ bool ignoreReport(SourceLocation SLoc, ReportOptions Opts, ErrorType ET);
GET_CALLER_PC_BP; \
ReportOptions Opts = {unrecoverable_handler, pc, bp}
+void GetStackTrace(BufferedStackTrace *stack, uptr max_depth, uptr pc, uptr bp,
+ void *context, bool fast);
+
/// \brief Instantiate this class before printing diagnostics in the error
/// report. This class ensures that reports from different threads and from
/// different sanitizers won't be mixed.
class ScopedReport {
+ struct Initializer {
+ Initializer();
+ };
+ Initializer initializer_;
+ ScopedErrorReportLock report_lock_;
+
ReportOptions Opts;
Location SummaryLoc;
ErrorType Type;
@@ -242,6 +251,8 @@ class ScopedReport {
public:
ScopedReport(ReportOptions Opts, Location SummaryLoc, ErrorType Type);
~ScopedReport();
+
+ static void CheckLocked() { ScopedErrorReportLock::CheckLocked(); }
};
void InitializeSuppressions();
diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_diag_standalone.cc b/contrib/compiler-rt/lib/ubsan/ubsan_diag_standalone.cc
index df8ed5fcdf6d..1f4a5bd4062b 100644
--- a/contrib/compiler-rt/lib/ubsan/ubsan_diag_standalone.cc
+++ b/contrib/compiler-rt/lib/ubsan/ubsan_diag_standalone.cc
@@ -26,9 +26,10 @@ void __sanitizer_print_stack_trace() {
if (request_fast_unwind)
__sanitizer::GetThreadStackTopAndBottom(false, &top, &bottom);
- GET_REPORT_OPTIONS(false);
+ GET_CURRENT_PC_BP_SP;
+ (void)sp;
BufferedStackTrace stack;
- stack.Unwind(kStackTraceMax, Opts.pc, Opts.bp, nullptr, top, bottom,
+ stack.Unwind(kStackTraceMax, pc, bp, nullptr, top, bottom,
request_fast_unwind);
stack.Print();
}
diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc b/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc
index 8e1f40885a58..c4fb278c8367 100644
--- a/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc
+++ b/contrib/compiler-rt/lib/ubsan/ubsan_flags.cc
@@ -18,6 +18,8 @@
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_flag_parser.h"
+#include <stdlib.h>
+
namespace __ubsan {
const char *MaybeCallUbsanDefaultOptions() {
@@ -45,7 +47,7 @@ void InitializeFlags() {
CommonFlags cf;
cf.CopyFrom(*common_flags());
cf.print_summary = false;
- cf.external_symbolizer_path = GetEnv("UBSAN_SYMBOLIZER_PATH");
+ cf.external_symbolizer_path = getenv("UBSAN_SYMBOLIZER_PATH");
OverrideCommonFlags(cf);
}
@@ -59,7 +61,7 @@ void InitializeFlags() {
// Override from user-specified string.
parser.ParseString(MaybeCallUbsanDefaultOptions());
// Override from environment variable.
- parser.ParseString(GetEnv("UBSAN_OPTIONS"));
+ parser.ParseString(getenv("UBSAN_OPTIONS"));
InitializeCommonFlags();
if (Verbosity()) ReportUnrecognizedFlags();
diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc
index 75a4490a1843..1112ce1cc5ff 100644
--- a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc
+++ b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.cc
@@ -437,6 +437,30 @@ void __ubsan::__ubsan_handle_load_invalid_value_abort(InvalidValueData *Data,
Die();
}
+static void handleInvalidBuiltin(InvalidBuiltinData *Data, ReportOptions Opts) {
+ SourceLocation Loc = Data->Loc.acquire();
+ ErrorType ET = ErrorType::InvalidBuiltin;
+
+ if (ignoreReport(Loc, Opts, ET))
+ return;
+
+ ScopedReport R(Opts, Loc, ET);
+
+ Diag(Loc, DL_Error,
+ "passing zero to %0, which is not a valid argument")
+ << ((Data->Kind == BCK_CTZPassedZero) ? "ctz()" : "clz()");
+}
+
+void __ubsan::__ubsan_handle_invalid_builtin(InvalidBuiltinData *Data) {
+ GET_REPORT_OPTIONS(true);
+ handleInvalidBuiltin(Data, Opts);
+}
+void __ubsan::__ubsan_handle_invalid_builtin_abort(InvalidBuiltinData *Data) {
+ GET_REPORT_OPTIONS(true);
+ handleInvalidBuiltin(Data, Opts);
+ Die();
+}
+
static void handleFunctionTypeMismatch(FunctionTypeMismatchData *Data,
ValueHandle Function,
ReportOptions Opts) {
@@ -628,16 +652,32 @@ static void handleCFIBadIcall(CFICheckFailData *Data, ValueHandle Function,
}
namespace __ubsan {
+
#ifdef UBSAN_CAN_USE_CXXABI
+
+#ifdef _WIN32
+
+extern "C" void __ubsan_handle_cfi_bad_type_default(CFICheckFailData *Data,
+ ValueHandle Vtable,
+ bool ValidVtable,
+ ReportOptions Opts) {
+ Die();
+}
+
+WIN_WEAK_ALIAS(__ubsan_handle_cfi_bad_type, __ubsan_handle_cfi_bad_type_default)
+#else
SANITIZER_WEAK_ATTRIBUTE
-void HandleCFIBadType(CFICheckFailData *Data, ValueHandle Vtable,
- bool ValidVtable, ReportOptions Opts);
+#endif
+void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable,
+ bool ValidVtable, ReportOptions Opts);
+
#else
-static void HandleCFIBadType(CFICheckFailData *Data, ValueHandle Vtable,
- bool ValidVtable, ReportOptions Opts) {
+void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable,
+ bool ValidVtable, ReportOptions Opts) {
Die();
}
#endif
+
} // namespace __ubsan
void __ubsan::__ubsan_handle_cfi_check_fail(CFICheckFailData *Data,
@@ -647,7 +687,7 @@ void __ubsan::__ubsan_handle_cfi_check_fail(CFICheckFailData *Data,
if (Data->CheckKind == CFITCK_ICall)
handleCFIBadIcall(Data, Value, Opts);
else
- HandleCFIBadType(Data, Value, ValidVtable, Opts);
+ __ubsan_handle_cfi_bad_type(Data, Value, ValidVtable, Opts);
}
void __ubsan::__ubsan_handle_cfi_check_fail_abort(CFICheckFailData *Data,
@@ -657,7 +697,7 @@ void __ubsan::__ubsan_handle_cfi_check_fail_abort(CFICheckFailData *Data,
if (Data->CheckKind == CFITCK_ICall)
handleCFIBadIcall(Data, Value, Opts);
else
- HandleCFIBadType(Data, Value, ValidVtable, Opts);
+ __ubsan_handle_cfi_bad_type(Data, Value, ValidVtable, Opts);
Die();
}
diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h
index 796321b81889..311776b9f22c 100644
--- a/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h
+++ b/contrib/compiler-rt/lib/ubsan/ubsan_handlers.h
@@ -122,6 +122,21 @@ struct InvalidValueData {
/// \brief Handle a load of an invalid value for the type.
RECOVERABLE(load_invalid_value, InvalidValueData *Data, ValueHandle Val)
+/// Known builtin check kinds.
+/// Keep in sync with the enum of the same name in CodeGenFunction.h
+enum BuiltinCheckKind : unsigned char {
+ BCK_CTZPassedZero,
+ BCK_CLZPassedZero,
+};
+
+struct InvalidBuiltinData {
+ SourceLocation Loc;
+ unsigned char Kind;
+};
+
+/// Handle a builtin called in an invalid way.
+RECOVERABLE(invalid_builtin, InvalidBuiltinData *Data)
+
struct FunctionTypeMismatchData {
SourceLocation Loc;
const TypeDescriptor &Type;
@@ -177,6 +192,13 @@ struct CFICheckFailData {
/// \brief Handle control flow integrity failures.
RECOVERABLE(cfi_check_fail, CFICheckFailData *Data, ValueHandle Function,
uptr VtableIsValid)
+
+struct ReportOptions;
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __ubsan_handle_cfi_bad_type(
+ CFICheckFailData *Data, ValueHandle Vtable, bool ValidVtable,
+ ReportOptions Opts);
+
}
#endif // UBSAN_HANDLERS_H
diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc b/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc
index d97ec4813ccd..e15abc64ecca 100644
--- a/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc
+++ b/contrib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cc
@@ -95,8 +95,8 @@ void __ubsan::__ubsan_handle_dynamic_type_cache_miss_abort(
}
namespace __ubsan {
-void HandleCFIBadType(CFICheckFailData *Data, ValueHandle Vtable,
- bool ValidVtable, ReportOptions Opts) {
+void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable,
+ bool ValidVtable, ReportOptions Opts) {
SourceLocation Loc = Data->Loc.acquire();
ErrorType ET = ErrorType::CFIBadType;
diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_init.cc b/contrib/compiler-rt/lib/ubsan/ubsan_init.cc
index 307bca37680e..32fc434ad2bc 100644
--- a/contrib/compiler-rt/lib/ubsan/ubsan_init.cc
+++ b/contrib/compiler-rt/lib/ubsan/ubsan_init.cc
@@ -27,11 +27,7 @@ const char *__ubsan::GetSanititizerToolName() {
return "UndefinedBehaviorSanitizer";
}
-static enum {
- UBSAN_MODE_UNKNOWN = 0,
- UBSAN_MODE_STANDALONE,
- UBSAN_MODE_PLUGIN
-} ubsan_mode;
+static bool ubsan_initialized;
static StaticSpinMutex ubsan_init_mu;
static void CommonInit() {
@@ -40,44 +36,30 @@ static void CommonInit() {
static void CommonStandaloneInit() {
SanitizerToolName = GetSanititizerToolName();
- InitializeFlags();
CacheBinaryName();
+ InitializeFlags();
__sanitizer_set_report_path(common_flags()->log_path);
AndroidLogInit();
InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
CommonInit();
- ubsan_mode = UBSAN_MODE_STANDALONE;
}
void __ubsan::InitAsStandalone() {
- if (SANITIZER_CAN_USE_PREINIT_ARRAY) {
- CHECK_EQ(UBSAN_MODE_UNKNOWN, ubsan_mode);
- CommonStandaloneInit();
- return;
- }
SpinMutexLock l(&ubsan_init_mu);
- CHECK_NE(UBSAN_MODE_PLUGIN, ubsan_mode);
- if (ubsan_mode == UBSAN_MODE_UNKNOWN)
+ if (!ubsan_initialized) {
CommonStandaloneInit();
-}
-
-void __ubsan::InitAsStandaloneIfNecessary() {
- if (SANITIZER_CAN_USE_PREINIT_ARRAY) {
- CHECK_NE(UBSAN_MODE_UNKNOWN, ubsan_mode);
- return;
+ ubsan_initialized = true;
}
- SpinMutexLock l(&ubsan_init_mu);
- if (ubsan_mode == UBSAN_MODE_UNKNOWN)
- CommonStandaloneInit();
}
+void __ubsan::InitAsStandaloneIfNecessary() { return InitAsStandalone(); }
+
void __ubsan::InitAsPlugin() {
-#if !SANITIZER_CAN_USE_PREINIT_ARRAY
SpinMutexLock l(&ubsan_init_mu);
-#endif
- CHECK_EQ(UBSAN_MODE_UNKNOWN, ubsan_mode);
- CommonInit();
- ubsan_mode = UBSAN_MODE_PLUGIN;
+ if (!ubsan_initialized) {
+ CommonInit();
+ ubsan_initialized = true;
+ }
}
#endif // CAN_SANITIZE_UB
diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_init_standalone.cc b/contrib/compiler-rt/lib/ubsan/ubsan_init_standalone.cc
index ff1a20efea3d..8bd500025cbc 100644
--- a/contrib/compiler-rt/lib/ubsan/ubsan_init_standalone.cc
+++ b/contrib/compiler-rt/lib/ubsan/ubsan_init_standalone.cc
@@ -18,18 +18,17 @@
#include "sanitizer_common/sanitizer_internal_defs.h"
#include "ubsan_init.h"
+#include "ubsan_signals_standalone.h"
+
+namespace __ubsan {
-#if SANITIZER_CAN_USE_PREINIT_ARRAY
-__attribute__((section(".preinit_array"), used))
-void (*__local_ubsan_preinit)(void) = __ubsan::InitAsStandalone;
-#else
-// Use a dynamic initializer.
class UbsanStandaloneInitializer {
public:
UbsanStandaloneInitializer() {
- __ubsan::InitAsStandalone();
+ InitAsStandalone();
+ InitializeDeadlySignals();
}
};
static UbsanStandaloneInitializer ubsan_standalone_initializer;
-#endif // SANITIZER_CAN_USE_PREINIT_ARRAY
+} // namespace __ubsan
diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc b/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc
index a69ca57cd7af..9beb3e2ff953 100644
--- a/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc
+++ b/contrib/compiler-rt/lib/ubsan/ubsan_interface.inc
@@ -11,14 +11,19 @@
INTERFACE_FUNCTION(__ubsan_handle_add_overflow)
INTERFACE_FUNCTION(__ubsan_handle_add_overflow_abort)
INTERFACE_FUNCTION(__ubsan_handle_builtin_unreachable)
+INTERFACE_FUNCTION(__ubsan_handle_cfi_bad_type)
INTERFACE_FUNCTION(__ubsan_handle_cfi_check_fail)
INTERFACE_FUNCTION(__ubsan_handle_cfi_check_fail_abort)
INTERFACE_FUNCTION(__ubsan_handle_divrem_overflow)
INTERFACE_FUNCTION(__ubsan_handle_divrem_overflow_abort)
+INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss)
+INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss_abort)
INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow)
INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow_abort)
INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch)
INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_abort)
+INTERFACE_FUNCTION(__ubsan_handle_invalid_builtin)
+INTERFACE_FUNCTION(__ubsan_handle_invalid_builtin_abort)
INTERFACE_FUNCTION(__ubsan_handle_load_invalid_value)
INTERFACE_FUNCTION(__ubsan_handle_load_invalid_value_abort)
INTERFACE_FUNCTION(__ubsan_handle_missing_return)
diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_platform.h b/contrib/compiler-rt/lib/ubsan/ubsan_platform.h
index 1a3bfd6afb81..72eb4199960e 100644
--- a/contrib/compiler-rt/lib/ubsan/ubsan_platform.h
+++ b/contrib/compiler-rt/lib/ubsan/ubsan_platform.h
@@ -14,12 +14,13 @@
#define UBSAN_PLATFORM_H
// Other platforms should be easy to add, and probably work as-is.
-#if (defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__)) && \
- (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || \
- defined(__aarch64__) || defined(__mips__) || defined(__powerpc64__) || \
- defined(__s390__))
+#if (defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__) || \
+ defined(__NetBSD__)) && \
+ (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || \
+ defined(__aarch64__) || defined(__mips__) || defined(__powerpc64__) || \
+ defined(__s390__)) || (defined(__sun__) && defined(__svr4__))
# define CAN_SANITIZE_UB 1
-#elif defined(_WIN32)
+#elif defined(_WIN32) || defined(__Fuchsia__)
# define CAN_SANITIZE_UB 1
#else
# define CAN_SANITIZE_UB 0
diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cc b/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cc
new file mode 100644
index 000000000000..4c6646dd8b0b
--- /dev/null
+++ b/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cc
@@ -0,0 +1,53 @@
+//=-- ubsan_signals_standalone.cc
+//------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Installs signal handlers and related interceptors for UBSan standalone.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ubsan_platform.h"
+#if CAN_SANITIZE_UB
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "ubsan_diag.h"
+#include "ubsan_init.h"
+
+#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
+#include "sanitizer_common/sanitizer_signal_interceptors.inc"
+
+namespace __ubsan {
+
+#if SANITIZER_FUCHSIA
+void InitializeDeadlySignals() {}
+#else
+static void OnStackUnwind(const SignalContext &sig, const void *,
+ BufferedStackTrace *stack) {
+ GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context,
+ common_flags()->fast_unwind_on_fatal);
+}
+
+static void UBsanOnDeadlySignal(int signo, void *siginfo, void *context) {
+ HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr);
+}
+
+static bool is_initialized = false;
+
+void InitializeDeadlySignals() {
+ if (is_initialized)
+ return;
+ is_initialized = true;
+ InitializeSignalInterceptors();
+ InstallDeadlySignalHandlers(&UBsanOnDeadlySignal);
+}
+#endif
+
+} // namespace __ubsan
+
+#endif // CAN_SANITIZE_UB
diff --git a/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.h b/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.h
new file mode 100644
index 000000000000..b29c29482ec8
--- /dev/null
+++ b/contrib/compiler-rt/lib/ubsan/ubsan_signals_standalone.h
@@ -0,0 +1,25 @@
+//=-- ubsan_signals_standalone.h
+//------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Installs signal handlers and related interceptors for UBSan standalone.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef UBSAN_SIGNALS_STANDALONE_H
+#define UBSAN_SIGNALS_STANDALONE_H
+
+namespace __ubsan {
+
+// Initializes signal handlers and interceptors.
+void InitializeDeadlySignals();
+
+} // namespace __ubsan
+
+#endif // UBSAN_SIGNALS_STANDALONE_H
diff --git a/contrib/compiler-rt/lib/ubsan_minimal/ubsan.syms.extra b/contrib/compiler-rt/lib/ubsan_minimal/ubsan.syms.extra
new file mode 100644
index 000000000000..7f8be694401a
--- /dev/null
+++ b/contrib/compiler-rt/lib/ubsan_minimal/ubsan.syms.extra
@@ -0,0 +1 @@
+__ubsan_*
diff --git a/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc b/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc
new file mode 100644
index 000000000000..5a5675c983fe
--- /dev/null
+++ b/contrib/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cc
@@ -0,0 +1,104 @@
+#include "sanitizer_common/sanitizer_atomic.h"
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+
+#ifdef KERNEL_USE
+extern "C" void ubsan_message(const char *msg);
+static void message(const char *msg) { ubsan_message(msg); }
+#else
+static void message(const char *msg) {
+ write(2, msg, strlen(msg));
+}
+#endif
+
+static const int kMaxCallerPcs = 20;
+static __sanitizer::atomic_uintptr_t caller_pcs[kMaxCallerPcs];
+// Number of elements in caller_pcs. A special value of kMaxCallerPcs + 1 means
+// that "too many errors" has already been reported.
+static __sanitizer::atomic_uint32_t caller_pcs_sz;
+
+__attribute__((noinline)) static bool report_this_error(void *caller_p) {
+ uintptr_t caller = reinterpret_cast<uintptr_t>(caller_p);
+ if (caller == 0) return false;
+ while (true) {
+ unsigned sz = __sanitizer::atomic_load_relaxed(&caller_pcs_sz);
+ if (sz > kMaxCallerPcs) return false; // early exit
+ // when sz==kMaxCallerPcs print "too many errors", but only when cmpxchg
+ // succeeds in order to not print it multiple times.
+ if (sz > 0 && sz < kMaxCallerPcs) {
+ uintptr_t p;
+ for (unsigned i = 0; i < sz; ++i) {
+ p = __sanitizer::atomic_load_relaxed(&caller_pcs[i]);
+ if (p == 0) break; // Concurrent update.
+ if (p == caller) return false;
+ }
+ if (p == 0) continue; // FIXME: yield?
+ }
+
+ if (!__sanitizer::atomic_compare_exchange_strong(
+ &caller_pcs_sz, &sz, sz + 1, __sanitizer::memory_order_seq_cst))
+ continue; // Concurrent update! Try again from the start.
+
+ if (sz == kMaxCallerPcs) {
+ message("ubsan: too many errors\n");
+ return false;
+ }
+ __sanitizer::atomic_store_relaxed(&caller_pcs[sz], caller);
+ return true;
+ }
+}
+
+#if defined(__ANDROID__)
+extern "C" __attribute__((weak)) void android_set_abort_message(const char *);
+static void abort_with_message(const char *msg) {
+ if (&android_set_abort_message) android_set_abort_message(msg);
+ abort();
+}
+#else
+static void abort_with_message(const char *) { abort(); }
+#endif
+
+#define INTERFACE extern "C" __attribute__((visibility("default")))
+
+// FIXME: add caller pc to the error message (possibly as "ubsan: error-type
+// @1234ABCD").
+#define HANDLER_RECOVER(name, msg) \
+ INTERFACE void __ubsan_handle_##name##_minimal() { \
+ if (!report_this_error(__builtin_return_address(0))) return; \
+ message("ubsan: " msg "\n"); \
+ }
+
+#define HANDLER_NORECOVER(name, msg) \
+ INTERFACE void __ubsan_handle_##name##_minimal_abort() { \
+ message("ubsan: " msg "\n"); \
+ abort_with_message("ubsan: " msg); \
+ }
+
+#define HANDLER(name, msg) \
+ HANDLER_RECOVER(name, msg) \
+ HANDLER_NORECOVER(name, msg)
+
+HANDLER(type_mismatch, "type-mismatch")
+HANDLER(add_overflow, "add-overflow")
+HANDLER(sub_overflow, "sub-overflow")
+HANDLER(mul_overflow, "mul-overflow")
+HANDLER(negate_overflow, "negate-overflow")
+HANDLER(divrem_overflow, "divrem-overflow")
+HANDLER(shift_out_of_bounds, "shift-out-of-bounds")
+HANDLER(out_of_bounds, "out-of-bounds")
+HANDLER_RECOVER(builtin_unreachable, "builtin-unreachable")
+HANDLER_RECOVER(missing_return, "missing-return")
+HANDLER(vla_bound_not_positive, "vla-bound-not-positive")
+HANDLER(float_cast_overflow, "float-cast-overflow")
+HANDLER(load_invalid_value, "load-invalid-value")
+HANDLER(invalid_builtin, "invalid-builtin")
+HANDLER(function_type_mismatch, "function-type-mismatch")
+HANDLER(nonnull_arg, "nonnull-arg")
+HANDLER(nonnull_return, "nonnull-return")
+HANDLER(nullability_arg, "nullability-arg")
+HANDLER(nullability_return, "nullability-return")
+HANDLER(pointer_overflow, "pointer-overflow")
+HANDLER(cfi_check_fail, "cfi-check-fail")
diff --git a/contrib/compiler-rt/lib/xray/weak_symbols.txt b/contrib/compiler-rt/lib/xray/weak_symbols.txt
new file mode 100644
index 000000000000..963fff2d697e
--- /dev/null
+++ b/contrib/compiler-rt/lib/xray/weak_symbols.txt
@@ -0,0 +1,4 @@
+___start_xray_fn_idx
+___start_xray_instr_map
+___stop_xray_fn_idx
+___stop_xray_instr_map
diff --git a/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc b/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc
index 7ba755ac3069..a0018f6b0cba 100644
--- a/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc
+++ b/contrib/compiler-rt/lib/xray/xray_buffer_queue.cc
@@ -13,28 +13,34 @@
//
//===----------------------------------------------------------------------===//
#include "xray_buffer_queue.h"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_libc.h"
-#include <cstdlib>
-#include <tuple>
-
using namespace __xray;
using namespace __sanitizer;
-BufferQueue::BufferQueue(std::size_t B, std::size_t N, bool &Success)
- : BufferSize(B), Buffers(N), Mutex(), OwnedBuffers(), Finalizing{0} {
- for (auto &T : Buffers) {
- void *Tmp = malloc(BufferSize);
+BufferQueue::BufferQueue(size_t B, size_t N, bool &Success)
+ : BufferSize(B), Buffers(new BufferRep[N]()), BufferCount(N), Finalizing{0},
+ OwnedBuffers(new void *[N]()), Next(Buffers), First(Buffers),
+ LiveBuffers(0) {
+ for (size_t i = 0; i < N; ++i) {
+ auto &T = Buffers[i];
+ void *Tmp = InternalAlloc(BufferSize, nullptr, 64);
if (Tmp == nullptr) {
Success = false;
return;
}
-
- auto &Buf = std::get<0>(T);
+ void *Extents = InternalAlloc(sizeof(BufferExtents), nullptr, 64);
+ if (Extents == nullptr) {
+ Success = false;
+ return;
+ }
+ auto &Buf = T.Buff;
Buf.Buffer = Tmp;
Buf.Size = B;
- OwnedBuffers.emplace(Tmp);
+ Buf.Extents = reinterpret_cast<BufferExtents *>(Extents);
+ OwnedBuffers[i] = Tmp;
}
Success = true;
}
@@ -42,27 +48,50 @@ BufferQueue::BufferQueue(std::size_t B, std::size_t N, bool &Success)
BufferQueue::ErrorCode BufferQueue::getBuffer(Buffer &Buf) {
if (__sanitizer::atomic_load(&Finalizing, __sanitizer::memory_order_acquire))
return ErrorCode::QueueFinalizing;
- __sanitizer::BlockingMutexLock Guard(&Mutex);
- if (Buffers.empty())
+ __sanitizer::SpinMutexLock Guard(&Mutex);
+ if (LiveBuffers == BufferCount)
return ErrorCode::NotEnoughMemory;
- auto &T = Buffers.front();
- auto &B = std::get<0>(T);
+
+ auto &T = *Next;
+ auto &B = T.Buff;
Buf = B;
- B.Buffer = nullptr;
- B.Size = 0;
- Buffers.pop_front();
+ T.Used = true;
+ ++LiveBuffers;
+
+ if (++Next == (Buffers + BufferCount))
+ Next = Buffers;
+
return ErrorCode::Ok;
}
BufferQueue::ErrorCode BufferQueue::releaseBuffer(Buffer &Buf) {
- if (OwnedBuffers.count(Buf.Buffer) == 0)
+ // Blitz through the buffers array to find the buffer.
+ bool Found = false;
+ for (auto I = OwnedBuffers, E = OwnedBuffers + BufferCount; I != E; ++I) {
+ if (*I == Buf.Buffer) {
+ Found = true;
+ break;
+ }
+ }
+ if (!Found)
return ErrorCode::UnrecognizedBuffer;
- __sanitizer::BlockingMutexLock Guard(&Mutex);
+
+ __sanitizer::SpinMutexLock Guard(&Mutex);
+
+ // This points to a semantic bug, we really ought to not be releasing more
+ // buffers than we actually get.
+ if (LiveBuffers == 0)
+ return ErrorCode::NotEnoughMemory;
// Now that the buffer has been released, we mark it as "used".
- Buffers.emplace(Buffers.end(), Buf, true /* used */);
+ First->Buff = Buf;
+ First->Used = true;
Buf.Buffer = nullptr;
Buf.Size = 0;
+ --LiveBuffers;
+ if (++First == (Buffers + BufferCount))
+ First = Buffers;
+
return ErrorCode::Ok;
}
@@ -74,8 +103,12 @@ BufferQueue::ErrorCode BufferQueue::finalize() {
}
BufferQueue::~BufferQueue() {
- for (auto &T : Buffers) {
- auto &Buf = std::get<0>(T);
- free(Buf.Buffer);
+ for (auto I = Buffers, E = Buffers + BufferCount; I != E; ++I) {
+ auto &T = *I;
+ auto &Buf = T.Buff;
+ InternalFree(Buf.Buffer);
+ InternalFree(Buf.Extents);
}
+ delete[] Buffers;
+ delete[] OwnedBuffers;
}
diff --git a/contrib/compiler-rt/lib/xray/xray_buffer_queue.h b/contrib/compiler-rt/lib/xray/xray_buffer_queue.h
index e051695a297b..1ceb58274616 100644
--- a/contrib/compiler-rt/lib/xray/xray_buffer_queue.h
+++ b/contrib/compiler-rt/lib/xray/xray_buffer_queue.h
@@ -15,11 +15,9 @@
#ifndef XRAY_BUFFER_QUEUE_H
#define XRAY_BUFFER_QUEUE_H
+#include <cstddef>
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_mutex.h"
-#include <deque>
-#include <unordered_set>
-#include <utility>
namespace __xray {
@@ -29,23 +27,50 @@ namespace __xray {
/// the "flight data recorder" (FDR) mode to support ongoing XRay function call
/// trace collection.
class BufferQueue {
-public:
+ public:
+ struct alignas(64) BufferExtents {
+ __sanitizer::atomic_uint64_t Size;
+ };
+
struct Buffer {
void *Buffer = nullptr;
size_t Size = 0;
+ BufferExtents* Extents;
+ };
+
+ private:
+ struct BufferRep {
+ // The managed buffer.
+ Buffer Buff;
+
+ // This is true if the buffer has been returned to the available queue, and
+ // is considered "used" by another thread.
+ bool Used = false;
};
-private:
+ // Size of each individual Buffer.
size_t BufferSize;
- // We use a bool to indicate whether the Buffer has been used in this
- // freelist implementation.
- std::deque<std::tuple<Buffer, bool>> Buffers;
- __sanitizer::BlockingMutex Mutex;
- std::unordered_set<void *> OwnedBuffers;
+ BufferRep *Buffers;
+ size_t BufferCount;
+
+ __sanitizer::SpinMutex Mutex;
__sanitizer::atomic_uint8_t Finalizing;
-public:
+ // Pointers to buffers managed/owned by the BufferQueue.
+ void **OwnedBuffers;
+
+ // Pointer to the next buffer to be handed out.
+ BufferRep *Next;
+
+ // Pointer to the entry in the array where the next released buffer will be
+ // placed.
+ BufferRep *First;
+
+ // Count of buffers that have been handed out through 'getBuffer'.
+ size_t LiveBuffers;
+
+ public:
enum class ErrorCode : unsigned {
Ok,
NotEnoughMemory,
@@ -56,16 +81,16 @@ public:
static const char *getErrorString(ErrorCode E) {
switch (E) {
- case ErrorCode::Ok:
- return "(none)";
- case ErrorCode::NotEnoughMemory:
- return "no available buffers in the queue";
- case ErrorCode::QueueFinalizing:
- return "queue already finalizing";
- case ErrorCode::UnrecognizedBuffer:
- return "buffer being returned not owned by buffer queue";
- case ErrorCode::AlreadyFinalized:
- return "queue already finalized";
+ case ErrorCode::Ok:
+ return "(none)";
+ case ErrorCode::NotEnoughMemory:
+ return "no available buffers in the queue";
+ case ErrorCode::QueueFinalizing:
+ return "queue already finalizing";
+ case ErrorCode::UnrecognizedBuffer:
+ return "buffer being returned not owned by buffer queue";
+ case ErrorCode::AlreadyFinalized:
+ return "queue already finalized";
}
return "unknown error";
}
@@ -82,15 +107,18 @@ public:
/// - BufferQueue is not finalising.
///
/// Returns:
- /// - std::errc::not_enough_memory on exceeding MaxSize.
- /// - no error when we find a Buffer.
- /// - std::errc::state_not_recoverable on finalising BufferQueue.
+ /// - ErrorCode::NotEnoughMemory on exceeding MaxSize.
+ /// - ErrorCode::Ok when we find a Buffer.
+ /// - ErrorCode::QueueFinalizing or ErrorCode::AlreadyFinalized on
+ /// a finalizing/finalized BufferQueue.
ErrorCode getBuffer(Buffer &Buf);
/// Updates |Buf| to point to nullptr, with size 0.
///
/// Returns:
- /// - ...
+ /// - ErrorCode::Ok when we successfully release the buffer.
+ /// - ErrorCode::UnrecognizedBuffer for when this BufferQueue does not own
+ /// the buffer being released.
ErrorCode releaseBuffer(Buffer &Buf);
bool finalizing() const {
@@ -107,17 +135,18 @@ public:
/// - All releaseBuffer operations will not fail.
///
/// After a call to finalize succeeds, all subsequent calls to finalize will
- /// fail with std::errc::state_not_recoverable.
+ /// fail with ErrorCode::QueueFinalizing.
ErrorCode finalize();
/// Applies the provided function F to each Buffer in the queue, only if the
/// Buffer is marked 'used' (i.e. has been the result of getBuffer(...) and a
- /// releaseBuffer(...) operation.
- template <class F> void apply(F Fn) {
- __sanitizer::BlockingMutexLock G(&Mutex);
- for (const auto &T : Buffers) {
- if (std::get<1>(T))
- Fn(std::get<0>(T));
+ /// releaseBuffer(...) operation).
+ template <class F>
+ void apply(F Fn) {
+ __sanitizer::SpinMutexLock G(&Mutex);
+ for (auto I = Buffers, E = Buffers + BufferCount; I != E; ++I) {
+ const auto &T = *I;
+ if (T.Used) Fn(T.Buff);
}
}
@@ -125,6 +154,6 @@ public:
~BufferQueue();
};
-} // namespace __xray
+} // namespace __xray
-#endif // XRAY_BUFFER_QUEUE_H
+#endif // XRAY_BUFFER_QUEUE_H
diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h b/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h
index 3d6d38892c76..324208db82ca 100644
--- a/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h
+++ b/contrib/compiler-rt/lib/xray/xray_fdr_log_records.h
@@ -30,7 +30,10 @@ struct alignas(16) MetadataRecord {
TSCWrap,
WalltimeMarker,
CustomEventMarker,
+ CallArgument,
+ BufferExtents,
};
+
// Use 7 bits to identify this record type.
/* RecordKinds */ uint8_t RecordKind : 7;
char Data[15];
diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc b/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc
index a7e1382c3865..1bfa10c21f5c 100644
--- a/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc
+++ b/contrib/compiler-rt/lib/xray/xray_fdr_logging.cc
@@ -15,15 +15,11 @@
//
//===----------------------------------------------------------------------===//
#include "xray_fdr_logging.h"
-#include <algorithm>
-#include <bitset>
-#include <cerrno>
-#include <cstring>
+#include <errno.h>
#include <sys/syscall.h>
#include <sys/time.h>
#include <time.h>
#include <unistd.h>
-#include <unordered_map>
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
@@ -39,7 +35,7 @@
namespace __xray {
// Global BufferQueue.
-std::shared_ptr<BufferQueue> BQ;
+BufferQueue *BQ = nullptr;
__sanitizer::atomic_sint32_t LogFlushStatus = {
XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING};
@@ -52,19 +48,31 @@ __sanitizer::SpinMutex FDROptionsMutex;
XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT {
if (__sanitizer::atomic_load(&LoggingStatus,
__sanitizer::memory_order_acquire) !=
- XRayLogInitStatus::XRAY_LOG_FINALIZED)
+ XRayLogInitStatus::XRAY_LOG_FINALIZED) {
+ if (__sanitizer::Verbosity())
+ Report("Not flushing log, implementation is not finalized.\n");
return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING;
+ }
s32 Result = XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING;
if (!__sanitizer::atomic_compare_exchange_strong(
&LogFlushStatus, &Result, XRayLogFlushStatus::XRAY_LOG_FLUSHING,
- __sanitizer::memory_order_release))
+ __sanitizer::memory_order_release)) {
+
+ if (__sanitizer::Verbosity())
+ Report("Not flushing log, implementation is still finalizing.\n");
return static_cast<XRayLogFlushStatus>(Result);
+ }
- // Make a copy of the BufferQueue pointer to prevent other threads that may be
- // resetting it from blowing away the queue prematurely while we're dealing
- // with it.
- auto LocalBQ = BQ;
+ if (BQ == nullptr) {
+ if (__sanitizer::Verbosity())
+ Report("Cannot flush when global buffer queue is null.\n");
+ return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING;
+ }
+
+ // We wait a number of milliseconds to allow threads to see that we've
+ // finalised before attempting to flush the log.
+ __sanitizer::SleepForMillis(flags()->xray_fdr_log_grace_period_ms);
// We write out the file in the following format:
//
@@ -95,24 +103,44 @@ XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT {
TSCSupported ? getTSCFrequency() : __xray::NanosecondsPerSecond;
XRayFileHeader Header;
- Header.Version = 1;
+
+ // Version 2 of the log writes the extents of the buffer, instead of relying
+ // on an end-of-buffer record.
+ Header.Version = 2;
Header.Type = FileTypes::FDR_LOG;
Header.CycleFrequency = CycleFrequency;
+
// FIXME: Actually check whether we have 'constant_tsc' and 'nonstop_tsc'
// before setting the values in the header.
Header.ConstantTSC = 1;
Header.NonstopTSC = 1;
- Header.FdrData = FdrAdditionalHeaderData{LocalBQ->ConfiguredBufferSize()};
+ Header.FdrData = FdrAdditionalHeaderData{BQ->ConfiguredBufferSize()};
retryingWriteAll(Fd, reinterpret_cast<char *>(&Header),
reinterpret_cast<char *>(&Header) + sizeof(Header));
- LocalBQ->apply([&](const BufferQueue::Buffer &B) {
- uint64_t BufferSize = B.Size;
- if (BufferSize > 0) {
+ BQ->apply([&](const BufferQueue::Buffer &B) {
+ // Starting at version 2 of the FDR logging implementation, we only write
+ // the records identified by the extents of the buffer. We use the Extents
+ // from the Buffer and write that out as the first record in the buffer.
+ // We still use a Metadata record, but fill in the extents instead for the
+ // data.
+ MetadataRecord ExtentsRecord;
+ auto BufferExtents = __sanitizer::atomic_load(
+ &B.Extents->Size, __sanitizer::memory_order_acquire);
+ assert(BufferExtents <= B.Size);
+ ExtentsRecord.Type = uint8_t(RecordType::Metadata);
+ ExtentsRecord.RecordKind =
+ uint8_t(MetadataRecord::RecordKinds::BufferExtents);
+ std::memcpy(ExtentsRecord.Data, &BufferExtents, sizeof(BufferExtents));
+ if (BufferExtents > 0) {
+ retryingWriteAll(Fd, reinterpret_cast<char *>(&ExtentsRecord),
+ reinterpret_cast<char *>(&ExtentsRecord) +
+ sizeof(MetadataRecord));
retryingWriteAll(Fd, reinterpret_cast<char *>(B.Buffer),
- reinterpret_cast<char *>(B.Buffer) + B.Size);
+ reinterpret_cast<char *>(B.Buffer) + BufferExtents);
}
});
+
__sanitizer::atomic_store(&LogFlushStatus,
XRayLogFlushStatus::XRAY_LOG_FLUSHED,
__sanitizer::memory_order_release);
@@ -124,8 +152,11 @@ XRayLogInitStatus fdrLoggingFinalize() XRAY_NEVER_INSTRUMENT {
if (!__sanitizer::atomic_compare_exchange_strong(
&LoggingStatus, &CurrentStatus,
XRayLogInitStatus::XRAY_LOG_FINALIZING,
- __sanitizer::memory_order_release))
+ __sanitizer::memory_order_release)) {
+ if (__sanitizer::Verbosity())
+ Report("Cannot finalize log, implementation not initialized.\n");
return static_cast<XRayLogInitStatus>(CurrentStatus);
+ }
// Do special things to make the log finalize itself, and not allow any more
// operations to be performed until re-initialized.
@@ -146,7 +177,8 @@ XRayLogInitStatus fdrLoggingReset() XRAY_NEVER_INSTRUMENT {
return static_cast<XRayLogInitStatus>(CurrentStatus);
// Release the in-memory buffer queue.
- BQ.reset();
+ delete BQ;
+ BQ = nullptr;
// Spin until the flushing status is flushed.
s32 CurrentFlushingStatus = XRayLogFlushStatus::XRAY_LOG_FLUSHED;
@@ -163,19 +195,22 @@ XRayLogInitStatus fdrLoggingReset() XRAY_NEVER_INSTRUMENT {
return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
}
-static std::tuple<uint64_t, unsigned char>
-getTimestamp() XRAY_NEVER_INSTRUMENT {
+struct TSCAndCPU {
+ uint64_t TSC = 0;
+ unsigned char CPU = 0;
+};
+
+static TSCAndCPU getTimestamp() XRAY_NEVER_INSTRUMENT {
// We want to get the TSC as early as possible, so that we can check whether
// we've seen this CPU before. We also do it before we load anything else, to
// allow for forward progress with the scheduling.
- unsigned char CPU;
- uint64_t TSC;
+ TSCAndCPU Result;
// Test once for required CPU features
static bool TSCSupported = probeRequiredCPUFeatures();
if (TSCSupported) {
- TSC = __xray::readTSC(CPU);
+ Result.TSC = __xray::readTSC(Result.CPU);
} else {
// FIXME: This code needs refactoring as it appears in multiple locations
timespec TS;
@@ -184,32 +219,35 @@ getTimestamp() XRAY_NEVER_INSTRUMENT {
Report("clock_gettime(2) return %d, errno=%d", result, int(errno));
TS = {0, 0};
}
- CPU = 0;
- TSC = TS.tv_sec * __xray::NanosecondsPerSecond + TS.tv_nsec;
+ Result.CPU = 0;
+ Result.TSC = TS.tv_sec * __xray::NanosecondsPerSecond + TS.tv_nsec;
}
- return std::make_tuple(TSC, CPU);
+ return Result;
}
void fdrLoggingHandleArg0(int32_t FuncId,
XRayEntryType Entry) XRAY_NEVER_INSTRUMENT {
- auto TSC_CPU = getTimestamp();
- __xray_fdr_internal::processFunctionHook(FuncId, Entry, std::get<0>(TSC_CPU),
- std::get<1>(TSC_CPU), clock_gettime,
- LoggingStatus, BQ);
+ auto TC = getTimestamp();
+ __xray_fdr_internal::processFunctionHook(FuncId, Entry, TC.TSC, TC.CPU, 0,
+ clock_gettime, BQ);
+}
+
+void fdrLoggingHandleArg1(int32_t FuncId, XRayEntryType Entry,
+ uint64_t Arg) XRAY_NEVER_INSTRUMENT {
+ auto TC = getTimestamp();
+ __xray_fdr_internal::processFunctionHook(FuncId, Entry, TC.TSC, TC.CPU, Arg,
+ clock_gettime, BQ);
}
void fdrLoggingHandleCustomEvent(void *Event,
std::size_t EventSize) XRAY_NEVER_INSTRUMENT {
using namespace __xray_fdr_internal;
- auto TSC_CPU = getTimestamp();
- auto &TSC = std::get<0>(TSC_CPU);
- auto &CPU = std::get<1>(TSC_CPU);
- thread_local bool Running = false;
+ auto TC = getTimestamp();
+ auto &TSC = TC.TSC;
+ auto &CPU = TC.CPU;
RecursionGuard Guard{Running};
- if (!Guard) {
- assert(Running && "RecursionGuard is buggy!");
+ if (!Guard)
return;
- }
if (EventSize > std::numeric_limits<int32_t>::max()) {
using Empty = struct {};
static Empty Once = [&] {
@@ -220,15 +258,16 @@ void fdrLoggingHandleCustomEvent(void *Event,
(void)Once;
}
int32_t ReducedEventSize = static_cast<int32_t>(EventSize);
- if (!isLogInitializedAndReady(LocalBQ, TSC, CPU, clock_gettime))
+ auto &TLD = getThreadLocalData();
+ if (!isLogInitializedAndReady(TLD.BQ, TSC, CPU, clock_gettime))
return;
// Here we need to prepare the log to handle:
// - The metadata record we're going to write. (16 bytes)
// - The additional data we're going to write. Currently, that's the size of
// the event we're going to dump into the log as free-form bytes.
- if (!prepareBuffer(clock_gettime, MetadataRecSize + EventSize)) {
- LocalBQ = nullptr;
+ if (!prepareBuffer(TSC, CPU, clock_gettime, MetadataRecSize + EventSize)) {
+ TLD.BQ = nullptr;
return;
}
@@ -240,27 +279,35 @@ void fdrLoggingHandleCustomEvent(void *Event,
CustomEvent.Type = uint8_t(RecordType::Metadata);
CustomEvent.RecordKind =
uint8_t(MetadataRecord::RecordKinds::CustomEventMarker);
- constexpr auto TSCSize = sizeof(std::get<0>(TSC_CPU));
+ constexpr auto TSCSize = sizeof(TC.TSC);
std::memcpy(&CustomEvent.Data, &ReducedEventSize, sizeof(int32_t));
std::memcpy(&CustomEvent.Data[sizeof(int32_t)], &TSC, TSCSize);
- std::memcpy(RecordPtr, &CustomEvent, sizeof(CustomEvent));
- RecordPtr += sizeof(CustomEvent);
- std::memcpy(RecordPtr, Event, ReducedEventSize);
+ std::memcpy(TLD.RecordPtr, &CustomEvent, sizeof(CustomEvent));
+ TLD.RecordPtr += sizeof(CustomEvent);
+ std::memcpy(TLD.RecordPtr, Event, ReducedEventSize);
+ incrementExtents(MetadataRecSize + EventSize);
endBufferIfFull();
}
XRayLogInitStatus fdrLoggingInit(std::size_t BufferSize, std::size_t BufferMax,
void *Options,
size_t OptionsSize) XRAY_NEVER_INSTRUMENT {
- if (OptionsSize != sizeof(FDRLoggingOptions))
+ if (OptionsSize != sizeof(FDRLoggingOptions)) {
+ if (__sanitizer::Verbosity())
+ Report("Cannot initialize FDR logging; wrong size for options: %d\n",
+ OptionsSize);
return static_cast<XRayLogInitStatus>(__sanitizer::atomic_load(
&LoggingStatus, __sanitizer::memory_order_acquire));
+ }
s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
if (!__sanitizer::atomic_compare_exchange_strong(
&LoggingStatus, &CurrentStatus,
XRayLogInitStatus::XRAY_LOG_INITIALIZING,
- __sanitizer::memory_order_release))
+ __sanitizer::memory_order_release)) {
+ if (__sanitizer::Verbosity())
+ Report("Cannot initialize already initialized implementation.\n");
return static_cast<XRayLogInitStatus>(CurrentStatus);
+ }
{
__sanitizer::SpinMutexLock Guard(&FDROptionsMutex);
@@ -268,12 +315,40 @@ XRayLogInitStatus fdrLoggingInit(std::size_t BufferSize, std::size_t BufferMax,
}
bool Success = false;
- BQ = std::make_shared<BufferQueue>(BufferSize, BufferMax, Success);
+
+ if (BQ != nullptr) {
+ delete BQ;
+ BQ = nullptr;
+ }
+
+ if (BQ == nullptr)
+ BQ = new BufferQueue(BufferSize, BufferMax, Success);
+
if (!Success) {
Report("BufferQueue init failed.\n");
+ if (BQ != nullptr) {
+ delete BQ;
+ BQ = nullptr;
+ }
return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
}
+ static bool UNUSED Once = [] {
+ pthread_key_create(&__xray_fdr_internal::Key, +[](void *) {
+ auto &TLD = __xray_fdr_internal::getThreadLocalData();
+ if (TLD.BQ == nullptr)
+ return;
+ auto EC = TLD.BQ->releaseBuffer(TLD.Buffer);
+ if (EC != BufferQueue::ErrorCode::Ok)
+ Report("At thread exit, failed to release buffer at %p; error=%s\n",
+ TLD.Buffer.Buffer, BufferQueue::getErrorString(EC));
+ });
+ return false;
+ }();
+
+ // Arg1 handler should go in first to avoid concurrent code accidentally
+ // falling back to arg0 when it should have ran arg1.
+ __xray_set_handler_arg1(fdrLoggingHandleArg1);
// Install the actual handleArg0 handler after initialising the buffers.
__xray_set_handler(fdrLoggingHandleArg0);
__xray_set_customevent_handler(fdrLoggingHandleCustomEvent);
@@ -281,20 +356,31 @@ XRayLogInitStatus fdrLoggingInit(std::size_t BufferSize, std::size_t BufferMax,
__sanitizer::atomic_store(&LoggingStatus,
XRayLogInitStatus::XRAY_LOG_INITIALIZED,
__sanitizer::memory_order_release);
- Report("XRay FDR init successful.\n");
+
+ if (__sanitizer::Verbosity())
+ Report("XRay FDR init successful.\n");
return XRayLogInitStatus::XRAY_LOG_INITIALIZED;
}
-} // namespace __xray
-
-static auto UNUSED Unused = [] {
+bool fdrLogDynamicInitializer() XRAY_NEVER_INSTRUMENT {
using namespace __xray;
- if (flags()->xray_fdr_log) {
- XRayLogImpl Impl{
- fdrLoggingInit, fdrLoggingFinalize, fdrLoggingHandleArg0,
- fdrLoggingFlush,
- };
+ XRayLogImpl Impl{
+ fdrLoggingInit,
+ fdrLoggingFinalize,
+ fdrLoggingHandleArg0,
+ fdrLoggingFlush,
+ };
+ auto RegistrationResult = __xray_log_register_mode("xray-fdr", Impl);
+ if (RegistrationResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK &&
+ __sanitizer::Verbosity())
+ Report("Cannot register XRay FDR mode to 'xray-fdr'; error = %d\n",
+ RegistrationResult);
+ if (flags()->xray_fdr_log ||
+ !__sanitizer::internal_strcmp(flags()->xray_mode, "xray-fdr"))
__xray_set_log_impl(Impl);
- }
return true;
-}();
+}
+
+} // namespace __xray
+
+static auto UNUSED Unused = __xray::fdrLogDynamicInitializer();
diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_logging.h b/contrib/compiler-rt/lib/xray/xray_fdr_logging.h
index 426b54dc7884..1639d550a44c 100644
--- a/contrib/compiler-rt/lib/xray/xray_fdr_logging.h
+++ b/contrib/compiler-rt/lib/xray/xray_fdr_logging.h
@@ -30,6 +30,7 @@ XRayLogInitStatus fdrLoggingInit(size_t BufferSize, size_t BufferMax,
void *Options, size_t OptionsSize);
XRayLogInitStatus fdrLoggingFinalize();
void fdrLoggingHandleArg0(int32_t FuncId, XRayEntryType Entry);
+void fdrLoggingHandleArg1(int32_t FuncId, XRayEntryType Entry, uint64_t Arg1);
XRayLogFlushStatus fdrLoggingFlush();
XRayLogInitStatus fdrLoggingReset();
diff --git a/contrib/compiler-rt/lib/xray/xray_fdr_logging_impl.h b/contrib/compiler-rt/lib/xray/xray_fdr_logging_impl.h
index 4a1d80fd0eba..59eab55b2573 100644
--- a/contrib/compiler-rt/lib/xray/xray_fdr_logging_impl.h
+++ b/contrib/compiler-rt/lib/xray/xray_fdr_logging_impl.h
@@ -18,13 +18,13 @@
#define XRAY_XRAY_FDR_LOGGING_IMPL_H
#include <cassert>
-#include <cstdint>
+#include <cstddef>
#include <cstring>
#include <limits>
-#include <memory>
-#include <string>
+#include <pthread.h>
#include <sys/syscall.h>
#include <time.h>
+#include <type_traits>
#include <unistd.h>
#include "sanitizer_common/sanitizer_common.h"
@@ -52,132 +52,120 @@ __sanitizer::atomic_sint32_t LoggingStatus = {
/// cooperation with xray_fdr_logging class, so be careful and think twice.
namespace __xray_fdr_internal {
-/// Writes the new buffer record and wallclock time that begin a buffer for a
-/// thread to MemPtr and increments MemPtr. Bypasses the thread local state
-/// machine and writes directly to memory without checks.
-static void writeNewBufferPreamble(pid_t Tid, timespec TS, char *&MemPtr);
+/// Writes the new buffer record and wallclock time that begin a buffer for the
+/// current thread.
+static void writeNewBufferPreamble(pid_t Tid, timespec TS);
-/// Write a metadata record to switch to a new CPU to MemPtr and increments
-/// MemPtr. Bypasses the thread local state machine and writes directly to
-/// memory without checks.
-static void writeNewCPUIdMetadata(uint16_t CPU, uint64_t TSC, char *&MemPtr);
-
-/// Writes an EOB metadata record to MemPtr and increments MemPtr. Bypasses the
-/// thread local state machine and writes directly to memory without checks.
-static void writeEOBMetadata(char *&MemPtr);
-
-/// Writes a TSC Wrap metadata record to MemPtr and increments MemPtr. Bypasses
-/// the thread local state machine and directly writes to memory without checks.
-static void writeTSCWrapMetadata(uint64_t TSC, char *&MemPtr);
-
-/// Writes a Function Record to MemPtr and increments MemPtr. Bypasses the
-/// thread local state machine and writes the function record directly to
-/// memory.
+/// Writes a Function Record to the buffer associated with the current thread.
static void writeFunctionRecord(int FuncId, uint32_t TSCDelta,
- XRayEntryType EntryType, char *&MemPtr);
+ XRayEntryType EntryType);
/// Sets up a new buffer in thread_local storage and writes a preamble. The
/// wall_clock_reader function is used to populate the WallTimeRecord entry.
static void setupNewBuffer(int (*wall_clock_reader)(clockid_t,
struct timespec *));
-/// Called to record CPU time for a new CPU within the current thread.
-static void writeNewCPUIdMetadata(uint16_t CPU, uint64_t TSC);
-
-/// Called to close the buffer when the thread exhausts the buffer or when the
-/// thread exits (via a thread local variable destructor).
-static void writeEOBMetadata();
-
/// TSC Wrap records are written when a TSC delta encoding scheme overflows.
static void writeTSCWrapMetadata(uint64_t TSC);
-/// Here's where the meat of the processing happens. The writer captures
-/// function entry, exit and tail exit points with a time and will create
-/// TSCWrap, NewCPUId and Function records as necessary. The writer might
-/// walk backward through its buffer and erase trivial functions to avoid
-/// polluting the log and may use the buffer queue to obtain or release a
-/// buffer.
-static void processFunctionHook(int32_t FuncId, XRayEntryType Entry,
- uint64_t TSC, unsigned char CPU,
- int (*wall_clock_reader)(clockid_t,
- struct timespec *),
- __sanitizer::atomic_sint32_t &LoggingStatus,
- const std::shared_ptr<BufferQueue> &BQ);
+// Group together thread-local-data in a struct, then hide it behind a function
+// call so that it can be initialized on first use instead of as a global. We
+// force the alignment to 64-bytes for x86 cache line alignment, as this
+// structure is used in the hot path of implementation.
+struct alignas(64) ThreadLocalData {
+ BufferQueue::Buffer Buffer;
+ char *RecordPtr = nullptr;
+ // The number of FunctionEntry records immediately preceding RecordPtr.
+ uint8_t NumConsecutiveFnEnters = 0;
-//-----------------------------------------------------------------------------|
-// The rest of the file is implementation. |
-//-----------------------------------------------------------------------------|
-// Functions are implemented in the header for inlining since we don't want |
-// to grow the stack when we've hijacked the binary for logging. |
-//-----------------------------------------------------------------------------|
+ // The number of adjacent, consecutive pairs of FunctionEntry, Tail Exit
+ // records preceding RecordPtr.
+ uint8_t NumTailCalls = 0;
-namespace {
+ // We use a thread_local variable to keep track of which CPUs we've already
+ // run, and the TSC times for these CPUs. This allows us to stop repeating the
+ // CPU field in the function records.
+ //
+ // We assume that we'll support only 65536 CPUs for x86_64.
+ uint16_t CurrentCPU = std::numeric_limits<uint16_t>::max();
+ uint64_t LastTSC = 0;
+ uint64_t LastFunctionEntryTSC = 0;
-thread_local BufferQueue::Buffer Buffer;
-thread_local char *RecordPtr = nullptr;
+ // Make sure a thread that's ever called handleArg0 has a thread-local
+ // live reference to the buffer queue for this particular instance of
+ // FDRLogging, and that we're going to clean it up when the thread exits.
+ BufferQueue *BQ = nullptr;
+};
-// The number of FunctionEntry records immediately preceding RecordPtr.
-thread_local uint8_t NumConsecutiveFnEnters = 0;
+static_assert(std::is_trivially_destructible<ThreadLocalData>::value,
+ "ThreadLocalData must be trivially destructible");
-// The number of adjacent, consecutive pairs of FunctionEntry, Tail Exit
-// records preceding RecordPtr.
-thread_local uint8_t NumTailCalls = 0;
+static constexpr auto MetadataRecSize = sizeof(MetadataRecord);
+static constexpr auto FunctionRecSize = sizeof(FunctionRecord);
-constexpr auto MetadataRecSize = sizeof(MetadataRecord);
-constexpr auto FunctionRecSize = sizeof(FunctionRecord);
+// Use a global pthread key to identify thread-local data for logging.
+static pthread_key_t Key;
-// We use a thread_local variable to keep track of which CPUs we've already
-// run, and the TSC times for these CPUs. This allows us to stop repeating the
-// CPU field in the function records.
+// This function will initialize the thread-local data structure used by the FDR
+// logging implementation and return a reference to it. The implementation
+// details require a bit of care to maintain.
//
-// We assume that we'll support only 65536 CPUs for x86_64.
-thread_local uint16_t CurrentCPU = std::numeric_limits<uint16_t>::max();
-thread_local uint64_t LastTSC = 0;
-thread_local uint64_t LastFunctionEntryTSC = 0;
-
-class ThreadExitBufferCleanup {
- std::shared_ptr<BufferQueue> &Buffers;
- BufferQueue::Buffer &Buffer;
-
-public:
- explicit ThreadExitBufferCleanup(std::shared_ptr<BufferQueue> &BQ,
- BufferQueue::Buffer &Buffer)
- XRAY_NEVER_INSTRUMENT : Buffers(BQ),
- Buffer(Buffer) {}
-
- ~ThreadExitBufferCleanup() noexcept XRAY_NEVER_INSTRUMENT {
- if (RecordPtr == nullptr)
- return;
+// First, some requirements on the implementation in general:
+//
+// - XRay handlers should not call any memory allocation routines that may
+// delegate to an instrumented implementation. This means functions like
+// malloc() and free() should not be called while instrumenting.
+//
+// - We would like to use some thread-local data initialized on first-use of
+// the XRay instrumentation. These allow us to implement unsynchronized
+// routines that access resources associated with the thread.
+//
+// The implementation here uses a few mechanisms that allow us to provide both
+// the requirements listed above. We do this by:
+//
+// 1. Using a thread-local aligned storage buffer for representing the
+// ThreadLocalData struct. This data will be uninitialized memory by
+// design.
+//
+// 2. Not requiring a thread exit handler/implementation, keeping the
+// thread-local as purely a collection of references/data that do not
+// require cleanup.
+//
+// We're doing this to avoid using a `thread_local` object that has a
+// non-trivial destructor, because the C++ runtime might call std::malloc(...)
+// to register calls to destructors. Deadlocks may arise when, for example, an
+// externally provided malloc implementation is XRay instrumented, and
+// initializing the thread-locals involves calling into malloc. A malloc
+// implementation that does global synchronization might be holding a lock for a
+// critical section, calling a function that might be XRay instrumented (and
+// thus in turn calling into malloc by virtue of registration of the
+// thread_local's destructor).
+static ThreadLocalData &getThreadLocalData() {
+ static_assert(alignof(ThreadLocalData) >= 64,
+ "ThreadLocalData must be cache line aligned.");
+ thread_local ThreadLocalData TLD;
+ thread_local bool UNUSED ThreadOnce = [] {
+ pthread_setspecific(Key, &TLD);
+ return false;
+ }();
+ return TLD;
+}
- // We make sure that upon exit, a thread will write out the EOB
- // MetadataRecord in the thread-local log, and also release the buffer to
- // the queue.
- assert((RecordPtr + MetadataRecSize) - static_cast<char *>(Buffer.Buffer) >=
- static_cast<ptrdiff_t>(MetadataRecSize));
- if (Buffers) {
- writeEOBMetadata();
- auto EC = Buffers->releaseBuffer(Buffer);
- if (EC != BufferQueue::ErrorCode::Ok)
- Report("Failed to release buffer at %p; error=%s\n", Buffer.Buffer,
- BufferQueue::getErrorString(EC));
- Buffers = nullptr;
- return;
- }
- }
-};
+//-----------------------------------------------------------------------------|
+// The rest of the file is implementation. |
+//-----------------------------------------------------------------------------|
+// Functions are implemented in the header for inlining since we don't want |
+// to grow the stack when we've hijacked the binary for logging. |
+//-----------------------------------------------------------------------------|
-// Make sure a thread that's ever called handleArg0 has a thread-local
-// live reference to the buffer queue for this particular instance of
-// FDRLogging, and that we're going to clean it up when the thread exits.
-thread_local std::shared_ptr<BufferQueue> LocalBQ = nullptr;
-thread_local ThreadExitBufferCleanup Cleanup(LocalBQ, Buffer);
+namespace {
class RecursionGuard {
- bool &Running;
+ volatile bool &Running;
const bool Valid;
public:
- explicit RecursionGuard(bool &R) : Running(R), Valid(!R) {
+ explicit RecursionGuard(volatile bool &R) : Running(R), Valid(!R) {
if (Valid)
Running = true;
}
@@ -195,34 +183,29 @@ public:
}
};
-inline bool loggingInitialized(
- const __sanitizer::atomic_sint32_t &LoggingStatus) XRAY_NEVER_INSTRUMENT {
- return __sanitizer::atomic_load(&LoggingStatus,
- __sanitizer::memory_order_acquire) ==
- XRayLogInitStatus::XRAY_LOG_INITIALIZED;
-}
-
} // namespace
-inline void writeNewBufferPreamble(pid_t Tid, timespec TS,
- char *&MemPtr) XRAY_NEVER_INSTRUMENT {
+static void writeNewBufferPreamble(pid_t Tid,
+ timespec TS) XRAY_NEVER_INSTRUMENT {
static constexpr int InitRecordsCount = 2;
- std::aligned_storage<sizeof(MetadataRecord)>::type Records[InitRecordsCount];
+ auto &TLD = getThreadLocalData();
+ MetadataRecord Metadata[InitRecordsCount];
{
// Write out a MetadataRecord to signify that this is the start of a new
// buffer, associated with a particular thread, with a new CPU. For the
// data, we have 15 bytes to squeeze as much information as we can. At this
// point we only write down the following bytes:
// - Thread ID (pid_t, 4 bytes)
- auto &NewBuffer = *reinterpret_cast<MetadataRecord *>(&Records[0]);
+ auto &NewBuffer = Metadata[0];
NewBuffer.Type = uint8_t(RecordType::Metadata);
NewBuffer.RecordKind = uint8_t(MetadataRecord::RecordKinds::NewBuffer);
std::memcpy(&NewBuffer.Data, &Tid, sizeof(pid_t));
}
+
// Also write the WalltimeMarker record.
{
static_assert(sizeof(time_t) <= 8, "time_t needs to be at most 8 bytes");
- auto &WalltimeMarker = *reinterpret_cast<MetadataRecord *>(&Records[1]);
+ auto &WalltimeMarker = Metadata[1];
WalltimeMarker.Type = uint8_t(RecordType::Metadata);
WalltimeMarker.RecordKind =
uint8_t(MetadataRecord::RecordKinds::WalltimeMarker);
@@ -235,26 +218,48 @@ inline void writeNewBufferPreamble(pid_t Tid, timespec TS,
std::memcpy(WalltimeMarker.Data, &Seconds, sizeof(Seconds));
std::memcpy(WalltimeMarker.Data + sizeof(Seconds), &Micros, sizeof(Micros));
}
- std::memcpy(MemPtr, Records, sizeof(MetadataRecord) * InitRecordsCount);
- MemPtr += sizeof(MetadataRecord) * InitRecordsCount;
- NumConsecutiveFnEnters = 0;
- NumTailCalls = 0;
+
+ TLD.NumConsecutiveFnEnters = 0;
+ TLD.NumTailCalls = 0;
+ if (TLD.BQ == nullptr || TLD.BQ->finalizing())
+ return;
+ std::memcpy(TLD.RecordPtr, Metadata, sizeof(Metadata));
+ TLD.RecordPtr += sizeof(Metadata);
+ // Since we write out the extents as the first metadata record of the
+ // buffer, we need to write out the extents including the extents record.
+ __sanitizer::atomic_store(&TLD.Buffer.Extents->Size, sizeof(Metadata),
+ __sanitizer::memory_order_release);
}
inline void setupNewBuffer(int (*wall_clock_reader)(
clockid_t, struct timespec *)) XRAY_NEVER_INSTRUMENT {
- RecordPtr = static_cast<char *>(Buffer.Buffer);
+ auto &TLD = getThreadLocalData();
+ auto &B = TLD.Buffer;
+ TLD.RecordPtr = static_cast<char *>(B.Buffer);
pid_t Tid = syscall(SYS_gettid);
timespec TS{0, 0};
// This is typically clock_gettime, but callers have injection ability.
wall_clock_reader(CLOCK_MONOTONIC, &TS);
- writeNewBufferPreamble(Tid, TS, RecordPtr);
- NumConsecutiveFnEnters = 0;
- NumTailCalls = 0;
+ writeNewBufferPreamble(Tid, TS);
+ TLD.NumConsecutiveFnEnters = 0;
+ TLD.NumTailCalls = 0;
+}
+
+static void incrementExtents(size_t Add) {
+ auto &TLD = getThreadLocalData();
+ __sanitizer::atomic_fetch_add(&TLD.Buffer.Extents->Size, Add,
+ __sanitizer::memory_order_acq_rel);
}
-inline void writeNewCPUIdMetadata(uint16_t CPU, uint64_t TSC,
- char *&MemPtr) XRAY_NEVER_INSTRUMENT {
+static void decrementExtents(size_t Subtract) {
+ auto &TLD = getThreadLocalData();
+ __sanitizer::atomic_fetch_sub(&TLD.Buffer.Extents->Size, Subtract,
+ __sanitizer::memory_order_acq_rel);
+}
+
+inline void writeNewCPUIdMetadata(uint16_t CPU,
+ uint64_t TSC) XRAY_NEVER_INSTRUMENT {
+ auto &TLD = getThreadLocalData();
MetadataRecord NewCPUId;
NewCPUId.Type = uint8_t(RecordType::Metadata);
NewCPUId.RecordKind = uint8_t(MetadataRecord::RecordKinds::NewCPUId);
@@ -265,34 +270,15 @@ inline void writeNewCPUIdMetadata(uint16_t CPU, uint64_t TSC,
// Total = 10 bytes.
std::memcpy(&NewCPUId.Data, &CPU, sizeof(CPU));
std::memcpy(&NewCPUId.Data[sizeof(CPU)], &TSC, sizeof(TSC));
- std::memcpy(MemPtr, &NewCPUId, sizeof(MetadataRecord));
- MemPtr += sizeof(MetadataRecord);
- NumConsecutiveFnEnters = 0;
- NumTailCalls = 0;
-}
-
-inline void writeNewCPUIdMetadata(uint16_t CPU,
- uint64_t TSC) XRAY_NEVER_INSTRUMENT {
- writeNewCPUIdMetadata(CPU, TSC, RecordPtr);
-}
-
-inline void writeEOBMetadata(char *&MemPtr) XRAY_NEVER_INSTRUMENT {
- MetadataRecord EOBMeta;
- EOBMeta.Type = uint8_t(RecordType::Metadata);
- EOBMeta.RecordKind = uint8_t(MetadataRecord::RecordKinds::EndOfBuffer);
- // For now we don't write any bytes into the Data field.
- std::memcpy(MemPtr, &EOBMeta, sizeof(MetadataRecord));
- MemPtr += sizeof(MetadataRecord);
- NumConsecutiveFnEnters = 0;
- NumTailCalls = 0;
+ std::memcpy(TLD.RecordPtr, &NewCPUId, sizeof(MetadataRecord));
+ TLD.RecordPtr += sizeof(MetadataRecord);
+ TLD.NumConsecutiveFnEnters = 0;
+ TLD.NumTailCalls = 0;
+ incrementExtents(sizeof(MetadataRecord));
}
-inline void writeEOBMetadata() XRAY_NEVER_INSTRUMENT {
- writeEOBMetadata(RecordPtr);
-}
-
-inline void writeTSCWrapMetadata(uint64_t TSC,
- char *&MemPtr) XRAY_NEVER_INSTRUMENT {
+inline void writeTSCWrapMetadata(uint64_t TSC) XRAY_NEVER_INSTRUMENT {
+ auto &TLD = getThreadLocalData();
MetadataRecord TSCWrap;
TSCWrap.Type = uint8_t(RecordType::Metadata);
TSCWrap.RecordKind = uint8_t(MetadataRecord::RecordKinds::TSCWrap);
@@ -301,58 +287,67 @@ inline void writeTSCWrapMetadata(uint64_t TSC,
// - Full TSC (uint64_t, 8 bytes)
// Total = 8 bytes.
std::memcpy(&TSCWrap.Data, &TSC, sizeof(TSC));
- std::memcpy(MemPtr, &TSCWrap, sizeof(MetadataRecord));
- MemPtr += sizeof(MetadataRecord);
- NumConsecutiveFnEnters = 0;
- NumTailCalls = 0;
+ std::memcpy(TLD.RecordPtr, &TSCWrap, sizeof(MetadataRecord));
+ TLD.RecordPtr += sizeof(MetadataRecord);
+ TLD.NumConsecutiveFnEnters = 0;
+ TLD.NumTailCalls = 0;
+ incrementExtents(sizeof(MetadataRecord));
}
-inline void writeTSCWrapMetadata(uint64_t TSC) XRAY_NEVER_INSTRUMENT {
- writeTSCWrapMetadata(TSC, RecordPtr);
+// Call Argument metadata records store the arguments to a function in the
+// order of their appearance; holes are not supported by the buffer format.
+static inline void writeCallArgumentMetadata(uint64_t A) XRAY_NEVER_INSTRUMENT {
+ auto &TLD = getThreadLocalData();
+ MetadataRecord CallArg;
+ CallArg.Type = uint8_t(RecordType::Metadata);
+ CallArg.RecordKind = uint8_t(MetadataRecord::RecordKinds::CallArgument);
+
+ std::memcpy(CallArg.Data, &A, sizeof(A));
+ std::memcpy(TLD.RecordPtr, &CallArg, sizeof(MetadataRecord));
+ TLD.RecordPtr += sizeof(MetadataRecord);
+ incrementExtents(sizeof(MetadataRecord));
}
-inline void writeFunctionRecord(int FuncId, uint32_t TSCDelta,
- XRayEntryType EntryType,
- char *&MemPtr) XRAY_NEVER_INSTRUMENT {
- std::aligned_storage<sizeof(FunctionRecord), alignof(FunctionRecord)>::type
- AlignedFuncRecordBuffer;
- auto &FuncRecord =
- *reinterpret_cast<FunctionRecord *>(&AlignedFuncRecordBuffer);
+static inline void
+writeFunctionRecord(int FuncId, uint32_t TSCDelta,
+ XRayEntryType EntryType) XRAY_NEVER_INSTRUMENT {
+ FunctionRecord FuncRecord;
FuncRecord.Type = uint8_t(RecordType::Function);
// Only take 28 bits of the function id.
FuncRecord.FuncId = FuncId & ~(0x0F << 28);
FuncRecord.TSCDelta = TSCDelta;
+ auto &TLD = getThreadLocalData();
switch (EntryType) {
case XRayEntryType::ENTRY:
- ++NumConsecutiveFnEnters;
+ ++TLD.NumConsecutiveFnEnters;
FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionEnter);
break;
case XRayEntryType::LOG_ARGS_ENTRY:
// We should not rewind functions with logged args.
- NumConsecutiveFnEnters = 0;
- NumTailCalls = 0;
+ TLD.NumConsecutiveFnEnters = 0;
+ TLD.NumTailCalls = 0;
FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionEnter);
break;
case XRayEntryType::EXIT:
// If we've decided to log the function exit, we will never erase the log
// before it.
- NumConsecutiveFnEnters = 0;
- NumTailCalls = 0;
+ TLD.NumConsecutiveFnEnters = 0;
+ TLD.NumTailCalls = 0;
FuncRecord.RecordKind = uint8_t(FunctionRecord::RecordKinds::FunctionExit);
break;
case XRayEntryType::TAIL:
// If we just entered the function we're tail exiting from or erased every
// invocation since then, this function entry tail pair is a candidate to
// be erased when the child function exits.
- if (NumConsecutiveFnEnters > 0) {
- ++NumTailCalls;
- NumConsecutiveFnEnters = 0;
+ if (TLD.NumConsecutiveFnEnters > 0) {
+ ++TLD.NumTailCalls;
+ TLD.NumConsecutiveFnEnters = 0;
} else {
// We will never be able to erase this tail call since we have logged
// something in between the function entry and tail exit.
- NumTailCalls = 0;
- NumConsecutiveFnEnters = 0;
+ TLD.NumTailCalls = 0;
+ TLD.NumConsecutiveFnEnters = 0;
}
FuncRecord.RecordKind =
uint8_t(FunctionRecord::RecordKinds::FunctionTailExit);
@@ -370,8 +365,9 @@ inline void writeFunctionRecord(int FuncId, uint32_t TSCDelta,
}
}
- std::memcpy(MemPtr, &AlignedFuncRecordBuffer, sizeof(FunctionRecord));
- MemPtr += sizeof(FunctionRecord);
+ std::memcpy(TLD.RecordPtr, &FuncRecord, sizeof(FunctionRecord));
+ TLD.RecordPtr += sizeof(FunctionRecord);
+ incrementExtents(sizeof(FunctionRecord));
}
static uint64_t thresholdTicks() {
@@ -387,23 +383,21 @@ static uint64_t thresholdTicks() {
// "Function Entry" record and any "Tail Call Exit" records after that.
static void rewindRecentCall(uint64_t TSC, uint64_t &LastTSC,
uint64_t &LastFunctionEntryTSC, int32_t FuncId) {
- using AlignedFuncStorage =
- std::aligned_storage<sizeof(FunctionRecord),
- alignof(FunctionRecord)>::type;
- RecordPtr -= FunctionRecSize;
- AlignedFuncStorage AlignedFuncRecordBuffer;
- const auto &FuncRecord = *reinterpret_cast<FunctionRecord *>(
- std::memcpy(&AlignedFuncRecordBuffer, RecordPtr, FunctionRecSize));
+ auto &TLD = getThreadLocalData();
+ TLD.RecordPtr -= FunctionRecSize;
+ decrementExtents(FunctionRecSize);
+ FunctionRecord FuncRecord;
+ std::memcpy(&FuncRecord, TLD.RecordPtr, FunctionRecSize);
assert(FuncRecord.RecordKind ==
uint8_t(FunctionRecord::RecordKinds::FunctionEnter) &&
"Expected to find function entry recording when rewinding.");
assert(FuncRecord.FuncId == (FuncId & ~(0x0F << 28)) &&
"Expected matching function id when rewinding Exit");
- --NumConsecutiveFnEnters;
+ --TLD.NumConsecutiveFnEnters;
LastTSC -= FuncRecord.TSCDelta;
// We unwound one call. Update the state and return without writing a log.
- if (NumConsecutiveFnEnters != 0) {
+ if (TLD.NumConsecutiveFnEnters != 0) {
LastFunctionEntryTSC -= FuncRecord.TSCDelta;
return;
}
@@ -413,22 +407,19 @@ static void rewindRecentCall(uint64_t TSC, uint64_t &LastTSC,
// exited from via this exit.
LastFunctionEntryTSC = 0;
auto RewindingTSC = LastTSC;
- auto RewindingRecordPtr = RecordPtr - FunctionRecSize;
- while (NumTailCalls > 0) {
- AlignedFuncStorage TailExitRecordBuffer;
+ auto RewindingRecordPtr = TLD.RecordPtr - FunctionRecSize;
+ while (TLD.NumTailCalls > 0) {
// Rewind the TSC back over the TAIL EXIT record.
- const auto &ExpectedTailExit =
- *reinterpret_cast<FunctionRecord *>(std::memcpy(
- &TailExitRecordBuffer, RewindingRecordPtr, FunctionRecSize));
+ FunctionRecord ExpectedTailExit;
+ std::memcpy(&ExpectedTailExit, RewindingRecordPtr, FunctionRecSize);
assert(ExpectedTailExit.RecordKind ==
uint8_t(FunctionRecord::RecordKinds::FunctionTailExit) &&
"Expected to find tail exit when rewinding.");
RewindingRecordPtr -= FunctionRecSize;
RewindingTSC -= ExpectedTailExit.TSCDelta;
- AlignedFuncStorage FunctionEntryBuffer;
- const auto &ExpectedFunctionEntry = *reinterpret_cast<FunctionRecord *>(
- std::memcpy(&FunctionEntryBuffer, RewindingRecordPtr, FunctionRecSize));
+ FunctionRecord ExpectedFunctionEntry;
+ std::memcpy(&ExpectedFunctionEntry, RewindingRecordPtr, FunctionRecSize);
assert(ExpectedFunctionEntry.RecordKind ==
uint8_t(FunctionRecord::RecordKinds::FunctionEnter) &&
"Expected to find function entry when rewinding tail call.");
@@ -437,80 +428,87 @@ static void rewindRecentCall(uint64_t TSC, uint64_t &LastTSC,
// This tail call exceeded the threshold duration. It will not be erased.
if ((TSC - RewindingTSC) >= thresholdTicks()) {
- NumTailCalls = 0;
+ TLD.NumTailCalls = 0;
return;
}
// We can erase a tail exit pair that we're exiting through since
// its duration is under threshold.
- --NumTailCalls;
+ --TLD.NumTailCalls;
RewindingRecordPtr -= FunctionRecSize;
RewindingTSC -= ExpectedFunctionEntry.TSCDelta;
- RecordPtr -= 2 * FunctionRecSize;
+ TLD.RecordPtr -= 2 * FunctionRecSize;
LastTSC = RewindingTSC;
+ decrementExtents(2 * FunctionRecSize);
}
}
-inline bool releaseThreadLocalBuffer(BufferQueue *BQ) {
- auto EC = BQ->releaseBuffer(Buffer);
+inline bool releaseThreadLocalBuffer(BufferQueue &BQArg) {
+ auto &TLD = getThreadLocalData();
+ auto EC = BQArg.releaseBuffer(TLD.Buffer);
if (EC != BufferQueue::ErrorCode::Ok) {
- Report("Failed to release buffer at %p; error=%s\n", Buffer.Buffer,
+ Report("Failed to release buffer at %p; error=%s\n", TLD.Buffer.Buffer,
BufferQueue::getErrorString(EC));
return false;
}
return true;
}
-inline bool prepareBuffer(int (*wall_clock_reader)(clockid_t,
+inline bool prepareBuffer(uint64_t TSC, unsigned char CPU,
+ int (*wall_clock_reader)(clockid_t,
struct timespec *),
size_t MaxSize) XRAY_NEVER_INSTRUMENT {
- char *BufferStart = static_cast<char *>(Buffer.Buffer);
- if ((RecordPtr + MaxSize) > (BufferStart + Buffer.Size - MetadataRecSize)) {
- writeEOBMetadata();
- if (!releaseThreadLocalBuffer(LocalBQ.get()))
+ auto &TLD = getThreadLocalData();
+ char *BufferStart = static_cast<char *>(TLD.Buffer.Buffer);
+ if ((TLD.RecordPtr + MaxSize) > (BufferStart + TLD.Buffer.Size)) {
+ if (!releaseThreadLocalBuffer(*TLD.BQ))
return false;
- auto EC = LocalBQ->getBuffer(Buffer);
+ auto EC = TLD.BQ->getBuffer(TLD.Buffer);
if (EC != BufferQueue::ErrorCode::Ok) {
Report("Failed to acquire a buffer; error=%s\n",
BufferQueue::getErrorString(EC));
return false;
}
setupNewBuffer(wall_clock_reader);
+
+ // Always write the CPU metadata as the first record in the buffer.
+ writeNewCPUIdMetadata(CPU, TSC);
}
return true;
}
-inline bool isLogInitializedAndReady(
- std::shared_ptr<BufferQueue> &LocalBQ, uint64_t TSC, unsigned char CPU,
- int (*wall_clock_reader)(clockid_t,
- struct timespec *)) XRAY_NEVER_INSTRUMENT {
+inline bool
+isLogInitializedAndReady(BufferQueue *LBQ, uint64_t TSC, unsigned char CPU,
+ int (*wall_clock_reader)(clockid_t, struct timespec *))
+ XRAY_NEVER_INSTRUMENT {
// Bail out right away if logging is not initialized yet.
// We should take the opportunity to release the buffer though.
auto Status = __sanitizer::atomic_load(&LoggingStatus,
__sanitizer::memory_order_acquire);
+ auto &TLD = getThreadLocalData();
if (Status != XRayLogInitStatus::XRAY_LOG_INITIALIZED) {
- if (RecordPtr != nullptr &&
+ if (TLD.RecordPtr != nullptr &&
(Status == XRayLogInitStatus::XRAY_LOG_FINALIZING ||
Status == XRayLogInitStatus::XRAY_LOG_FINALIZED)) {
- writeEOBMetadata();
- if (!releaseThreadLocalBuffer(LocalBQ.get()))
+ if (!releaseThreadLocalBuffer(*LBQ))
return false;
- RecordPtr = nullptr;
- LocalBQ = nullptr;
+ TLD.RecordPtr = nullptr;
return false;
}
return false;
}
- if (!loggingInitialized(LoggingStatus) || LocalBQ->finalizing()) {
- writeEOBMetadata();
- if (!releaseThreadLocalBuffer(LocalBQ.get()))
+ if (__sanitizer::atomic_load(&LoggingStatus,
+ __sanitizer::memory_order_acquire) !=
+ XRayLogInitStatus::XRAY_LOG_INITIALIZED ||
+ LBQ->finalizing()) {
+ if (!releaseThreadLocalBuffer(*LBQ))
return false;
- RecordPtr = nullptr;
+ TLD.RecordPtr = nullptr;
}
- if (Buffer.Buffer == nullptr) {
- auto EC = LocalBQ->getBuffer(Buffer);
+ if (TLD.Buffer.Buffer == nullptr) {
+ auto EC = LBQ->getBuffer(TLD.Buffer);
if (EC != BufferQueue::ErrorCode::Ok) {
auto LS = __sanitizer::atomic_load(&LoggingStatus,
__sanitizer::memory_order_acquire);
@@ -522,51 +520,100 @@ inline bool isLogInitializedAndReady(
}
setupNewBuffer(wall_clock_reader);
+
+ // Always write the CPU metadata as the first record in the buffer.
+ writeNewCPUIdMetadata(CPU, TSC);
}
- if (CurrentCPU == std::numeric_limits<uint16_t>::max()) {
+ if (TLD.CurrentCPU == std::numeric_limits<uint16_t>::max()) {
// This means this is the first CPU this thread has ever run on. We set
// the current CPU and record this as the first TSC we've seen.
- CurrentCPU = CPU;
+ TLD.CurrentCPU = CPU;
writeNewCPUIdMetadata(CPU, TSC);
}
return true;
} // namespace __xray_fdr_internal
+// Compute the TSC difference between the time of measurement and the previous
+// event. There are a few interesting situations we need to account for:
+//
+// - The thread has migrated to a different CPU. If this is the case, then
+// we write down the following records:
+//
+// 1. A 'NewCPUId' Metadata record.
+// 2. A FunctionRecord with a 0 for the TSCDelta field.
+//
+// - The TSC delta is greater than the 32 bits we can store in a
+// FunctionRecord. In this case we write down the following records:
+//
+// 1. A 'TSCWrap' Metadata record.
+// 2. A FunctionRecord with a 0 for the TSCDelta field.
+//
+// - The TSC delta is representable within the 32 bits we can store in a
+// FunctionRecord. In this case we write down just a FunctionRecord with
+// the correct TSC delta.
+inline uint32_t writeCurrentCPUTSC(ThreadLocalData &TLD, uint64_t TSC,
+ uint8_t CPU) {
+ if (CPU != TLD.CurrentCPU) {
+ // We've moved to a new CPU.
+ writeNewCPUIdMetadata(CPU, TSC);
+ return 0;
+ }
+ // If the delta is greater than the range for a uint32_t, then we write out
+ // the TSC wrap metadata entry with the full TSC, and the TSC for the
+ // function record be 0.
+ uint64_t Delta = TSC - TLD.LastTSC;
+ if (Delta <= std::numeric_limits<uint32_t>::max())
+ return Delta;
+
+ writeTSCWrapMetadata(TSC);
+ return 0;
+}
+
inline void endBufferIfFull() XRAY_NEVER_INSTRUMENT {
- auto BufferStart = static_cast<char *>(Buffer.Buffer);
- if ((RecordPtr + MetadataRecSize) - BufferStart == MetadataRecSize) {
- writeEOBMetadata();
- if (!releaseThreadLocalBuffer(LocalBQ.get()))
+ auto &TLD = getThreadLocalData();
+ auto BufferStart = static_cast<char *>(TLD.Buffer.Buffer);
+ if ((TLD.RecordPtr + MetadataRecSize) - BufferStart <=
+ ptrdiff_t{MetadataRecSize}) {
+ if (!releaseThreadLocalBuffer(*TLD.BQ))
return;
- RecordPtr = nullptr;
+ TLD.RecordPtr = nullptr;
}
}
-inline void processFunctionHook(
- int32_t FuncId, XRayEntryType Entry, uint64_t TSC, unsigned char CPU,
- int (*wall_clock_reader)(clockid_t, struct timespec *),
- __sanitizer::atomic_sint32_t &LoggingStatus,
- const std::shared_ptr<BufferQueue> &BQ) XRAY_NEVER_INSTRUMENT {
+thread_local volatile bool Running = false;
+
+/// Here's where the meat of the processing happens. The writer captures
+/// function entry, exit and tail exit points with a time and will create
+/// TSCWrap, NewCPUId and Function records as necessary. The writer might
+/// walk backward through its buffer and erase trivial functions to avoid
+/// polluting the log and may use the buffer queue to obtain or release a
+/// buffer.
+inline void processFunctionHook(int32_t FuncId, XRayEntryType Entry,
+ uint64_t TSC, unsigned char CPU, uint64_t Arg1,
+ int (*wall_clock_reader)(clockid_t,
+ struct timespec *),
+ BufferQueue *BQ) XRAY_NEVER_INSTRUMENT {
// Prevent signal handler recursion, so in case we're already in a log writing
// mode and the signal handler comes in (and is also instrumented) then we
// don't want to be clobbering potentially partial writes already happening in
// the thread. We use a simple thread_local latch to only allow one on-going
// handleArg0 to happen at any given time.
- thread_local bool Running = false;
RecursionGuard Guard{Running};
if (!Guard) {
assert(Running == true && "RecursionGuard is buggy!");
return;
}
+ auto &TLD = getThreadLocalData();
+
// In case the reference has been cleaned up before, we make sure we
// initialize it to the provided BufferQueue.
- if (LocalBQ == nullptr)
- LocalBQ = BQ;
+ if (TLD.BQ == nullptr)
+ TLD.BQ = BQ;
- if (!isLogInitializedAndReady(LocalBQ, TSC, CPU, wall_clock_reader))
+ if (!isLogInitializedAndReady(TLD.BQ, TSC, CPU, wall_clock_reader))
return;
// Before we go setting up writing new function entries, we need to be really
@@ -579,10 +626,10 @@ inline void processFunctionHook(
// - The least number of bytes we will ever write is 8
// (sizeof(FunctionRecord)) only if the delta between the previous entry
// and this entry is within 32 bits.
- // - The most number of bytes we will ever write is 8 + 16 = 24. This is
- // computed by:
+ // - The most number of bytes we will ever write is 8 + 16 + 16 = 40.
+ // This is computed by:
//
- // sizeof(FunctionRecord) + sizeof(MetadataRecord)
+ // MaxSize = sizeof(FunctionRecord) + 2 * sizeof(MetadataRecord)
//
// These arise in the following cases:
//
@@ -596,77 +643,39 @@ inline void processFunctionHook(
// FunctionRecord.
// 3. When we learn about a new CPU ID, we need to write down a "new cpu
// id" MetadataRecord before writing out the actual FunctionRecord.
+ // 4. The second MetadataRecord is the optional function call argument.
//
- // - An End-of-Buffer (EOB) MetadataRecord is 16 bytes.
- //
- // So the math we need to do is to determine whether writing 24 bytes past the
- // current pointer leaves us with enough bytes to write the EOB
- // MetadataRecord. If we don't have enough space after writing as much as 24
- // bytes in the end of the buffer, we need to write out the EOB, get a new
- // Buffer, set it up properly before doing any further writing.
- //
- if (!prepareBuffer(wall_clock_reader, FunctionRecSize + MetadataRecSize)) {
- LocalBQ = nullptr;
+ // So the math we need to do is to determine whether writing 40 bytes past the
+ // current pointer exceeds the buffer's maximum size. If we don't have enough
+ // space to write 40 bytes in the buffer, we need get a new Buffer, set it up
+ // properly before doing any further writing.
+ size_t MaxSize = FunctionRecSize + 2 * MetadataRecSize;
+ if (!prepareBuffer(TSC, CPU, wall_clock_reader, MaxSize)) {
+ TLD.BQ = nullptr;
return;
}
- // By this point, we are now ready to write at most 24 bytes (one metadata
- // record and one function record).
- assert((RecordPtr + (MetadataRecSize + FunctionRecSize)) -
- static_cast<char *>(Buffer.Buffer) >=
+ // By this point, we are now ready to write up to 40 bytes (explained above).
+ assert((TLD.RecordPtr + MaxSize) - static_cast<char *>(TLD.Buffer.Buffer) >=
static_cast<ptrdiff_t>(MetadataRecSize) &&
"Misconfigured BufferQueue provided; Buffer size not large enough.");
- // Here we compute the TSC Delta. There are a few interesting situations we
- // need to account for:
- //
- // - The thread has migrated to a different CPU. If this is the case, then
- // we write down the following records:
- //
- // 1. A 'NewCPUId' Metadata record.
- // 2. A FunctionRecord with a 0 for the TSCDelta field.
- //
- // - The TSC delta is greater than the 32 bits we can store in a
- // FunctionRecord. In this case we write down the following records:
- //
- // 1. A 'TSCWrap' Metadata record.
- // 2. A FunctionRecord with a 0 for the TSCDelta field.
- //
- // - The TSC delta is representable within the 32 bits we can store in a
- // FunctionRecord. In this case we write down just a FunctionRecord with
- // the correct TSC delta.
- //
- uint32_t RecordTSCDelta = 0;
- if (CPU != CurrentCPU) {
- // We've moved to a new CPU.
- writeNewCPUIdMetadata(CPU, TSC);
- } else {
- // If the delta is greater than the range for a uint32_t, then we write out
- // the TSC wrap metadata entry with the full TSC, and the TSC for the
- // function record be 0.
- auto Delta = TSC - LastTSC;
- if (Delta > (1ULL << 32) - 1)
- writeTSCWrapMetadata(TSC);
- else
- RecordTSCDelta = Delta;
- }
-
- LastTSC = TSC;
- CurrentCPU = CPU;
+ auto RecordTSCDelta = writeCurrentCPUTSC(TLD, TSC, CPU);
+ TLD.LastTSC = TSC;
+ TLD.CurrentCPU = CPU;
switch (Entry) {
case XRayEntryType::ENTRY:
case XRayEntryType::LOG_ARGS_ENTRY:
// Update the thread local state for the next invocation.
- LastFunctionEntryTSC = TSC;
+ TLD.LastFunctionEntryTSC = TSC;
break;
case XRayEntryType::TAIL:
- break;
case XRayEntryType::EXIT:
// Break out and write the exit record if we can't erase any functions.
- if (NumConsecutiveFnEnters == 0 ||
- (TSC - LastFunctionEntryTSC) >= thresholdTicks())
+ if (TLD.NumConsecutiveFnEnters == 0 ||
+ (TSC - TLD.LastFunctionEntryTSC) >= thresholdTicks())
break;
- rewindRecentCall(TSC, LastTSC, LastFunctionEntryTSC, FuncId);
+ rewindRecentCall(TSC, TLD.LastTSC, TLD.LastFunctionEntryTSC, FuncId);
return; // without writing log.
case XRayEntryType::CUSTOM_EVENT: {
// This is a bug in patching, so we'll report it once and move on.
@@ -681,7 +690,9 @@ inline void processFunctionHook(
}
}
- writeFunctionRecord(FuncId, RecordTSCDelta, Entry, RecordPtr);
+ writeFunctionRecord(FuncId, RecordTSCDelta, Entry);
+ if (Entry == XRayEntryType::LOG_ARGS_ENTRY)
+ writeCallArgumentMetadata(Arg1);
// If we've exhausted the buffer by this time, we then release the buffer to
// make sure that other threads may start using this buffer.
diff --git a/contrib/compiler-rt/lib/xray/xray_flags.h b/contrib/compiler-rt/lib/xray/xray_flags.h
index f4e30283b8de..3ed5b8844cb4 100644
--- a/contrib/compiler-rt/lib/xray/xray_flags.h
+++ b/contrib/compiler-rt/lib/xray/xray_flags.h
@@ -16,6 +16,7 @@
#define XRAY_FLAGS_H
#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
namespace __xray {
diff --git a/contrib/compiler-rt/lib/xray/xray_flags.inc b/contrib/compiler-rt/lib/xray/xray_flags.inc
index 7ddce78eb413..29f1fce7d7f4 100644
--- a/contrib/compiler-rt/lib/xray/xray_flags.inc
+++ b/contrib/compiler-rt/lib/xray/xray_flags.inc
@@ -16,12 +16,34 @@
XRAY_FLAG(bool, patch_premain, false,
"Whether to patch instrumentation points before main.")
-XRAY_FLAG(bool, xray_naive_log, true,
- "Whether to install the naive log implementation.")
XRAY_FLAG(const char *, xray_logfile_base, "xray-log.",
"Filename base for the xray logfile.")
+XRAY_FLAG(const char *, xray_mode, "", "Mode to install by default.")
+XRAY_FLAG(uptr, xray_page_size_override, 0,
+ "Override the default page size for the system, in bytes. The size "
+ "should be a power-of-two.")
+
+// Basic (Naive) Mode logging options.
+XRAY_FLAG(bool, xray_naive_log, false,
+ "DEPRECATED: Use xray_mode=xray-basic instead.")
+XRAY_FLAG(int, xray_naive_log_func_duration_threshold_us, 5,
+ "Naive logging will try to skip functions that execute for fewer "
+ "microseconds than this threshold.")
+XRAY_FLAG(int, xray_naive_log_max_stack_depth, 64,
+ "Naive logging will keep track of at most this deep a call stack, "
+ "any more and the recordings will be droppped.")
+XRAY_FLAG(int, xray_naive_log_thread_buffer_size, 1024,
+ "The number of entries to keep on a per-thread buffer.")
+
+// FDR (Flight Data Recorder) Mode logging options.
XRAY_FLAG(bool, xray_fdr_log, false,
- "Whether to install the flight data recorder logging implementation.")
+ "DEPRECATED: Use xray_mode=xray-fdr instead.")
XRAY_FLAG(int, xray_fdr_log_func_duration_threshold_us, 5,
"FDR logging will try to skip functions that execute for fewer "
"microseconds than this threshold.")
+XRAY_FLAG(int, xray_fdr_log_grace_period_us, 0,
+ "DEPRECATED: use xray_fdr_log_grace_period_ms instead.")
+XRAY_FLAG(int, xray_fdr_log_grace_period_ms, 100,
+ "FDR logging will wait this much time in microseconds before "
+ "actually flushing the log; this gives a chance for threads to "
+ "notice that the log has been finalized and clean up.")
diff --git a/contrib/compiler-rt/lib/xray/xray_init.cc b/contrib/compiler-rt/lib/xray/xray_init.cc
index aa660baa9920..11892cb8b7a3 100644
--- a/contrib/compiler-rt/lib/xray/xray_init.cc
+++ b/contrib/compiler-rt/lib/xray/xray_init.cc
@@ -44,12 +44,31 @@ __sanitizer::atomic_uint8_t XRayInitialized{0};
__sanitizer::SpinMutex XRayInstrMapMutex;
XRaySledMap XRayInstrMap;
+// Global flag to determine whether the flags have been initialized.
+__sanitizer::atomic_uint8_t XRayFlagsInitialized{0};
+
+// A mutex to allow only one thread to initialize the XRay data structures.
+__sanitizer::SpinMutex XRayInitMutex;
+
// __xray_init() will do the actual loading of the current process' memory map
// and then proceed to look for the .xray_instr_map section/segment.
void __xray_init() XRAY_NEVER_INSTRUMENT {
- initializeFlags();
+ __sanitizer::SpinMutexLock Guard(&XRayInitMutex);
+ // Short-circuit if we've already initialized XRay before.
+ if (__sanitizer::atomic_load(&XRayInitialized,
+ __sanitizer::memory_order_acquire))
+ return;
+
+ if (!__sanitizer::atomic_load(&XRayFlagsInitialized,
+ __sanitizer::memory_order_acquire)) {
+ initializeFlags();
+ __sanitizer::atomic_store(&XRayFlagsInitialized, true,
+ __sanitizer::memory_order_release);
+ }
+
if (__start_xray_instr_map == nullptr) {
- Report("XRay instrumentation map missing. Not initializing XRay.\n");
+ if (Verbosity())
+ Report("XRay instrumentation map missing. Not initializing XRay.\n");
return;
}
@@ -63,9 +82,21 @@ void __xray_init() XRAY_NEVER_INSTRUMENT {
__sanitizer::atomic_store(&XRayInitialized, true,
__sanitizer::memory_order_release);
+#ifndef XRAY_NO_PREINIT
if (flags()->patch_premain)
__xray_patch();
+#endif
}
+#if !defined(XRAY_NO_PREINIT) && SANITIZER_CAN_USE_PREINIT_ARRAY
+// Only add the preinit array initialization if the sanitizers can.
__attribute__((section(".preinit_array"),
used)) void (*__local_xray_preinit)(void) = __xray_init;
+#else
+// If we cannot use the .preinit_array section, we should instead use dynamic
+// initialisation.
+static bool UNUSED __local_xray_dyninit = [] {
+ __xray_init();
+ return true;
+}();
+#endif
diff --git a/contrib/compiler-rt/lib/xray/xray_inmemory_log.cc b/contrib/compiler-rt/lib/xray/xray_inmemory_log.cc
index 83aecfaf7700..a27ffbcbd12e 100644
--- a/contrib/compiler-rt/lib/xray/xray_inmemory_log.cc
+++ b/contrib/compiler-rt/lib/xray/xray_inmemory_log.cc
@@ -16,80 +16,85 @@
//===----------------------------------------------------------------------===//
#include <cassert>
+#include <cstring>
+#include <errno.h>
#include <fcntl.h>
-#include <mutex>
+#include <pthread.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/types.h>
-#include <thread>
+#include <time.h>
#include <unistd.h>
+#include "sanitizer_common/sanitizer_allocator_internal.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "xray/xray_records.h"
#include "xray_defs.h"
#include "xray_flags.h"
+#include "xray_inmemory_log.h"
#include "xray_interface_internal.h"
#include "xray_tsc.h"
#include "xray_utils.h"
-// __xray_InMemoryRawLog will use a thread-local aligned buffer capped to a
-// certain size (32kb by default) and use it as if it were a circular buffer for
-// events. We store simple fixed-sized entries in the log for external analysis.
-
-extern "C" {
-void __xray_InMemoryRawLog(int32_t FuncId,
- XRayEntryType Type) XRAY_NEVER_INSTRUMENT;
-}
-
namespace __xray {
-std::mutex LogMutex;
+__sanitizer::SpinMutex LogMutex;
-class ThreadExitFlusher {
- int Fd;
- XRayRecord *Start;
- size_t &Offset;
+// We use elements of this type to record the entry TSC of every function ID we
+// see as we're tracing a particular thread's execution.
+struct alignas(16) StackEntry {
+ int32_t FuncId;
+ uint16_t Type;
+ uint8_t CPU;
+ uint8_t Padding;
+ uint64_t TSC;
+};
-public:
- explicit ThreadExitFlusher(int Fd, XRayRecord *Start,
- size_t &Offset) XRAY_NEVER_INSTRUMENT
- : Fd(Fd),
- Start(Start),
- Offset(Offset) {}
+static_assert(sizeof(StackEntry) == 16, "Wrong size for StackEntry");
- ~ThreadExitFlusher() XRAY_NEVER_INSTRUMENT {
- std::lock_guard<std::mutex> L(LogMutex);
- if (Fd > 0 && Start != nullptr) {
- retryingWriteAll(Fd, reinterpret_cast<char *>(Start),
- reinterpret_cast<char *>(Start + Offset));
- // Because this thread's exit could be the last one trying to write to the
- // file and that we're not able to close out the file properly, we sync
- // instead and hope that the pending writes are flushed as the thread
- // exits.
- fsync(Fd);
- }
- }
+struct alignas(64) ThreadLocalData {
+ void *InMemoryBuffer = nullptr;
+ size_t BufferSize = 0;
+ size_t BufferOffset = 0;
+ void *ShadowStack = nullptr;
+ size_t StackSize = 0;
+ size_t StackEntries = 0;
+ int Fd = -1;
+ pid_t TID = 0;
};
-} // namespace __xray
+static pthread_key_t PThreadKey;
+
+static __sanitizer::atomic_uint8_t BasicInitialized{0};
+
+BasicLoggingOptions GlobalOptions;
-using namespace __xray;
+thread_local volatile bool RecursionGuard = false;
+
+static uint64_t thresholdTicks() XRAY_NEVER_INSTRUMENT {
+ static uint64_t TicksPerSec = probeRequiredCPUFeatures()
+ ? getTSCFrequency()
+ : __xray::NanosecondsPerSecond;
+ static const uint64_t ThresholdTicks =
+ TicksPerSec * GlobalOptions.DurationFilterMicros / 1000000;
+ return ThresholdTicks;
+}
-static int __xray_OpenLogFile() XRAY_NEVER_INSTRUMENT {
+static int openLogFile() XRAY_NEVER_INSTRUMENT {
int F = getLogFD();
if (F == -1)
return -1;
// Test for required CPU features and cache the cycle frequency
static bool TSCSupported = probeRequiredCPUFeatures();
- static uint64_t CycleFrequency = TSCSupported ? getTSCFrequency()
- : __xray::NanosecondsPerSecond;
+ static uint64_t CycleFrequency =
+ TSCSupported ? getTSCFrequency() : __xray::NanosecondsPerSecond;
// Since we're here, we get to write the header. We set it up so that the
// header will only be written once, at the start, and let the threads
// logging do writes which just append.
XRayFileHeader Header;
- Header.Version = 1;
+ Header.Version = 2; // Version 2 includes tail exit records.
Header.Type = FileTypes::NAIVE_LOG;
Header.CycleFrequency = CycleFrequency;
@@ -102,47 +107,210 @@ static int __xray_OpenLogFile() XRAY_NEVER_INSTRUMENT {
return F;
}
+int getGlobalFd() XRAY_NEVER_INSTRUMENT {
+ static int Fd = openLogFile();
+ return Fd;
+}
+
+ThreadLocalData &getThreadLocalData() XRAY_NEVER_INSTRUMENT {
+ thread_local ThreadLocalData TLD;
+ thread_local bool UNUSED TOnce = [] {
+ if (GlobalOptions.ThreadBufferSize == 0) {
+ if (__sanitizer::Verbosity())
+ Report("Not initializing TLD since ThreadBufferSize == 0.\n");
+ return false;
+ }
+ TLD.TID = __sanitizer::GetTid();
+ pthread_setspecific(PThreadKey, &TLD);
+ TLD.Fd = getGlobalFd();
+ TLD.InMemoryBuffer = reinterpret_cast<XRayRecord *>(
+ InternalAlloc(sizeof(XRayRecord) * GlobalOptions.ThreadBufferSize,
+ nullptr, alignof(XRayRecord)));
+ TLD.BufferSize = GlobalOptions.ThreadBufferSize;
+ TLD.BufferOffset = 0;
+ if (GlobalOptions.MaxStackDepth == 0) {
+ if (__sanitizer::Verbosity())
+ Report("Not initializing the ShadowStack since MaxStackDepth == 0.\n");
+ TLD.StackSize = 0;
+ TLD.StackEntries = 0;
+ TLD.ShadowStack = nullptr;
+ return false;
+ }
+ TLD.ShadowStack = reinterpret_cast<StackEntry *>(
+ InternalAlloc(sizeof(StackEntry) * GlobalOptions.MaxStackDepth, nullptr,
+ alignof(StackEntry)));
+ TLD.StackSize = GlobalOptions.MaxStackDepth;
+ TLD.StackEntries = 0;
+ if (__sanitizer::Verbosity() >= 2) {
+ static auto UNUSED Once = [] {
+ auto ticks = thresholdTicks();
+ Report("Ticks threshold: %d\n", ticks);
+ return false;
+ }();
+ }
+ return false;
+ }();
+ return TLD;
+}
+
template <class RDTSC>
-void __xray_InMemoryRawLog(int32_t FuncId, XRayEntryType Type,
- RDTSC ReadTSC) XRAY_NEVER_INSTRUMENT {
- using Buffer =
- std::aligned_storage<sizeof(XRayRecord), alignof(XRayRecord)>::type;
- static constexpr size_t BuffLen = 1024;
- thread_local static Buffer InMemoryBuffer[BuffLen] = {};
- thread_local static size_t Offset = 0;
- static int Fd = __xray_OpenLogFile();
+void InMemoryRawLog(int32_t FuncId, XRayEntryType Type,
+ RDTSC ReadTSC) XRAY_NEVER_INSTRUMENT {
+ auto &TLD = getThreadLocalData();
+ auto &InMemoryBuffer = TLD.InMemoryBuffer;
+ int Fd = getGlobalFd();
if (Fd == -1)
return;
- thread_local __xray::ThreadExitFlusher Flusher(
- Fd, reinterpret_cast<__xray::XRayRecord *>(InMemoryBuffer), Offset);
- thread_local pid_t TId = syscall(SYS_gettid);
- // First we get the useful data, and stuff it into the already aligned buffer
- // through a pointer offset.
- auto &R = reinterpret_cast<__xray::XRayRecord *>(InMemoryBuffer)[Offset];
+ // Use a simple recursion guard, to handle cases where we're already logging
+ // and for one reason or another, this function gets called again in the same
+ // thread.
+ if (RecursionGuard)
+ return;
+ RecursionGuard = true;
+ auto ExitGuard = __sanitizer::at_scope_exit([] { RecursionGuard = false; });
+
+ uint8_t CPU = 0;
+ uint64_t TSC = ReadTSC(CPU);
+
+ switch (Type) {
+ case XRayEntryType::ENTRY:
+ case XRayEntryType::LOG_ARGS_ENTRY: {
+ // Short circuit if we've reached the maximum depth of the stack.
+ if (TLD.StackEntries++ >= TLD.StackSize)
+ return;
+
+ // When we encounter an entry event, we keep track of the TSC and the CPU,
+ // and put it in the stack.
+ StackEntry E;
+ E.FuncId = FuncId;
+ E.CPU = CPU;
+ E.Type = Type;
+ E.TSC = TSC;
+ auto StackEntryPtr = static_cast<char *>(TLD.ShadowStack) +
+ (sizeof(StackEntry) * (TLD.StackEntries - 1));
+ __sanitizer::internal_memcpy(StackEntryPtr, &E, sizeof(StackEntry));
+ break;
+ }
+ case XRayEntryType::EXIT:
+ case XRayEntryType::TAIL: {
+ if (TLD.StackEntries == 0)
+ break;
+
+ if (--TLD.StackEntries >= TLD.StackSize)
+ return;
+
+ // When we encounter an exit event, we check whether all the following are
+ // true:
+ //
+ // - The Function ID is the same as the most recent entry in the stack.
+ // - The CPU is the same as the most recent entry in the stack.
+ // - The Delta of the TSCs is less than the threshold amount of time we're
+ // looking to record.
+ //
+ // If all of these conditions are true, we pop the stack and don't write a
+ // record and move the record offset back.
+ StackEntry StackTop;
+ auto StackEntryPtr = static_cast<char *>(TLD.ShadowStack) +
+ (sizeof(StackEntry) * TLD.StackEntries);
+ __sanitizer::internal_memcpy(&StackTop, StackEntryPtr, sizeof(StackEntry));
+ if (StackTop.FuncId == FuncId && StackTop.CPU == CPU &&
+ StackTop.TSC < TSC) {
+ auto Delta = TSC - StackTop.TSC;
+ if (Delta < thresholdTicks()) {
+ assert(TLD.BufferOffset > 0);
+ TLD.BufferOffset -= StackTop.Type == XRayEntryType::ENTRY ? 1 : 2;
+ return;
+ }
+ }
+ break;
+ }
+ default:
+ // Should be unreachable.
+ assert(false && "Unsupported XRayEntryType encountered.");
+ break;
+ }
+
+ // First determine whether the delta between the function's enter record and
+ // the exit record is higher than the threshold.
+ __xray::XRayRecord R;
R.RecordType = RecordTypes::NORMAL;
- R.TSC = ReadTSC(R.CPU);
- R.TId = TId;
+ R.CPU = CPU;
+ R.TSC = TSC;
+ R.TId = TLD.TID;
R.Type = Type;
R.FuncId = FuncId;
- ++Offset;
- if (Offset == BuffLen) {
- std::lock_guard<std::mutex> L(LogMutex);
+ auto EntryPtr = static_cast<char *>(InMemoryBuffer) +
+ (sizeof(__xray::XRayRecord) * TLD.BufferOffset);
+ __sanitizer::internal_memcpy(EntryPtr, &R, sizeof(R));
+ if (++TLD.BufferOffset == TLD.BufferSize) {
+ __sanitizer::SpinMutexLock L(&LogMutex);
+ auto RecordBuffer = reinterpret_cast<__xray::XRayRecord *>(InMemoryBuffer);
+ retryingWriteAll(Fd, reinterpret_cast<char *>(RecordBuffer),
+ reinterpret_cast<char *>(RecordBuffer + TLD.BufferOffset));
+ TLD.BufferOffset = 0;
+ TLD.StackEntries = 0;
+ }
+}
+
+template <class RDTSC>
+void InMemoryRawLogWithArg(int32_t FuncId, XRayEntryType Type, uint64_t Arg1,
+ RDTSC ReadTSC) XRAY_NEVER_INSTRUMENT {
+ auto &TLD = getThreadLocalData();
+ auto &InMemoryBuffer = TLD.InMemoryBuffer;
+ auto &Offset = TLD.BufferOffset;
+ const auto &BuffLen = TLD.BufferSize;
+ int Fd = getGlobalFd();
+ if (Fd == -1)
+ return;
+
+ // First we check whether there's enough space to write the data consecutively
+ // in the thread-local buffer. If not, we first flush the buffer before
+ // attempting to write the two records that must be consecutive.
+ if (Offset + 2 > BuffLen) {
+ __sanitizer::SpinMutexLock L(&LogMutex);
auto RecordBuffer = reinterpret_cast<__xray::XRayRecord *>(InMemoryBuffer);
retryingWriteAll(Fd, reinterpret_cast<char *>(RecordBuffer),
reinterpret_cast<char *>(RecordBuffer + Offset));
Offset = 0;
+ TLD.StackEntries = 0;
+ }
+
+ // Then we write the "we have an argument" record.
+ InMemoryRawLog(FuncId, Type, ReadTSC);
+
+ if (RecursionGuard)
+ return;
+ RecursionGuard = true;
+ auto ExitGuard = __sanitizer::at_scope_exit([] { RecursionGuard = false; });
+
+ // And from here on write the arg payload.
+ __xray::XRayArgPayload R;
+ R.RecordType = RecordTypes::ARG_PAYLOAD;
+ R.FuncId = FuncId;
+ R.TId = TLD.TID;
+ R.Arg = Arg1;
+ auto EntryPtr =
+ &reinterpret_cast<__xray::XRayArgPayload *>(&InMemoryBuffer)[Offset];
+ std::memcpy(EntryPtr, &R, sizeof(R));
+ if (++Offset == BuffLen) {
+ __sanitizer::SpinMutexLock L(&LogMutex);
+ auto RecordBuffer = reinterpret_cast<__xray::XRayRecord *>(InMemoryBuffer);
+ retryingWriteAll(Fd, reinterpret_cast<char *>(RecordBuffer),
+ reinterpret_cast<char *>(RecordBuffer + Offset));
+ Offset = 0;
+ TLD.StackEntries = 0;
}
}
-void __xray_InMemoryRawLogRealTSC(int32_t FuncId,
- XRayEntryType Type) XRAY_NEVER_INSTRUMENT {
- __xray_InMemoryRawLog(FuncId, Type, __xray::readTSC);
+void basicLoggingHandleArg0RealTSC(int32_t FuncId,
+ XRayEntryType Type) XRAY_NEVER_INSTRUMENT {
+ InMemoryRawLog(FuncId, Type, __xray::readTSC);
}
-void __xray_InMemoryEmulateTSC(int32_t FuncId,
- XRayEntryType Type) XRAY_NEVER_INSTRUMENT {
- __xray_InMemoryRawLog(FuncId, Type, [](uint8_t &CPU) XRAY_NEVER_INSTRUMENT {
+void basicLoggingHandleArg0EmulateTSC(int32_t FuncId, XRayEntryType Type)
+ XRAY_NEVER_INSTRUMENT {
+ InMemoryRawLog(FuncId, Type, [](uint8_t &CPU) XRAY_NEVER_INSTRUMENT {
timespec TS;
int result = clock_gettime(CLOCK_REALTIME, &TS);
if (result != 0) {
@@ -154,13 +322,150 @@ void __xray_InMemoryEmulateTSC(int32_t FuncId,
});
}
-static auto UNUSED Unused = [] {
- auto UseRealTSC = probeRequiredCPUFeatures();
- if (!UseRealTSC)
+void basicLoggingHandleArg1RealTSC(int32_t FuncId, XRayEntryType Type,
+ uint64_t Arg1) XRAY_NEVER_INSTRUMENT {
+ InMemoryRawLogWithArg(FuncId, Type, Arg1, __xray::readTSC);
+}
+
+void basicLoggingHandleArg1EmulateTSC(int32_t FuncId, XRayEntryType Type,
+ uint64_t Arg1) XRAY_NEVER_INSTRUMENT {
+ InMemoryRawLogWithArg(
+ FuncId, Type, Arg1, [](uint8_t &CPU) XRAY_NEVER_INSTRUMENT {
+ timespec TS;
+ int result = clock_gettime(CLOCK_REALTIME, &TS);
+ if (result != 0) {
+ Report("clock_gettimg(2) return %d, errno=%d.", result, int(errno));
+ TS = {0, 0};
+ }
+ CPU = 0;
+ return TS.tv_sec * __xray::NanosecondsPerSecond + TS.tv_nsec;
+ });
+}
+
+static void TLDDestructor(void *P) XRAY_NEVER_INSTRUMENT {
+ ThreadLocalData &TLD = *reinterpret_cast<ThreadLocalData *>(P);
+ auto ExitGuard = __sanitizer::at_scope_exit([&TLD] {
+ // Clean up dynamic resources.
+ if (TLD.InMemoryBuffer)
+ InternalFree(TLD.InMemoryBuffer);
+ if (TLD.ShadowStack)
+ InternalFree(TLD.ShadowStack);
+ if (__sanitizer::Verbosity())
+ Report("Cleaned up log for TID: %d\n", TLD.TID);
+ });
+
+ if (TLD.Fd == -1 || TLD.BufferOffset == 0) {
+ if (__sanitizer::Verbosity())
+ Report("Skipping buffer for TID: %d; Fd = %d; Offset = %llu\n", TLD.TID,
+ TLD.Fd, TLD.BufferOffset);
+ return;
+ }
+
+ {
+ __sanitizer::SpinMutexLock L(&LogMutex);
+ retryingWriteAll(TLD.Fd, reinterpret_cast<char *>(TLD.InMemoryBuffer),
+ reinterpret_cast<char *>(TLD.InMemoryBuffer) +
+ (sizeof(__xray::XRayRecord) * TLD.BufferOffset));
+ }
+
+ // Because this thread's exit could be the last one trying to write to
+ // the file and that we're not able to close out the file properly, we
+ // sync instead and hope that the pending writes are flushed as the
+ // thread exits.
+ fsync(TLD.Fd);
+}
+
+XRayLogInitStatus basicLoggingInit(size_t BufferSize, size_t BufferMax,
+ void *Options,
+ size_t OptionsSize) XRAY_NEVER_INSTRUMENT {
+ static bool UNUSED Once = [] {
+ pthread_key_create(&PThreadKey, TLDDestructor);
+ return false;
+ }();
+
+ uint8_t Expected = 0;
+ if (!__sanitizer::atomic_compare_exchange_strong(
+ &BasicInitialized, &Expected, 1, __sanitizer::memory_order_acq_rel)) {
+ if (__sanitizer::Verbosity())
+ Report("Basic logging already initialized.\n");
+ return XRayLogInitStatus::XRAY_LOG_INITIALIZED;
+ }
+
+ if (OptionsSize != sizeof(BasicLoggingOptions)) {
+ Report("Invalid options size, potential ABI mismatch; expected %d got %d",
+ sizeof(BasicLoggingOptions), OptionsSize);
+ return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
+ }
+
+ static auto UseRealTSC = probeRequiredCPUFeatures();
+ if (!UseRealTSC && __sanitizer::Verbosity())
Report("WARNING: Required CPU features missing for XRay instrumentation, "
"using emulation instead.\n");
- if (flags()->xray_naive_log)
- __xray_set_handler(UseRealTSC ? __xray_InMemoryRawLogRealTSC
- : __xray_InMemoryEmulateTSC);
+
+ GlobalOptions = *reinterpret_cast<BasicLoggingOptions *>(Options);
+ __xray_set_handler_arg1(UseRealTSC ? basicLoggingHandleArg1RealTSC
+ : basicLoggingHandleArg1EmulateTSC);
+ __xray_set_handler(UseRealTSC ? basicLoggingHandleArg0RealTSC
+ : basicLoggingHandleArg0EmulateTSC);
+ __xray_remove_customevent_handler();
+
+ return XRayLogInitStatus::XRAY_LOG_INITIALIZED;
+}
+
+XRayLogInitStatus basicLoggingFinalize() XRAY_NEVER_INSTRUMENT {
+ uint8_t Expected = 0;
+ if (!__sanitizer::atomic_compare_exchange_strong(
+ &BasicInitialized, &Expected, 0, __sanitizer::memory_order_acq_rel) &&
+ __sanitizer::Verbosity())
+ Report("Basic logging already finalized.\n");
+
+ // Nothing really to do aside from marking state of the global to be
+ // uninitialized.
+
+ return XRayLogInitStatus::XRAY_LOG_FINALIZED;
+}
+
+XRayLogFlushStatus basicLoggingFlush() XRAY_NEVER_INSTRUMENT {
+ // This really does nothing, since flushing the logs happen at the end of a
+ // thread's lifetime, or when the buffers are full.
+ return XRayLogFlushStatus::XRAY_LOG_FLUSHED;
+}
+
+// This is a handler that, effectively, does nothing.
+void basicLoggingHandleArg0Empty(int32_t, XRayEntryType) XRAY_NEVER_INSTRUMENT {
+}
+
+bool basicLogDynamicInitializer() XRAY_NEVER_INSTRUMENT {
+ XRayLogImpl Impl{
+ basicLoggingInit,
+ basicLoggingFinalize,
+ basicLoggingHandleArg0Empty,
+ basicLoggingFlush,
+ };
+ auto RegistrationResult = __xray_log_register_mode("xray-basic", Impl);
+ if (RegistrationResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK &&
+ __sanitizer::Verbosity())
+ Report("Cannot register XRay Basic Mode to 'xray-basic'; error = %d\n",
+ RegistrationResult);
+ if (flags()->xray_naive_log ||
+ !__sanitizer::internal_strcmp(flags()->xray_mode, "xray-basic")) {
+ __xray_set_log_impl(Impl);
+ BasicLoggingOptions Options;
+ Options.DurationFilterMicros =
+ flags()->xray_naive_log_func_duration_threshold_us;
+ Options.MaxStackDepth = flags()->xray_naive_log_max_stack_depth;
+ Options.ThreadBufferSize = flags()->xray_naive_log_thread_buffer_size;
+ __xray_log_init(flags()->xray_naive_log_thread_buffer_size, 0, &Options,
+ sizeof(BasicLoggingOptions));
+ static auto UNUSED Once = [] {
+ static auto UNUSED &TLD = getThreadLocalData();
+ __sanitizer::Atexit(+[] { TLDDestructor(&TLD); });
+ return false;
+ }();
+ }
return true;
-}();
+}
+
+} // namespace __xray
+
+static auto UNUSED Unused = __xray::basicLogDynamicInitializer();
diff --git a/contrib/compiler-rt/lib/xray/xray_inmemory_log.h b/contrib/compiler-rt/lib/xray/xray_inmemory_log.h
new file mode 100644
index 000000000000..e4fcb8ca5ffd
--- /dev/null
+++ b/contrib/compiler-rt/lib/xray/xray_inmemory_log.h
@@ -0,0 +1,44 @@
+//===-- xray_inmemory_log.h
+//------------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a function call tracing system.
+//
+//===----------------------------------------------------------------------===//
+#ifndef XRAY_XRAY_INMEMORY_LOG_H
+#define XRAY_XRAY_INMEMORY_LOG_H
+
+#include "xray/xray_log_interface.h"
+
+/// Basic (Naive) Mode
+/// ==================
+///
+/// This implementation hooks in through the XRay logging implementation
+/// framework. The Basic Mode implementation will keep appending to a file as
+/// soon as the thread-local buffers are full. It keeps minimal in-memory state
+/// and does the minimum filtering required to keep log files smaller.
+
+namespace __xray {
+
+XRayLogInitStatus basicLoggingInit(size_t BufferSize, size_t BufferMax,
+ void *Options, size_t OptionsSize);
+XRayLogInitStatus basicLoggingFinalize();
+
+void basicLoggingHandleArg0RealTSC(int32_t FuncId, XRayEntryType Entry);
+void basicLoggingHandleArg0EmulateTSC(int32_t FuncId, XRayEntryType Entry);
+void basicLoggingHandleArg1RealTSC(int32_t FuncId, XRayEntryType Entry,
+ uint64_t Arg1);
+void basicLoggingHandleArg1EmulateTSC(int32_t FuncId, XRayEntryType Entry,
+ uint64_t Arg1);
+XRayLogFlushStatus basicLoggingFlush();
+XRayLogInitStatus basicLoggingReset();
+
+} // namespace __xray
+
+#endif // XRAY_XRAY_INMEMORY_LOG_H
diff --git a/contrib/compiler-rt/lib/xray/xray_interface.cc b/contrib/compiler-rt/lib/xray/xray_interface.cc
index 694d34c0102b..766313e85c58 100644
--- a/contrib/compiler-rt/lib/xray/xray_interface.cc
+++ b/contrib/compiler-rt/lib/xray/xray_interface.cc
@@ -23,12 +23,15 @@
#include "sanitizer_common/sanitizer_common.h"
#include "xray_defs.h"
+#include "xray_flags.h"
+
+extern __sanitizer::SpinMutex XRayInstrMapMutex;
+extern __sanitizer::atomic_uint8_t XRayInitialized;
+extern __xray::XRaySledMap XRayInstrMap;
namespace __xray {
#if defined(__x86_64__)
-// FIXME: The actual length is 11 bytes. Why was length 12 passed to mprotect()
-// ?
static const int16_t cSledLength = 12;
#elif defined(__aarch64__)
static const int16_t cSledLength = 32;
@@ -53,6 +56,10 @@ __sanitizer::atomic_uintptr_t XRayArgLogger{0};
// This is the function to call when we encounter a custom event log call.
__sanitizer::atomic_uintptr_t XRayPatchedCustomEvent{0};
+// This is the global status to determine whether we are currently
+// patching/unpatching.
+__sanitizer::atomic_uint8_t XRayPatching{0};
+
// MProtectHelper is an RAII wrapper for calls to mprotect(...) that will undo
// any successful mprotect(...) changes. This is used to make a page writeable
// and executable, and upon destruction if it was successful in doing so returns
@@ -88,85 +95,10 @@ public:
}
};
-} // namespace __xray
-
-extern __sanitizer::SpinMutex XRayInstrMapMutex;
-extern __sanitizer::atomic_uint8_t XRayInitialized;
-extern __xray::XRaySledMap XRayInstrMap;
-
-int __xray_set_handler(void (*entry)(int32_t,
- XRayEntryType)) XRAY_NEVER_INSTRUMENT {
- if (__sanitizer::atomic_load(&XRayInitialized,
- __sanitizer::memory_order_acquire)) {
-
- __sanitizer::atomic_store(&__xray::XRayPatchedFunction,
- reinterpret_cast<uintptr_t>(entry),
- __sanitizer::memory_order_release);
- return 1;
- }
- return 0;
-}
-
-int __xray_set_customevent_handler(void (*entry)(void *, size_t))
- XRAY_NEVER_INSTRUMENT {
- if (__sanitizer::atomic_load(&XRayInitialized,
- __sanitizer::memory_order_acquire)) {
- __sanitizer::atomic_store(&__xray::XRayPatchedCustomEvent,
- reinterpret_cast<uintptr_t>(entry),
- __sanitizer::memory_order_release);
- return 1;
- }
- return 0;
-}
-
-
-int __xray_remove_handler() XRAY_NEVER_INSTRUMENT {
- return __xray_set_handler(nullptr);
-}
-
-int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT {
- return __xray_set_customevent_handler(nullptr);
-}
-
-__sanitizer::atomic_uint8_t XRayPatching{0};
-
-using namespace __xray;
-
-// FIXME: Figure out whether we can move this class to sanitizer_common instead
-// as a generic "scope guard".
-template <class Function> class CleanupInvoker {
- Function Fn;
-
-public:
- explicit CleanupInvoker(Function Fn) XRAY_NEVER_INSTRUMENT : Fn(Fn) {}
- CleanupInvoker(const CleanupInvoker &) XRAY_NEVER_INSTRUMENT = default;
- CleanupInvoker(CleanupInvoker &&) XRAY_NEVER_INSTRUMENT = default;
- CleanupInvoker &
- operator=(const CleanupInvoker &) XRAY_NEVER_INSTRUMENT = delete;
- CleanupInvoker &operator=(CleanupInvoker &&) XRAY_NEVER_INSTRUMENT = delete;
- ~CleanupInvoker() XRAY_NEVER_INSTRUMENT { Fn(); }
-};
-
-template <class Function>
-CleanupInvoker<Function> scopeCleanup(Function Fn) XRAY_NEVER_INSTRUMENT {
- return CleanupInvoker<Function>{Fn};
-}
-
-inline bool patchSled(const XRaySledEntry &Sled, bool Enable,
- int32_t FuncId) XRAY_NEVER_INSTRUMENT {
- // While we're here, we should patch the nop sled. To do that we mprotect
- // the page containing the function to be writeable.
- const uint64_t PageSize = GetPageSizeCached();
- void *PageAlignedAddr =
- reinterpret_cast<void *>(Sled.Address & ~(PageSize - 1));
- std::size_t MProtectLen = (Sled.Address + cSledLength) -
- reinterpret_cast<uint64_t>(PageAlignedAddr);
- MProtectHelper Protector(PageAlignedAddr, MProtectLen);
- if (Protector.MakeWriteable() == -1) {
- printf("Failed mprotect: %d\n", errno);
- return XRayPatchingStatus::FAILED;
- }
+namespace {
+bool patchSled(const XRaySledEntry &Sled, bool Enable,
+ int32_t FuncId) XRAY_NEVER_INSTRUMENT {
bool Success = false;
switch (Sled.Kind) {
case XRayEntryType::ENTRY:
@@ -191,6 +123,55 @@ inline bool patchSled(const XRaySledEntry &Sled, bool Enable,
return Success;
}
+XRayPatchingStatus patchFunction(int32_t FuncId,
+ bool Enable) XRAY_NEVER_INSTRUMENT {
+ if (!__sanitizer::atomic_load(&XRayInitialized,
+ __sanitizer::memory_order_acquire))
+ return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
+
+ uint8_t NotPatching = false;
+ if (!__sanitizer::atomic_compare_exchange_strong(
+ &XRayPatching, &NotPatching, true, __sanitizer::memory_order_acq_rel))
+ return XRayPatchingStatus::ONGOING; // Already patching.
+
+ // Next, we look for the function index.
+ XRaySledMap InstrMap;
+ {
+ __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex);
+ InstrMap = XRayInstrMap;
+ }
+
+ // If we don't have an index, we can't patch individual functions.
+ if (InstrMap.Functions == 0)
+ return XRayPatchingStatus::NOT_INITIALIZED;
+
+ // FuncId must be a positive number, less than the number of functions
+ // instrumented.
+ if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
+ Report("Invalid function id provided: %d\n", FuncId);
+ return XRayPatchingStatus::FAILED;
+ }
+
+ // Now we patch ths sleds for this specific function.
+ auto SledRange = InstrMap.SledsIndex[FuncId - 1];
+ auto *f = SledRange.Begin;
+ auto *e = SledRange.End;
+
+ bool SucceedOnce = false;
+ while (f != e)
+ SucceedOnce |= patchSled(*f++, Enable, FuncId);
+
+ __sanitizer::atomic_store(&XRayPatching, false,
+ __sanitizer::memory_order_release);
+
+ if (!SucceedOnce) {
+ Report("Failed patching any sled for function '%d'.", FuncId);
+ return XRayPatchingStatus::FAILED;
+ }
+
+ return XRayPatchingStatus::SUCCESS;
+}
+
// controlPatching implements the common internals of the patching/unpatching
// implementation. |Enable| defines whether we're enabling or disabling the
// runtime XRay instrumentation.
@@ -205,14 +186,13 @@ XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
return XRayPatchingStatus::ONGOING; // Already patching.
uint8_t PatchingSuccess = false;
- auto XRayPatchingStatusResetter = scopeCleanup([&PatchingSuccess] {
- if (!PatchingSuccess)
- __sanitizer::atomic_store(&XRayPatching, false,
- __sanitizer::memory_order_release);
- });
+ auto XRayPatchingStatusResetter =
+ __sanitizer::at_scope_exit([&PatchingSuccess] {
+ if (!PatchingSuccess)
+ __sanitizer::atomic_store(&XRayPatching, false,
+ __sanitizer::memory_order_release);
+ });
- // Step 1: Compute the function id, as a unique identifier per function in the
- // instrumentation map.
XRaySledMap InstrMap;
{
__sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex);
@@ -221,16 +201,47 @@ XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
if (InstrMap.Entries == 0)
return XRayPatchingStatus::NOT_INITIALIZED;
- const uint64_t PageSize = GetPageSizeCached();
+ uint32_t FuncId = 1;
+ uint64_t CurFun = 0;
+
+ // First we want to find the bounds for which we have instrumentation points,
+ // and try to get as few calls to mprotect(...) as possible. We're assuming
+ // that all the sleds for the instrumentation map are contiguous as a single
+ // set of pages. When we do support dynamic shared object instrumentation,
+ // we'll need to do this for each set of page load offsets per DSO loaded. For
+ // now we're assuming we can mprotect the whole section of text between the
+ // minimum sled address and the maximum sled address (+ the largest sled
+ // size).
+ auto MinSled = InstrMap.Sleds[0];
+ auto MaxSled = InstrMap.Sleds[InstrMap.Entries - 1];
+ for (std::size_t I = 0; I < InstrMap.Entries; I++) {
+ const auto &Sled = InstrMap.Sleds[I];
+ if (Sled.Address < MinSled.Address)
+ MinSled = Sled;
+ if (Sled.Address > MaxSled.Address)
+ MaxSled = Sled;
+ }
+
+ const size_t PageSize = flags()->xray_page_size_override > 0
+ ? flags()->xray_page_size_override
+ : GetPageSizeCached();
if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
Report("System page size is not a power of two: %lld\n", PageSize);
return XRayPatchingStatus::FAILED;
}
- uint32_t FuncId = 1;
- uint64_t CurFun = 0;
- for (std::size_t I = 0; I < InstrMap.Entries; I++) {
- auto Sled = InstrMap.Sleds[I];
+ void *PageAlignedAddr =
+ reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1));
+ size_t MProtectLen =
+ (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength;
+ MProtectHelper Protector(PageAlignedAddr, MProtectLen);
+ if (Protector.MakeWriteable() == -1) {
+ Report("Failed mprotect: %d\n", errno);
+ return XRayPatchingStatus::FAILED;
+ }
+
+ for (std::size_t I = 0; I < InstrMap.Entries; ++I) {
+ auto &Sled = InstrMap.Sleds[I];
auto F = Sled.Function;
if (CurFun == 0)
CurFun = F;
@@ -246,36 +257,14 @@ XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT {
return XRayPatchingStatus::SUCCESS;
}
-XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT {
- return controlPatching(true);
-}
-
-XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT {
- return controlPatching(false);
-}
-
-XRayPatchingStatus patchFunction(int32_t FuncId,
- bool Enable) XRAY_NEVER_INSTRUMENT {
- if (!__sanitizer::atomic_load(&XRayInitialized,
- __sanitizer::memory_order_acquire))
- return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized.
-
- uint8_t NotPatching = false;
- if (!__sanitizer::atomic_compare_exchange_strong(
- &XRayPatching, &NotPatching, true, __sanitizer::memory_order_acq_rel))
- return XRayPatchingStatus::ONGOING; // Already patching.
-
- // Next, we look for the function index.
+XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId,
+ bool Enable) XRAY_NEVER_INSTRUMENT {
XRaySledMap InstrMap;
{
__sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex);
InstrMap = XRayInstrMap;
}
- // If we don't have an index, we can't patch individual functions.
- if (InstrMap.Functions == 0)
- return XRayPatchingStatus::NOT_INITIALIZED;
-
// FuncId must be a positive number, less than the number of functions
// instrumented.
if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) {
@@ -283,33 +272,98 @@ XRayPatchingStatus patchFunction(int32_t FuncId,
return XRayPatchingStatus::FAILED;
}
- // Now we patch ths sleds for this specific function.
+ const size_t PageSize = flags()->xray_page_size_override > 0
+ ? flags()->xray_page_size_override
+ : GetPageSizeCached();
+ if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) {
+ Report("Provided page size is not a power of two: %lld\n", PageSize);
+ return XRayPatchingStatus::FAILED;
+ }
+
+ // Here we compute the minumum sled and maximum sled associated with a
+ // particular function ID.
auto SledRange = InstrMap.SledsIndex[FuncId - 1];
auto *f = SledRange.Begin;
auto *e = SledRange.End;
+ auto MinSled = *f;
+ auto MaxSled = *(SledRange.End - 1);
+ while (f != e) {
+ if (f->Address < MinSled.Address)
+ MinSled = *f;
+ if (f->Address > MaxSled.Address)
+ MaxSled = *f;
+ ++f;
+ }
- bool SucceedOnce = false;
- while (f != e)
- SucceedOnce |= patchSled(*f++, Enable, FuncId);
+ void *PageAlignedAddr =
+ reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1));
+ size_t MProtectLen =
+ (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength;
+ MProtectHelper Protector(PageAlignedAddr, MProtectLen);
+ if (Protector.MakeWriteable() == -1) {
+ Report("Failed mprotect: %d\n", errno);
+ return XRayPatchingStatus::FAILED;
+ }
+ return patchFunction(FuncId, Enable);
+}
- __sanitizer::atomic_store(&XRayPatching, false,
- __sanitizer::memory_order_release);
+} // namespace
- if (!SucceedOnce) {
- Report("Failed patching any sled for function '%d'.", FuncId);
- return XRayPatchingStatus::FAILED;
+} // namespace __xray
+
+using namespace __xray;
+
+// The following functions are declared `extern "C" {...}` in the header, hence
+// they're defined in the global namespace.
+
+int __xray_set_handler(void (*entry)(int32_t,
+ XRayEntryType)) XRAY_NEVER_INSTRUMENT {
+ if (__sanitizer::atomic_load(&XRayInitialized,
+ __sanitizer::memory_order_acquire)) {
+
+ __sanitizer::atomic_store(&__xray::XRayPatchedFunction,
+ reinterpret_cast<uintptr_t>(entry),
+ __sanitizer::memory_order_release);
+ return 1;
}
+ return 0;
+}
- return XRayPatchingStatus::SUCCESS;
+int __xray_set_customevent_handler(void (*entry)(void *, size_t))
+ XRAY_NEVER_INSTRUMENT {
+ if (__sanitizer::atomic_load(&XRayInitialized,
+ __sanitizer::memory_order_acquire)) {
+ __sanitizer::atomic_store(&__xray::XRayPatchedCustomEvent,
+ reinterpret_cast<uintptr_t>(entry),
+ __sanitizer::memory_order_release);
+ return 1;
+ }
+ return 0;
+}
+
+int __xray_remove_handler() XRAY_NEVER_INSTRUMENT {
+ return __xray_set_handler(nullptr);
+}
+
+int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT {
+ return __xray_set_customevent_handler(nullptr);
+}
+
+XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT {
+ return controlPatching(true);
+}
+
+XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT {
+ return controlPatching(false);
}
XRayPatchingStatus __xray_patch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
- return patchFunction(FuncId, true);
+ return mprotectAndPatchFunction(FuncId, true);
}
XRayPatchingStatus
__xray_unpatch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
- return patchFunction(FuncId, false);
+ return mprotectAndPatchFunction(FuncId, false);
}
int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) {
@@ -331,7 +385,7 @@ uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT {
__sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex);
if (FuncId <= 0 || static_cast<size_t>(FuncId) > XRayInstrMap.Functions)
return 0;
- return XRayInstrMap.SledsIndex[FuncId - 1].Begin->Address
+ return XRayInstrMap.SledsIndex[FuncId - 1].Begin->Function
// On PPC, function entries are always aligned to 16 bytes. The beginning of a
// sled might be a local entry, which is always +8 based on the global entry.
// Always return the global entry.
diff --git a/contrib/compiler-rt/lib/xray/xray_interface_internal.h b/contrib/compiler-rt/lib/xray/xray_interface_internal.h
index 4a2784612fcb..5811e2b7300a 100644
--- a/contrib/compiler-rt/lib/xray/xray_interface_internal.h
+++ b/contrib/compiler-rt/lib/xray/xray_interface_internal.h
@@ -28,13 +28,15 @@ struct XRaySledEntry {
uint64_t Function;
unsigned char Kind;
unsigned char AlwaysInstrument;
- unsigned char Padding[14]; // Need 32 bytes
+ unsigned char Version;
+ unsigned char Padding[13]; // Need 32 bytes
#elif SANITIZER_WORDSIZE == 32
uint32_t Address;
uint32_t Function;
unsigned char Kind;
unsigned char AlwaysInstrument;
- unsigned char Padding[6]; // Need 16 bytes
+ unsigned char Version;
+ unsigned char Padding[5]; // Need 16 bytes
#else
#error "Unsupported word size."
#endif
diff --git a/contrib/compiler-rt/lib/xray/xray_log_interface.cc b/contrib/compiler-rt/lib/xray/xray_log_interface.cc
index ee14ae4b1b62..783f004d292a 100644
--- a/contrib/compiler-rt/lib/xray/xray_log_interface.cc
+++ b/contrib/compiler-rt/lib/xray/xray_log_interface.cc
@@ -12,35 +12,84 @@
//===----------------------------------------------------------------------===//
#include "xray/xray_log_interface.h"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_mutex.h"
#include "xray/xray_interface.h"
#include "xray_defs.h"
-#include <memory>
-
__sanitizer::SpinMutex XRayImplMutex;
-std::unique_ptr<XRayLogImpl> GlobalXRayImpl;
+XRayLogImpl CurrentXRayImpl{nullptr, nullptr, nullptr, nullptr};
+XRayLogImpl *GlobalXRayImpl = nullptr;
+
+// We use a linked list of Mode to XRayLogImpl mappings. This is a linked list
+// when it should be a map because we're avoiding having to depend on C++
+// standard library data structures at this level of the implementation.
+struct ModeImpl {
+ ModeImpl *Next;
+ const char *Mode;
+ XRayLogImpl Impl;
+};
+
+ModeImpl SentinelModeImpl{
+ nullptr, nullptr, {nullptr, nullptr, nullptr, nullptr}};
+ModeImpl *ModeImpls = &SentinelModeImpl;
+
+XRayLogRegisterStatus
+__xray_log_register_mode(const char *Mode,
+ XRayLogImpl Impl) XRAY_NEVER_INSTRUMENT {
+ if (Impl.flush_log == nullptr || Impl.handle_arg0 == nullptr ||
+ Impl.log_finalize == nullptr || Impl.log_init == nullptr)
+ return XRayLogRegisterStatus::XRAY_INCOMPLETE_IMPL;
+
+ __sanitizer::SpinMutexLock Guard(&XRayImplMutex);
+ // First, look for whether the mode already has a registered implementation.
+ for (ModeImpl *it = ModeImpls; it != &SentinelModeImpl; it = it->Next) {
+ if (!__sanitizer::internal_strcmp(Mode, it->Mode))
+ return XRayLogRegisterStatus::XRAY_DUPLICATE_MODE;
+ }
+ auto *NewModeImpl =
+ static_cast<ModeImpl *>(__sanitizer::InternalAlloc(sizeof(ModeImpl)));
+ NewModeImpl->Next = ModeImpls;
+ NewModeImpl->Mode = __sanitizer::internal_strdup(Mode);
+ NewModeImpl->Impl = Impl;
+ ModeImpls = NewModeImpl;
+ return XRayLogRegisterStatus::XRAY_REGISTRATION_OK;
+}
+
+XRayLogRegisterStatus
+__xray_log_select_mode(const char *Mode) XRAY_NEVER_INSTRUMENT {
+ __sanitizer::SpinMutexLock Guard(&XRayImplMutex);
+ for (ModeImpl *it = ModeImpls; it != &SentinelModeImpl; it = it->Next) {
+ if (!__sanitizer::internal_strcmp(Mode, it->Mode)) {
+ CurrentXRayImpl = it->Impl;
+ GlobalXRayImpl = &CurrentXRayImpl;
+ __xray_set_handler(it->Impl.handle_arg0);
+ return XRayLogRegisterStatus::XRAY_REGISTRATION_OK;
+ }
+ }
+ return XRayLogRegisterStatus::XRAY_MODE_NOT_FOUND;
+}
void __xray_set_log_impl(XRayLogImpl Impl) XRAY_NEVER_INSTRUMENT {
if (Impl.log_init == nullptr || Impl.log_finalize == nullptr ||
Impl.handle_arg0 == nullptr || Impl.flush_log == nullptr) {
__sanitizer::SpinMutexLock Guard(&XRayImplMutex);
- GlobalXRayImpl.reset();
+ GlobalXRayImpl = nullptr;
__xray_remove_handler();
__xray_remove_handler_arg1();
return;
}
__sanitizer::SpinMutexLock Guard(&XRayImplMutex);
- GlobalXRayImpl.reset(new XRayLogImpl);
- *GlobalXRayImpl = Impl;
+ CurrentXRayImpl = Impl;
+ GlobalXRayImpl = &CurrentXRayImpl;
__xray_set_handler(Impl.handle_arg0);
}
void __xray_remove_log_impl() XRAY_NEVER_INSTRUMENT {
__sanitizer::SpinMutexLock Guard(&XRayImplMutex);
- GlobalXRayImpl.reset();
+ GlobalXRayImpl = nullptr;
__xray_remove_handler();
__xray_remove_handler_arg1();
}
diff --git a/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S b/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S
index b59eedc4bb1b..350afd9265fd 100644
--- a/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S
+++ b/contrib/compiler-rt/lib/xray/xray_trampoline_x86_64.S
@@ -14,10 +14,13 @@
//===----------------------------------------------------------------------===//
#include "../builtins/assembly.h"
+#include "../sanitizer_common/sanitizer_asm.h"
+
+
.macro SAVE_REGISTERS
subq $192, %rsp
- .cfi_def_cfa_offset 200
+ CFI_DEF_CFA_OFFSET(200)
// At this point, the stack pointer should be aligned to an 8-byte boundary,
// because any call instructions that come after this will add another 8
// bytes and therefore align it to 16-bytes.
@@ -57,63 +60,82 @@
movq 8(%rsp), %r8
movq 0(%rsp), %r9
addq $192, %rsp
- .cfi_def_cfa_offset 8
+ CFI_DEF_CFA_OFFSET(8)
+.endm
+
+.macro ALIGNED_CALL_RAX
+ // Call the logging handler, after aligning the stack to a 16-byte boundary.
+ // The approach we're taking here uses additional stack space to stash the
+ // stack pointer twice before aligning the pointer to 16-bytes. If the stack
+ // was 8-byte aligned, it will become 16-byte aligned -- when restoring the
+ // pointer, we can always look -8 bytes from the current position to get
+ // either of the values we've stashed in the first place.
+ pushq %rsp
+ pushq (%rsp)
+ andq $-0x10, %rsp
+ callq *%rax
+ movq 8(%rsp), %rsp
.endm
.text
+#if !defined(__APPLE__)
+ .section .text
+#else
+ .section __TEXT,__text
+#endif
.file "xray_trampoline_x86.S"
//===----------------------------------------------------------------------===//
- .globl __xray_FunctionEntry
+ .globl ASM_SYMBOL(__xray_FunctionEntry)
.align 16, 0x90
- .type __xray_FunctionEntry,@function
-
-__xray_FunctionEntry:
- .cfi_startproc
+ ASM_TYPE_FUNCTION(__xray_FunctionEntry)
+ASM_SYMBOL(__xray_FunctionEntry):
+ CFI_STARTPROC
SAVE_REGISTERS
// This load has to be atomic, it's concurrent with __xray_patch().
// On x86/amd64, a simple (type-aligned) MOV instruction is enough.
- movq _ZN6__xray19XRayPatchedFunctionE(%rip), %rax
+ movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax
testq %rax, %rax
je .Ltmp0
// The patched function prolog puts its xray_instr_map index into %r10d.
movl %r10d, %edi
xor %esi,%esi
- callq *%rax
+ ALIGNED_CALL_RAX
+
.Ltmp0:
RESTORE_REGISTERS
retq
-.Ltmp1:
- .size __xray_FunctionEntry, .Ltmp1-__xray_FunctionEntry
- .cfi_endproc
+ ASM_SIZE(__xray_FunctionEntry)
+ CFI_ENDPROC
//===----------------------------------------------------------------------===//
- .globl __xray_FunctionExit
+ .globl ASM_SYMBOL(__xray_FunctionExit)
.align 16, 0x90
- .type __xray_FunctionExit,@function
-__xray_FunctionExit:
- .cfi_startproc
+ ASM_TYPE_FUNCTION(__xray_FunctionExit)
+ASM_SYMBOL(__xray_FunctionExit):
+ CFI_STARTPROC
// Save the important registers first. Since we're assuming that this
// function is only jumped into, we only preserve the registers for
// returning.
subq $56, %rsp
- .cfi_def_cfa_offset 64
+ CFI_DEF_CFA_OFFSET(64)
movq %rbp, 48(%rsp)
movupd %xmm0, 32(%rsp)
movupd %xmm1, 16(%rsp)
movq %rax, 8(%rsp)
movq %rdx, 0(%rsp)
- movq _ZN6__xray19XRayPatchedFunctionE(%rip), %rax
+ movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax
testq %rax,%rax
je .Ltmp2
movl %r10d, %edi
movl $1, %esi
- callq *%rax
+ ALIGNED_CALL_RAX
+
.Ltmp2:
// Restore the important registers.
movq 48(%rsp), %rbp
@@ -122,111 +144,94 @@ __xray_FunctionExit:
movq 8(%rsp), %rax
movq 0(%rsp), %rdx
addq $56, %rsp
- .cfi_def_cfa_offset 8
+ CFI_DEF_CFA_OFFSET(8)
retq
-.Ltmp3:
- .size __xray_FunctionExit, .Ltmp3-__xray_FunctionExit
- .cfi_endproc
+ ASM_SIZE(__xray_FunctionExit)
+ CFI_ENDPROC
//===----------------------------------------------------------------------===//
- .global __xray_FunctionTailExit
+ .globl ASM_SYMBOL(__xray_FunctionTailExit)
.align 16, 0x90
- .type __xray_FunctionTailExit,@function
-__xray_FunctionTailExit:
- .cfi_startproc
- // Save the important registers as in the entry trampoline, but indicate that
- // this is an exit. In the future, we will introduce a new entry type that
- // differentiates between a normal exit and a tail exit, but we'd have to do
- // this and increment the version number for the header.
+ ASM_TYPE_FUNCTION(__xray_FunctionTailExit)
+ASM_SYMBOL(__xray_FunctionTailExit):
+ CFI_STARTPROC
SAVE_REGISTERS
- movq _ZN6__xray19XRayPatchedFunctionE(%rip), %rax
+ movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax
testq %rax,%rax
je .Ltmp4
movl %r10d, %edi
- movl $1, %esi
- callq *%rax
+ movl $2, %esi
+
+ ALIGNED_CALL_RAX
.Ltmp4:
RESTORE_REGISTERS
retq
-.Ltmp5:
- .size __xray_FunctionTailExit, .Ltmp5-__xray_FunctionTailExit
- .cfi_endproc
+ ASM_SIZE(__xray_FunctionTailExit)
+ CFI_ENDPROC
//===----------------------------------------------------------------------===//
- .globl __xray_ArgLoggerEntry
+ .globl ASM_SYMBOL(__xray_ArgLoggerEntry)
.align 16, 0x90
- .type __xray_ArgLoggerEntry,@function
-__xray_ArgLoggerEntry:
- .cfi_startproc
+ ASM_TYPE_FUNCTION(__xray_ArgLoggerEntry)
+ASM_SYMBOL(__xray_ArgLoggerEntry):
+ CFI_STARTPROC
SAVE_REGISTERS
// Again, these function pointer loads must be atomic; MOV is fine.
- movq _ZN6__xray13XRayArgLoggerE(%rip), %rax
+ movq ASM_SYMBOL(_ZN6__xray13XRayArgLoggerE)(%rip), %rax
testq %rax, %rax
jne .Larg1entryLog
// If [arg1 logging handler] not set, defer to no-arg logging.
- movq _ZN6__xray19XRayPatchedFunctionE(%rip), %rax
+ movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax
testq %rax, %rax
je .Larg1entryFail
.Larg1entryLog:
- // First argument will become the third
+ // First argument will become the third
movq %rdi, %rdx
- // XRayEntryType::ENTRY into the second
- xorq %rsi, %rsi
+ // XRayEntryType::LOG_ARGS_ENTRY into the second
+ mov $0x3, %esi
// 32-bit function ID becomes the first
movl %r10d, %edi
- callq *%rax
+ ALIGNED_CALL_RAX
.Larg1entryFail:
RESTORE_REGISTERS
retq
-
-.Larg1entryEnd:
- .size __xray_ArgLoggerEntry, .Larg1entryEnd-__xray_ArgLoggerEntry
- .cfi_endproc
+ ASM_SIZE(__xray_ArgLoggerEntry)
+ CFI_ENDPROC
//===----------------------------------------------------------------------===//
- .global __xray_CustomEvent
+ .global ASM_SYMBOL(__xray_CustomEvent)
.align 16, 0x90
- .type __xray_CustomEvent,@function
-__xray_CustomEvent:
- .cfi_startproc
- subq $16, %rsp
- .cfi_def_cfa_offset 24
- movq %rbp, 8(%rsp)
- movq %rax, 0(%rsp)
+ ASM_TYPE_FUNCTION(__xray_CustomEvent)
+ASM_SYMBOL(__xray_CustomEvent):
+ CFI_STARTPROC
+ SAVE_REGISTERS
// We take two arguments to this trampoline, which should be in rdi and rsi
// already. We also make sure that we stash %rax because we use that register
// to call the logging handler.
- movq _ZN6__xray22XRayPatchedCustomEventE(%rip), %rax
+ movq ASM_SYMBOL(_ZN6__xray22XRayPatchedCustomEventE)(%rip), %rax
testq %rax,%rax
je .LcustomEventCleanup
- // At this point we know that rcx and rdx already has the data, so we just
- // call the logging handler.
- callq *%rax
+ ALIGNED_CALL_RAX
.LcustomEventCleanup:
- movq 0(%rsp), %rax
- movq 8(%rsp), %rbp
- addq $16, %rsp
- .cfi_def_cfa_offset 8
+ RESTORE_REGISTERS
retq
-
-.Ltmp8:
- .size __xray_CustomEvent, .Ltmp8-__xray_CustomEvent
- .cfi_endproc
+ ASM_SIZE(__xray_CustomEvent)
+ CFI_ENDPROC
NO_EXEC_STACK_DIRECTIVE
diff --git a/contrib/compiler-rt/lib/xray/xray_utils.cc b/contrib/compiler-rt/lib/xray/xray_utils.cc
index b9a38d1b98eb..cf800d3aeaf8 100644
--- a/contrib/compiler-rt/lib/xray/xray_utils.cc
+++ b/contrib/compiler-rt/lib/xray/xray_utils.cc
@@ -117,7 +117,8 @@ int getLogFD() XRAY_NEVER_INSTRUMENT {
TmpFilename);
return -1;
}
- Report("XRay: Log file in '%s'\n", TmpFilename);
+ if (__sanitizer::Verbosity())
+ Report("XRay: Log file in '%s'\n", TmpFilename);
return Fd;
}
diff --git a/contrib/compiler-rt/lib/xray/xray_x86_64.cc b/contrib/compiler-rt/lib/xray/xray_x86_64.cc
index e34806fa1cea..e17f00ac3a62 100644
--- a/contrib/compiler-rt/lib/xray/xray_x86_64.cc
+++ b/contrib/compiler-rt/lib/xray/xray_x86_64.cc
@@ -76,6 +76,7 @@ static constexpr uint8_t CallOpCode = 0xe8;
static constexpr uint16_t MovR10Seq = 0xba41;
static constexpr uint16_t Jmp9Seq = 0x09eb;
static constexpr uint16_t Jmp20Seq = 0x14eb;
+static constexpr uint16_t Jmp15Seq = 0x0feb;
static constexpr uint8_t JmpOpCode = 0xe9;
static constexpr uint8_t RetOpCode = 0xc3;
static constexpr uint16_t NopwSeq = 0x9066;
@@ -207,8 +208,10 @@ bool patchCustomEvent(const bool Enable, const uint32_t FuncId,
const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
// Here we do the dance of replacing the following sled:
//
+ // In Version 0:
+ //
// xray_sled_n:
- // jmp +19 // 2 bytes
+ // jmp +20 // 2 bytes
// ...
//
// With the following:
@@ -216,24 +219,35 @@ bool patchCustomEvent(const bool Enable, const uint32_t FuncId,
// nopw // 2 bytes*
// ...
//
- // We need to do this in the following order:
//
- // 1. Overwrite the 5-byte nop with the call (relative), where (relative) is
- // the relative offset to the __xray_CustomEvent trampoline.
- // 2. Do a two-byte atomic write over the 'jmp +24' to turn it into a 'nopw'.
- // This allows us to "enable" this code once the changes have committed.
+ // The "unpatch" should just turn the 'nopw' back to a 'jmp +20'.
+ //
+ // ---
//
- // The "unpatch" should just turn the 'nopw' back to a 'jmp +24'.
+ // In Version 1:
+ //
+ // The jump offset is now 15 bytes (0x0f), so when restoring the nopw back
+ // to a jmp, use 15 bytes instead.
//
if (Enable) {
std::atomic_store_explicit(
reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), NopwSeq,
std::memory_order_release);
} else {
- std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), Jmp20Seq,
- std::memory_order_release);
- }
+ switch (Sled.Version) {
+ case 1:
+ std::atomic_store_explicit(
+ reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), Jmp15Seq,
+ std::memory_order_release);
+ break;
+ case 0:
+ default:
+ std::atomic_store_explicit(
+ reinterpret_cast<std::atomic<uint16_t> *>(Sled.Address), Jmp20Seq,
+ std::memory_order_release);
+ break;
+ }
+ }
return false;
}
@@ -244,9 +258,9 @@ bool probeRequiredCPUFeatures() XRAY_NEVER_INSTRUMENT {
// We check whether rdtscp support is enabled. According to the x86_64 manual,
// level should be set at 0x80000001, and we should have a look at bit 27 in
- // EDX. That's 0x8000000 (or 1u << 26).
+ // EDX. That's 0x8000000 (or 1u << 27).
__get_cpuid(0x80000001, &EAX, &EBX, &ECX, &EDX);
- if (!(EDX & (1u << 26))) {
+ if (!(EDX & (1u << 27))) {
Report("Missing rdtscp support.\n");
return false;
}