1 files changed, 379 insertions, 111 deletions

diff --git a/lib/profile/InstrProfilingFile.c b/lib/profile/InstrProfilingFile.c
index 68d088a1956ad..32762d14eef2e 100644
--- a/lib/profile/InstrProfilingFile.c
+++ b/lib/profile/InstrProfilingFile.c
@@ -14,13 +14,77 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-
-#define UNCONST(ptr) ((void *)(uintptr_t)(ptr))
-
 #ifdef _MSC_VER
-#define snprintf _snprintf
+/* For _alloca. */
+#include <malloc.h>
+#endif
+#if defined(_WIN32)
+#include "WindowsMMap.h"
+/* For _chsize_s */
+#include <io.h>
+#else
+#include <sys/file.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#if defined(__linux__)
+#include <sys/types.h>
+#endif
 #endif
 
+/* From where is profile name specified.
+ * The order the enumerators define their
+ * precedence. Re-order them may lead to
+ * runtime behavior change. */ 
+typedef enum ProfileNameSpecifier {
+  PNS_unknown = 0,
+  PNS_default,
+  PNS_command_line,
+  PNS_environment,
+  PNS_runtime_api
+} ProfileNameSpecifier;
+
+static const char *getPNSStr(ProfileNameSpecifier PNS) {
+  switch (PNS) {
+  case PNS_default:
+    return "default setting";
+  case PNS_command_line:
+    return "command line";
+  case PNS_environment:
+    return "environment variable";
+  case PNS_runtime_api:
+    return "runtime API";
+  default:
+    return "Unknown";
+  }
+}
+
+#define MAX_PID_SIZE 16
+/* Data structure holding the result of parsed filename pattern.  */
+typedef struct lprofFilename {
+  /* File name string possibly with %p or %h specifiers. */
+  const char *FilenamePat;
+  char PidChars[MAX_PID_SIZE];
+  char Hostname[COMPILER_RT_MAX_HOSTLEN];
+  unsigned NumPids;
+  unsigned NumHosts;
+  /* When in-process merging is enabled, this parameter specifies
+   * the total number of profile data files shared by all the processes
+   * spawned from the same binary. By default the value is 1. If merging
+   * is not enabled, its value should be 0. This parameter is specified
+   * by the %[0-9]m specifier. For instance %2m enables merging using
+   * 2 profile data files. %1m is equivalent to %m. Also %m specifier
+   * can only appear once at the end of the name pattern. */
+  unsigned MergePoolSize;
+  ProfileNameSpecifier PNS;
+} lprofFilename;
+
+lprofFilename lprofCurFilename = {0, {0}, {0}, 0, 0, 0, PNS_unknown};
+
+int getpid(void);
+static int getCurFilenameLength();
+static const char *getCurFilename(char *FilenameBuf);
+static unsigned doMerging() { return lprofCurFilename.MergePoolSize; }
+
 /* Return 1 if there is an error, otherwise return  0.  */
 static uint32_t fileWriter(ProfDataIOVec *IOVecs, uint32_t NumIOVecs,
                            void **WriterCtx) {
@@ -35,59 +99,144 @@ static uint32_t fileWriter(ProfDataIOVec *IOVecs, uint32_t NumIOVecs,
 }
 
 COMPILER_RT_VISIBILITY ProfBufferIO *
-llvmCreateBufferIOInternal(void *File, uint32_t BufferSz) {
-  CallocHook = calloc;
-  FreeHook = free;
-  return llvmCreateBufferIO(fileWriter, File, BufferSz);
+lprofCreateBufferIOInternal(void *File, uint32_t BufferSz) {
+  FreeHook = &free;
+  DynamicBufferIOBuffer = (uint8_t *)calloc(BufferSz, 1);
+  VPBufferSize = BufferSz;
+  return lprofCreateBufferIO(fileWriter, File);
 }
 
-static int writeFile(FILE *File) {
+static void setupIOBuffer() {
   const char *BufferSzStr = 0;
-  uint64_t ValueDataSize = 0;
-  struct ValueProfData **ValueDataArray =
-      __llvm_profile_gather_value_data(&ValueDataSize);
-  FreeHook = &free;
-  CallocHook = &calloc;
   BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE");
-  if (BufferSzStr && BufferSzStr[0])
+  if (BufferSzStr && BufferSzStr[0]) {
     VPBufferSize = atoi(BufferSzStr);
-  return llvmWriteProfData(fileWriter, File, ValueDataArray, ValueDataSize);
+    DynamicBufferIOBuffer = (uint8_t *)calloc(VPBufferSize, 1);
+  }
 }
 
-static int writeFileWithName(const char *OutputName) {
+/* Read profile data in \c ProfileFile and merge with in-memory
+   profile counters. Returns -1 if there is fatal error, otheriwse
+   0 is returned.
+*/
+static int doProfileMerging(FILE *ProfileFile) {
+  uint64_t ProfileFileSize;
+  char *ProfileBuffer;
+
+  if (fseek(ProfileFile, 0L, SEEK_END) == -1) {
+    PROF_ERR("Unable to merge profile data, unable to get size: %s\n",
+             strerror(errno));
+    return -1;
+  }
+  ProfileFileSize = ftell(ProfileFile);
+
+  /* Restore file offset.  */
+  if (fseek(ProfileFile, 0L, SEEK_SET) == -1) {
+    PROF_ERR("Unable to merge profile data, unable to rewind: %s\n",
+             strerror(errno));
+    return -1;
+  }
+
+  /* Nothing to merge.  */
+  if (ProfileFileSize < sizeof(__llvm_profile_header)) {
+    if (ProfileFileSize)
+      PROF_WARN("Unable to merge profile data: %s\n",
+                "source profile file is too small.");
+    return 0;
+  }
+
+  ProfileBuffer = mmap(NULL, ProfileFileSize, PROT_READ, MAP_SHARED | MAP_FILE,
+                       fileno(ProfileFile), 0);
+  if (ProfileBuffer == MAP_FAILED) {
+    PROF_ERR("Unable to merge profile data, mmap failed: %s\n",
+             strerror(errno));
+    return -1;
+  }
+
+  if (__llvm_profile_check_compatibility(ProfileBuffer, ProfileFileSize)) {
+    (void)munmap(ProfileBuffer, ProfileFileSize);
+    PROF_WARN("Unable to merge profile data: %s\n",
+              "source profile file is not compatible.");
+    return 0;
+  }
+
+  /* Now start merging */
+  __llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize);
+  (void)munmap(ProfileBuffer, ProfileFileSize);
+
+  return 0;
+}
+
+/* Open the profile data for merging. It opens the file in r+b mode with
+ * file locking.  If the file has content which is compatible with the
+ * current process, it also reads in the profile data in the file and merge
+ * it with in-memory counters. After the profile data is merged in memory,
+ * the original profile data is truncated and gets ready for the profile
+ * dumper. With profile merging enabled, each executable as well as any of
+ * its instrumented shared libraries dump profile data into their own data file.
+*/
+static FILE *openFileForMerging(const char *ProfileFileName) {
+  FILE *ProfileFile;
+  int rc;
+
+  ProfileFile = lprofOpenFileEx(ProfileFileName);
+  if (!ProfileFile)
+    return NULL;
+
+  rc = doProfileMerging(ProfileFile);
+  if (rc || COMPILER_RT_FTRUNCATE(ProfileFile, 0L) ||
+      fseek(ProfileFile, 0L, SEEK_SET) == -1) {
+    PROF_ERR("Profile Merging of file %s failed: %s\n", ProfileFileName,
+             strerror(errno));
+    fclose(ProfileFile);
+    return NULL;
+  }
+  fseek(ProfileFile, 0L, SEEK_SET);
+  return ProfileFile;
+}
+
+/* Write profile data to file \c OutputName.  */
+static int writeFile(const char *OutputName) {
   int RetVal;
   FILE *OutputFile;
-  if (!OutputName || !OutputName[0])
-    return -1;
 
-  /* Append to the file to support profiling multiple shared objects. */
-  OutputFile = fopen(OutputName, "ab");
+  if (!doMerging())
+    OutputFile = fopen(OutputName, "ab");
+  else
+    OutputFile = openFileForMerging(OutputName);
+
   if (!OutputFile)
     return -1;
 
-  RetVal = writeFile(OutputFile);
+  FreeHook = &free;
+  setupIOBuffer();
+  RetVal = lprofWriteData(fileWriter, OutputFile, lprofGetVPDataReader());
 
   fclose(OutputFile);
   return RetVal;
 }
 
-COMPILER_RT_WEAK int __llvm_profile_OwnsFilename = 0;
-COMPILER_RT_WEAK const char *__llvm_profile_CurrentFilename = NULL;
-
 static void truncateCurrentFile(void) {
   const char *Filename;
+  char *FilenameBuf;
   FILE *File;
+  int Length;
 
-  Filename = __llvm_profile_CurrentFilename;
-  if (!Filename || !Filename[0])
+  Length = getCurFilenameLength();
+  FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
+  Filename = getCurFilename(FilenameBuf);
+  if (!Filename)
     return;
 
   /* Create the directory holding the file, if needed. */
-  if (strchr(Filename, '/')) {
-    char *Copy = malloc(strlen(Filename) + 1);
-    strcpy(Copy, Filename);
+  if (strchr(Filename, DIR_SEPARATOR)
+#if defined(DIR_SEPARATOR_2)
+      || strchr(Filename, DIR_SEPARATOR_2)
+#endif
+          ) {
+    char *Copy = (char *)COMPILER_RT_ALLOCA(Length + 1);
+    strncpy(Copy, Filename, Length + 1);
     __llvm_profile_recursive_mkdir(Copy);
-    free(Copy);
   }
 
   /* Truncate the file.  Later we'll reopen and append. */
@@ -97,137 +246,254 @@ static void truncateCurrentFile(void) {
   fclose(File);
 }
 
-static void setFilename(const char *Filename, int OwnsFilename) {
-  /* Check if this is a new filename and therefore needs truncation. */
-  int NewFile = !__llvm_profile_CurrentFilename ||
-      (Filename && strcmp(Filename, __llvm_profile_CurrentFilename));
-  if (__llvm_profile_OwnsFilename)
-    free(UNCONST(__llvm_profile_CurrentFilename));
+static const char *DefaultProfileName = "default.profraw";
+static void resetFilenameToDefault(void) {
+  memset(&lprofCurFilename, 0, sizeof(lprofCurFilename));
+  lprofCurFilename.FilenamePat = DefaultProfileName;
+  lprofCurFilename.PNS = PNS_default;
+}
+
+static int containsMergeSpecifier(const char *FilenamePat, int I) {
+  return (FilenamePat[I] == 'm' ||
+          (FilenamePat[I] >= '1' && FilenamePat[I] <= '9' &&
+           /* If FilenamePat[I] is not '\0', the next byte is guaranteed
+            * to be in-bound as the string is null terminated. */
+           FilenamePat[I + 1] == 'm'));
+}
 
-  __llvm_profile_CurrentFilename = Filename;
-  __llvm_profile_OwnsFilename = OwnsFilename;
+/* Parses the pattern string \p FilenamePat and stores the result to
+ * lprofcurFilename structure. */
+static int parseFilenamePattern(const char *FilenamePat) {
+  int NumPids = 0, NumHosts = 0, I;
+  char *PidChars = &lprofCurFilename.PidChars[0];
+  char *Hostname = &lprofCurFilename.Hostname[0];
+  int MergingEnabled = 0;
 
-  /* If not a new file, append to support profiling multiple shared objects. */
-  if (NewFile)
-    truncateCurrentFile();
+  lprofCurFilename.FilenamePat = FilenamePat;
+  /* Check the filename for "%p", which indicates a pid-substitution. */
+  for (I = 0; FilenamePat[I]; ++I)
+    if (FilenamePat[I] == '%') {
+      if (FilenamePat[++I] == 'p') {
+        if (!NumPids++) {
+          if (snprintf(PidChars, MAX_PID_SIZE, "%d", getpid()) <= 0) {
+            PROF_WARN(
+                "Unable to parse filename pattern %s. Using the default name.",
+                FilenamePat);
+            return -1;
+          }
+        }
+      } else if (FilenamePat[I] == 'h') {
+        if (!NumHosts++)
+          if (COMPILER_RT_GETHOSTNAME(Hostname, COMPILER_RT_MAX_HOSTLEN)) {
+            PROF_WARN(
+                "Unable to parse filename pattern %s. Using the default name.",
+                FilenamePat);
+            return -1;
+          }
+      } else if (containsMergeSpecifier(FilenamePat, I)) {
+        if (MergingEnabled) {
+          PROF_WARN("%%m specifier can only be specified once in %s.\n",
+                    FilenamePat);
+          return -1;
+        }
+        MergingEnabled = 1;
+        if (FilenamePat[I] == 'm')
+          lprofCurFilename.MergePoolSize = 1;
+        else {
+          lprofCurFilename.MergePoolSize = FilenamePat[I] - '0';
+          I++; /* advance to 'm' */
+        }
+      }
+    }
+
+  lprofCurFilename.NumPids = NumPids;
+  lprofCurFilename.NumHosts = NumHosts;
+  return 0;
 }
 
-static void resetFilenameToDefault(void) { setFilename("default.profraw", 0); }
+static void parseAndSetFilename(const char *FilenamePat,
+                                ProfileNameSpecifier PNS) {
 
-int getpid(void);
-static int setFilenamePossiblyWithPid(const char *Filename) {
-#define MAX_PID_SIZE 16
-  char PidChars[MAX_PID_SIZE] = {0};
-  int NumPids = 0, PidLength = 0;
-  char *Allocated;
-  int I, J;
+  const char *OldFilenamePat = lprofCurFilename.FilenamePat;
+  ProfileNameSpecifier OldPNS = lprofCurFilename.PNS;
 
-  /* Reset filename on NULL, except with env var which is checked by caller. */
-  if (!Filename) {
+  if (PNS < OldPNS)
+    return;
+
+  if (!FilenamePat)
+    FilenamePat = DefaultProfileName;
+
+  /* When -fprofile-instr-generate=<path> is specified on the
+   * command line, each module will be instrumented with runtime
+   * init call to __llvm_profile_init function which calls
+   * __llvm_profile_override_default_filename. In most of the cases,
+   * the path will be identical, so bypass the parsing completely.
+   */
+  if (OldFilenamePat && !strcmp(OldFilenamePat, FilenamePat)) {
+    lprofCurFilename.PNS = PNS;
+    return;
+  }
+
+  /* When PNS >= OldPNS, the last one wins. */
+  if (!FilenamePat || parseFilenamePattern(FilenamePat))
     resetFilenameToDefault();
-    return 0;
+  lprofCurFilename.PNS = PNS;
+
+  if (!OldFilenamePat) {
+    PROF_NOTE("Set profile file path to \"%s\" via %s.\n",
+              lprofCurFilename.FilenamePat, getPNSStr(PNS));
+  } else {
+    PROF_NOTE("Override old profile path \"%s\" via %s to \"%s\" via %s.\n",
+              OldFilenamePat, getPNSStr(OldPNS), lprofCurFilename.FilenamePat,
+              getPNSStr(PNS));
   }
 
-  /* Check the filename for "%p", which indicates a pid-substitution. */
-  for (I = 0; Filename[I]; ++I)
-    if (Filename[I] == '%' && Filename[++I] == 'p')
-      if (!NumPids++) {
-        PidLength = snprintf(PidChars, MAX_PID_SIZE, "%d", getpid());
-        if (PidLength <= 0)
-          return -1;
-      }
-  if (!NumPids) {
-    setFilename(Filename, 0);
+  if (!lprofCurFilename.MergePoolSize)
+    truncateCurrentFile();
+}
+
+/* Return buffer length that is required to store the current profile
+ * filename with PID and hostname substitutions. */
+/* The length to hold uint64_t followed by 2 digit pool id including '_' */
+#define SIGLEN 24
+static int getCurFilenameLength() {
+  int Len;
+  if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
     return 0;
-  }
 
-  /* Allocate enough space for the substituted filename. */
-  Allocated = malloc(I + NumPids*(PidLength - 2) + 1);
-  if (!Allocated)
-    return -1;
+  if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
+        lprofCurFilename.MergePoolSize))
+    return strlen(lprofCurFilename.FilenamePat);
+
+  Len = strlen(lprofCurFilename.FilenamePat) +
+        lprofCurFilename.NumPids * (strlen(lprofCurFilename.PidChars) - 2) +
+        lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname) - 2);
+  if (lprofCurFilename.MergePoolSize)
+    Len += SIGLEN;
+  return Len;
+}
+
+/* Return the pointer to the current profile file name (after substituting
+ * PIDs and Hostnames in filename pattern. \p FilenameBuf is the buffer
+ * to store the resulting filename. If no substitution is needed, the
+ * current filename pattern string is directly returned. */
+static const char *getCurFilename(char *FilenameBuf) {
+  int I, J, PidLength, HostNameLength;
+  const char *FilenamePat = lprofCurFilename.FilenamePat;
+
+  if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
+    return 0;
+
+  if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
+        lprofCurFilename.MergePoolSize))
+    return lprofCurFilename.FilenamePat;
 
+  PidLength = strlen(lprofCurFilename.PidChars);
+  HostNameLength = strlen(lprofCurFilename.Hostname);
   /* Construct the new filename. */
-  for (I = 0, J = 0; Filename[I]; ++I)
-    if (Filename[I] == '%') {
-      if (Filename[++I] == 'p') {
-        memcpy(Allocated + J, PidChars, PidLength);
+  for (I = 0, J = 0; FilenamePat[I]; ++I)
+    if (FilenamePat[I] == '%') {
+      if (FilenamePat[++I] == 'p') {
+        memcpy(FilenameBuf + J, lprofCurFilename.PidChars, PidLength);
         J += PidLength;
+      } else if (FilenamePat[I] == 'h') {
+        memcpy(FilenameBuf + J, lprofCurFilename.Hostname, HostNameLength);
+        J += HostNameLength;
+      } else if (containsMergeSpecifier(FilenamePat, I)) {
+        char LoadModuleSignature[SIGLEN];
+        int S;
+        int ProfilePoolId = getpid() % lprofCurFilename.MergePoolSize;
+        S = snprintf(LoadModuleSignature, SIGLEN, "%" PRIu64 "_%d",
+                     lprofGetLoadModuleSignature(), ProfilePoolId);
+        if (S == -1 || S > SIGLEN)
+          S = SIGLEN;
+        memcpy(FilenameBuf + J, LoadModuleSignature, S);
+        J += S;
+        if (FilenamePat[I] != 'm')
+          I++;
       }
       /* Drop any unknown substitutions. */
     } else
-      Allocated[J++] = Filename[I];
-  Allocated[J] = 0;
+      FilenameBuf[J++] = FilenamePat[I];
+  FilenameBuf[J] = 0;
 
-  /* Use the computed name. */
-  setFilename(Allocated, 1);
-  return 0;
+  return FilenameBuf;
 }
 
-static int setFilenameFromEnvironment(void) {
+/* Returns the pointer to the environment variable
+ * string. Returns null if the env var is not set. */
+static const char *getFilenamePatFromEnv(void) {
   const char *Filename = getenv("LLVM_PROFILE_FILE");
-
   if (!Filename || !Filename[0])
-    return -1;
-
-  return setFilenamePossiblyWithPid(Filename);
-}
-
-static void setFilenameAutomatically(void) {
-  if (!setFilenameFromEnvironment())
-    return;
-
-  resetFilenameToDefault();
+    return 0;
+  return Filename;
 }
 
+/* This method is invoked by the runtime initialization hook
+ * InstrProfilingRuntime.o if it is linked in. Both user specified
+ * profile path via -fprofile-instr-generate= and LLVM_PROFILE_FILE
+ * environment variable can override this default value. */
 COMPILER_RT_VISIBILITY
 void __llvm_profile_initialize_file(void) {
-  /* Check if the filename has been initialized. */
-  if (__llvm_profile_CurrentFilename)
-    return;
+  const char *FilenamePat;
 
-  /* Detect the filename and truncate. */
-  setFilenameAutomatically();
+  FilenamePat = getFilenamePatFromEnv();
+  parseAndSetFilename(FilenamePat, FilenamePat ? PNS_environment : PNS_default);
 }
 
+/* This API is directly called by the user application code. It has the
+ * highest precedence compared with LLVM_PROFILE_FILE environment variable
+ * and command line option -fprofile-instr-generate=<profile_name>.
+ */
 COMPILER_RT_VISIBILITY
-void __llvm_profile_set_filename(const char *Filename) {
-  setFilenamePossiblyWithPid(Filename);
+void __llvm_profile_set_filename(const char *FilenamePat) {
+  parseAndSetFilename(FilenamePat, PNS_runtime_api);
 }
 
+/*
+ * This API is invoked by the global initializers emitted by Clang/LLVM when
+ * -fprofile-instr-generate=<..> is specified (vs -fprofile-instr-generate
+ *  without an argument). This option has lower precedence than the
+ *  LLVM_PROFILE_FILE environment variable.
+ */
 COMPILER_RT_VISIBILITY
-void __llvm_profile_override_default_filename(const char *Filename) {
-  /* If the env var is set, skip setting filename from argument. */
-  const char *Env_Filename = getenv("LLVM_PROFILE_FILE");
-  if (Env_Filename && Env_Filename[0])
-    return;
-  setFilenamePossiblyWithPid(Filename);
+void __llvm_profile_override_default_filename(const char *FilenamePat) {
+  parseAndSetFilename(FilenamePat, PNS_command_line);
 }
 
+/* The public API for writing profile data into the file with name
+ * set by previous calls to __llvm_profile_set_filename or
+ * __llvm_profile_override_default_filename or
+ * __llvm_profile_initialize_file. */
 COMPILER_RT_VISIBILITY
 int __llvm_profile_write_file(void) {
-  int rc;
+  int rc, Length;
+  const char *Filename;
+  char *FilenameBuf;
+
+  Length = getCurFilenameLength();
+  FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
+  Filename = getCurFilename(FilenameBuf);
 
-  GetEnvHook = &getenv;
   /* Check the filename. */
-  if (!__llvm_profile_CurrentFilename) {
-    PROF_ERR("LLVM Profile: Failed to write file : %s\n", "Filename not set");
+  if (!Filename) {
+    PROF_ERR("Failed to write file : %s\n", "Filename not set");
     return -1;
   }
 
   /* Check if there is llvm/runtime version mismatch.  */
   if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
-    PROF_ERR("LLVM Profile: runtime and instrumentation version mismatch : "
+    PROF_ERR("Runtime and instrumentation version mismatch : "
              "expected %d, but get %d\n",
              INSTR_PROF_RAW_VERSION,
              (int)GET_VERSION(__llvm_profile_get_version()));
     return -1;
   }
 
-  /* Write the file. */
-  rc = writeFileWithName(__llvm_profile_CurrentFilename);
+  /* Write profile data to the file. */
+  rc = writeFile(Filename);
   if (rc)
-    PROF_ERR("LLVM Profile: Failed to write file \"%s\": %s\n",
-            __llvm_profile_CurrentFilename, strerror(errno));
+    PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
   return rc;
 }
 
@@ -240,6 +506,8 @@ int __llvm_profile_register_write_file_atexit(void) {
   if (HasBeenRegistered)
     return 0;
 
+  lprofSetupValueProfiler();
+
   HasBeenRegistered = 1;
   return atexit(writeFileWithoutReturn);
 }