Merge branch 'master' into merge_master

Author: davidwrighton

.gitattributes

@@ -69,8 +69,10 @@ src/pal/tests/palsuite/paltestlist_to_be_reviewed.txt text eol=lf
 
 tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regexdna/regexdna-input25.txt text eol=lf
 tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regexdna/regexdna-input25000.txt text eol=lf
-tests/src/JIT/Performance/CodeQuality/BenchmarksGame/revcomp/revcomp-input25.txt text eol=lf
-tests/src/JIT/Performance/CodeQuality/BenchmarksGame/revcomp/revcomp-input25000.txt text eol=lf
+tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regex-redux/regexdna-input25.txt text eol=lf
+tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regex-redux/regexdna-input25000.txt text eol=lf
+tests/src/JIT/Performance/CodeQuality/BenchmarksGame/reverse-complement/revcomp-input25.txt text eol=lf
+tests/src/JIT/Performance/CodeQuality/BenchmarksGame/reverse-complement/revcomp-input25000.txt text eol=lf
 tests/src/JIT/Performance/CodeQuality/BenchmarksGame/k-nucleotide/knucleotide-input.txt text eol=lf
 tests/src/JIT/Performance/CodeQuality/BenchmarksGame/k-nucleotide/knucleotide-input-big.txt text eol=lf
 

CMakeLists.txt

@@ -43,7 +43,7 @@ endif()
 # Ensure other tools are present
 if (WIN32)
     if(CLR_CMAKE_HOST_ARCH STREQUAL arm)
-      
+
       # Confirm that Windows SDK is present
       if(NOT DEFINED CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION OR CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION STREQUAL "" )
 	      message(FATAL_ERROR "Windows SDK is required for the Arm32 build.")
@@ -57,9 +57,9 @@ if (WIN32)
       else()
         file(TO_CMAKE_PATH "$ENV{VCToolsInstallDir}\\bin\\HostX86\\arm\\armasm.exe" CMAKE_ASM_COMPILER)
       endif()
-      
+
       set(CMAKE_ASM_MASM_COMPILER ${CMAKE_ASM_COMPILER})
-      message("CMAKE_ASM_MASM_COMPILER explicitly set to: ${CMAKE_ASM_MASM_COMPILER}") 
+      message("CMAKE_ASM_MASM_COMPILER explicitly set to: ${CMAKE_ASM_MASM_COMPILER}")
 
       # Enable generic assembly compilation to avoid CMake generate VS proj files that explicitly
       # use ml[64].exe as the assembler.
@@ -75,13 +75,13 @@ if (WIN32)
     endif()
 
     if (CLR_CMAKE_HOST_ARCH STREQUAL arm64)
-      # CMAKE_CXX_COMPILER will default to the compiler installed with 
-      # Visual studio. Overwrite it to the compiler on the path. 
-      # TODO, remove when cmake generator supports Arm64 as a target.  
-      find_program(PATH_CXX_COMPILER cl) 
-      set(CMAKE_CXX_COMPILER ${PATH_CXX_COMPILER})  
-      message("Overwriting the CMAKE_CXX_COMPILER.") 
-      message(CMAKE_CXX_COMPILER found:${CMAKE_CXX_COMPILER}) 
+      # CMAKE_CXX_COMPILER will default to the compiler installed with
+      # Visual studio. Overwrite it to the compiler on the path.
+      # TODO, remove when cmake generator supports Arm64 as a target.
+      find_program(PATH_CXX_COMPILER cl)
+      set(CMAKE_CXX_COMPILER ${PATH_CXX_COMPILER})
+      message("Overwriting the CMAKE_CXX_COMPILER.")
+      message(CMAKE_CXX_COMPILER found:${CMAKE_CXX_COMPILER})
     endif()
 
 else (WIN32)
@@ -97,7 +97,7 @@ else (WIN32)
     # but the generated executables won't work on a system where PAX is set
     # to prevent applications to create executable memory mappings.
     find_program(PAXCTL paxctl)
- 
+
     if (CMAKE_SYSTEM_NAME STREQUAL Darwin)
 
       # Ensure that dsymutil and strip are present
@@ -155,7 +155,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL Linux)
         endif()
     else()
         # CMAKE_SYSTEM_PROCESSOR returns the value of `uname -p` on target.
-        # For the AMD/Intel 64bit architecure two different strings are common.
+        # For the AMD/Intel 64bit architecture two different strings are common.
         # Linux and Darwin identify it as "x86_64" while FreeBSD and netbsd uses the
         # "amd64" string. Accept either of the two here.
         if(CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL amd64)
@@ -330,13 +330,13 @@ if (WIN32)
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /MANIFEST:NO") #Do not create Side-by-Side Assembly Manifest
 
   if (CLR_CMAKE_PLATFORM_ARCH_ARM)
-    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /SUBSYSTEM:WINDOWS,6.02") #windows subsystem - arm minimum is 6.02  
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /SUBSYSTEM:WINDOWS,6.02") #windows subsystem - arm minimum is 6.02
   elseif(CLR_CMAKE_PLATFORM_ARCH_ARM64)
-    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /SUBSYSTEM:WINDOWS,6.03") #windows subsystem - arm64 minimum is 6.03  
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /SUBSYSTEM:WINDOWS,6.03") #windows subsystem - arm64 minimum is 6.03
   else ()
     set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /SUBSYSTEM:WINDOWS,6.01") #windows subsystem
   endif ()
-    
+
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /LARGEADDRESSAWARE") # can handle addresses larger than 2 gigabytes
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /NXCOMPAT") #Compatible with Data Execution Prevention
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DYNAMICBASE") #Use address space layout randomization
@@ -379,11 +379,11 @@ if (WIN32)
     set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /machine:arm64")
   endif(CLR_CMAKE_PLATFORM_ARCH_ARM64)
 
-  # Force uCRT to be dynamically linked for Release build  
-  set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib")  
-  set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib")  
-  set(CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO} /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib")  
-  set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib")  
+  # Force uCRT to be dynamically linked for Release build
+  set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib")
+  set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib")
+  set(CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO} /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib")
+  set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib")
 
 elseif (CLR_CMAKE_PLATFORM_UNIX)
   # Set the values to display when interactively configuring CMAKE_BUILD_TYPE
@@ -446,11 +446,11 @@ elseif (CLR_CMAKE_PLATFORM_UNIX)
   endif(UPPERCASE_CMAKE_BUILD_TYPE STREQUAL DEBUG OR UPPERCASE_CMAKE_BUILD_TYPE STREQUAL CHECKED)
 endif(WIN32)
 
-if(CLR_CMAKE_PLATFORM_LINUX)  
+if(CLR_CMAKE_PLATFORM_LINUX)
   set(CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -Wa,--noexecstack")
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--build-id=sha1")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id=sha1")
-endif(CLR_CMAKE_PLATFORM_LINUX)  
+endif(CLR_CMAKE_PLATFORM_LINUX)
 
 #------------------------------------
 # Definitions (for platform)
@@ -505,9 +505,9 @@ if (CLR_CMAKE_PLATFORM_UNIX)
     message("Detected FreeBSD amd64")
   endif(CLR_CMAKE_PLATFORM_FREEBSD)
 
-  if(CLR_CMAKE_PLATFORM_NETBSD)  
-    message("Detected NetBSD amd64")  
-  endif(CLR_CMAKE_PLATFORM_NETBSD)  
+  if(CLR_CMAKE_PLATFORM_NETBSD)
+    message("Detected NetBSD amd64")
+  endif(CLR_CMAKE_PLATFORM_NETBSD)
 endif(CLR_CMAKE_PLATFORM_UNIX)
 
 if (WIN32)
@@ -578,7 +578,7 @@ endif()
 
 # Above projects do not build with these compile options
 # All of the compiler options are specified in file compileoptions.cmake
-# Do not add any new options here. They shoul be added in compileoptions.cmake
+# Do not add any new options here. They should be added in compileoptions.cmake
 if(WIN32)
   add_compile_options(/FIWarningControl.h) # force include of WarningControl.h
   add_compile_options(/Zl) # omit default library name in .OBJ

Documentation/botr/clr-abi.md

@@ -75,7 +75,7 @@ NOTE: This optimization is now disabled for all platforms (`IsStructRequiringSta
 
 ## Hidden parameters
 
-*Stub dispatch* - when a virtual call uses a VSD stub, rather than back-patching the calling code (or disassembling it), the JIT must place the address of the stub used to load the call target, the "stub indirection cell", in (x86) `EAX` / (AMD64) `R11` / (ARM) `R4` / (ARM64) `R11`. In the JIT, this is `REG_VIRTUAL_STUB_PARAM`.
+*Stub dispatch* - when a virtual call uses a VSD stub, rather than back-patching the calling code (or disassembling it), the JIT must place the address of the stub used to load the call target, the "stub indirection cell", in (x86) `EAX` / (AMD64) `R11` / (AMD64 CoreRT ABI) `R10` / (ARM) `R4` / (ARM CoreRT ABI) `R12` / (ARM64) `R11`. In the JIT, this is encapsulated in the `VirtualStubParamInfo` class.
 
 AMD64-only: Fast Pinvoke - The VM wants a conservative estimate of the size of the stack arguments placed in `R11`. (This is consumed by callout stubs used in SQL hosting).
 
@@ -119,7 +119,7 @@ For IL stubs only, the per-frame initialization includes setting `Thread->m_pFra
 2. For JIT64/AMD64 only: Next for non-IL stubs, the InlinedCallFrame is 'pushed' by setting `Thread->m_pFrame` to point to the InlinedCallFrame (recall that the per-frame initialization already set `InlinedCallFrame->m_pNext` to point to the previous top). For IL stubs this step is accomplished in the per-frame initialization.
 3. The Frame is made active by setting `InlinedCallFrame->m_pCallerReturnAddress`.
 4. The code then toggles the GC mode by setting `Thread->m_fPreemptiveGCDisabled = 0`.
-5. Starting now, no GC pointers may be live in registers.
+5. Starting now, no GC pointers may be live in registers. RyuJit LSRA meets this requirement by adding special refPositon `RefTypeKillGCRefs` before unmanaged calls and special helpers.
 6. Then comes the actual call/PInvoke.
 7. The GC mode is set back by setting `Thread->m_fPreemptiveGCDisabled = 1`.
 8. Then we check to see if `g_TrapReturningThreads` is set (non-zero). If it is, we call `CORINFO_HELP_STOP_FOR_GC`.

Documentation/building/cross-building.md

@@ -1,3 +1,21 @@
+Cross Compilation for ARM on Windows
+==================================
+
+Building ARM for Windows can be done using cross compilation.
+
+Requirements
+------------
+
+Install the ARM tools and Windows SDK, as described [here](https://github.com/dotnet/coreclr/blob/master/Documentation/building/windows-instructions.md).
+
+Cross compiling CoreCLR
+-----------------------
+
+Build using "-arm" as the architecture. For example:
+
+    C:\coreclr> build.cmd -arm -debug
+
+
 Cross Compilation for ARM on Linux
 ==================================
 

Documentation/building/windows-instructions.md

@@ -45,7 +45,8 @@ For Visual Studio 2017:
     * Visual Studio C++ core features
   * Under "SDKs, libraries, and frameworks":
     * Windows 10 SDK or Windows 8.1 SDK
-* To build for Arm32, Make sure that you have the Windows 10 SDK installed (or selected to be installed as part of VS installation). To explicitly install Windows SDK, download it from here: [Windows SDK for Windows 10](https://developer.microsoft.com/en-us/windows/downloads)
+* To build for Arm32, Make sure that you have the Windows 10 SDK installed (or selected to be installed as part of VS installation). To explicitly install Windows SDK, download it from here: [Windows SDK for Windows 10](https://developer.microsoft.com/en-us/windows/downloads).
+  * In addition, ensure you install the ARM tools. In the "Individual components" window, in the "Compilers, build tools, and runtimes" section, check the box for "Visual C++ compilers and libraries for ARM".
 * **Important:** You must have the `msdia120.dll` COM Library registered in order to build the repository.
   * This binary is registered by default when installing the "VC++ Tools" with Visual Studio 2015
   * You can also manually register the binary by launching the "Developer Command Prompt for VS2017" with Administrative privileges and running `regsvr32.exe "%VSINSTALLDIR%\Common7\IDE\msdia120.dll"`

Documentation/project-docs/linux-performance-tracing.md

@@ -1,4 +1,4 @@
-Performance Tracing on Linux
+Performance Tracing on Linux
 ============================
 
 When a performance problem is encountered on Linux, these instructions can be used to gather detailed information about what was happening on the machine at the time of the performance problem.
@@ -81,6 +81,26 @@ Follow these steps to prepare your machine to collect a performance trace.
 
 	The compressed trace file is now stored in the current working directory.
 
+# Resolving Framework Symbols #
+Framework symbols need to be manually generated at the time the trace is collected.  They are different than app-level symbols because the framework is pre-compiled while apps are just-in-time-compiled.
+
+The easiest way to generate framework symbols is to have perfcollect do it for you automatically when the trace is collected.  To do this, crossgen must be present on the collection machine and located next to libcoreclr.so.  This can be done manually using the following steps:
+
+1. Download the CoreCLR nuget package.  A simple way to do this is to create a self-contained version of your application using the dotnet CLI:
+	> ```bash
+	> dotnet publish --self-contained -r linux-x64
+	> ```	
+
+	This will create a bin/*/netcoreapp2.0/Linux-x64/publish directory which contains all of the files required to run your app including the .NET runtime and framework.  As a side-effect, the dotnet CLI downloads and extracts the CoreCLR nuget package.  You should be able to find crossgen at:
+
+	> ```bash
+	> ~/packages/runtime.linux-x64.microsoft.netcore.app/2.0.0/tools/crossgen
+	> ``` 
+
+2. Copy crossgen next to libcoreclr.so.  If you run your application out of the publish directory, you can copy it into the directory that you just created during step # 1.  If you run your application using the dotnet CLI, then you likely need to copy it to /usr/share/dotnet/shared/Microsoft.NETCore.App/<Version> where <Version> is the version number of CoreCLR.  This should match the version number in the path to crossgen from step # 1.
+
+Now, traces containing apps that were run on the updated runtime should contain framework-level symbols.  To view the trace with framework-level symbols, you will need to use PerfView.  Details on using PerfView to view a trace are available below in this document.
+
 # Collecting in a Docker Container #
 Perfcollect can be used to collect data for an application running inside a Docker container.  The main thing to know is that collecting a trace requires elevated privileges because the [default seccomp profile](https://docs.docker.com/engine/security/seccomp/) blocks a required syscall - perf_events_open.
 

build-test.sh

@@ -37,6 +37,10 @@ initTargetDistroRid()
             export __DistroRid="osx-$__BuildArch"
         fi
     fi
+
+   if [ "$ID.$VERSION_ID" == "ubuntu.16.04" ]; then
+     export __DistroRid="ubuntu.14.04-$__BuildArch"
+   fi
 }
 
 isMSBuildOnNETCoreSupported()

build.cmd

@@ -273,9 +273,9 @@ if not exist "%__BinDir%"           md "%__BinDir%"
 if not exist "%__IntermediatesDir%" md "%__IntermediatesDir%"
 if not exist "%__LogsDir%"          md "%__LogsDir%"
 
-REM It is convinient to have your Nuget search path include the location where the build
-REM will plass packages.  However nuget used during the build will fail if that directory 
-REM does not exist.   Avoid this in at least one case by agressively creating the directory. 
+REM It is convenient to have your Nuget search path include the location where the build
+REM will place packages.  However nuget used during the build will fail if that directory
+REM does not exist.   Avoid this in at least one case by aggressively creating the directory.
 if not exist "%__BinDir%\.nuget\pkg"           md "%__BinDir%\.nuget\pkg"
 
 echo %__MsgPrefix%Commencing CoreCLR Repo build
@@ -296,7 +296,7 @@ echo %__MsgPrefix%Using environment: "%__VSToolsRoot%\VsDevCmd.bat"
 call                                 "%__VSToolsRoot%\VsDevCmd.bat"
 @if defined _echo @echo on
 
-@call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\build.proj -generateHeaderWindows -NativeVersionHeaderFile="%__RootBinDir%\obj\_version.h" %__RunArgs% %__UnprocessedBuildArgs% 
+@call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\build.proj -generateHeaderWindows -NativeVersionHeaderFile="%__RootBinDir%\obj\_version.h" %__RunArgs% %__UnprocessedBuildArgs%
 
 REM =========================================================================================
 REM ===
@@ -342,7 +342,7 @@ if %__BuildNative% EQU 1 (
     set nativePlatfromArgs=-platform=%__BuildArch%
     if /i "%__BuildArch%" == "arm64" ( set nativePlatfromArgs=-useEnv )
 
-    if /i "%__BuildArch%" == "arm64" ( 
+    if /i "%__BuildArch%" == "arm64" (
         rem arm64 builds currently use private toolset which has not been released yet
         REM TODO, remove once the toolset is open.
         call :PrivateToolSet
@@ -354,7 +354,7 @@ if %__BuildNative% EQU 1 (
     if /i "%__BuildArch%" == "x86" ( set __VCBuildArch=x86 )
     if /i "%__BuildArch%" == "arm" (
         set __VCBuildArch=x86_arm
-        
+
         REM Make CMake pick the highest installed version in the 10.0.* range
         set ___SDKVersion="-DCMAKE_SYSTEM_VERSION=10.0"
     )
@@ -474,7 +474,7 @@ if /i "%__DoCrossArchBuild%"=="1" (
         echo     !__BuildWrn!
         echo     !__BuildErr!
         exit /b 1
-    )    
+    )
 
 :SkipCrossCompBuild
     REM } Scope environment changes end
@@ -632,7 +632,7 @@ if %__BuildTests% EQU 1 (
 
     rem arm64 builds currently use private toolset which has not been released yet
     REM TODO, remove once the toolset is open.
-    if /i "%__BuildArch%" == "arm64" call :PrivateToolSet 
+    if /i "%__BuildArch%" == "arm64" call :PrivateToolSet
 
     set NEXTCMD=call %__ProjectDir%\build-test.cmd %__BuildArch% %__BuildType% %__UnprocessedBuildArgs%
     echo %__MsgPrefix%!NEXTCMD!
@@ -642,7 +642,7 @@ if %__BuildTests% EQU 1 (
         REM buildtest.cmd has already emitted an error message and mentioned the build log file to examine.
         exit /b 1
     )
-) 
+)
 
 REM =========================================================================================
 REM ===