test-release.sh: Update to fetch source from GitHub

Summary: This also changes the test-release.sh script to build using the monorepo layout instead of copying sub-projects into llvm/tools or llvm/projects. Reviewers: jdoerfert, hans Reviewed By: hans Subscribers: hans, llvm-commits Tags: #llvm Differential Revision: https://reviews.llvm.org/D70353

Author: tstellar

llvm/utils/release/test-release.sh

@@ -43,6 +43,7 @@ do_polly="yes"
 BuildDir="`pwd`"
 ExtraConfigureFlags=""
 ExportBranch=""
+git_ref=""
 
 function usage() {
  echo "usage: `basename $0` -release X.Y.Z -rc NUM [OPTIONS]"
@@ -60,8 +61,7 @@ function usage() {
  echo " -use-gzip Use gzip instead of xz."
  echo " -use-ninja Use ninja instead of make/gmake."
  echo " -configure-flags FLAGS Extra flags to pass to the configure step."
- echo " -svn-path DIR Use the specified DIR instead of a release."
- echo " For example -svn-path trunk or -svn-path branches/release_37"
+ echo " -git-ref sha Use the specified git ref for testing instead of a release."
  echo " -no-rt Disable check-out & build Compiler-RT"
  echo " -no-libs Disable check-out & build libcxx/libcxxabi/libunwind"
  echo " -no-libcxxabi Disable check-out & build libcxxabi"
@@ -88,13 +88,14 @@ while [ $# -gt 0 ]; do
  -final | --final )
  RC=final
  ;;
- -svn-path | --svn-path )
+ -git-ref | --git-ref )
  shift
  Release="test"
  Release_no_dot="test"
  ExportBranch="$1"
  RC="`echo $ExportBranch | sed -e 's,/,_,g'`"
- echo "WARNING: Using the branch $ExportBranch instead of a release tag"
+ git_ref="$1"
+ echo "WARNING: Using the ref $git_ref instead of a release tag"
  echo " This is intended to aid new packagers in trialing "
  echo " builds without requiring a tag to be created first"
  ;;
@@ -196,6 +197,17 @@ if [ -z "$Triple" ]; then
  exit 1
 fi
 
+if [ "$Release" != "test" ]; then
+ if [ -n "$git_ref" ]; then
+ echo "error: can't specify both -release and -git-ref"
+ exit 1
+ fi
+ git_ref=llvmorg-$Release
+ if [ "$RC" != "final" ]; then
+ git_ref="$git_ref-$RC"
+ fi
+fi
+
 # Figure out how many make processes to run.
 if [ -z "$NumJobs" ]; then
  NumJobs=`sysctl -n hw.activecpu 2> /dev/null || true`
@@ -211,7 +223,7 @@ if [ -z "$NumJobs" ]; then
 fi
 
 # Projects list
-projects="llvm cfe clang-tools-extra"
+projects="llvm clang clang-tools-extra"
 if [ $do_rt = "yes" ]; then
  projects="$projects compiler-rt"
 fi
@@ -288,60 +300,37 @@ fi
 
 check_program_exists ${MAKE}
 
-# Make sure that the URLs are valid.
-function check_valid_urls() {
- for proj in $projects ; do
- echo "# Validating $proj SVN URL"
-
- if ! svn ls $Base_url/$proj/$ExportBranch > /dev/null 2>&1 ; then
- echo "$proj does not have a $ExportBranch branch/tag!"
- exit 1
- fi
- done
-}
-
 # Export sources to the build directory.
 function export_sources() {
- check_valid_urls
-
- for proj in $projects ; do
- case $proj in
- llvm)
- projsrc=$proj.src
- ;;
- cfe)
- projsrc=llvm.src/tools/clang
- ;;
- lld|lldb|polly)
- projsrc=llvm.src/tools/$proj
- ;;
- clang-tools-extra)
- projsrc=llvm.src/tools/clang/tools/extra
- ;;
- compiler-rt|libcxx|libcxxabi|libunwind|openmp)
- projsrc=llvm.src/projects/$proj
- ;;
- test-suite)
- projsrc=$proj.src
- ;;
- *)
- echo "error: unknown project $proj"
- exit 1
- ;;
- esac
-
- if [ -d $projsrc ]; then
- echo "# Reusing $proj $Release-$RC sources in $projsrc"
- continue
- fi
- echo "# Exporting $proj $Release-$RC sources to $projsrc"
- if ! svn export -q $Base_url/$proj/$ExportBranch $projsrc ; then
- echo "error: failed to export $proj project"
- exit 1
- fi
- done
+ SrcDir=$BuildDir/llvm-project
+ mkdir -p $SrcDir
+ echo "# Using git ref: $git_ref"
+
+ # GitHub allows you to download a tarball of any commit using the URL:
+ # https://github.com/$organization/$repo/archive/$ref.tar.gz
+ curl -L https://github.com/llvm/llvm-project/archive/$git_ref.tar.gz | \
+ tar -C $SrcDir --strip-components=1 -xzf -
+
+ if [ "$do_test_suite" = "yes" ]; then
+ TestSuiteSrcDir=$BuildDir/llvm-test-suite
+ mkdir -p $TestSuiteSrcDir
+
+ # We can only use named refs, like branches and tags, that exist in
+ # both the llvm-project and test-suite repos if we want to run the
+ # test suite.
+ # If the test-suite fails to download assume we are using a ref that
+ # doesn't exist in the test suite and disable it.
+ set +e
+ curl -L https://github.com/llvm/test-suite/archive/$git_ref.tar.gz | \
+ tar -C $TestSuiteSrcDir --strip-components=1 -xzf -
+ if [ $? -ne -0 ]; then
+ echo "$git_ref not found in test-suite repo, test-suite disabled."
+ do_test_suite="no"
+ fi
+ set -e
+ fi
 
-  cd $BuildDir
+ cd $BuildDir
 }
 
 function configure_llvmCore() {
@@ -369,6 +358,7 @@ function configure_llvmCore() {
  ;;
  esac
 
+ project_list=${projects// /;}
  echo "# Using C compiler: $c_compiler"
  echo "# Using C++ compiler: $cxx_compiler"
 
@@ -378,12 +368,14 @@ function configure_llvmCore() {
  echo "#" env CC="$c_compiler" CXX="$cxx_compiler" \
  cmake -G "$generator" \
  -DCMAKE_BUILD_TYPE=$BuildType -DLLVM_ENABLE_ASSERTIONS=$Assertions \
- $ExtraConfigureFlags $BuildDir/llvm.src \
+ -DLLVM_ENABLE_PROJECTS="$project_list" \
+ $ExtraConfigureFlags $BuildDir/llvm-project/llvm \
  2>&1 | tee $LogDir/llvm.configure-Phase$Phase-$Flavor.log
  env CC="$c_compiler" CXX="$cxx_compiler" \
  cmake -G "$generator" \
  -DCMAKE_BUILD_TYPE=$BuildType -DLLVM_ENABLE_ASSERTIONS=$Assertions \
- $ExtraConfigureFlags $BuildDir/llvm.src \
+ -DLLVM_ENABLE_PROJECTS="$project_list" \
+ $ExtraConfigureFlags $BuildDir/llvm-project/llvm \
  2>&1 | tee $LogDir/llvm.configure-Phase$Phase-$Flavor.log
 
  cd $BuildDir
@@ -494,10 +486,10 @@ if [ $do_test_suite = "yes" ]; then
  SandboxDir="$BuildDir/sandbox"
  Lit=$SandboxDir/bin/lit
  TestSuiteBuildDir="$BuildDir/test-suite-build"
- TestSuiteSrcDir="$BuildDir/test-suite.src"
+ TestSuiteSrcDir="$BuildDir/llvm-test-suite"
 
  virtualenv $SandboxDir
- $SandboxDir/bin/python $BuildDir/llvm.src/utils/lit/setup.py install
+ $SandboxDir/bin/python $BuildDir/llvm-project/llvm/utils/lit/setup.py install
  mkdir -p $TestSuiteBuildDir
 fi