[LV] Add -scalable-vectorization=<option> flag.

This patch adds a new option to the LoopVectorizer to control how scalable vectors can be used. Initially, this suggests three levels to control scalable vectorization, although other more aggressive options can be added in the future. The possible options are: - Disabled: Disables vectorization with scalable vectors. - Enabled: Vectorize loops using scalable vectors or fixed-width vectors, but favors fixed-width vectors when the cost is a tie. - Preferred: Like 'Enabled', but favoring scalable vectors when the cost-model is inconclusive. Reviewed By: paulwalker-arm, vkmr Differential Revision: https://reviews.llvm.org/D101945

Author: sdesmalen-arm

llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h

@@ -96,6 +96,20 @@ class LoopVectorizeHints {
  FK_Enabled = 1, ///< Forcing enabled.
  };
 
+ enum ScalableForceKind {
+ /// Not selected.
+ SK_Unspecified = -1,
+ /// Disables vectorization with scalable vectors.
+ SK_FixedWidthOnly = 0,
+ /// Vectorize loops using scalable vectors or fixed-width vectors, but favor
+ /// scalable vectors when the cost-model is inconclusive. This is the
+ /// default when the scalable.enable hint is enabled through a pragma.
+ SK_PreferScalable = 1,
+ /// Vectorize loops using scalable vectors or fixed-width vectors, but
+ /// favor fixed-width vectors when the cost is inconclusive.
+ SK_PreferFixedWidth = 2,
+ };
+
  LoopVectorizeHints(const Loop *L, bool InterleaveOnlyWhenForced,
  OptimizationRemarkEmitter &ORE);
 
@@ -109,7 +123,8 @@ class LoopVectorizeHints {
  void emitRemarkWithHints() const;
 
  ElementCount getWidth() const {
- return ElementCount::get(Width.Value, isScalable());
+ return ElementCount::get(Width.Value,
+ isScalableVectorizationExplicitlyEnabled());
  }
  unsigned getInterleave() const {
  if (Interleave.Value)
@@ -129,7 +144,16 @@ class LoopVectorizeHints {
  return (ForceKind)Force.Value;
  }
 
- bool isScalable() const { return Scalable.Value; }
+ /// \return true if scalable vectorization has been explicitly enabled.
+ bool isScalableVectorizationExplicitlyEnabled() const {
+ return Scalable.Value == SK_PreferFixedWidth ||
+ Scalable.Value == SK_PreferScalable;
+ }
+
+ /// \return true if scalable vectorization has been explicitly disabled.
+ bool isScalableVectorizationDisabled() const {
+ return Scalable.Value == SK_FixedWidthOnly;
+ }
 
  /// If hints are provided that force vectorization, use the AlwaysPrint
  /// pass name to force the frontend to print the diagnostic.

llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp

@@ -48,6 +48,23 @@ static cl::opt<unsigned> PragmaVectorizeSCEVCheckThreshold(
  cl::desc("The maximum number of SCEV checks allowed with a "
  "vectorize(enable) pragma"));
 
+// FIXME: When scalable vectorization is stable enough, change the default
+// to SK_PreferFixedWidth.
+static cl::opt<LoopVectorizeHints::ScalableForceKind> ScalableVectorization(
+ "scalable-vectorization", cl::init(LoopVectorizeHints::SK_FixedWidthOnly),
+ cl::Hidden,
+ cl::desc("Control whether the compiler can use scalable vectors to "
+ "vectorize a loop"),
+ cl::values(
+ clEnumValN(LoopVectorizeHints::SK_FixedWidthOnly, "off",
+ "Scalable vectorization is disabled."),
+ clEnumValN(LoopVectorizeHints::SK_PreferFixedWidth, "on",
+ "Scalable vectorization is available, but favor fixed-width "
+ "vectorization when the cost is inconclusive."),
+ clEnumValN(LoopVectorizeHints::SK_PreferScalable, "preferred",
+ "Scalable vectorization is available and favored when the "
+ "cost is inconclusive.")));
+
 /// Maximum vectorization interleave count.
 static const unsigned MaxInterleaveFactor = 16;
 
@@ -77,15 +94,25 @@ LoopVectorizeHints::LoopVectorizeHints(const Loop *L,
  Force("vectorize.enable", FK_Undefined, HK_FORCE),
  IsVectorized("isvectorized", 0, HK_ISVECTORIZED),
  Predicate("vectorize.predicate.enable", FK_Undefined, HK_PREDICATE),
- Scalable("vectorize.scalable.enable", false, HK_SCALABLE), TheLoop(L),
- ORE(ORE) {
+ Scalable("vectorize.scalable.enable", SK_Unspecified, HK_SCALABLE),
+ TheLoop(L), ORE(ORE) {
  // Populate values with existing loop metadata.
  getHintsFromMetadata();
 
  // force-vector-interleave overrides DisableInterleaving.
  if (VectorizerParams::isInterleaveForced())
  Interleave.Value = VectorizerParams::VectorizationInterleave;
 
+ if ((LoopVectorizeHints::ScalableForceKind)Scalable.Value == SK_Unspecified)
+ // If the width is set, but the metadata says nothing about the scalable
+ // property, then assume it concerns only a fixed-width UserVF.
+ // If width is not set, the flag takes precedence.
+ Scalable.Value = Width.Value ? SK_FixedWidthOnly : ScalableVectorization;
+ else if (ScalableVectorization == SK_FixedWidthOnly)
+ // If the flag is set to disable any use of scalable vectors, override the
+ // loop hint.
+ Scalable.Value = SK_FixedWidthOnly;
+
  if (IsVectorized.Value != 1)
  // If the vectorization width and interleaving count are both 1 then
  // consider the loop to have been already vectorized because there's

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -5648,6 +5648,12 @@ LoopVectorizationCostModel::getMaxLegalScalableVF(unsigned MaxSafeElements) {
  return ElementCount::getScalable(0);
  }
 
+ if (Hints->isScalableVectorizationDisabled()) {
+ reportVectorizationInfo("Scalable vectorization is explicitly disabled",
+ "ScalableVectorizationDisabled", ORE, TheLoop);
+ return ElementCount::getScalable(0);
+ }
+
  auto MaxScalableVF = ElementCount::getScalable(
  std::numeric_limits<ElementCount::ScalarTy>::max());
 

llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll

@@ -1,4 +1,4 @@
-; RUN: opt -loop-vectorize -force-target-instruction-cost=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S < %s 2>&1 | FileCheck %s
+; RUN: opt -loop-vectorize -scalable-vectorization=on -force-target-instruction-cost=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S < %s 2>&1 | FileCheck %s
 
 ; This test currently fails when the LV calculates a maximums safe
 ; distance for scalable vectors, because the code to eliminate the tail is

llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence.ll

@@ -1,5 +1,5 @@
-; RUN: opt -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S < %s | FileCheck %s --check-prefix=CHECK-VF4UF1
-; RUN: opt -loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S < %s | FileCheck %s --check-prefix=CHECK-VF4UF2
+; RUN: opt -loop-vectorize -scalable-vectorization=on -force-vector-width=4 -force-vector-interleave=1 -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S < %s | FileCheck %s --check-prefix=CHECK-VF4UF1
+; RUN: opt -loop-vectorize -scalable-vectorization=on -force-vector-width=4 -force-vector-interleave=2 -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S < %s | FileCheck %s --check-prefix=CHECK-VF4UF2
 
 ; We vectorize this first order recurrence, with a set of insertelements for
 ; each unrolled part. Make sure these insertelements are generated in-order,

llvm/test/Transforms/LoopVectorize/AArch64/masked-op-cost.ll

@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -loop-vectorize -force-vector-interleave=1 -S -debug < %s 2>%t | FileCheck %s
+; RUN: opt -loop-vectorize -scalable-vectorization=on -force-vector-interleave=1 -S -debug-only=loop-vectorize < %s 2>%t | FileCheck %s
 ; RUN: cat %t | FileCheck %s --check-prefix=CHECK-COST
 
 target triple = "aarch64-unknown-linux-gnu"

llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll

@@ -1,4 +1,4 @@
-; RUN: opt -S -loop-vectorize -force-vector-interleave=1 -instcombine -mattr=+sve -mtriple aarch64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: opt -S -loop-vectorize -force-vector-interleave=1 -instcombine -mattr=+sve -mtriple aarch64-unknown-linux-gnu -scalable-vectorization=on < %s | FileCheck %s
 
 define void @vec_load(i64 %N, double* nocapture %a, double* nocapture readonly %b) {
 ; CHECK-LABEL: @vec_load

llvm/test/Transforms/LoopVectorize/AArch64/scalable-reductions.ll

@@ -1,4 +1,4 @@
-; RUN: opt < %s -loop-vectorize -pass-remarks=loop-vectorize -pass-remarks-analysis=loop-vectorize -pass-remarks-missed=loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S 2>%t | FileCheck %s -check-prefix=CHECK
+; RUN: opt < %s -loop-vectorize -pass-remarks=loop-vectorize -pass-remarks-analysis=loop-vectorize -pass-remarks-missed=loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S -scalable-vectorization=on 2>%t | FileCheck %s -check-prefix=CHECK
 ; RUN: cat %t | FileCheck %s -check-prefix=CHECK-REMARK
 
 ; Reduction can be vectorized

llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll

@@ -1,4 +1,4 @@
-; RUN: opt < %s -loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -enable-strict-reductions -S | FileCheck %s -check-prefix=CHECK
+; RUN: opt < %s -loop-vectorize -scalable-vectorization=on -mtriple aarch64-unknown-linux-gnu -mattr=+sve -enable-strict-reductions -S | FileCheck %s -check-prefix=CHECK
 
 define float @fadd_strict(float* noalias nocapture readonly %a, i64 %n) {
 ; CHECK-LABEL: @fadd_strict

llvm/test/Transforms/LoopVectorize/AArch64/scalable-vf-analysis.ll

@@ -1,13 +1,17 @@
 ; REQUIRES: asserts
-; RUN: opt -mtriple=aarch64-none-linux-gnu -mattr=+sve -force-target-instruction-cost=1 -loop-vectorize -S -debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK_SCALABLE_ON
-; RUN: opt -mtriple=aarch64-none-linux-gnu -mattr=+sve -force-target-instruction-cost=1 -loop-vectorize -S -debug-only=loop-vectorize -vectorizer-maximize-bandwidth < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK_SCALABLE_ON_MAXBW
+; RUN: opt -mtriple=aarch64-none-linux-gnu -mattr=+sve -force-target-instruction-cost=1 -loop-vectorize -S -debug-only=loop-vectorize -scalable-vectorization=on < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK_SCALABLE_ON
+; RUN: opt -mtriple=aarch64-none-linux-gnu -mattr=+sve -force-target-instruction-cost=1 -loop-vectorize -S -debug-only=loop-vectorize -scalable-vectorization=preferred < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK_SCALABLE_PREFERRED
+; RUN: opt -mtriple=aarch64-none-linux-gnu -mattr=+sve -force-target-instruction-cost=1 -loop-vectorize -S -debug-only=loop-vectorize -scalable-vectorization=off < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK_SCALABLE_DISABLED
+; RUN: opt -mtriple=aarch64-none-linux-gnu -mattr=+sve -force-target-instruction-cost=1 -loop-vectorize -S -debug-only=loop-vectorize -vectorizer-maximize-bandwidth -scalable-vectorization=on < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK_SCALABLE_ON_MAXBW
 
 ; Test that the MaxVF for the following loop, that has no dependence distances,
 ; is calculated as vscale x 4 (max legal SVE vector size) or vscale x 16
 ; (maximized bandwidth for i8 in the loop).
 define void @test0(i32* %a, i8* %b, i32* %c) {
 ; CHECK: LV: Checking a loop in "test0"
 ; CHECK_SCALABLE_ON: LV: Found feasible scalable VF = vscale x 4
+; CHECK_SCALABLE_PREFERRED: LV: Found feasible scalable VF = vscale x 4
+; CHECK_SCALABLE_DISABLED-NOT: LV: Found feasible scalable VF
 ; CHECK_SCALABLE_ON_MAXBW: LV: Found feasible scalable VF = vscale x 16
 entry:
  br label %loop
@@ -35,6 +39,8 @@ exit:
 define void @test1(i32* %a, i8* %b) {
 ; CHECK: LV: Checking a loop in "test1"
 ; CHECK_SCALABLE_ON: LV: Found feasible scalable VF = vscale x 4
+; CHECK_SCALABLE_PREFERRED: LV: Found feasible scalable VF = vscale x 4
+; CHECK_SCALABLE_DISABLED-NOT: LV: Found feasible scalable VF
 ; CHECK_SCALABLE_ON_MAXBW: LV: Found feasible scalable VF = vscale x 4
 entry:
  br label %loop
@@ -63,6 +69,8 @@ exit:
 define void @test2(i32* %a, i8* %b) {
 ; CHECK: LV: Checking a loop in "test2"
 ; CHECK_SCALABLE_ON: LV: Found feasible scalable VF = vscale x 2
+; CHECK_SCALABLE_PREFERRED: LV: Found feasible scalable VF = vscale x 2
+; CHECK_SCALABLE_DISABLED-NOT: LV: Found feasible scalable VF
 ; CHECK_SCALABLE_ON_MAXBW: LV: Found feasible scalable VF = vscale x 2
 entry:
  br label %loop
@@ -91,6 +99,8 @@ exit:
 define void @test3(i32* %a, i8* %b) {
 ; CHECK: LV: Checking a loop in "test3"
 ; CHECK_SCALABLE_ON: LV: Found feasible scalable VF = vscale x 1
+; CHECK_SCALABLE_PREFERRED: LV: Found feasible scalable VF = vscale x 1
+; CHECK_SCALABLE_DISABLED-NOT: LV: Found feasible scalable VF
 ; CHECK_SCALABLE_ON_MAXBW: LV: Found feasible scalable VF = vscale x 1
 entry:
  br label %loop
@@ -120,6 +130,8 @@ exit:
 define void @test4(i32* %a, i32* %b) {
 ; CHECK: LV: Checking a loop in "test4"
 ; CHECK_SCALABLE_ON-NOT: LV: Found feasible scalable VF
+; CHECK_SCALABLE_PREFERRED-NOT: LV: Found feasible scalable VF
+; CHECK_SCALABLE_DISABLED-NOT: LV: Found feasible scalable VF
 ; CHECK_SCALABLE_ON_MAXBW-NOT: LV: Found feasible scalable VF
 entry:
  br label %loop