Test local cache in the CI

.github/workflows/ci.yaml

@@ -26,10 +26,10 @@ on:
  - cron: '0 3 * * *' # Every day at 3 AM
  workflow_dispatch:
 
-# Cancels any in-progress runs within the same group identified by workflow name and GH reference (branch or tag) 
+# Cancels any in-progress runs within the same group identified by workflow name and GH reference (branch or tag)
 # For example it would:
 # - terminate previous PR CI execution after pushing more changes to the same PR branch
-# - terminate previous on-push CI run after merging new PR to main 
+# - terminate previous on-push CI run after merging new PR to main
 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
@@ -53,12 +53,65 @@ env:
 # text on stderr and so can break tests which check the output of a program).
 
 jobs:
+ build-non-bootstrapped:
+ name: Build Non-Bootstrapped Compiler
+ runs-on: [self-hosted, Linux]
+ container:
+ image: lampepfl/dotty:2024-10-18
+ options: --cpu-shares 4096
+ volumes:
+ # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
+ - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
+ - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
+ - ${{ github.workspace }}/../../cache/general:/root/.cache
+ steps:
+ - name: Set JDK 17 as default
+ run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH
+ - name: Checkout cleanup script
+ uses: actions/checkout@v4
+ - name: Cleanup
+ run: .github/workflows/cleanup.sh
+ - name: Git Checkout
+ uses: actions/checkout@v4
+ - name: Add SBT proxy repositories
+ run: cp -vf .github/workflows/repositories /root/.sbt/ ; true
+ - name: Compile Non-Bootstrapped Dotty
+ run: ./project/scripts/sbt "; clean; compile"
+
+ build-bootstrapped:
+ name: Build Bootstrapped Compiler
+ needs: build-non-bootstrapped
+ runs-on: [self-hosted, Linux]
+ container:
+ image: lampepfl/dotty:2024-10-18
+ options: --cpu-shares 4096
+ volumes:
+ # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
+ - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
+ - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
+ - ${{ github.workspace }}/../../cache/general:/root/.cache
+ steps:
+ - name: Set JDK 17 as default
+ run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH
+ - name: Checkout cleanup script
+ uses: actions/checkout@v4
+ - name: Cleanup
+ run: .github/workflows/cleanup.sh
+ - name: Git Checkout
+ uses: actions/checkout@v4
+ - name: Add SBT proxy repositories
+ run: cp -vf .github/workflows/repositories /root/.sbt/ ; true
+ - name: Compile Bootstrapped Dotty
+ run: ./project/scripts/sbt "; clean; scala3-bootstrapped/compile"
+
  test_non_bootstrapped:
+ needs: build-non-bootstrapped
  runs-on: [self-hosted, Linux]
  container:
  image: lampepfl/dotty:2024-10-18
  options: --cpu-shares 4096
  volumes:
+ # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
  - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
  - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
  - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -106,11 +159,13 @@ jobs:
  ./project/scripts/cmdTests
 
  test:
+ needs: build-bootstrapped
  runs-on: [self-hosted, Linux]
  container:
  image: lampepfl/dotty:2024-10-18
  options: --cpu-shares 4096
  volumes:
+ # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
  - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
  - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
  - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -166,11 +221,13 @@ jobs:
  run: ./project/scripts/sbt ";set ThisBuild/Build.scala2Library := Build.Scala2LibraryCCTasty; scala2-library-cc/compile; scala2-library-cc-tasty/compile; scala3-bootstrapped/testCompilation i3"
 
  test_scala2_library_tasty:
+ needs: build-bootstrapped
  runs-on: [self-hosted, Linux]
  container:
  image: lampepfl/dotty:2024-10-18
  options: --cpu-shares 4096
  volumes:
+ # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
  - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
  - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
  - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -293,11 +350,13 @@ jobs:
 
  mima:
  name: MiMa
+ needs: build-bootstrapped
  runs-on: [self-hosted, Linux]
  container:
  image: lampepfl/dotty:2024-10-18
  options: --cpu-shares 4096
  volumes:
+ # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
  - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
  - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
  - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -344,11 +403,13 @@ jobs:
  ./project/scripts/scala2-library-tasty-mima.sh
 
  community_build_a:
+ needs: build-bootstrapped
  runs-on: [self-hosted, Linux]
  container:
  image: lampepfl/dotty:2024-10-18
  options: --cpu-shares 4096
  volumes:
+ # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
  - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
  - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
  - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -401,11 +462,13 @@ jobs:
  run: cat community-build/dotty-community-build-deps || true
 
  community_build_b:
+ needs: build-bootstrapped
  runs-on: [self-hosted, Linux]
  container:
  image: lampepfl/dotty:2024-10-18
  options: --cpu-shares 4096
  volumes:
+ # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
  - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
  - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
  - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -458,11 +521,13 @@ jobs:
  run: cat community-build/dotty-community-build-deps || true
 
  community_build_c:
+ needs: build-bootstrapped
  runs-on: [self-hosted, Linux]
  container:
  image: lampepfl/dotty:2024-10-18
  options: --cpu-shares 4096
  volumes:
+ # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
  - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
  - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
  - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -515,11 +580,13 @@ jobs:
  run: cat community-build/dotty-community-build-deps || true
 
  test_sbt:
+ needs: build-bootstrapped
  runs-on: [self-hosted, Linux]
  container:
  image: lampepfl/dotty:2024-10-18
  options: --cpu-shares 4096
  volumes:
+ # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
  - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
  - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
  - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -565,6 +632,7 @@ jobs:
  image: lampepfl/dotty:2024-10-18
  options: --cpu-shares 4096
  volumes:
+ # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
  - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
  - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
  - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -625,6 +693,7 @@ jobs:
  image: lampepfl/dotty:2024-10-18
  options: --cpu-shares 4096
  volumes:
+ # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
  - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
  - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
  - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -696,6 +765,7 @@ jobs:
  image: lampepfl/dotty:2024-10-18
  options: --cpu-shares 4096
  volumes:
+ # - /nfs-cache/.develocity/build-cache:/root/.sbt/1.0/.develocity/build-cache
  - ${{ github.workspace }}/../../cache/sbt:/root/.sbt
  - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
  - ${{ github.workspace }}/../../cache/general:/root/.cache
@@ -855,7 +925,7 @@ jobs:
  scala3-${{ env.RELEASE_TAG }}*.zip \
  scala3-${{ env.RELEASE_TAG }}*.tar.gz \
  scala3-${{ env.RELEASE_TAG }}*.sha256 \
- scala3-${{ env.RELEASE_TAG }}.msi 
+ scala3-${{ env.RELEASE_TAG }}.msi
 
  - name: Publish Release
  run: ./project/scripts/sbtPublish ";project scala3-bootstrapped ;publishSigned ;sonatypeBundleUpload"

.sbtopts

@@ -0,0 +1,2 @@
+# read build from develocity cache
+-Ddevelocity.scan.metaBuild

project/Build.scala

@@ -339,24 +339,28 @@ object Build {
  buildScan
  .withPublishing(Publishing.onlyIf(_.authenticated))
  .withBackgroundUpload(!isInsideCI)
- .tag(if (isInsideCI) "CI" else "Local")
+ .withTag(if (isInsideCI) "CI" else "Local")
  .withLinks(buildScan.links ++ GithubEnv.develocityLinks)
  .withValues(buildScan.values ++ GithubEnv.develocityValues)
  .withObfuscation(buildScan.obfuscation.withIpAddresses(_.map(_ => "0.0.0.0")))
  )
  .withBuildCache(
  buildCache
- .withLocal(buildCache.local.withEnabled(false))
- .withRemote(buildCache.remote.withEnabled(false))
+ .withLocal(buildCache.local.withEnabled(true).withStoreEnabled(true))
+ .withRemote(buildCache.remote.withEnabled(true).withStoreEnabled(isInsideCI))
+ .withRequireClean(!isInsideCI) // always cache in CI
  )
- .withTestRetryConfiguration(
- config.testRetryConfiguration
+ .withTestRetry(
+ config.testRetry
  .withFlakyTestPolicy(FlakyTestPolicy.Fail)
  .withMaxRetries(if (isInsideCI) 1 else 0)
  .withMaxFailures(10)
  .withClassesFilter((className, _) => !noRetryTestClasses.contains(className))
  )
- }
+ },
+ // Deactivate Develocity's test caching because it caches all tests or nothing.
+ // Also at the moment, it does not take compilation files as inputs.
+ Test / develocityBuildCacheClient := None,
  )
 
  // Settings shared globally (scoped in Global). Used in build.sbt
@@ -1015,10 +1019,6 @@ object Build {
  sjsSources
  } (Set(scalaJSIRSourcesJar)).toSeq
  }.taskValue,
-
- // Develocity's Build Cache does not work with our compilation tests
- // at the moment: it does not take compilation files as inputs.
- Test / develocityBuildCacheClient := None,
  )
 
  def insertClasspathInArgs(args: List[String], cp: String): List[String] = {
@@ -1117,7 +1117,7 @@ object Build {
  libraryDependencies += "org.scala-lang" % "scala-library" % stdlibVersion,
  (Compile / scalacOptions) ++= Seq(
  // Needed so that the library sources are visible when `dotty.tools.dotc.core.Definitions#init` is called
- "-sourcepath", (Compile / sourceDirectories).value.map(_.getAbsolutePath).distinct.mkString(File.pathSeparator),
+ "-sourcepath", (Compile / sourceDirectories).value.map(_.getCanonicalPath).distinct.mkString(File.pathSeparator),
  "-Yexplicit-nulls",
  ),
  (Compile / doc / scalacOptions) ++= ScaladocConfigs.DefaultGenerationSettings.value.settings,

project/build.sbt

@@ -2,3 +2,21 @@
 libraryDependencies += "org.eclipse.jgit" % "org.eclipse.jgit" % "4.11.0.201803080745-r"
 
 libraryDependencies += Dependencies.`jackson-databind`
+
+// Configuration to publish build compilation to local and remote cache
+ThisBuild / develocityConfiguration := {
+ val isInsideCI = insideCI.value
+ val config = develocityConfiguration.value
+ val buildScan = config.buildScan
+ val buildCache = config.buildCache
+ config
+ .withProjectId(ProjectId("scala3-build"))
+ .withServer(config.server.withUrl(Some(url("https://develocity.scala-lang.org"))))
+ .withBuildScan(buildScan.withPublishing(Publishing.onlyIf(_ => false)))
+ .withBuildCache(
+ buildCache
+ .withLocal(buildCache.local.withEnabled(true).withStoreEnabled(true))
+ .withRemote(buildCache.remote.withEnabled(true).withStoreEnabled(isInsideCI))
+ .withRequireClean(!isInsideCI) // always cache inside CI
+ )
+}

project/plugins.sbt

@@ -22,4 +22,7 @@ addSbtPlugin("ch.epfl.scala" % "sbt-tasty-mima" % "1.0.0")
 
 addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.10.0")
 
-addSbtPlugin("com.gradle" % "sbt-develocity" % "1.1.1")
+resolvers +=
+ "Develocity Artifactory" at "https://repo.grdev.net/artifactory/public/"
+
+addSbtPlugin("com.gradle" % "sbt-develocity" % "1.2-rc-2")

project/project/plugins.sbt

@@ -0,0 +1,3 @@
+resolvers += "Develocity Artifactory" at "https://repo.grdev.net/artifactory/public/"
+
+addSbtPlugin("com.gradle" % "sbt-develocity" % "1.2-rc-2")

project/scripts/sbt

@@ -10,4 +10,6 @@ CMD="${1:?Missing sbt command}"
 sbt -J-XX:ReservedCodeCacheSize=512m \
  -DSBT_PGP_USE_GPG=false \
  -no-colors \
+ -Ddevelocity.internal.cache.verbose=true \
+ -Ddevelocity.internal.cache.defaultLogLevel=info \
  "$CMD"