Spring/spring-aot-smoke-tests
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Try to improve GitHub Actions cache efficiency

Browse Source 
Previously, every individual smoke test would write to and read from
the GitHub Actions cache. This quickly exhausted to 10GB limit,
resulting in cache misses and time spent downloading Gradle and
dependencies.

This commit configures each smoke test's workflow to only read from
the GitHub Actions cache and to never write to it. To populate the
cache, it introduces a new Warm Caches workflow that has read-write
access to the cache. It runs once per branch per day before the
branch's smoke tests run. The intent is that the Warm Caches workflow
will populate the GitHub Actions cache and that each of the
individual smoke tests will then reuse this cache. With each branch
only have a single workflow that writes to the cache, the 10GB limit
should not be exhausted, increasing cache hits.
This commit is contained in:
Andy Wilkinson
2025-01-17 11:16:17 +00:00
parent 5b511adc55
commit c57e2dfcea
13 changed files with 215 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.github/workflows/3.0.x-warm-caches.yml vendored Normal file
View File

@@ -0,0 +1,12 @@
name: 3.0.x | Warm Caches
on:
schedule:
- cron : '0 0 * * *'
workflow_dispatch:
jobs:
warm_caches:
uses: ./.github/workflows/warm-caches.yml
secrets: inherit
with:
checkout_repository: spring-projects/spring-aot-smoke-tests
checkout_ref: 3.0.x

12
.github/workflows/3.1.x-warm-caches.yml vendored Normal file
View File

@@ -0,0 +1,12 @@
name: 3.1.x | Warm Caches
on:
schedule:
- cron : '10 0 * * *'
workflow_dispatch:
jobs:
warm_caches:
uses: ./.github/workflows/warm-caches.yml
secrets: inherit
with:
checkout_repository: spring-projects/spring-aot-smoke-tests
checkout_ref: 3.1.x

12
.github/workflows/3.2.x-warm-caches.yml vendored Normal file
View File

@@ -0,0 +1,12 @@
name: 3.2.x | Warm Caches
on:
schedule:
- cron : '20 0 * * *'
workflow_dispatch:
jobs:
warm_caches:
uses: ./.github/workflows/warm-caches.yml
secrets: inherit
with:
checkout_repository: spring-projects/spring-aot-smoke-tests
checkout_ref: 3.2.x

12
.github/workflows/3.3.x-warm-caches.yml vendored Normal file
View File

@@ -0,0 +1,12 @@
name: 3.3.x | Warm Caches
on:
schedule:
- cron : '30 0 * * *'
workflow_dispatch:
jobs:
warm_caches:
uses: ./.github/workflows/warm-caches.yml
secrets: inherit
with:
checkout_repository: spring-projects/spring-aot-smoke-tests
checkout_ref: 3.3.x

12
.github/workflows/3.4.x-warm-caches.yml vendored Normal file
View File

@@ -0,0 +1,12 @@
name: 3.4.x | Warm Caches
on:
schedule:
- cron : '40 0 * * *'
workflow_dispatch:
jobs:
warm_caches:
uses: ./.github/workflows/warm-caches.yml
secrets: inherit
with:
checkout_repository: spring-projects/spring-aot-smoke-tests
checkout_ref: main

2
.github/workflows/smoke-test.yml vendored
View File

@@ -52,6 +52,8 @@ jobs:
distribution: 'liberica'
- name: Set up Gradle
uses: gradle/actions/setup-gradle@db19848a5fa7950289d3668fb053140cf3028d43 # v3.3.2
with:
cache-read-only: true
- name: Configure Gradle user.name
run: |
mkdir -p ~/.gradle

37
.github/workflows/warm-caches.yml vendored Normal file
View File

@@ -0,0 +1,37 @@
name: Smoke Test
on:
workflow_call:
inputs:
checkout_repository:
required: true
type: string
checkout_ref:
required: true
type: string
jobs:
warm_caches:
name: ${{ inputs.task }}
runs-on: ubuntu-latest
steps:
- name: Check out
uses: actions/checkout@v4
with:
repository: ${{ inputs.checkout_repository }}
ref: ${{ inputs.checkout_ref }}
- name: Set up Java
uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1
with:
java-version: ${{ inputs.java_version }}
distribution: 'liberica'
- name: Set up Gradle
uses: gradle/actions/setup-gradle@db19848a5fa7950289d3668fb053140cf3028d43 # v3.3.2
- name: Configure Gradle user.name
run: |
mkdir -p ~/.gradle
echo 'systemProp.user.name=spring-builds+github' >> ~/.gradle/gradle.properties
- name: Warm Caches
env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
REPO_SPRING_IO_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }}
REPO_SPRING_IO_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
run: ./gradlew warmCaches

5
build.gradle
View File

@@ -5,23 +5,18 @@ plugins {
smokeTests {
'3.0.x' {
location = "../3.0.x"
cronSchedule = "10 0 * * *"
}
'3.1.x' {
location = "../3.1.x"
cronSchedule = "20 0 * * *"
}
'3.2.x' {
location = "../3.2.x"
cronSchedule = "30 0 * * *"
}
'3.3.x' {
location = "../3.3.x"
cronSchedule = "40 0 * * *"
}
'3.4.x' {
branch = "main"
location = "../main"
cronSchedule = "50 0 * * *"
}
}

47
gradle/plugins/aot-smoke-test-ci-plugin/src/main/java/org/springframework/aot/gradle/AotSmokeTestCiPlugin.java
View File

@@ -50,8 +50,12 @@ public class AotSmokeTestCiPlugin implements Plugin<Project> {
syncFromClasspath("smoke-test-jvm.yml", sync);
syncFromClasspath("smoke-test-native.yml", sync);
syncFromClasspath("validate-gradle-wrapper.yml", sync);
syncFromClasspath("warm-caches.yml", sync);
});
CronSchedule cronSchedule = new CronSchedule();
smokeTests.configureEach((tests) -> {
String warmCachesSchedule = cronSchedule.warmCaches();
String runTestsSchedule = cronSchedule.runTests();
TaskProvider<Exec> describeSmokeTestsForBranch = project.getTasks()
.register("describeSmokeTestsFor" + tests.getName(), Exec.class);
describeSmokeTestsForBranch.configure((task) -> {
@@ -67,9 +71,11 @@ public class AotSmokeTestCiPlugin implements Plugin<Project> {
task.getGitBranch().set(tests.getBranch());
}
task.getSmokeTests().set(project.provider(() -> loadSmokeTests(tests.getLocation())));
task.getCronSchedule().set(tests.getCronSchedule());
task.getWarmCachesCronSchedule().set(warmCachesSchedule);
task.getCronSchedule().set(runTestsSchedule);
});
syncWorkflows.configure((sync) -> sync.from(generateWorkflowsForBranch));
cronSchedule.nextBatch();
});
}
@@ -94,4 +100,43 @@ public class AotSmokeTestCiPlugin implements Plugin<Project> {
(spec) -> spec.rename((temp) -> name));
}
static final class CronSchedule {
private int minute;
private int hour;
private CronSchedule() {
this(0, 0);
}
private CronSchedule(int minute, int hour) {
this.minute = minute;
this.hour = hour;
}
String warmCaches() {
return asString();
}
String runTests() {
CronSchedule offsetSchedule = new CronSchedule(this.minute, this.hour);
offsetSchedule.nextBatch();
return offsetSchedule.asString();
}
private String asString() {
return "%d %d * * *".formatted(this.minute, this.hour);
}
void nextBatch() {
this.minute += 10;
if (this.minute == 60) {
this.minute = 0;
this.hour += 1;
}
}
}
}

32
gradle/plugins/aot-smoke-test-ci-plugin/src/main/java/org/springframework/aot/gradle/GenerateGitHubActionsWorkflows.java
View File

@@ -48,6 +48,9 @@ public abstract class GenerateGitHubActionsWorkflows extends DefaultTask {
@Input
public abstract Property<String> getGitBranch();
@Input
public abstract Property<String> getWarmCachesCronSchedule();
@Input
public abstract Property<String> getCronSchedule();
@@ -63,10 +66,35 @@ public abstract class GenerateGitHubActionsWorkflows extends DefaultTask {
@TaskAction
void generateWorkflows() {
getProject().delete(getOutputDirectory());
getSmokeTests().get().forEach(this::generateWorkflow);
generateWarmCachesWorkflow();
getSmokeTests().get().forEach(this::generateSmokeTestWorkflow);
}
void generateWorkflow(SmokeTest smokeTest) {
private void generateWarmCachesWorkflow() {
String springBootGeneration = getSpringBootGeneration().get();
File workflowFile = getOutputDirectory().file(springBootGeneration + "-warm-caches.yml").get().getAsFile();
workflowFile.getParentFile().mkdirs();
String workflowName = springBootGeneration + " | Warm Caches";
try (PrintWriter writer = new PrintWriter(new FileWriter(workflowFile))) {
writer.println("name: " + workflowName);
writer.println("on:");
writer.println(" schedule:");
writer.println(" - cron : '" + getWarmCachesCronSchedule().get() + "'");
writer.println(" workflow_dispatch:");
writer.println("jobs:");
writer.println(" warm_caches:");
writer.println(" uses: ./.github/workflows/warm-caches.yml");
writer.println(" secrets: inherit");
writer.println(" with:");
writer.println(" checkout_repository: " + GITHUB_REPOSITORY);
writer.println(" checkout_ref: " + getGitBranch().get());
}
catch (IOException ex) {
throw new GradleException("Failed to write workflow file '" + workflowFile + "'", ex);
}
}
private void generateSmokeTestWorkflow(SmokeTest smokeTest) {
String springBootGeneration = getSpringBootGeneration().get();
File workflowFile = getOutputDirectory()
.file(springBootGeneration + "-" + smokeTest.group() + "-" + smokeTest.name() + ".yml")

12
gradle/plugins/aot-smoke-test-ci-plugin/src/main/java/org/springframework/aot/gradle/SmokeTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2022-2024 the original author or authors.
* Copyright 2022-2025 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -29,8 +29,6 @@ public class SmokeTests {
private String location;
private String cronSchedule;
public SmokeTests(String name) {
this.name = name;
}
@@ -55,12 +53,4 @@ public class SmokeTests {
this.location = location;
}
public String getCronSchedule() {
return this.cronSchedule;
}
public void setCronSchedule(String cronSchedule) {
this.cronSchedule = cronSchedule;
}
}

2
gradle/plugins/aot-smoke-test-ci-plugin/src/main/resources/smoke-test.yml
View File

@@ -52,6 +52,8 @@ jobs:
distribution: 'liberica'
- name: Set up Gradle
uses: gradle/actions/setup-gradle@db19848a5fa7950289d3668fb053140cf3028d43 # v3.3.2
with:
cache-read-only: true
- name: Configure Gradle user.name
run: |
mkdir -p ~/.gradle

37
gradle/plugins/aot-smoke-test-ci-plugin/src/main/resources/warm-caches.yml Normal file
View File

@@ -0,0 +1,37 @@
name: Smoke Test
on:
workflow_call:
inputs:
checkout_repository:
required: true
type: string
checkout_ref:
required: true
type: string
jobs:
warm_caches:
name: ${{ inputs.task }}
runs-on: ubuntu-latest
steps:
- name: Check out
uses: actions/checkout@v4
with:
repository: ${{ inputs.checkout_repository }}
ref: ${{ inputs.checkout_ref }}
- name: Set up Java
uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1
with:
java-version: ${{ inputs.java_version }}
distribution: 'liberica'
- name: Set up Gradle
uses: gradle/actions/setup-gradle@db19848a5fa7950289d3668fb053140cf3028d43 # v3.3.2
- name: Configure Gradle user.name
run: |
mkdir -p ~/.gradle
echo 'systemProp.user.name=spring-builds+github' >> ~/.gradle/gradle.properties
- name: Warm Caches
env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
REPO_SPRING_IO_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }}
REPO_SPRING_IO_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
run: ./gradlew warmCaches