Repository navigation
102 lines (89 loc) · 4.5 KB
/
Copy pathbench.yml
File metadata and controls
102 lines (89 loc) · 4.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
name: Benchmarks
# csharpc vs the .NET JIT vs NativeAOT on a clean hosted runner (benchmarks/run.sh), plus csharpc's
# executed-instruction counts (benchmarks/instructions.sh), which don't depend on runner noise.
# Wall-clock numbers vary a few percent between runs; compare engines within one run.
# Builds csharpc from this commit (not the published tool): the suite measures the tree under test.
on:
workflow_dispatch:
schedule:
- cron: "0 6 * * 1" # Mondays 06:00 UTC
push:
# Only the branches that exist to measure speed or size: ci/perf*, ci/bench* and the M7 branches
# (ci/m7-*, docs/20-status.md). Any other ci/** branch is a correctness or flake-hunting branch;
# its push touches src and used to start two 120-minute benchmark jobs nobody read. Dispatch the
# workflow (or name the branch ci/bench-<topic>) to measure one of those. A push still has to
# touch src, benchmarks, or this file. main is not here: a merge to main used to start two 120-minute
# jobs that competed with the merge-bar jobs of every open PR; the weekly schedule above measures main.
branches: ["ci/perf*", "ci/bench*", "ci/m7-*"]
paths: ["src/**", "benchmarks/**", ".github/workflows/bench.yml"]
permissions:
contents: read
concurrency:
group: bench-${{ github.ref }}
cancel-in-progress: true
jobs:
bench:
name: bench (${{ matrix.arch }})
strategy:
fail-fast: false
matrix:
include:
- arch: x86-64
runner: ubuntu-24.04
- arch: arm64
runner: ubuntu-24.04-arm
runs-on: ${{ matrix.runner }}
timeout-minutes: 120
env:
DOTNET_CLI_TELEMETRY_OPTOUT: 1
DOTNET_NOLOGO: 1
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install LLVM 18, clang, valgrind and NativeAOT prerequisites
run: sudo apt-get update && sudo apt-get install -y llvm-18 clang-18 libllvm18 lld-18 valgrind zlib1g-dev time && sudo ln -sf /usr/bin/clang-18 /usr/local/bin/clang && sudo ln -sf /usr/bin/ld.lld-18 /usr/local/bin/ld.lld
- name: Install .NET 11 SDK
run: |
curl -sSLf --retry 5 --retry-all-errors --retry-delay 5 https://dot.net/v1/dotnet-install.sh -o dotnet-install.sh
bash dotnet-install.sh --channel 11.0.1xx --install-dir "$HOME/.dotnet11"
echo "$HOME/.dotnet11" >> "$GITHUB_PATH"
- name: Runner
run: |
{
echo "## ${{ matrix.arch }} runner"
echo '```'
lscpu | grep -E 'Model name|^CPU\(s\)|Thread|MHz' || true
free -h | head -2
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Throughput, size and compile time (best of 5)
run: |
RUNS=5 benchmarks/run.sh | tee bench.md
{ echo "## Throughput, size, compile time (${{ matrix.arch }})"; cat bench.md; } >> "$GITHUB_STEP_SUMMARY"
- name: Executed instructions (deterministic)
run: |
benchmarks/instructions.sh | tee instructions.md
{ echo "## Executed instructions, csharpc -O2 (${{ matrix.arch }})"; cat instructions.md; } >> "$GITHUB_STEP_SUMMARY"
- name: Executed instructions with the concurrent marker's barrier (docs/adr/0020)
run: |
CSHARPC_GC_CONCURRENT=1 benchmarks/instructions.sh | tee instructions-concurrent.md
{ echo "## Executed instructions, csharpc -O2 --gc-concurrent (${{ matrix.arch }})"; cat instructions-concurrent.md; } >> "$GITHUB_STEP_SUMMARY"
- name: GC throughput, pauses and RSS
run: |
GC_BENCH_RUNS=3 benchmarks/gc/gc.sh | tee gc.md
{ echo "## GC latency (${{ matrix.arch }})"; cat gc.md; } >> "$GITHUB_STEP_SUMMARY"
- name: Throughput and GC latency with concurrent marking (--gc-concurrent, docs/adr/0020)
run: |
CSHARPC_GC_CONCURRENT=1 NO_AOT=1 RUNS=5 benchmarks/run.sh | tee bench-concurrent.md
CSHARPC_GC_CONCURRENT=1 GC_BENCH_RUNS=3 benchmarks/gc/gc.sh | tee gc-concurrent.md
{ echo "## Throughput with --gc-concurrent (${{ matrix.arch }})"; cat bench-concurrent.md;
echo "## GC latency with --gc-concurrent (${{ matrix.arch }})"; cat gc-concurrent.md; } >> "$GITHUB_STEP_SUMMARY"
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: bench-${{ matrix.arch }}
path: |
bench.md
instructions.md
instructions-concurrent.md
bench-concurrent.md
gc.md
gc-concurrent.md