Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/scripts/bench_ab.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
# and each bench here is pinned to one core), and `sla` and `multiplicities` (plain `main`s that
# print no divan table).
DEFAULT_BENCHES = ('shakespeare cities sparse_keys binary_keys superdense_keys act_paths zipper_head_owned product_zipper '
'write_zipper graft_masked_branches restrict prune oeis byte_mask serde pointcloud')
'write_zipper graft_masked_branches restrict convex prune oeis byte_mask serde pointcloud')
THRESH = 0.05 # a case is listed in the summary when |change| exceeds this


Expand Down
26 changes: 21 additions & 5 deletions .github/scripts/fuzz_ab.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,13 @@ def tail(path, n=30):
return '\n'.join(Path(path).read_text().splitlines()[-n:])


def selector_count(src):
"""Read the append-only operation table, including pre-versioned baselines."""
harness = (src / 'differential' / 'src' / 'harness.rs').read_text()
match = re.search(r'pub const NOPS: usize = (\d+);', harness)
return int(match[1]) if match else 56


class Fuzz:
def __init__(self, base, head):
self.repo = Path.cwd()
Expand Down Expand Up @@ -107,16 +114,18 @@ def prepare_base(self):
git('clean', '-fdq', '--', 'differential', 'lean', cwd=self.base_src)
return 'base-harness' if self.build_side('base', self.base_src) else 'none'

def run_side(self, side, src, label, n, flags):
def run_side(self, side, src, label, n, flags, selectors):
env = dict(os.environ,
TMPDIR=str(self.out / f'fails-{side}-{label}'),
PATHMAP_ORACLE=str(src / 'lean' / '.lake' / 'build' / 'bin' / 'pathmap-oracle'),
PATHMAP_TRACE=str(self.target / f'fuzz-{side}' / 'release' / 'pathmap_trace'),
PATHMAP_ACT_TRACE=str(self.target / f'fuzz-{side}' / 'release' / 'act_trace'))
Path(env['TMPDIR']).mkdir(parents=True, exist_ok=True)
log(f'== fuzz {label} {side}: {n} inputs, seed {self.seed}')
outf = self.out / f'fuzz-{label}-{side}.txt'
with open(outf, 'w') as f:
subprocess.run([str(src / 'lean' / 'differential.py'), '--random', str(n), '--seed', self.seed,
subprocess.run([str(self.repo / 'lean' / 'differential.py'), '--random', str(n), '--seed', self.seed,
'--selectors', str(selectors),
'--maxlen', '300', '--max-fails', '0', '-j', self.jobs, *flags],
cwd=src, env=env, stdout=f, stderr=subprocess.STDOUT)
text = outf.read_text()
Expand Down Expand Up @@ -189,7 +198,7 @@ def summarize(self, baseline):
L += ['**No baseline**: base could not be built with either harness, so only head was run and nothing is gated.', '']
elif baseline == 'base-harness':
L += ["Base was built with its own harness and model (head's did not build against it), "
'so harness changes may show up as differences.', '']
'so harness changes may show up as differences. Random inputs use the shared selector range.', '']
new_total, unfinished = 0, 0
for label, n, _ in self.modes:
hf, hs, hk = self.parse(label, 'head')
Expand Down Expand Up @@ -272,10 +281,17 @@ def main(self):
if not self.build_side('head', self.repo):
sys.exit(1)
baseline = self.prepare_base()
# Always use head's generator with one recorded selector count.
# A fallback baseline may not implement newly appended operations;
# those are exercised separately by the strict convex CI sweep.
selectors = selector_count(self.repo)
if baseline == 'base-harness':
selectors = min(selectors, selector_count(self.base_src))
log(f'== shared random input encoding: {selectors} selectors')
for label, n, flags in self.modes:
self.run_side('head', self.repo, label, n, flags)
self.run_side('head', self.repo, label, n, flags, selectors)
if baseline != 'none':
self.run_side('base', self.base_src, label, n, flags)
self.run_side('base', self.base_src, label, n, flags, selectors)
new_total, unfinished = self.summarize(baseline)
return 1 if unfinished or (self.strict and new_total) else 0
finally:
Expand Down
45 changes: 39 additions & 6 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -196,12 +196,20 @@ jobs:
cd lean
lake build 2>&1 | tee ../lake.log
test "${PIPESTATUS[0]}" -eq 0
- name: generated algebraic laws are current
run: |
cd lean
lake build LawExport.Checks
lake env lean --run ZippyLaws.lean > "$RUNNER_TEMP/zippy_laws.rs"
lake env lean --run PathSetLaws.lean > "$RUNNER_TEMP/path_set_laws.rs"
diff -u ../differential/src/algebraic/zippy_laws.rs "$RUNNER_TEMP/zippy_laws.rs"
diff -u ../differential/src/algebraic/path_set_laws.rs "$RUNNER_TEMP/path_set_laws.rs"
- name: no sorry, no unexpected axioms
run: |
if grep -n 'declaration uses .sorry.' lake.log; then
echo "::error::a declaration uses sorry"; exit 1
fi
if grep -rn "\bsorry\b\|^axiom " lean/PathMapModel lean/Main.lean; then
if grep -rn "\bsorry\b\|^axiom " lean/PathMapModel lean/PrunedModel lean/Main.lean lean/PrunedMain.lean; then
echo "::error::sorry or axiom in the model sources"; exit 1
fi
- name: summary
Expand All @@ -212,9 +220,9 @@ jobs:
echo
echo "| file | theorems | guards |"
echo "|---|---:|---:|"
for f in lean/PathMapModel/*.lean; do
for f in lean/PathMapModel/*.lean lean/PrunedModel/*.lean lean/PrunedModel/Convex/*.lean; do
t=$(grep -c '^theorem\|^lemma' "$f" || true); g=$(grep -c '#guard' "$f" || true)
[ "$t$g" = "00" ] || echo "| $(basename "$f") | $t | $g |"
[ "$t$g" = "00" ] || echo "| ${f#lean/} | $t | $g |"
done
echo
echo "$(grep -c 'declaration uses' lake.log || true) declaration(s) use sorry; $(grep -c '^warning' lake.log || true) warning(s) from lake build."
Expand All @@ -225,7 +233,8 @@ jobs:
# Regression check: head and base run on identical inputs; an input on which head diverges
# from the model but base did not is reported as a warning annotation (master has known
# divergences). Set FUZZ_STRICT=1 below to make that fail the job instead.
# Ordered after the tests but not gated on them: findings are warnings, and a PR whose
# Focused convex and path-set programs have no known-bug exemptions: every divergence fails.
# Ordered after the tests but not gated on them: baseline findings are warnings, and a PR whose
# tests fail is exactly one whose divergences are worth seeing.
needs: test
if: ${{ !cancelled() && (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') }}
Expand Down Expand Up @@ -270,6 +279,30 @@ jobs:
FUZZ_OUT: ${{ runner.temp }}/fuzz-out
run: |
.github/scripts/fuzz_ab.py "${{ steps.base.outputs.sha }}" "${{ github.sha }}"
- name: fuzzer encoding tests
run: |
cargo test --release -p differential --lib --target-dir "$CARGO_TARGET_DIR/fuzz-head"
python3 -m unittest discover -s lean -p 'test_input_format.py'
- name: convex and path-set programs against Lean
shell: bash
env:
PATHMAP_TRACE: ${{ env.CARGO_TARGET_DIR }}/fuzz-head/release/pathmap_trace
TMPDIR: ${{ runner.temp }}/fuzz-out/fails-convex
run: |
mkdir -p "$TMPDIR"
if python3 lean/differential.py --convex 2000 --seed 7 --strict --max-fails 0 -j 16 | tee "${{ runner.temp }}/fuzz-out/fuzz-convex-head.txt"; then
convex_status=0
else
convex_status=$?
fi
{
printf '\n## Convex and path-set programs\n\n'
tail -n 1 "${{ runner.temp }}/fuzz-out/fuzz-convex-head.txt"
} >> "${{ runner.temp }}/fuzz-out/summary.md"
if [ "$convex_status" -ne 0 ]; then
printf 'Convex and path-set program check failed\n' >> "${{ runner.temp }}/fuzz-out/findings"
fi
exit "$convex_status"
- name: comment on the PR
# One comment per PR for this job (separate from the bench one): verdict, per-mode tables,
# newly diverging inputs and their shrunk Rust reproducers. Created only when there is
Expand All @@ -281,7 +314,7 @@ jobs:
run: |
python3 .github/scripts/pr_comment.py --id fuzz --title "Differential fuzz vs Lean model" \
--dir "${{ runner.temp }}/fuzz-out" --create-only-if "${{ runner.temp }}/fuzz-out/findings" \
"${{ github.event.pull_request.number }}" "${{ steps.fuzz.outcome }}" || true
"${{ github.event.pull_request.number }}" "${{ job.status }}" || true
- name: summary
if: always()
run: |
Expand Down Expand Up @@ -326,7 +359,7 @@ jobs:
rustc --version && cargo --version
- name: harness gate
# recogniser invariants, a short sweep, and route coverage; no corpus in CI
run: cargo test --release -p differential --test algebraic
run: cargo test --release -p differential --lib --test algebraic
- name: sweep (release)
if: ${{ !cancelled() }}
env:
Expand Down
4 changes: 4 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,10 @@ harness = false
name = "restrict"
harness = false

[[bench]]
name = "convex"
harness = false

[[bench]]
name = "write_zipper"
harness = false
Expand Down
Loading