benchmark_commits.py

import argparse
import os
import re
import subprocess
import sys
from datetime import datetime

# NOTE: mostly generated by GPT-4

FASTMARK_PY = os.path.join(os.path.dirname(__file__), "fastmark.py")

BENCHMARK_COMMAND = f"PYTHON_GIL=0 PYTHONHASHSEED=0 taskset -c 12 perf stat ./python {FASTMARK_PY}"

def run_command(command, cwd=None):
    """Run a shell command and return the output, exit code, and error message."""
    result = subprocess.run(command, shell=True, cwd=cwd, text=True, capture_output=True)
    return result.stdout.strip(), result.returncode, result.stderr.strip()

def parse_git_log(revisions):
    """Parse the git log to get commit hashes and messages."""
    log_output, _, _ = run_command(f"git log --oneline {revisions}")
    commits = []
    for line in log_output.splitlines():
        parts = line.split(" ", 1)
        if len(parts) == 2:
            commit_hash, message = parts
            commits.append((commit_hash, message))
    return commits

def build_commit():
    """Build the project for the current commit."""
    configure_command = "./configure -C --disable-gil"
    make_clean_command = "make clean"
    make_command = "make -j"

    # Run configure, retry if it fails
    _, configure_exit_code, _ = run_command(configure_command)
    if configure_exit_code != 0:
        run_command("rm -f config.cache")
        stdout, configure_exit_code, stderr = run_command(configure_command)
        if configure_exit_code != 0:
            print(stdout)
            print(stderr)
            return False

    # Run make clean and make
    _, make_clean_exit_code, _ = run_command(make_clean_command)
    stdout, make_exit_code, stderr = run_command(make_command)
    if make_clean_exit_code != 0 or make_exit_code != 0:
        print(stdout, stderr)
        print(make_clean_exit_code, make_exit_code)
    return make_clean_exit_code == 0 and make_exit_code == 0

def benchmark_commit():
    """Run the benchmark and return the score and instruction count."""

    # Run the benchmark twice and ignore the first result
    run_command(BENCHMARK_COMMAND)
    benchmark_output, benchmark_exit_code, perf_output = run_command(BENCHMARK_COMMAND)

    if benchmark_exit_code != 0:
        print(benchmark_output, perf_output)
        return None, None

    # Parse the output for score and instruction count
    score_match = re.search(r"Score:\s+([\d.]+)", benchmark_output)
    instructions_match = re.search(r"([\d.]+)\s+instructions", perf_output)

    score = float(score_match.group(1)) if score_match else None
    instructions = int(instructions_match.group(1).replace(',', '')) if instructions_match else None

    if score is None or instructions is None:
        print(benchmark_output, perf_output)
        return None, None

    return score, instructions

def main(options):
    cwd = os.getcwd()

    os.chdir(options.dir)

    commits = parse_git_log(options.revisions)
    base_instructions = None

    # format filename with benchmark_results-<date>.txt
    results_path = os.path.join(cwd, "benchmark_results", f"results-{datetime.now().strftime('%Y-%m-%d_%H-%M')}.txt")
    if not os.path.exists(os.path.dirname(results_path)):
        os.mkdir(os.path.dirname(results_path))

    with open(results_path, "w") as f:
        for commit_hash, message in commits:
            print(f"Benchmarking commit: {commit_hash} - {message}")

            # Checkout the commit
            _, checkout_exit_code, _ = run_command(f"git checkout {commit_hash}")
            if checkout_exit_code != 0:
                print(f"Failed to checkout commit {commit_hash}. Skipping.")
                continue

            # Build the commit
            if not build_commit():
                print(f"Failed to build commit {commit_hash}. Skipping.")
                continue

            # Benchmark the commit
            score, instructions = benchmark_commit()
            if score is None or instructions is None:
                print(f"Failed to benchmark commit {commit_hash}. Skipping.")
                continue

            # Write the result
            if base_instructions is None:
                base_instructions = instructions

            percent_change = ((instructions - base_instructions) / base_instructions) * 100
            result_line = f"{score} {instructions:14} {percent_change:5.2f}% {commit_hash} {message}\n"
            f.write(result_line)
            f.flush()  # Ensure the result is written immediately

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("dir",
                        help="path to python directory")
    parser.add_argument("revisions",
                        help="git revisions")
    options = parser.parse_args()
    main(options)