Références : analyses de coups, de videau, chance et niveaux, comparées à gnubg
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Pierre MARTIN
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
34c3eaeca5
commit
242a2433d2
+122
-3
@@ -138,6 +138,123 @@ def compare_position(report, cli, path):
|
||||
return len(position["plies"])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Analyses (analysis/<id>.json, voir README.md, section « Analyses »).
|
||||
|
||||
# Seuils de gnubg (arSkillLevel et arLuckLevel) : une appréciation ou une
|
||||
# chance n'est comparée strictement que loin de ces seuils.
|
||||
SKILL_THRESHOLDS = (0.03, 0.06, 0.12)
|
||||
LUCK_THRESHOLDS = (0.3, 0.6)
|
||||
|
||||
|
||||
def run_cli(cli, args):
|
||||
command = shlex.split(cli) + args
|
||||
output = subprocess.run(command, capture_output=True, text=True)
|
||||
if output.returncode != 0:
|
||||
raise SystemExit("%s a échoué : %s" % (" ".join(command), output.stdout + output.stderr))
|
||||
return json.loads(output.stdout)
|
||||
|
||||
|
||||
def near_threshold(value, thresholds, tolerance):
|
||||
return any(abs(abs(value) - t) <= tolerance for t in thresholds)
|
||||
|
||||
|
||||
def same_label(report, where, field, expected, actual, value, thresholds):
|
||||
"""Même appréciation ou chance, sauf si la valeur de gnubg est à moins de
|
||||
la tolérance d'un seuil : remarque sans échec."""
|
||||
if expected == actual:
|
||||
return
|
||||
if near_threshold(value, thresholds, report.tolerance):
|
||||
report.notes.append("%s : %s %s pour gnubg, %s pour le moteur ; valeur %.4f proche "
|
||||
"d'un seuil" % (where, field, expected, actual, value))
|
||||
else:
|
||||
report.same(where, field, expected, actual)
|
||||
|
||||
|
||||
def compare_move_analysis(report, cli, case):
|
||||
dice = "%d%d" % tuple(case["dice"])
|
||||
args = [case["position_id"], case["match_id"], dice]
|
||||
luck = run_cli(cli, ["luck"] + args)
|
||||
where = "%s chance" % case["id"]
|
||||
report.number(where, "luck", case["luck"]["value"], luck["luck"])
|
||||
same_label(report, where, "rating", case["luck"]["rating"], luck["rating"],
|
||||
case["luck"]["value"], LUCK_THRESHOLDS)
|
||||
for ply, expected in sorted(case["plies"].items()):
|
||||
actual = run_cli(cli, ["--ply", ply, "analysemove"] + args
|
||||
+ [expected["played_position_id"]])
|
||||
where = "%s à %s pli(s)" % (case["id"], ply)
|
||||
report.same(where, "move_count", expected["move_count"], actual["move_count"])
|
||||
report.same(where, "coup joué", expected["played_move"], actual["played"]["move"])
|
||||
# Rang : le même, sauf si les coups entre les deux rangs ont, pour
|
||||
# gnubg, la même équité que le coup joué à la tolérance près.
|
||||
if expected["rank"] != actual["rank"]:
|
||||
low, high = sorted((expected["rank"], actual["rank"]))
|
||||
between = expected["ranking"][low - 1:high]
|
||||
tied = len(between) == high - low + 1 and all(
|
||||
abs(m["equity"] - expected["played_equity"]) <= report.tolerance for m in between)
|
||||
if tied:
|
||||
report.notes.append("%s : rang %d pour gnubg, %d pour le moteur ; égalité"
|
||||
% (where, expected["rank"], actual["rank"]))
|
||||
else:
|
||||
report.same(where, "rank", expected["rank"], actual["rank"])
|
||||
if expected["best_move"] != actual["best"]["move"]:
|
||||
ranking = expected["ranking"]
|
||||
tied = len(ranking) > 1 and ranking[0]["equity"] - ranking[1]["equity"] <= report.tolerance
|
||||
if tied:
|
||||
report.notes.append("%s : meilleur coup %s pour gnubg, %s pour le moteur ; égalité"
|
||||
% (where, expected["best_move"], actual["best"]["move"]))
|
||||
else:
|
||||
report.same(where, "meilleur coup", expected["best_move"], actual["best"]["move"])
|
||||
report.number(where, "played_equity", expected["played_equity"], actual["played"]["equity"])
|
||||
report.number(where, "best_equity", expected["best_equity"], actual["best"]["equity"])
|
||||
report.number(where, "equity_loss", expected["equity_loss"], actual["equity_loss"])
|
||||
same_label(report, where, "skill", expected["skill"], actual["skill"],
|
||||
expected["equity_loss"], SKILL_THRESHOLDS)
|
||||
|
||||
|
||||
def compare_cube_analysis(report, cli, case):
|
||||
for ply, expected in sorted(case["plies"].items()):
|
||||
actual = run_cli(cli, ["--ply", ply, "analysecube", case["position_id"],
|
||||
case["match_id"], case["action"]])
|
||||
where = "%s à %s pli(s)" % (case["id"], ply)
|
||||
report.same(where, "analysed", expected["analysed"], actual["analysed"])
|
||||
if expected["analysed"] and actual["analysed"]:
|
||||
compare_cube(report, where, expected["cube"], actual["cube"])
|
||||
report.number(where, "equity_loss", expected["equity_loss"], actual["equity_loss"])
|
||||
same_label(report, where, "skill", expected["skill"], actual["skill"],
|
||||
expected["equity_loss"], SKILL_THRESHOLDS)
|
||||
|
||||
|
||||
def compare_levels(report, cli, case, settings):
|
||||
for name, expected in sorted(case["levels"].items()):
|
||||
where = "%s niveau %s" % (case["id"], name)
|
||||
report.same(where, "réglages", (expected["plies"], expected["noise"]),
|
||||
(settings[name]["plies"], round(settings[name]["noise"], 6)))
|
||||
if case["dice"]:
|
||||
actual = run_cli(cli, ["--level", name, "bestmove", case["position_id"],
|
||||
case["match_id"], "%d%d" % tuple(case["dice"])])
|
||||
report.same(where, "coup choisi", expected["move"], actual["best_move"]["move"])
|
||||
else:
|
||||
actual = run_cli(cli, ["--level", name, "cube", case["position_id"], case["match_id"]])
|
||||
compare_cube(report, where, expected, actual)
|
||||
|
||||
|
||||
def compare_analyses(report, cli, folder):
|
||||
"""Compare toutes les analyses ; renvoie leur nombre."""
|
||||
settings = {level["name"]: level for level in run_cli(cli, ["levels"])["levels"]}
|
||||
paths = sorted(glob.glob(os.path.join(folder, "*.json")))
|
||||
for path in paths:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
case = json.load(f)
|
||||
if case["type"] == "move":
|
||||
compare_move_analysis(report, cli, case)
|
||||
elif case["type"] == "cube":
|
||||
compare_cube_analysis(report, cli, case)
|
||||
else:
|
||||
compare_levels(report, cli, case, settings)
|
||||
return len(paths)
|
||||
|
||||
|
||||
def print_table(lines):
|
||||
header = ("position et niveau", "champ", "attendu", "obtenu", "écart")
|
||||
rows = [header] + lines
|
||||
@@ -152,6 +269,7 @@ def main():
|
||||
parser.add_argument("--cli", default="build/engine-cli",
|
||||
help="programme interrogé, éventuellement avec ses arguments")
|
||||
parser.add_argument("--results", default=os.path.join(here, "results"))
|
||||
parser.add_argument("--analysis", default=os.path.join(here, "analysis"))
|
||||
parser.add_argument("--tolerance", type=float, default=TOLERANCE)
|
||||
parser.add_argument("--details", action="store_true",
|
||||
help="affiche tous les nombres comparés")
|
||||
@@ -160,14 +278,15 @@ def main():
|
||||
report = Report(args.tolerance)
|
||||
paths = sorted(glob.glob(os.path.join(args.results, "*.json")))
|
||||
levels = sum(compare_position(report, args.cli, path) for path in paths)
|
||||
analyses = compare_analyses(report, args.cli, args.analysis)
|
||||
|
||||
if args.details:
|
||||
print_table(report.lines)
|
||||
for note in report.notes:
|
||||
print(" Remarque : " + note)
|
||||
print(" %d positions, %d niveaux, %d nombres comparés ; écart maximal %.2e (%s), "
|
||||
"tolérance %.0e." % (len(paths), levels, len(report.lines), report.max_gap,
|
||||
report.max_gap_where, args.tolerance))
|
||||
print(" %d positions, %d niveaux, %d analyses, %d nombres comparés ; écart maximal %.2e "
|
||||
"(%s), tolérance %.0e." % (len(paths), levels, analyses, len(report.lines),
|
||||
report.max_gap, report.max_gap_where, args.tolerance))
|
||||
if report.failures:
|
||||
print(" %d écart(s) hors critères :" % len(report.failures))
|
||||
print_table(report.failures)
|
||||
|
||||
Reference in New Issue
Block a user