commit 7ea50e51c76aa8dc9820569d4b104bb6e33348a2
parent 1d997a4501521a73405ffb99416e5f855db6bf6a
Author: n0tr1v <n0tr1v@protonmail.com>
Date: Tue, 13 Jun 2023 12:55:30 -0700
testing accuracy algo
Diffstat:
2 files changed, 285 insertions(+), 0 deletions(-)
diff --git a/pkg/web/handlers/interceptors/chess.go b/pkg/web/handlers/interceptors/chess.go
@@ -14,10 +14,13 @@ import (
"github.com/google/uuid"
"github.com/labstack/echo"
"github.com/notnil/chess"
+ "github.com/notnil/chess/uci"
+ "github.com/sirupsen/logrus"
"html/template"
"image"
"image/color"
"image/png"
+ "math"
"sort"
"strconv"
"strings"
@@ -717,3 +720,201 @@ func CalcAdvantage(position *chess.Position) (string, string, string, string) {
}
return whiteAdvantage, whiteScoreLbl, blackAdvantage, blackScoreLbl
}
+
+func Test() {
+ // lichess: 86% 97% (game against stockfish)
+ // Lila: 26.036024039104852 25.007112871751964
+ // Ours: 28.07242196565701 25.629909658576725
+ // 35, -29, 37, -42, 31, -15, 28, -16, 16, -4, 30, -24, 13, -14, 0, -17, 9, 6, -17, 8, -3, -5, -1, 5, -13, 20, -21, 90, -59, 77, -55, 70, -84, 116, -97, 68, -50, 46, -54, 93, -53, 57, -62, 61, -18, 11, -17, 41, -15, 13, -19, 8, 0, 22, -76, 44, -46, 87, -62, 62, -56, 59, -28, 69, -50, 70, -63, 63, -27, 30, -53, 14, -65, 43, -17, 398, -430, 565, -597, 793, -802, 837, -821, 908, -915, 957, -984, 1073, -981, 1285
+ pgn := "1. d4 Nf6 2. Nf3 e6 3. Bf4 { A46 Indian Defense: London System } c5 4. e3 d5 5. c3 Bd6 6. Bg3 O-O " +
+ "7. Bd3 Qc7 8. Bxd6 Qxd6 9. O-O Nbd7 10. Nbd2 e5 11. dxe5 Nxe5 12. Nxe5 Qxe5 13. Nf3 Qe7 14. b3 Rd8 " +
+ "15. Rc1 Bg4 16. Be2 Bf5 17. Bd3 Ne4 18. Qc2 Rd6 19. Rfd1 Re8 20. Bb5 Rc8 21. Bd3 Rcd8 22. Nd2 h6 " +
+ "23. Nxe4 dxe4 24. Bc4 b6 25. Rxd6 Qxd6 26. h3 a5 27. Qe2 Qd2 28. Qxd2 Rxd2 29. a4 Be6 30. Bxe6 fxe6 " +
+ "31. Rb1 Kf7 32. Kf1 Rc2 33. c4 h5 34. g4 hxg4 35. hxg4 Kf6 36. Kg2 Ke5 37. Kg3 Rd2 38. f3 Re2 39. f4+ Kf6 " +
+ "40. g5+ Kf5 41. Rh1 Rxe3+ 42. Kh4 Rxb3 43. g6 Kxf4 44. Kh5 Ra3 45. Rf1+ Rf3 46. Rxf3+ exf3 47. Kh4 f2 " +
+ "48. Kh3 Kf3 49. Kh2 f1=R 50. Kh3 Rh1# { Black wins by checkmate. } 0-1"
+ pgnOpt, _ := chess.PGN(strings.NewReader(pgn))
+ g := chess.NewGame(pgnOpt)
+ positions := g.Positions()
+
+ eng, err := uci.New("stockfish")
+ if err != nil {
+ logrus.Error(err)
+ return
+ }
+ if err := eng.Run(uci.CmdUCI, uci.CmdIsReady, uci.CmdUCINewGame); err != nil {
+ logrus.Error(err)
+ return
+ }
+ defer eng.Close()
+
+ cps := make([]int, 0)
+
+ for idx, position := range positions {
+ cmdPos := uci.CmdPosition{Position: position}
+ cmdGo := uci.CmdGo{MoveTime: time.Second / 20, Depth: 22}
+ if err := eng.Run(cmdPos, cmdGo); err != nil {
+ logrus.Fatal(err)
+ }
+ res := eng.SearchResults()
+ cp := res.Info.Score.CP
+ cps = append(cps, cp)
+ fmt.Printf("%d/%d %d %d\n", idx, len(positions), idx%2, cp)
+ }
+
+ s := make([]string, 0)
+ for _, c := range cps {
+ s = append(s, strconv.Itoa(c))
+ }
+ fmt.Println(strings.Join(s, ", "))
+ fmt.Println(gameAccuracy(cps))
+}
+
+func standardDeviation(num []WinPercent) float64 {
+ nb := len(num)
+ var sum, mean, sd float64
+ for i := 0; i < nb; i++ {
+ sum += float64(num[i])
+ }
+ mean = sum / float64(nb)
+ for j := 0; j < nb; j++ {
+ sd += math.Pow(float64(num[j])-mean, 2)
+ }
+ sd = math.Sqrt(sd) / float64(nb)
+ return sd
+}
+
+// https://www.scribbr.com/statistics/standard-deviation/
+//// using population variance
+//def standardDeviation(a: Iterable[Double]): Option[Double] =
+//mean(a) map { mean =>
+//Math.sqrt:
+//a.foldLeft(0d) { (sum, x) =>
+//sum + Math.pow(x - mean, 2)
+//} / a.size
+//}
+
+type WinPercent float64
+
+type Cp int
+
+func fromCentiPawns(cp Cp) WinPercent {
+ return 50 + 50*winningChances(cp)
+}
+
+func winningChances(cp Cp) WinPercent {
+ const MULTIPLIER = -0.00368208 // https://github.com/lichess-org/lila/pull/11148
+ res := 2/(1+math.Exp(MULTIPLIER*float64(cp))) - 1
+ out := WinPercent(math.Max(math.Min(res, 1), -1))
+ return out
+}
+
+func fromWinPercents(before, after float64) (accuracy float64) {
+ if after >= before {
+ return 100
+ }
+ winDiff := before - after
+ raw := 103.1668100711649*math.Exp(-0.04354415386753951*winDiff) + -3.166924740191411
+ raw += 1
+ return math.Min(math.Max(raw, 0), 100)
+}
+
+func calcWindows(allWinPercents []WinPercent, windowSize int) (out [][]WinPercent) {
+ start := allWinPercents[:windowSize]
+ tmp := windowSize
+ if tmp > len(allWinPercents) {
+ tmp = len(allWinPercents)
+ }
+ for i := 0; i < tmp-2; i++ {
+ out = append(out, start)
+ }
+
+ for i := 0; i < len(allWinPercents)-(windowSize-1); i++ {
+ curr := make([]WinPercent, 0)
+ for j := 0; j < windowSize; j++ {
+ curr = append(curr, allWinPercents[i+j])
+ }
+ out = append(out, curr)
+ }
+ return
+}
+
+func calcWeights(windows [][]WinPercent) (out []float64) {
+ for _, w := range windows {
+ out = append(out, math.Min(math.Max(standardDeviation(w), 0.5), 12))
+ }
+ return
+}
+
+func calcWeightedAccuracies(allWinPercents []WinPercent, weights []float64) (float64, float64) {
+ sw := calcWindows(allWinPercents, 2)
+ whites := make([][2]float64, 0)
+ blacks := make([][2]float64, 0)
+ for i := 0; i < len(sw); i++ {
+ prev, next := sw[i][0], sw[i][1]
+ acc := prev
+ acc1 := next
+ if i%2 != 0 {
+ acc, acc1 = acc1, acc
+ }
+ accuracy := fromWinPercents(float64(acc), float64(acc1))
+
+ if i%2 == 0 {
+ whites = append(whites, [2]float64{accuracy, weights[i]})
+ } else {
+ blacks = append(blacks, [2]float64{accuracy, weights[i]})
+ }
+ }
+
+ www1 := weightedMean(whites)
+ www2 := harmonicMean(whites)
+ bbb1 := weightedMean(blacks)
+ bbb2 := harmonicMean(blacks)
+ return (www1 + www2) / 2, (bbb1 + bbb2) / 2
+}
+
+func harmonicMean(arr [][2]float64) float64 {
+ vs := make([]float64, 0)
+ for _, v := range arr {
+ vs = append(vs, v[0])
+ }
+ sm := 0.0
+ for i := 0; i < len(vs); i++ {
+ sm = sm + 1/vs[i]
+ }
+ return float64(len(vs)) / sm
+}
+
+func weightedMean(a [][2]float64) float64 {
+ vs := make([]float64, 0)
+ ws := make([]float64, 0)
+
+ for _, v := range a {
+ vs = append(vs, v[0])
+ ws = append(ws, v[1])
+ }
+
+ sumWeight, avg := 0.0, 0.0
+ for i, v := range vs {
+ if v == 0 {
+ continue
+ }
+ sumWeight += ws[i]
+ avg += float64(v) * ws[i]
+ }
+ avg /= sumWeight
+ return avg
+}
+
+func gameAccuracy(cps []int) (float64, float64) {
+ cps = append([]int{15}, cps...)
+ var allWinPercents []WinPercent
+ for _, cp := range cps {
+ allWinPercents = append(allWinPercents, fromCentiPawns(Cp(cp)))
+ }
+ windowSize := int(math.Min(math.Max(float64(len(cps)/10), 2), 8))
+ windows := calcWindows(allWinPercents, windowSize)
+ weights := calcWeights(windows)
+ wa, ba := calcWeightedAccuracies(allWinPercents, weights)
+ return wa, ba
+}
diff --git a/pkg/web/handlers/interceptors/chess_test.go b/pkg/web/handlers/interceptors/chess_test.go
@@ -0,0 +1,84 @@
+package interceptors
+
+import (
+ "github.com/stretchr/testify/assert"
+ "math"
+ "testing"
+)
+
+func isCloseTo(a, b, delta float64) bool {
+ return math.Abs(a-b) <= delta
+}
+
+func fill(n, v int) []int {
+ out := make([]int, n)
+ for i := 0; i < n; i++ {
+ out[i] = v
+ }
+ return out
+}
+
+func fill1(n int, v []int) []int {
+ out := make([]int, n*len(v))
+ for i := 0; i < n*len(v); i += len(v) {
+ for j := range v {
+ out[i+j] = v[j]
+ }
+ }
+ return out
+}
+
+func Test_gameAccuracy(t *testing.T) {
+ // two good moves
+ w, b := gameAccuracy([]int{15, 15})
+ assert.True(t, isCloseTo(w, 100, 1))
+ assert.True(t, isCloseTo(b, 100, 1))
+ // white blunders on first move
+ w, b = gameAccuracy([]int{-900, -900})
+ assert.True(t, isCloseTo(w, 10, 5))
+ assert.True(t, isCloseTo(b, 100, 1))
+ // black blunders on first move
+ w, b = gameAccuracy([]int{15, 900})
+ assert.True(t, isCloseTo(w, 100, 1))
+ assert.True(t, isCloseTo(b, 10, 5))
+ // both blunder on first move
+ w, b = gameAccuracy([]int{-900, 0})
+ assert.True(t, isCloseTo(w, 10, 5))
+ assert.True(t, isCloseTo(b, 10, 5))
+ // 20 perfect moves
+ w, b = gameAccuracy(fill(20, 15))
+ assert.True(t, isCloseTo(w, 100, 1))
+ assert.True(t, isCloseTo(b, 100, 1))
+ // 20 perfect moves and a white blunder
+ cps := fill(20, 15)
+ cps = append(cps, -900)
+ w, b = gameAccuracy(cps)
+ assert.True(t, isCloseTo(w, 50, 5))
+ assert.True(t, isCloseTo(b, 100, 1))
+ // 21 perfect moves and a black blunder
+ cps = fill(21, 15)
+ cps = append(cps, 900)
+ w, b = gameAccuracy(cps)
+ assert.True(t, isCloseTo(w, 100, 1))
+ assert.True(t, isCloseTo(b, 50, 5))
+ // 5 average moves (65 cpl) on each side
+ cps = []int{-50, 15, -50, 15, -50, 15, -50, 15, -50, 15}
+ w, b = gameAccuracy(cps)
+ assert.True(t, isCloseTo(w, 76, 8))
+ assert.True(t, isCloseTo(b, 76, 8))
+ // 50 average moves (65 cpl) on each side
+ cps = fill1(50, []int{-50, 15})
+ w, b = gameAccuracy(cps)
+ assert.True(t, isCloseTo(w, 76, 8))
+ assert.True(t, isCloseTo(b, 76, 8))
+ // 50 mediocre moves (150 cpl) on each side
+ cps = fill1(50, []int{-135, 15})
+ w, b = gameAccuracy(cps)
+ assert.True(t, isCloseTo(w, 54, 8))
+ assert.True(t, isCloseTo(b, 54, 8))
+ // 50 terrible moves (500 cpl) on each side
+ cps = fill1(50, []int{-435, 15})
+ w, b = gameAccuracy(cps)
+ assert.True(t, isCloseTo(w, 20, 8))
+ assert.True(t, isCloseTo(b, 20, 8))
+}
