Here is a test suite to estimate the chess playing strength (ELO-rating) of a Chess Engine.

This suite was developed by Hubert Bednorz and Fred Toennissen to measure the tactical capability of chess engines, as opposed to strategic/positional strength.

How does that work? A chess engine is given 15 minutes (900 seconds) to analyze each position.

• If a position is solved, the solution time is recorded in seconds.
• It does not count as a solution if the engine finds the move and then changes its mind.
• If the engine finds the move, changes its mind then finds the move again, that 2nd time is used.
• Any solution that is not found scores as 900 seconds.
• The 30 times are averaged and subtracted from 2450 to give the estimated ELO rating.
• So, if no solution is found, the estimated ELO rating will be 2450-900=1550.
• If the average time is 8 minutes (480 seconds), then the estimated ELO rating is 2450-480=1970.

The suite has 30 test positions.

BT2450.pgn

[Event "Short - Ribli"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-01"]
[Black "Nxg7"]
[Result "*"]
[FEN "rq2r1k1/5pp1/p7/4bNP1/1p2P2P/5Q2/PP4K1/5R1R w - - 0 1"]

1.Nxg7 *

[Event "Hedinger"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-02"]
[Black "Bxb6"]
[Result "*"]
[FEN "6k1/2b2p1p/ppP3p1/4p3/PP1B4/5PP1/7P/7K w - - 0 1"]

1.Bxb6 *

[Event "Kwinones"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-03"]
[Black "Re6"]
[Result "*"]
[FEN "5r1k/p1q2pp1/1pb4p/n3R1NQ/7P/3B1P2/2P3P1/7K w - - 0 1"]

1.Re6 *

[Event "Harding Test"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-04"]
[Black "Qf7"]
[Result "*"]
[FEN "5r1k/1P4pp/3P1p2/4p3/1P5P/3q2P1/Q2b2K1/B3R3 w - - 0 1"]

1.Qf7 *

[Event "Judowitsch"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-05"]
[Black "Ka6"]
[Result "*"]
[FEN "3B4/8/2B5/1K6/8/8/3p4/3k4 w - - 0 1"]

1.Ka6 { Only winning move for white } *

[Event "Bernd Feustel"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-06"]
[Black "e3"]
[Result "*"]
[FEN "1k1r4/1pp4p/2n5/P6R/2R1p1r1/2P2p2/1PP2B1P/4K3 b - - 0 1"]

1...e3 *

[Event "Schulz - Buecker"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-07"]
[Black "Rd6"]
[Result "*"]
[FEN "6k1/p3q2p/1nr3pB/8/3Q1P2/6P1/PP5P/3R2K1 b - - 0 1"]

1...Rd6 { Black forces a draw } *

[Event "Source unkown"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-08"]
[Black "Rxc6+"]
[Result "*"]
[FEN "2krr3/1p4pp/p1bRpp1n/2p5/P1B1PP2/8/1PP3PP/R1K3B1 w - - 0 1"]

1.Rxc6+ *

[Event "Reitz - Bednorz"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-09"]
[Black "g5"]
[Result "*"]
[FEN "r5k1/pp2p1bp/6p1/n1p1P3/2qP1NP1/2PQB3/P5PP/R4K2 b - - 0 1"]

1...g5 *

[Event "Kasparov (Blindfold !!) 1985"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-10"]
[Black "Rxg7+"]
[Result "*"]
[FEN "2r3k1/1qr1b1p1/p2pPn2/nppPp3/8/1PP1B2P/P1BQ1P2/5KRR w - - 0 1"]

1.Rxg7+ *

[Event "Karpov - Chandler"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-11"]
[Black "Qxh2+"]
[Result "*"]
[FEN "1br3k1/p4p2/2p1r3/3p1b2/3Bn1p1/1P2P1Pq/P3Q1BP/2R1NRK1 b - - 0 1"]

1...Qxh2+ *

[Event "Source unkown"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-12"]
[Black "Qe4"]
[Result "*"]
[FEN "8/pp3k2/2p1qp2/2P5/5P2/1R2p1rp/PP2R3/4K2Q b - - 0 1"]

1...Qe4 *

[Event "Source unkown"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-13"]
[Black "Nb4"]
[Result "*"]
[FEN "3b2k1/1pp2rpp/r2n1p1B/p2N1q2/3Q4/6R1/PPP2PPP/4R1K1 w - - 0 1"]

1.Nb4 *

[Event "Mader -Mephisto MMII"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-14"]
[Black "Rxh7"]
[Result "*"]
[FEN "3r1rk1/1p3pnp/p3pBp1/1qPpP3/1P1P2R1/P2Q3R/6PP/6K1 w - - 0 1"]

1.Rxh7 { Mate in 7 moves } *

[Event "Source unknown"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-15"]
[Black "Rg6"]
[Result "*"]
[FEN "4k1rr/ppp5/3b1p1p/4pP1P/3pP2N/3P3P/PPP5/2KR2R1 w kq - 0 1"]

1.Rg6 *

[Event "Source unkown"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-16"]
[Black "g6"]
[Result "*"]
[FEN "r1b3k1/ppp3pp/2qpp3/2r3N1/2R5/8/P1Q2PPP/2B3K1 b - - 0 1"]

1...g6 *

[Event "Colditz Test"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-17"]
[Black "Qxf4"]
[Result "*"]
[FEN "4r1k1/p1qr1p2/2pb1Bp1/1p5p/3P1n1R/3B1P2/PP3PK1/2Q4R w - - 0 1"]

1.Qxf4 *

[Event "Judowitsch"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-18"]
[Black "d6"]
[Result "*"]
[FEN "8/4p3/8/3P3p/P2pK3/6P1/7b/3k4 w - - 0 1"]

1.d6 *

[Event "Baumann Test"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-19"]
[Black "f3"]
[Result "*"]
[FEN "3r2k1/pp4B1/6pp/PP1Np2n/2Pp1p2/3P2Pq/3QPPbP/R4RK1 b - - 0 1"]

1...f3 *

[Event "Agsamow - Tringov"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-20"]
[Black "Ra2"]
[Result "*"]
[FEN "r4rk1/5p2/1n4pQ/2p5/p5P1/P4N2/1qb1BP1P/R3R1K1 w - - 0 1"]

1.Ra2 *

[Event "Hort - Wockenfuss"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-21"]
[Black "Re6"]
[Result "*"]
[FEN "r4rk1/pb3p2/1pp4p/2qn2p1/2B5/6BP/PPQ2PP1/3RR1K1 w - - 0 1"]

1.Re6 *

[Event "Hort - Miles"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-22"]
[Black "a3"]
[Result "*"]
[FEN "rnb1k2r/pp2qppp/3p1n2/2pp2B1/1bP5/2N1P3/PP2NPPP/R2QKB1R w KQkq - 0 1"]

1.a3 *

[Event "Gaprindaschwili - Servati"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-23"]
[Black "Qf6"]
[Result "*"]
[FEN "r1b2rk1/pp1p1pBp/6p1/8/2PQ4/8/PP1KBP1P/q7 w - - 0 1"]

1.Qf6 *

[Event "Kubbel"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-24"]
[Black "g6"]
[Result "*"]
[FEN "R7/3p3p/8/3P2P1/3k4/1p5p/1P1NKP1P/7q w - - 0 1"]

1.g6 *

[Event "Source unknown"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-25"]
[Black "Nd3"]
[Result "*"]
[FEN "8/8/3k1p2/p2BnP2/4PN2/1P2K1p1/8/5b2 b - - 0 1"]

1...Nd3 *

[Event "Zuckertort - Blackburne"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-26"]
[Black "f5"]
[Result "*"]
[FEN "2r3k1/pbr1q2p/1p2pnp1/3p4/3P1P2/1P1BR3/PB1Q2PP/5RK1 w - - 0 1"]

1.f5 *

[Event "Rubenstein - Keller"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-27"]
[Black "Bb4"]
[Result "*"]
[FEN "3r2k1/p2r2p1/1p1B2Pp/4PQ1P/2b1p3/P3P3/7K/8 w - - 0 1"]

1.Bb4 *

[Event "Baumann Test"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-28"]
[Black "Ne4"]
[Result "*"]
[FEN "b2r1rk1/2q2ppp/p1nbpn2/1p6/1P6/P1N1PN2/1B2QPPP/1BR2RK1 w - - 0 1"]

1.Ne4 *

[Event "Baumann Test"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-29"]
[Black "Ke1"]
[Result "*"]
[FEN "r1b4Q/p4k1p/1pp1ppqn/8/1nP5/8/PP1KBPPP/3R2NR w - - 0 1"]

1.Ke1 *

[Event "Fischer"]
[Site "?"]
[Date "1994.04.16"]
[Round "?"]
[White "BT2450-30"]
[Black "f4"]
[Result "*"]
[FEN "2k5/2p3Rp/p1pb4/1p2p3/4P3/PN1P1P2/1P2KP1r/8 w - - 0 1"]

1.f4 *

ELO can be calculated using Ordo