[pull] master from QueensGambit:master

.codacy.yml

@@ -0,0 +1,3 @@
+exclude_paths:
+  - '*.md'
+  - 'engine/3rdparty/*'

.github/workflows/build.yml

@@ -0,0 +1,60 @@
+name: build
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    # The CMake configure and build commands are platform agnostic and should work equally
+    # well on Windows or Mac.
+    name: ${{ matrix.config.name }}
+    runs-on: ${{ matrix.config.os }}
+    env:
+      COMPILER: ${{ matrix.config.compiler }}
+      COMP: ${{ matrix.config.comp }}
+      CXXFLAGS: "-Werror"
+      # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
+      BUILD_TYPE: Debug
+
+    strategy:
+      matrix:
+        config:
+          - name: "Ubuntu-Latest-GCC"
+            os: ubuntu-latest
+            compiler: g++
+            comp: gcc
+          - name: "Ubuntu-Latest-Clang"
+            os: ubuntu-latest
+            compiler: clang++
+            comp: clang
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Install dependencies
+      working-directory: ${{github.workspace}}/engine/3rdparty/blaze
+      run: |
+        # install blaze
+        cmake -DCMAKE_INSTALL_PREFIX=/usr/local/
+        sudo make install
+
+    - name: Configure CMake
+      working-directory: ${{github.workspace}}/engine
+      # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
+      # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
+      run: cmake -B ./build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DUSE_PROFILING=0 -DUSE_RL=0 -DBACKEND_TENSORRT=0 -DBACKEND_MXNET=0 -DBACKEND_TORCH=0 -DUSE_960=0 -DBUILD_TESTS=1 -DMODE_CRAZYHOUSE=1 -DMODE_CHESS=0 -DMODE_LICHESS=0 .
+
+    - name: Build
+      # Build your program with the given configuration
+      run: cmake --build ${{github.workspace}}/engine/build --config ${{env.BUILD_TYPE}}
+
+    - name: Test
+      working-directory: ${{github.workspace}}/engine/build
+      # Execute tests defined by the CMake configuration.  
+      # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
+      run: ./CrazyAra

.github/workflows/variants.yml

@@ -0,0 +1,68 @@
+name: variants
+
+on: [ pull_request, workflow_dispatch ]
+env:
+  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
+  BUILD_TYPE: Debug
+
+jobs:
+  build:
+    # The CMake configure and build commands are platform agnostic and should work equally
+    # well on Windows or Mac.  You can convert this to a matrix build if you need
+    # cross-platform coverage.
+    # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Install dependencies
+      working-directory: ${{github.workspace}}/engine/3rdparty/blaze
+      run: |
+        # install blaze
+        cmake -DCMAKE_INSTALL_PREFIX=/usr/local/
+        sudo make install
+
+    - name: Install open_spiel
+      working-directory: ${{github.workspace}}/engine/3rdparty/open_spiel
+      run: |
+        # install open_spiel
+        pwd
+        chmod +x install.sh
+        sudo ./install.sh
+    - name: Configure CMake
+      working-directory: ${{github.workspace}}/engine
+      # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
+      # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
+      run: |
+        mkdir build
+        cmake -B ./build/chess -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DUSE_PROFILING=0 -DUSE_RL=0 -DBACKEND_TENSORRT=0 -DBACKEND_MXNET=0 -DBACKEND_TORCH=0 -DUSE_960=0 -DBUILD_TESTS=1 -DMODE_CRAZYHOUSE=0 -DMODE_CHESS=1 -DMODE_LICHESS=0 -DMODE_XIANGQI=0 -DMODE_STRATEGO=0 -DMODE_OPEN_SPIEL=0 .
+        cmake -B ./build/lichess -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DUSE_PROFILING=0 -DUSE_RL=0 -DBACKEND_TENSORRT=0 -DBACKEND_MXNET=0 -DBACKEND_TORCH=0 -DUSE_960=0 -DBUILD_TESTS=1 -DMODE_CRAZYHOUSE=0 -DMODE_CHESS=0 -DMODE_LICHESS=1 -DMODE_XIANGQI=0 -DMODE_STRATEGO=0 -DMODE_OPEN_SPIEL=0  .
+        cmake -B ./build/xiangqi -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DUSE_PROFILING=0 -DUSE_RL=0 -DBACKEND_TENSORRT=0 -DBACKEND_MXNET=0 -DBACKEND_TORCH=0 -DUSE_960=0 -DBUILD_TESTS=1 -DMODE_CRAZYHOUSE=0 -DMODE_CHESS=0 -DMODE_LICHESS=0 -DMODE_XIANGQI=1 -DMODE_STRATEGO=0 -DMODE_OPEN_SPIEL=0 .
+        cmake -B ./build/stratego -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DUSE_PROFILING=0 -DUSE_RL=0 -DBACKEND_TENSORRT=0 -DBACKEND_MXNET=0 -DBACKEND_TORCH=0 -DUSE_960=0 -DBUILD_TESTS=0 -DMODE_CRAZYHOUSE=0 -DMODE_CHESS=0 -DMODE_LICHESS=0 -DMODE_XIANGQI=0 -DMODE_STRATEGO=1 -DMODE_OPEN_SPIEL=0 .
+        cmake -B ./build/hex -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DUSE_PROFILING=0 -DUSE_RL=0 -DBACKEND_TENSORRT=0 -DBACKEND_MXNET=0 -DBACKEND_TORCH=0 -DUSE_960=0 -DBUILD_TESTS=0 -DMODE_CRAZYHOUSE=0 -DMODE_CHESS=0 -DMODE_LICHESS=0 -DMODE_XIANGQI=0 -DMODE_STRATEGO=0 -DMODE_OPEN_SPIEL=1 .
+        cmake -B ./build/boardgames -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DUSE_PROFILING=0 -DUSE_RL=0 -DBACKEND_TENSORRT=0 -DBACKEND_MXNET=0 -DBACKEND_TORCH=0 -DUSE_960=0 -DBUILD_TESTS=1 -DMODE_CRAZYHOUSE=0 -DMODE_CHESS=0 -DMODE_LICHESS=0 -DMODE_XIANGQI=0 -DMODE_STRATEGO=0 -DMODE_OPEN_SPIEL=0 -DMODE_BOARDGAMES=1 .
+
+    - name: Build
+      # Build your program with the given configuration
+      run: |
+        cmake --build ${{github.workspace}}/engine/build/chess --config ${{env.BUILD_TYPE}}
+        cmake --build ${{github.workspace}}/engine/build/lichess --config ${{env.BUILD_TYPE}}
+        cmake --build ${{github.workspace}}/engine/build/xiangqi --config ${{env.BUILD_TYPE}}
+        cmake --build ${{github.workspace}}/engine/build/stratego --config ${{env.BUILD_TYPE}}
+        cmake --build ${{github.workspace}}/engine/build/hex --config ${{env.BUILD_TYPE}}
+        cmake --build ${{github.workspace}}/engine/build/boardgames --config ${{env.BUILD_TYPE}}
+
+    - name: Test
+      working-directory: ${{github.workspace}}/engine
+      # Execute tests defined by the CMake configuration.
+      # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
+      run: |
+        ./build/chess/ClassicAra
+        ./build/lichess/MultiAra
+        ./build/xiangqi/XiangqiAra
+        ./build/stratego/StrategoAra
+        ./build/hex/OpenSpielAra
+        ./build/boardgames/BoardAra

.gitignore

@@ -123,9 +123,6 @@ venv.bak/
 # avoid pushing any lock files
 *.~lock
 
-# avoid pushing the main_config.py file
-main_config.py
-
 # avoid pushing log-files generated by uci-communication
 CrazyAra-log.txt
 score-log.txt
@@ -136,3 +133,9 @@ crazyara_lichess_dataset_stats.csv
 
 # avoid pushing datasets
 sf_vs_sf_*.pgn
+
+# Build
+build-engine-Desktop_Qt*/
+
+# user
+*.txt.user

.gitmodules

@@ -1,3 +1,12 @@
 [submodule "engine/3rdparty/Stockfish"]
 	path = engine/3rdparty/Stockfish
 	url = https://github.com/QueensGambit/Stockfish.git
+[submodule "engine/3rdparty/Fairy-Stockfish"]
+	path = engine/3rdparty/Fairy-Stockfish
+	url = https://github.com/xsr7qsr/Fairy-Stockfish.git
+[submodule "engine/3rdparty/blaze"]
+	path = engine/3rdparty/blaze
+	url = https://bitbucket.org/blaze-lib/blaze.git
+[submodule "engine/3rdparty/open_spiel"]
+	path = engine/3rdparty/open_spiel
+	url = https://github.com/BluemlJ/open_spiel_stratego_hex.git

AUTHORS

@@ -0,0 +1,64 @@
+# CrazyAra authors (including ClassicAra, XiangqiAra, MultiAra, ...)
+
+# Main author
+Johannes Czech (QueensGambit)
+
+# Founders of the CrazyAra project
+Johannes Czech (QueensGambit)
+Moritz Willig (MoritzWillig)
+Alena Beyer
+
+# Development of MultiAra (support for all lichess variants) including major refactoring
+Maximilian Alexander Gehrke (maxalexger)
+
+# Integration of Fairy-Stockfish back-end and founder of XiangqiAra (#93)
+Maximilian Langer (xsr7qsr)
+
+# Students which contributed to the development by writing a B.Sc- or M.Sc.-thesis or seminar paper on a related topic
+Patrick Korus
+Daniel Siersleben (DanielSiersleben)
+Maximilian Langer (xsr7qsr)
+Maximilian Alexander Gehrke (maxalexger)
+Jannis Blüml (BluemlJ)
+Felix Helfenstein (HelpstoneX)
+
+# All other authors of the code in alphabetical order
+AtlanticBot - created test case for 3-fold repetition
+Mateus (Matuiss2) - code refactoring in python, testing
+noelben - initial time management in python version
+RainRat - typo fixes
+
+# Alpha-(Go)-Zero authors for project inspiration
+David Silver, Thomas Hubert, Julian Schrittwieser, Ioannis Antonoglou, Matthew Lai, Arthur Guez, Marc Lanctot,
+ Laurent Sifre, Dharshan Kumaran, Thore Graepel, Timothy Lillicrap, Karen Simonyan, Demis Hassabis.
+
+# Project or thesis supervision
+Prof. Kristian Kersting
+Prof. Johannes Fürnkranz
+Karl Stelzner
+
+# Honorable mentions for valuable discussions or support
+Fabian Fichter (ianfab)
+Daniel Dugovic (ddugovic)
+FTdiscovery (Gordon)
+Bajusz Tamás (gbtami)
+Justin Tan (JannLee)
+TheFinnisher
+IM gsvc
+okei
+catask
+
+# Acknowledgments for the 3rd party libraries being used
+Niklas Fiekas (niklasf) - python-chess
+Daniel Dugovic (ddugovic) - Multi-Variant Stockfish
+Fabian Fichter (ianfab) - Fairy-Stockfish
+Klaus Iglberger (eagle42) and others - Blaze
+Alistair Miles (alimanfoo) and others - Zarr
+Constantin Pape (constantinpape ) and others - z5
+Phil Nash (horenmar), Martin Hořeňovský (horenmar) and others - Catch2
+Wolf Vollprecht (wolfv) and others - xtensor
+Rajeev Rao (rajeevsrao), Kevin Chen (kevinch-nv) and others - TensorRT
+Chiyuan Zhang (pluskid), Eric Junyuan Xie (piiswrong), Sheng Zha (szha) and others - incubator-mxnet
+bddppq, Lu Fang (houseroad), Ke Zhang (linkerzhang) and others - onnx
+Adrian Alic (m2q), tomatenbrei - pomcpp
+lanctot, michalsustr, Edward Lockhart (elkhrt) and others - open_spiel

DeepCrazyhouse/configs/README.md

@@ -1,9 +1,9 @@
 ### Configuration files for CrazyAra
 
+This is the configuration directory of CrazyAra for both supervised training and reinforcement learning.
+
 If you want to test sample MCTS predictions in [MCTS_eval_demo.ipynb](https://github.com/QueensGambit/CrazyAra/blob/master/DeepCrazyhouse/src/samples/MCTS_eval_demo.ipynb),
  then follow these steps:
-
-*   First rename `main_config_template.py` into `main_config.py`
 
 *   Specify the fields `main_config["model_architecture_dir"]` and `main_config["model_weights_dir"]` in the file
     `main_config.py` to the appropriate paths of your system. Make sure that the path has a "/" at the end of the path.

DeepCrazyhouse/configs/main_config.py

@@ -0,0 +1,86 @@
+"""
+@file: main_config.py
+Created on 24.09.18
+@project: crazyara
+@author: queensgambit
+
+Main Config definition file used for the project.
+Adjust the paths accordingly.
+"""
+
+
+# define the default dir where the training data in plane representation is located
+# e.g. for supervised learning default_dir = "/data/planes/"
+default_dir = "/data/kingbase2019_lite_pgn_months/"
+phase = None  # current phase to use, set to None to treat everything as a single phase. Otherwise use integer e.g. 0, 1, 2 to define the phase to use.
+# type of phase definition, either "lichess" or "movecountX" with X determining the number of phases
+phase_definition = "lichess"
+
+if default_dir[-1] != "/":
+    default_dir = default_dir + "/"
+
+
+main_config = {
+    # Crazyhouse - LichessDB
+    # The training directory includes games from the months:        2016-01 - 2018-07 (excluding 2018-04 and 2018-08)
+    # The validation directory includes games from the month:       2018-04
+    # The test directory includes games from the month:             2018-08
+    # The mate_in_one directory includes games from the month:      2018-04
+
+    # Chess - KingBaseLite2019
+    # The training directory includes games from the months:        2000-01 - 2018-12 (excluding 2012-09 and 2017-05)
+    # The validation directory includes games from the month:       2012-09
+    # The test directory includes games from the month:             2017-05
+    # The mate_in_one directory includes games from the month:      lichess_db_standard_rated_2015-08.pgn
+
+    "phase": phase,
+    "phase_definition": phase_definition,
+    "default_dir": default_dir,
+
+    # The pgn directories contain all files which are converted to plane representation
+    "pgn_train_dir": default_dir + "pgn/train/",
+    "pgn_val_dir": default_dir + "pgn/val/",
+    "pgn_test_dir": default_dir + "pgn/test/",
+    "pgn_mate_in_one_dir": default_dir + "pgn/mate_in_one/",
+    # The plane directories contain the plane representation of the converted board state
+    #  (.zip files which have been compressed by  the python zarr library)
+
+    "planes_train_dir": default_dir + f"planes/{phase_definition}/phase{phase}/train/",
+    "planes_val_dir": default_dir + f"planes/{phase_definition}/phase{phase}/val/",
+    "planes_test_dir": default_dir + f"planes/{phase_definition}/phase{phase}/test/",
+    "planes_mate_in_one_dir": default_dir + f"planes/{phase_definition}/phase{phase}/mate_in_one/",
+
+    # The rec directory contains the plane representation which are used in the training loop of the network
+    # use the notebook create_rec_dataset to generate the .rec files:
+    # (Unfortunately when trying to start training with the big dataset a memory overflow occurred.
+    # therefore the old working solution was used to train the latest model by loading the dataset via batch files)
+    #  "train.idx", "val.idx", "test.idx", "mate_in_one.idx", "train.rec", "val.rec", "test.rec", "mate_in_one.rec"
+
+    "rec_dir": default_dir + "rec/",
+    # The architecture dir contains the architecture definition of the network in mxnet .symbol format
+    # These directories are used for inference
+    #"model_architecture_dir": "/home/demo_user/models/Crazyhouse/symbol/",
+    "model_architecture_dir": "/DeepCrazyhouse/models/Classic/symbol/",
+
+    # the weight directory contains the of the network in mxnet .params format
+    #"model_weights_dir": "/home/demo_user/models/Crazyhouse/params/",
+    "model_weights_dir": "/DeepCrazyhouse/models/Classic/params/",
+
+    # layer name of the value output layer (e.g. value_tanh0 for legacy crazyhouse networks and value_out for newer
+    # networks)
+    "value_output": "value_out",
+    # layer name of the policy output layer without softmax applied (e.g. flatten0 for legacy crazyhouse networks
+    # policy_out for newer networks)
+    "policy_output": "policy_out",
+    "auxiliary_output": "auxiliary_out",
+    "wdl_output": "wdl_out",
+    "plys_to_end_output": "plys_to_end_out",
+
+    # Active mode for different input & output representations.
+    # Each mode is only compatible with a certain network input-/output representation:
+    # Available modes:  0: MODE_CRAZYHOUSE    (crazyhouse only mode, no 960) available versions [1, 2, 3]
+    #                   1: MODE_LICHESS       (all available lichess variants) available versions [1, 2 (last_moves), 3 (last_moves+fx-features)]
+    #                   2: MODE_CHESS         (chess only mode, with 960) available versions [1, 2, 3]
+    "mode": 2,
+    "version": 3,
+}