crawld/crawld.conf.sample at master · DevMine/crawld · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
{
    "database": {
        "hostname": "localhost",
        "port": 5432,
        "username": "devmine",
        "password": "devmine",
        "dbname": "devmine",
        "ssl_mode": "disable"
    },
    "clone_dir": "/var/crawld",
    "crawling_time_interval": "12h",
    "fetch_time_interval": "10m",
    "fetch_languages": [
        "go",
        "ruby"
    ],
    "tar_repositories": true,
    "tmp_dir": "/ramdisk",
    "tmp_dir_file_size_limit": 2.0,
    "max_fetcher_workers": 4,
    "throttler_wait_time": 900,
    "throttler_sliding_window_size": 60,
    "throttler_leak_interval": 1000,
    "crawlers": [
        {
            "type": "github",
            "languages": [
                "go",
                "ruby"
            ],
            "limit": 0,
            "since_id": 0,
            "fork": false,
            "oauth_access_token": "token here",
            "use_search_api": false
        }
    ]
}