hyphe/config/config.json.example at ae1489773e1779ba8297c9dd913846ddc13150d9 · medialab/hyphe · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
{
  "mongo-scrapy": {
    "host": "localhost",
    "mongo_port": 27017,
    "proxy_host": "",
    "proxy_port": 3128,
    "db_name": "hyphe",
    "scrapy_port": 6800,
    "max_depth": 3,
    "download_delay": 1,
    "store_crawled_html_content": false,
    "max_simul_requests": 12,
    "max_simul_requests_per_host": 1
  },
  "traph": {
    "keepalive": 1800,
    "data_path": "##HYPHEPATH##/traph-data",
    "max_simul_pages_indexing": 250
  },
  "core_api_port": 6978,
  "defaultStartpagesMode": ["homepage", "prefixes", "pages-5"],
  "defaultCreationRule": "domain",
  "creationRules": {
    "twitter.com": "path-1",
    "facebook.com": "path-1",
    "facebook.com/pages": "path-2",
    "facebook.com/groups": "path-2",
    "facebook.com/people": "path-2",
    "plus.google.com": "path-1",
    "linkedin.com": "path-2",
    "viadeo.com/user": "path-2",
    "ello.co": "path-1",
    "pinterest.com": "path-1",
    "over-blog.com": "subdomain",
    "tumblr.com": "subdomain",
    "wordpress.com": "subdomain",
    "vimeo.com/user": "path-2",
    "dailymotion.com/user": "path-2",
    "youtube.com/user": "path-2"
  },
  "discoverPrefixes": [
    "bit.ly",
    "t.co",
    "tinyurl.com",
    "goo.gl",
    "fb.me",
    "on.fb.me",
    "l.facebook.com",
    "www.facebook.com/l.php",
    "ur1.ca",
    "j.mp",
    "wp.me",
    "ow.ly"
  ],
  "headless": {
    "autoretry": true,
    "timeout": 150,
    "idle_timeout": 20,
    "ajax_timeout": 15,
    "whitelist_domains": [
      "facebook.com",
      "twitter.com"
    ]
  },
  "ADMIN_PASSWORD": null,
  "OPEN_CORS_API": false,
  "DEBUG": 0
}