wikidata/crawl.js at master · lobbywatch/wikidata · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
const fetch = require('node-fetch')
const fs = require('fs')
const wdk = require('wikidata-sdk')
const { createApolloFetch } = require('apollo-fetch')

// fetch('http://ws-old.parlament.ch/councillors/basicdetails', {
//   headers: {
//     'Accept': 'text/json'
//   }
// })
//   .then(response => response.json())
//   .then(json => fs.writeFileSync(
//     'parlament.json',
//     JSON.stringify(json, null, 2),
//     'utf8'
//   ))

const sleep = ms => new Promise(resolve => {
  setTimeout(resolve, ms)
})

const getHistoric = async (page = 1, prev = []) => {
  console.log('fetch historic', page, prev.length)

  await sleep(200)

  return fetch(
    `http://ws-old.parlament.ch/councillors/historic?pageNumber=${page}`,
    {
      headers: {
        'Accept': 'text/json'
      }
    }
  )
    .then(response => response.json())
    .then(json => prev.concat(json))
    .then(json => json[json.length - 1].hasMorePages
      ? getHistoric(page + 1, json)
      : json
    )
}

// getHistoric()
//   .then(json => fs.writeFileSync(
//     'parlamentHistoric.json',
//     JSON.stringify(json, null, 2),
//     'utf8'
//   ))

// const apolloFetch = createApolloFetch({
//   uri: 'https://lobbywatch.ch/graphql'
// })

// apolloFetch({
//   query: `
//     {
//       parliamentarians(locale: de) {
//         id
//         name
//         parliamentId
//         council
//       }
//     }
//   `
// })
//   .then(response => fs.writeFileSync(
//     'lobbywatch.json',
//     JSON.stringify(response, null, 2),
//     'utf8'
//   ))

// const nationalCouncil = 'Q18510612'
// const councilOfStates = 'Q18510613'
// const federalAssembly = 'Q18515554'

// fetch(wdk.sparqlQuery(`
// SELECT DISTINCT ?human ?parliamentId ?humanLabel
// WHERE {
//   {
//     {?human wdt:P39 wd:${nationalCouncil}.}
//     UNION {?human wdt:P39 wd:${councilOfStates}.}
//     UNION {?human wdt:P39 wd:${federalAssembly}.}

//     SERVICE wikibase:label { bd:serviceParam wikibase:language "de". }
//     OPTIONAL { ?human wdt:P1307 ?parliamentId. }
//   }
// }
// ORDER BY ASC(xsd:integer(?parliamentId)) ASC(?humanLabel)
// `))
//   .then(response => response.json())
//   .then(json => fs.writeFileSync(
//     'wikidata.json',
//     JSON.stringify(json, null, 2),
//     'utf8'
//   ))

fetch(wdk.sparqlQuery(`
SELECT ?human ?parliamentId ?humanLabel ?positionHeld
WHERE {
  {
    ?human wdt:P1307 ?parliamentId.

    SERVICE wikibase:label { bd:serviceParam wikibase:language "de". }
    OPTIONAL { ?human wdt:P39 ?positionHeld. }
  }
}
ORDER BY ASC(xsd:integer(?parliamentId)) ASC(?humanLabel)
`))
  .then(response => response.json())
  .then(json => fs.writeFileSync(
    'wikidata.json',
    JSON.stringify(json, null, 2),
    'utf8'
  ))