blob: 067a16cccc5c9864711bc8641f65c296bd3445bc (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
|
#!/usr/bin/env python3
import fileinput
import os
from urllib.parse import quote_plus
from urllib.request import urlopen, Request
OUTDIR = 'data'
ERRORFILE = 'errors'
def get(dbpediaid):
outfile = os.path.join(OUTDIR, quote_plus(dbpediaid) + '.json')
if os.path.isfile(outfile):
return
url = 'http://api.nordlys.cc/ec/lookup_id/{}'.format(
quote_plus(dbpediaid.replace('/', '___SLASH')).replace('___SLASH', '/'))
print(url)
result = urlopen(Request(url,
headers={'User-Agent': 'Radboud University'})).read()
with open(outfile, 'w') as f:
f.write(result.decode(encoding='UTF-8'))
def scrape(line):
index, query, dbpediaid, relevance = line.split('\t')
try:
get(dbpediaid)
except Exception as e:
with open(ERRORFILE, 'a') as f:
f.write(dbpediaid + '\t' + e + '\n')
if __name__ == '__main__':
for line in fileinput.input():
scrape(line)
|
