Skip to content

Commit 72b9ea4

Browse files
bitsgalorebitsgalore
committed
extended GGC properties, TODO get properties by type
1 parent 018c022 commit 72b9ea4

14 files changed

Lines changed: 677 additions & 1 deletion

File tree

‎build/lib.linux-x86_64-2.7/kb/__init__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
__version__ = '0.1.5'
2+
__all__ = ['nl']
3+

‎build/lib.linux-x86_64-2.7/kb/nl/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
__version__ = '0.1.5'
2+
__all__ = ['api', 'collections', 'helpers']

‎build/lib.linux-x86_64-2.7/kb/nl/api/__init__.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
__version__ = '0.1.5'
2+
3+
__all__ = ['oai', 'sru']
4+
__author__ = 'WillemJan Faber <[email protected]'
5+
6+
from oai import oai
7+
from sru import sru
8+
9+
oai = oai()
10+
sru = sru()

‎build/lib.linux-x86_64-2.7/kb/nl/api/oai.py‎

Lines changed: 363 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,363 @@
1+
import sys
2+
import requests
3+
4+
from kb.nl.collections import SETS
5+
from kb.nl.helpers import etree
6+
7+
OAI_BASEURL = 'http://services.kb.nl/mdo/oai'
8+
9+
10+
class oai():
11+
"""
12+
OAI interface to the National Library of the Netherlands.
13+
For more information on the OAI protocol, visit:
14+
http://www.openarchives.org/OAI/openarchivesprotocol.html
15+
16+
This specific implementation does not strafe to implement a
17+
perfect OAI client. However it does expose collectionm data from the
18+
National Library of the Netherlands in Pyhton for easy usage.
19+
20+
Example usage:
21+
22+
from kb.nl.api import oai
23+
from kb.nl.helpers import alto_to_text
24+
25+
records = oai.list_records("DPO")
26+
record = oai.get(records.identifiers[0])
27+
alto_records = record.alto
28+
29+
for alto_record in alto_records:
30+
print(alto_to_text(alto_record))
31+
32+
"""
33+
current_set = False
34+
oai_sets = SETS
35+
resumptiontoken = False
36+
DEBUG = False
37+
38+
def __init__(self, current_set = False):
39+
if current_set:
40+
self.current_set = current_set
41+
42+
def list_sets(self):
43+
"""
44+
Shows a list of pre-defined OAI-sets.
45+
"""
46+
return sorted(self.oai_sets.keys())
47+
48+
def list_records(self, setname, resumptiontoken):
49+
"""
50+
Retrieves a list of records from the OAI server,
51+
and returns the list of records as an xml object.
52+
53+
:param setname: OAI setname
54+
"""
55+
if setname not in self.oai_sets:
56+
raise Exception('Error unknown setname')
57+
58+
if resumptiontoken:
59+
self.resumptiontoken = resumptiontoken
60+
61+
if not setname == self.current_set:
62+
self.current_set = setname
63+
if not resumptiontoken:
64+
self.resumptiontoken = False
65+
66+
url = OAI_BASEURL
67+
url += '?verb=ListRecords'
68+
url += '&metadataPrefix=' + self.oai_sets[setname]['metadataPrefix']
69+
url += '&set=' + self.oai_sets[setname]['setname']
70+
71+
if self.resumptiontoken:
72+
url += '&resumptionToken=' + self.resumptiontoken
73+
74+
if self.DEBUG:
75+
sys.stdout.write(url + '\n')
76+
77+
response = requests.get(url)
78+
79+
if not response.status_code == 200:
80+
raise Exception('Error while getting data from %s' % url)
81+
82+
records_data = etree.fromstring(response.content)
83+
84+
resumptiontoken = [i.text for i in records_data.iter()
85+
if i.tag.endswith('resumptionToken')][0]
86+
87+
self.resumptiontoken = resumptiontoken
88+
return records(records_data)
89+
90+
def get(self, identifier):
91+
"""
92+
Retrieves a record from the OAI server,
93+
and returns the requested record as an xml object.
94+
95+
:param identifier: Identifier to get from OAI server
96+
"""
97+
if self.current_set == "ANP":
98+
identifier = identifier.replace(':mpeg21', '')
99+
identifier = identifier.split(':')[0] + ':' + identifier
100+
identifier += ':' + 'mpeg21'
101+
if self.current_set == 'SGD':
102+
identifier = identifier.replace('sgd:register', 'SGD:sgd')
103+
identifier = ":".join(identifier.split(':')[:5])
104+
if self.current_set == 'DPO':
105+
identifier = 'DPO:' + identifier
106+
if self.current_set == 'BYVANCK':
107+
identifier = 'BYVANCK:' + identifier
108+
identifier = identifier.replace('BYVANCK', 'ByvanckB')
109+
110+
url = OAI_BASEURL
111+
url += '?verb=GetRecord'
112+
url += '&identifier=' + identifier
113+
114+
if self.current_set == 'BYVANCK':
115+
url += '&metadataPrefix=dcx'
116+
else:
117+
url += '&metadataPrefix=didl'
118+
119+
if self.DEBUG:
120+
sys.stdout.write(url + '\n')
121+
122+
response = requests.get(url)
123+
124+
if not response.status_code == 200:
125+
raise Exception('Error while getting data from %s' % url)
126+
127+
response = record(etree.fromstring(response.content),
128+
self.current_set, self.DEBUG)
129+
return response
130+
131+
132+
class records():
133+
"""
134+
Class for parsing xml output from OAI server,
135+
to usable objects.
136+
"""
137+
records_data = False
138+
DEBUG = False
139+
140+
def __init__(self, records_data, debug=False):
141+
"""
142+
:param records_data: ElementTree object
143+
:param debug: Shows debugging info
144+
"""
145+
self.records_data = records_data
146+
147+
@property
148+
def identifiers(self):
149+
"""
150+
Return a list of record Identifiers.
151+
"""
152+
ids = [i.text for i in self.records_data.iter() if
153+
i.tag.endswith('recordIdentifier') and
154+
i.text.find(':') > -1]
155+
return ids
156+
157+
@property
158+
def deleted_identifiers(self):
159+
"""
160+
Return al list of deleted record Identifiers.
161+
"""
162+
deleted = []
163+
for item in self.records_data[2]:
164+
if item.tag.endswith('record'):
165+
if item[0].tag.endswith('header') and item[0].attrib.get('status') == 'deleted':
166+
deleted.append(item[0][0].text)
167+
return deleted
168+
169+
170+
class record():
171+
"""
172+
Class for parsing XML output from OAI server,
173+
to more human readable form. This class
174+
is a generic record level object.
175+
176+
A record has a bunch of properties, most of them
177+
are exposed, for example:
178+
179+
>>> record.alto # Returns the alto files for a record.
180+
181+
>>> record.image # Returns the image for a record.
182+
# Mostly .jpg's
183+
"""
184+
record_data = False
185+
DEBUG = False
186+
current_set = False
187+
188+
def __init__(self, record_data, current_set=False, debug=False):
189+
"""
190+
:param record_data: XML object of the wanted record.
191+
:param debug: enable or disable debuging
192+
"""
193+
self.record_data = record_data
194+
self.current_set = current_set
195+
196+
if debug:
197+
self.DEBUG = debug
198+
199+
@property
200+
def alto(self):
201+
"""
202+
Return the ALTO(s) for the current record.
203+
For more information on ALTO see:
204+
205+
https://en.wikipedia.org/wiki/ALTO_(XML)
206+
207+
For parsing alto data back to text, use:
208+
209+
>>> from kb.nl.helpers import alto_to_text
210+
>>> alto_to_text(alto)
211+
"""
212+
if self.current_set == "BYVANCK":
213+
return False
214+
215+
alto_url_list = []
216+
alto_url = False
217+
218+
for item in self.record_data.iter():
219+
if item.attrib and \
220+
item.attrib.get('ref') and \
221+
item.attrib['ref'].lower().endswith(':alto'):
222+
223+
alto_url = item.attrib['ref']
224+
225+
if alto_url not in alto_url_list:
226+
alto_url_list.append(alto_url)
227+
228+
if not alto_url:
229+
for item in self.record_data.iter():
230+
if item.attrib and \
231+
item.attrib.get('ref') and \
232+
item.attrib['ref'].lower().endswith('.xml'):
233+
234+
alto_url = item.attrib['ref']
235+
alto_url_list.append(alto_url)
236+
break
237+
238+
# The result is either one ALTO file,
239+
# or a bunch of them, if more then one,
240+
# fetch the results and return a list.
241+
if len(alto_url_list) <= 1:
242+
alto_url = alto_url_list[0]
243+
244+
if self.DEBUG:
245+
sys.stdout.write(alto_url)
246+
247+
response = requests.get(alto_url)
248+
249+
if not response.status_code == 200:
250+
raise Exception('Error while getting data from %s' % alto_url)
251+
252+
return [response.text]
253+
else:
254+
alto_list = []
255+
256+
for alto_url in alto_url_list:
257+
response = requests.get(alto_url)
258+
259+
if self.DEBUG:
260+
sys.stdout.write(alto_url + " ")
261+
262+
if not response.status_code == 200:
263+
raise Exception('Error while getting data from %s' % alto_url)
264+
265+
alto_list.append(response.text)
266+
267+
return alto_list
268+
269+
@property
270+
def image(self):
271+
"""
272+
Retrieve the image for the current record,
273+
and return the bits.
274+
"""
275+
img_url = False
276+
277+
if self.current_set == "BYVANCK":
278+
img_url = 'http://imageviewer.kb.nl/'
279+
img_url += 'ImagingService/imagingService?id='
280+
img_url += self.record_data[2][0][1][0][1].text.split('=')[1]
281+
else:
282+
for item in self.record_data.iter():
283+
if item.attrib and \
284+
item.attrib.get('ref') and \
285+
item.attrib.get('ref').startswith('http://') and \
286+
(item.attrib['ref'].endswith(':image') or
287+
item.attrib['ref'].endswith('.jpg')):
288+
289+
img_url = item.attrib['ref']
290+
break
291+
292+
if not img_url:
293+
return False
294+
295+
if self.DEBUG:
296+
sys.stdout.write(img_url)
297+
298+
response = requests.get(img_url)
299+
300+
if not response.status_code == 200:
301+
raise Exception('Error while getting data from %s' % img_url)
302+
303+
return response.content
304+
305+
@property
306+
def title(self):
307+
"""
308+
Get corresponding title.
309+
"""
310+
for item in self.record_data.iter():
311+
if item.tag.endswith('title'):
312+
return item.text
313+
return False
314+
315+
@property
316+
def annotation(self):
317+
"""
318+
Get corresponding annotation.
319+
"""
320+
for item in self.record_data.iter():
321+
if item.tag.endswith('annotation'):
322+
return item.text
323+
return False
324+
325+
@property
326+
def date(self):
327+
"""
328+
Get timeperiod of creation.
329+
"""
330+
for item in self.record_data.iter():
331+
if item.tag.endswith('date'):
332+
return item.text
333+
return False
334+
335+
@property
336+
def creator(self):
337+
"""
338+
Get corresponding creator.
339+
"""
340+
for item in self.record_data.iter():
341+
if item.tag.endswith('creator'):
342+
return item.text
343+
return False
344+
345+
@property
346+
def contributor(self):
347+
"""
348+
Get contributors to this record.
349+
"""
350+
for item in self.record_data.iter():
351+
if item.tag.endswith('contributor'):
352+
return item.text
353+
return False
354+
355+
@property
356+
def publisher(self):
357+
"""
358+
Get publisher information for this record.
359+
"""
360+
for item in self.record_data.iter():
361+
if item.tag.endswith('publisher'):
362+
return item.text
363+
return False

0 commit comments

Comments
 (0)