Skip to content

Commit 077f8a9

Browse files
daviddrysdaledaviddrysdale
committed
Add support for flat (not per-locale) prefix data, and for multiple values.
1 parent a623ded commit 077f8a9

1 file changed

Lines changed: 51 additions & 13 deletions

File tree

‎tools/python/buildprefixdata.py‎

Lines changed: 51 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,15 @@
22
"""Script to read the libphonenumber per-prefix metadata and generate Python code.
33
44
Invocation:
5-
buildprefixdata.py indir outfile
5+
buildprefixdata.py [options] input outfile
66
77
Processes all of the per-prefix data under the given input directory and emit
88
generated Python code.
9+
10+
Options:
11+
--var XXX : use this prefix for variable names in generated code
12+
--flat : don't do per-locale processing
13+
--sep C : expect metadata to be a list with C as separator
914
"""
1015

1116
# Based on original metadata data files from libphonenumber:
@@ -54,7 +59,12 @@ def u(s):
5459
DATA_LINE_RE = re.compile(r'^\+?(?P<prefix>\d+)\|(?P<stringdata>.*)$', re.UNICODE)
5560

5661
# Boilerplate header
57-
PREFIXDATA_FILE_PROLOG = '''"""Per-prefix data, mapping each prefix to a dict of locale:name.
62+
PREFIXDATA_LOCALE_FILE_PROLOG = '''"""Per-prefix data, mapping each prefix to a dict of locale:name.
63+
64+
Auto-generated file, do not edit by hand.
65+
"""
66+
'''
67+
PREFIXDATA_FILE_PROLOG = '''"""Per-prefix data, mapping each prefix to a name.
5868
5969
Auto-generated file, do not edit by hand.
6070
"""
@@ -77,14 +87,21 @@ def u(s):
7787
""" % datetime.datetime.now().year
7888

7989

80-
def load_locale_prefixdata_file(prefixdata, filename, locale, overall_prefix):
90+
def load_locale_prefixdata_file(prefixdata, filename, locale=None, overall_prefix=None, separator=None):
8191
"""Load per-prefix data from the given file, for the given locale and prefix.
8292
8393
We assume that this file:
8494
- is encoded in UTF-8
8595
- may have comment lines (starting with #) and blank lines
8696
- has data lines of the form '<prefix>|<stringdata>'
8797
- contains only data for prefixes that are extensions of the filename.
98+
99+
If overall_prefix is specified, lines are checked to ensure their prefix falls within this value.
100+
101+
If locale is specified, prefixdata[prefix][locale] is filled in; otherwise, just prefixdata[prefix].
102+
103+
If separator is specified, the string data will be split on this separator, and the output values
104+
in the dict will be tuples of strings rather than strings.
88105
"""
89106
with open(filename, "rb") as infile:
90107
lineno = 0
@@ -98,12 +115,17 @@ def load_locale_prefixdata_file(prefixdata, filename, locale, overall_prefix):
98115
if stringdata != stringdata.rstrip():
99116
print ("%s:%d: Warning: stripping trailing whitespace" % (filename, lineno))
100117
stringdata = stringdata.rstrip()
101-
if not prefix.startswith(overall_prefix):
118+
if overall_prefix is not None and not prefix.startswith(overall_prefix):
102119
raise Exception("%s:%d: Prefix %s is not within %s" %
103120
(filename, lineno, prefix, overall_prefix))
121+
if separator is not None:
122+
stringdata = tuple(stringdata.split(separator))
104123
if prefix not in prefixdata:
105124
prefixdata[prefix] = {}
106-
prefixdata[prefix][locale] = stringdata
125+
if locale is not None:
126+
prefixdata[prefix][locale] = stringdata
127+
else:
128+
prefixdata[prefix] = stringdata
107129
elif BLANK_LINE_RE.match(uline):
108130
pass
109131
elif COMMENT_LINE_RE.match(uline):
@@ -113,7 +135,7 @@ def load_locale_prefixdata_file(prefixdata, filename, locale, overall_prefix):
113135
(filename, lineno, line))
114136

115137

116-
def load_locale_prefixdata(indir):
138+
def load_locale_prefixdata(indir, separator=None):
117139
"""Load per-prefix data from the given top-level directory.
118140
119141
Prefix data is assumed to be held in files <indir>/<locale>/<prefix>.txt.
@@ -126,7 +148,7 @@ def load_locale_prefixdata(indir):
126148
continue
127149
for filename in glob.glob(os.path.join(indir, locale, "*%s" % PREFIXDATA_SUFFIX)):
128150
overall_prefix, ext = os.path.splitext(os.path.basename(filename))
129-
load_locale_prefixdata_file(prefixdata, filename, locale, overall_prefix)
151+
load_locale_prefixdata_file(prefixdata, filename, locale, overall_prefix, separator)
130152
return prefixdata
131153

132154

@@ -138,26 +160,34 @@ def _stable_dict_repr(strdict):
138160
return "{%s}" % ", ".join(lines)
139161

140162

141-
def output_prefixdata_code(prefixdata, outfilename, varprefix):
163+
def output_prefixdata_code(prefixdata, outfilename, varprefix, per_locale):
142164
"""Output the per-prefix data in Python form to the given file """
143165
with open(outfilename, "w") as outfile:
144166
longest_prefix = 0
145-
prnt(PREFIXDATA_FILE_PROLOG, file=outfile)
167+
if per_locale:
168+
prnt(PREFIXDATA_LOCALE_FILE_PROLOG, file=outfile)
169+
else:
170+
prnt(PREFIXDATA_FILE_PROLOG, file=outfile)
146171
prnt(COPYRIGHT_NOTICE, file=outfile)
147172
prnt("%s_DATA = {" % varprefix, file=outfile)
148173
for prefix in sorted(prefixdata.keys()):
149174
if len(prefix) > longest_prefix:
150175
longest_prefix = len(prefix)
151-
prnt(" '%s':%s," % (prefix, _stable_dict_repr(prefixdata[prefix])), file=outfile)
176+
if per_locale:
177+
prnt(" '%s':%s," % (prefix, _stable_dict_repr(prefixdata[prefix])), file=outfile)
178+
else:
179+
prnt(" '%s':%r," % (prefix, prefixdata[prefix]), file=outfile)
152180
prnt("}", file=outfile)
153181
prnt("%s_LONGEST_PREFIX = %d" % (varprefix, longest_prefix), file=outfile)
154182

155183

156184
def _standalone(argv):
157185
"""Parse the given input directory and emit generated code."""
158186
varprefix = "GEOCODE"
187+
per_locale = True
188+
separator = None
159189
try:
160-
opts, args = getopt.getopt(argv, "hv:", ("help", "var="))
190+
opts, args = getopt.getopt(argv, "hv:fs:", ("help", "var=", "flat", "sep="))
161191
except getopt.GetoptError:
162192
prnt(__doc__, file=sys.stderr)
163193
sys.exit(1)
@@ -167,15 +197,23 @@ def _standalone(argv):
167197
sys.exit(1)
168198
elif opt in ("-v", "--var"):
169199
varprefix = arg
200+
elif opt in ("-f", "--flat"):
201+
per_locale = False
202+
elif opt in ("-s", "--sep"):
203+
separator = arg
170204
else:
171205
prnt("Unknown option %s" % opt, file=sys.stderr)
172206
prnt(__doc__, file=sys.stderr)
173207
sys.exit(1)
174208
if len(args) != 2:
175209
prnt(__doc__, file=sys.stderr)
176210
sys.exit(1)
177-
prefixdata = load_locale_prefixdata(args[0])
178-
output_prefixdata_code(prefixdata, args[1], varprefix)
211+
if per_locale:
212+
prefixdata = load_locale_prefixdata(args[0], separator=separator)
213+
else:
214+
prefixdata = {}
215+
load_locale_prefixdata_file(prefixdata, args[0], separator=separator)
216+
output_prefixdata_code(prefixdata, args[1], varprefix, per_locale)
179217

180218

181219
if __name__ == "__main__":

0 commit comments

Comments
 (0)