forked from remind101/stacker-python-bug-squash
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathaws_lambda.py
More file actions
321 lines (241 loc) · 10.8 KB
/
Copy pathaws_lambda.py
File metadata and controls
321 lines (241 loc) · 10.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
import os
import os.path
import stat
import logging
import hashlib
from StringIO import StringIO
from zipfile import ZipFile, ZIP_DEFLATED
import botocore
import formic
"""Mask to retrieve only UNIX file permissions from the external attributes
field of a ZIP entry.
"""
ZIP_PERMS_MASK = (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) << 16
logger = logging.getLogger(__name__)
def _zip_files(files, root):
"""Generates a ZIP file in-memory from a list of files.
Files will be stored in the archive with relative names, and have their
UNIX permissions forced to 755 or 644 (depending on whether they are
user-executable in the source filesystem).
Args:
files (list[str]): file names to add to the archive, relative to
``root``.
root (str): base directory to retrieve files from.
Returns:
str: content of the ZIP file as a byte string.
"""
zip_data = StringIO()
with ZipFile(zip_data, 'w', ZIP_DEFLATED) as zip_file:
for fname in files:
zip_file.write(os.path.join(root, fname), fname)
# Fix file permissions to avoid any issues - only care whether a file
# is executable or not, choosing between modes 755 and 644 accordingly.
for zip_entry in zip_file.filelist:
perms = (zip_entry.external_attr & ZIP_PERMS_MASK) >> 16
if perms & stat.S_IXUSR != 0:
new_perms = 0755
else:
new_perms = 0644
if new_perms != perms:
logger.debug("lambda: fixing perms: %s: %o => %o",
zip_entry.filename, perms, new_perms)
new_attr = ((zip_entry.external_attr & ~ZIP_PERMS_MASK) |
(new_perms << 16))
zip_entry.external_attr = new_attr
contents = zip_data.getvalue()
zip_data.close()
return contents
def _find_files(root, includes, excludes):
"""List files inside a directory based on include and exclude rules.
This is a more advanced version of `glob.glob`, that accepts multiple
complex patterns.
Args:
root (str): base directory to list files from.
includes (list[str]): inclusion patterns. Only files matching those
patterns will be included in the result.
includes (list[str]): exclusion patterns. Files matching those
patterns will be excluded from the result. Exclusions take
precedence over inclusions.
Yields:
str: a file name relative to the root.
Note:
Documentation for the patterns can be found at
http://www.aviser.asia/formic/doc/index.html
"""
root = os.path.abspath(root)
file_set = formic.FileSet(directory=root, include=includes,
exclude=excludes)
for filename in file_set.qualified_files(absolute=False):
yield filename
def _zip_from_file_patterns(root, includes, excludes):
"""Generates a ZIP file in-memory from file search patterns.
Args:
root (str): base directory to list files from.
includes (list[str]): inclusion patterns. Only files matching those
patterns will be included in the result.
includes (list[str]): exclusion patterns. Files matching those
patterns will be excluded from the result. Exclusions take
precedence over inclusions.
See Also:
:func:`_zip_files`, :func:`_find_files`.
Raises:
RuntimeError: when the generated archive would be empty.
"""
logger.info('lambda: base directory: %s', root)
files = list(_find_files(root, includes, excludes))
if not files:
raise RuntimeError('Empty list of files for Lambda payload. Check '
'your include/exclude options for errors.')
logger.info('lambda: adding %d files:', len(files))
for fname in files:
logger.debug('lambda: + %s', fname)
return _zip_files(files, root)
def _head_object(s3_conn, bucket, key):
"""Retrieve information about an object in S3 if it exists.
Args:
s3_conn (:class:`botocore.client.S3`): S3 connection to use for
operations.
bucket (str): name of the bucket containing the key.
key (str): name of the key to lookup.
Returns:
dict: S3 object information, or None if the object does not exist.
See the AWS documentation for explanation of the contents.
Raises:
botocore.exceptions.ClientError: any error from boto3 other than key
not found is passed through.
"""
try:
return s3_conn.head_object(Bucket=bucket, Key=key)
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == '404':
return None
else:
raise
def _ensure_bucket(s3_conn, bucket):
"""Create an S3 bucket if it does not already exist.
Args:
s3_conn (:class:`botocore.client.S3`): S3 connection to use for
operations.
bucket (str): name of the bucket to create.
Returns:
dict: S3 object information. See the AWS documentation for explanation
of the contents.
Raises:
botocore.exceptions.ClientError: any error from boto3 is passed
through.
"""
try:
s3_conn.head_bucket(Bucket=bucket)
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == '404':
logger.info('Creating bucket %s.', bucket)
s3_conn.create_bucket(Bucket=bucket)
elif e.response['Error']['Code'] in ('401', '403'):
logger.exception('Access denied for bucket %s.', bucket)
raise
else:
logger.exception('Error creating bucket %s. Error %s', bucket,
e.response)
raise
def _upload_code(s3_conn, bucket_name, name, contents):
"""Upload a ZIP file to S3 for use by Lambda.
The key used for the upload will be unique based on the checksum of the
contents. No changes will be made if the contents in S3 already match the
expected contents.
Args:
s3_conn (:class:`botocore.client.S3`): S3 connection to use for
operations.
bucket (str): name of the bucket to create.
prefix (str): S3 prefix to prepend to the constructed key name for
the uploaded file
name (str): desired name of the Lambda function. Will be used to
construct a key name for the uploaded file.
contents (str): byte string with the content of the file upload.
Returns:
troposphere.awslambda.Code: CloudFormation Lambda Code object,
pointing to the uploaded payload in S3.
Raises:
botocore.exceptions.ClientError: any error from boto3 is passed
through.
"""
hsh = hashlib.md5(contents)
logger.debug('lambda: ZIP hash: %s', hsh.hexdigest())
key = 'lambda-{}-{}.zip'.format(name, hsh.hexdigest())
info = _head_object(s3_conn, bucket_name, key)
expected_etag = '"{}"'.format(hsh.hexdigest())
if info and info['ETag'] == expected_etag:
logger.info('lambda: object %s already exists, not uploading', key)
else:
logger.info('lambda: uploading object %s', key)
s3_conn.put_object(Bucket=bucket_name, Key=key, Body=contents,
ContentType='application/zip',
ACL='authenticated-read')
return {"bucket": bucket_name, "key": key}
def _check_pattern_list(patterns, key, default=None):
"""Validates file search patterns from user configuration.
Acceptable input is a string (which will be converted to a singleton list),
a list of strings, or anything falsy (such as None or an empty dictionary).
Empty or unset input will be converted to a default.
Args:
patterns: input from user configuration (YAML).
key (str): name of the configuration key the input came from,
used for error display purposes.
Keyword Args:
default: value to return in case the input is empty or unset.
Returns:
list[str]: validated list of patterns
Raises:
ValueError: if the input is unacceptable.
"""
if not patterns:
return default
if isinstance(patterns, basestring):
return [patterns]
if isinstance(patterns, list):
if all(isinstance(p, basestring) for p in patterns):
return patterns
raise ValueError("Invalid file patterns in key '{}': must be a string or "
'list of strings'.format(key))
def _upload_function(s3_conn, bucket_name, function_name, path,
include=None, exclude=None):
"""Builds a Lambda payload from user configuration and uploads it to S3.
Args:
s3_conn (:class:`botocore.client.S3`): S3 connection to use for
operations.
bucket_name (str): name of the bucket to upload to.
function_name (str): desired name of the Lambda function. Will be used
to construct a key name for the uploaded file.
path (str): base path to retrieve files from (mandatory).
include (list): file patterns to include in the payload (optional).
exclude (list): file patterns to exclude from the payload (optional).
Returns:
dict: A dictionary with the bucket & key where the code is located.
"""
root = os.path.expanduser(path)
includes = _check_pattern_list(include, 'include', default=['**'])
excludes = _check_pattern_list(exclude, 'exclude', default=[])
logger.debug('lambda: processing function %s', function_name)
zip_contents = _zip_from_file_patterns(root, includes, excludes)
return _upload_code(s3_conn, bucket_name, function_name, zip_contents)
def upload_lambda_functions(s3_conn, bucket_name, function_name, path,
include=None, exclude=None):
"""Builds Lambda payloads from user configuration and uploads them to S3.
Constructs ZIP archives containing files matching specified patterns for
each function, uploads the result to Amazon S3, then returns the bucket
and key name where the function is stored.
Args:
s3_conn (:class:`botocore.client.S3`): S3 connection to use for
operations.
bucket_name (str): name of the bucket to upload to.
function_name (str): desired name of the Lambda function. Will be used
to construct a key name for the uploaded file.
path (str): base path to retrieve files from (mandatory).
include (list): file patterns to include in the payload (optional).
exclude (list): file patterns to exclude from the payload (optional).
Returns:
dict: A dictionary with the bucket & key where the code is located.
"""
_ensure_bucket(s3_conn, bucket_name)
results = _upload_function(s3_conn, bucket_name, function_name, path,
include, exclude)
return results