forked from dgleebits/PythonSystemAdminTools
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathSimpleApp.py
More file actions
29 lines (24 loc) · 733 Bytes
/
Copy pathSimpleApp.py
File metadata and controls
29 lines (24 loc) · 733 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
"""
auth log analysis with spark
SimpleApp.py
"""
import sys
from operator import add
from time import strftime
from pyspark import SparkContext
outFile = "counts" + strftime("%Y-%m-%d")
logFile = "auth.log"
destination = "local"
appName = "Simple App"
sc = SparkContext(destination, appName)
logData = sc.textFile(logFile).cache()
failedData = logData.filter(lambda x: 'Failed' in x)
rootData = failedData.filter(lambda s: 'root' in s)
def splitville(line):
return line.split("from ")[1].split()[0]
ipAddresses = rootData.map(splitville)
counts = ipAddresses.map(lambda x: (x, 1)).reduceByKey(add)
output = counts.collect()
for (word, count) in output:
print "%s: %i" % (word, count)
counts.saveAsTextFile(outFile)