Repository navigation
Expand file tree
/
Copy pathdfxinfo.py
More file actions
125 lines (102 loc) · 3.82 KB
/
Copy pathdfxinfo.py
File metadata and controls
125 lines (102 loc) · 3.82 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
#!/usr/bin/env python
# coding=UTF-8
"""dfxinfo.py:
Generates a report about what's up with a DFXML file.
"""
import platform
import os
import os.path
import sys
import time
import xml
import xml.etree
import xml.etree.ElementTree as ET
from collections import defaultdict
import dfxml
import dfxml.fiwalk as fiwalk
import dfxml.histogram as histogram
__version__='0.2.0'
MAXSIZE = 1024*1024*16
class DiskSet:
"""DiskSet maintains a database of the file objects within a disk.
The entire database is maintained in memory."""
def __init__(self):
self.ext_histogram = histogram()
self.ext_histogram_distinct = histogram()
self.fi_by_md5 = dict() # a dictionary of lists
def uniques(self):
return len(self.fi_by_md5)
def pass1(self,fi):
if fi.is_virtual(): return
if fi.is_file():
self.fi_by_md5.setdefault(fi.md5(),[]).append(fi)
def print_dups_report(self):
print("Duplicates:")
# First extract the dups, then sort them
dups = filter(lambda a:len(a[1])>1,self.fi_by_md5.items(),)
dup_bytes = 0
for (md5hash,fis) in sorted(dups,key=lambda a:a[1][0].filesize(),reverse=True):
for fi in fis:
print("{:>16,} {:32} {}".format(fi.filesize(),fi.md5(),fi.filename()))
print()
dup_bytes += fis[0].filesize() * (len(fis)-1)
print("Total duplicate bytes: {:,}".format(dup_bytes))
def get_text(tree,tag):
try:
return tree.find(tag).text
except AttributeError as e:
return ""
def dfxml_info(fn):
import statistics
try:
tree = ET.parse(fn)
except xml.etree.ElementTree.ParseError as e:
print("corrupt: {}".format(fn))
return
command_line = get_text(tree,".//command_line")
elapsed_seconds = get_text(tree,".//elapsed_seconds")
if '.' in elapsed_seconds:
elapsed_seconds = elapsed_seconds[0:elapsed_seconds.find('.')+2]
elapsed_seconds += 's' if elapsed_seconds else ' '
start_time = get_text(tree,".//start_time")
if '.' in start_time:
start_time = start_time[0:start_time.find('.')]
start_time = start_time.replace("T"," ")
memory_usage = ''
maxrss = tree.findall(".//rusage[@who='RUSAGE_SELF']/maxrss")
if maxrss:
memory_usage = '{} MiB'.format(int(maxrss[0].text)//1024)
print("{} {} {:>7} {:>10}".format(command_line,start_time,elapsed_seconds,memory_usage))
# See if there are timers
timers = tree.findall("timer")
times = defaultdict(list)
for timer in timers:
try:
name = timer.attrib['name']
elapsed = float(timer.attrib['elapsed'])
times[name].append(elapsed)
except KeyError as e:
continue
if timers:
print("Timers:")
fmt = " {:>6} {:>5} {:>9.4} {:>9.4} {:>9.4}"
print(fmt.format("name","#","min","med.","max"))
for name in times:
data = times[name]
print(fmt.format(name, len(data), min(data), statistics.median(data), max(data)))
if __name__=="__main__":
from argparse import ArgumentParser
from copy import deepcopy
parser = ArgumentParser(description='Report information about a DFXML file')
parser.add_argument('xmlfiles',help='XML files to process',nargs='+')
parser.add_argument("--files", help="Report on file objects that the DFXML file contains", action='store_true')
parser.add_argument("--imagefile",help="specifies imagefile to examine; automatically runs fiwalk",nargs='+')
args = parser.parse_args()
ds = DiskSet()
if args.files:
dfxml.read_dfxml(xmlfile=open(fn,'rb'),callback=ds.pass1)
if ds.uniques()>0:
ds.print_dups_report()
exit(0)
for fn in args.xmlfiles:
dfxml_info(fn)