Repository navigation
Expand file tree
/
Copy pathHuge_Insert.java
More file actions
115 lines (89 loc) · 3.36 KB
/
Copy pathHuge_Insert.java
File metadata and controls
115 lines (89 loc) · 3.36 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
package examples;
import org.mapdb.*;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
/**
* Demonstrate how-to create large BTreeMap using data pump.
* Typical usage is to import data set from external source.
*
* @author Jan Kotek
*/
public class Huge_Insert {
public static void main(String[] args) throws IOException {
/** max number of elements to import */
final long max = (int) 1e6;
/**
* Open database in temporary directory
*/
File dbFile = File.createTempFile("mapdb","temp");
DB db = DBMaker
.newFileDB(dbFile)
/** disabling Write Ahead Log makes import much faster */
.transactionDisable()
.make();
long time = System.currentTimeMillis();
/**
* Source of data which randomly generates strings.
* In real world this would return data from file.
*/
Iterator<String> source = new Iterator<String>() {
long counter = 0;
@Override public boolean hasNext() {
return counter<max;
}
@Override
public String next() {
counter++;
return randomString(10);
}
@Override public void remove() {}
};
/**
* BTreeMap Data Pump requires data source to be presorted in reverse order (highest to lowest).
* There is method in Data Pump we can use to sort data. It uses temporarly files and can handle fairly
* large data sets.
*/
source = Pump.sort(source,
true, 100000,
Collections.reverseOrder(BTreeMap.COMPARABLE_COMPARATOR), //reverse order comparator
db.getDefaultSerializer()
);
/**
* Disk space used by serialized keys should be minimised.
* Keys are sorted, so only difference between consequential keys is stored.
* This method is called delta-packing and typically saves 60% of disk space.
*/
BTreeKeySerializer<String> keySerializer = BTreeKeySerializer.STRING;
/**
* Translates Map Key into Map Value.
*/
Fun.Function1<Integer,String> valueExtractor = new Fun.Function1<Integer, String>() {
@Override public Integer run(String s) {
return s.hashCode();
}
};
/**
* Create BTreeMap and fill it with data
*/
Map<String,Integer> map = db.createTreeMap("map")
.pumpSource(source,valueExtractor)
//.pumpPresort(100000) // for presorting data we could also use this method
.keySerializer(keySerializer)
.make();
System.out.println("Finished; total time: "+(System.currentTimeMillis()-time)/1000+"s; there are "+map.size()+" items in map");
db.close();
}
public static String randomString(int size) {
String chars = "0123456789abcdefghijklmnopqrstuvwxyz !@#$%^&*()_+=-{}[]:\",./<>?|\\";
StringBuilder b = new StringBuilder(size);
Random r = new Random();
for(int i=0;i<size;i++){
b.append(chars.charAt(r.nextInt(chars.length())));
}
return b.toString();
}
}