-
Notifications
You must be signed in to change notification settings - Fork 11
Expand file tree
/
Copy pathbinaryfile.cpp
More file actions
114 lines (90 loc) · 2.89 KB
/
Copy pathbinaryfile.cpp
File metadata and controls
114 lines (90 loc) · 2.89 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
#include <blosc.h>
#include <cstring>
#include "../dataio/binaryfile.h"
static bool initialized = false;
void BinaryFile::init() {
blosc_init();
int result = blosc_set_compressor("zstd");
assert(result >= 0);
initialized = true;
}
void BinaryFile::finish() {
blosc_destroy();
}
BinaryFile::BinaryFile(const string& dataFile, const string& metaFile, int compressionLevel)
:compressionLevel(compressionLevel)
{
dataStream = new std::fstream(dataFile.c_str(), std::ios::out | std::ios::binary);
metaStream = new ofstream(metaFile);
outBuf = NULL;
outBufLen = 0;
compressedIndex = 0;
}
BinaryFile::~BinaryFile() {
dataStream->close();
metaStream->close();
delete dataStream;
delete metaStream;
delete[] outBuf;
}
/* Obtain a size of buffer guaranteed not to fail */
size_t BinaryFile::updateBufferSize(size_t nrows, size_t ncols, size_t sizeOfElt) {
size_t size = nrows * ncols * sizeOfElt + BLOSC_MAX_OVERHEAD;
if(outBuf == NULL || size > outBufLen) {
if(outBuf != NULL)
delete[] outBuf;
outBuf = new char[size];
outBufLen = size;
}
}
void BinaryFile::write(const void* buf, size_t nrows, size_t ncols, size_t sizeOfElt) {
size_t size = nrows * ncols * sizeOfElt;
if(size <= 0)
return;
size_t compressedSize;
if(compressionLevel <= 0) {
dataStream->write((char*)buf, size);
compressedSize = size;
}
else {
assert(size < (size_t)(1 << 31) - BLOSC_MAX_OVERHEAD);
updateBufferSize(nrows,ncols,sizeOfElt);
int doshuffle = BLOSC_BITSHUFFLE;
int typesize = (int)sizeOfElt;
int result = blosc_compress(compressionLevel,doshuffle,typesize,size,buf,outBuf,outBufLen);
assert(result > 0);
compressedSize = (size_t)result;
dataStream->write((char*)outBuf, compressedSize);
}
dataStream->flush();
(*metaStream) << nrows << " " << ncols << " " << sizeOfElt << " " << compressionLevel << " " << compressedIndex << " " << compressedSize << "\n";
metaStream->flush();
compressedIndex += compressedSize;
}
BinaryFileAutoChunking::BinaryFileAutoChunking(const string& dataFile, const string& metaFile, int compressionLevel, size_t nrows, size_t ncols, size_t sizeOfElt)
:nrows(nrows),ncols(ncols),sizeOfElt(sizeOfElt)
{
rowLen = ncols * sizeOfElt;
buf = new char[nrows * ncols * sizeOfElt];
curNumRows = 0;
binaryFile = new BinaryFile(dataFile,metaFile,compressionLevel);
}
BinaryFileAutoChunking::~BinaryFileAutoChunking() {
if(curNumRows > 0) {
binaryFile->write(buf,curNumRows,ncols,sizeOfElt);
curNumRows = 0;
}
delete[] buf;
delete binaryFile;
}
void BinaryFileAutoChunking::writeRow(const void* row, size_t nc, size_t soe) {
assert(curNumRows >= 0 && curNumRows < nrows);
assert(ncols = nc);
assert(sizeOfElt == soe);
std::memcpy(buf + (curNumRows * rowLen), row, rowLen);
curNumRows++;
if(curNumRows >= nrows) {
binaryFile->write(buf,nrows,ncols,sizeOfElt);
curNumRows = 0;
}
}