-
Notifications
You must be signed in to change notification settings - Fork 11
Expand file tree
/
Copy pathbinaryfile.h
More file actions
72 lines (55 loc) · 2.15 KB
/
Copy pathbinaryfile.h
File metadata and controls
72 lines (55 loc) · 2.15 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
#ifndef BINARYFILE_H
#define BINARYFILE_H
#include <fstream>
#include "../core/global.h"
/*
Write raw binary data to disk.
dataFile format:
Consists of concatenated chunks of raw binary data
Each chunk is an (optionally compressed via blosc) c-layout matrix of shape [nrows][ncols] where each entry has sizeOfElt bytes.
(for integer elements, endianness is whatever it was on the writing machine)
metaFile format:
Consists of ascii lines of "{nrows} {ncols} {sizeofElt} {compressionLevel} {starting byte} {compressed byte len}" indicating the shape of each
chunk and where it's located in the dataFile.
NOTE: Currently there is a limitation in blosc that prevents chunks from being larger than approximately 2^31 pre-compressed.
*/
class BinaryFile {
public:
/* Global initialization - Call this prior to any other functions here */
static void init();
/* Global cleanup - Call at the end of the program to clean up */
static void finish();
BinaryFile(const string& dataFile, const string& metaFile, int compressionLevel);
~BinaryFile();
BinaryFile(const BinaryFile&) = delete;
BinaryFile& operator=(const BinaryFile&) = delete;
/* Write one chunk of binary data compressed using blosc compression. */
void write(const void* buf, size_t nrows, size_t ncols, size_t sizeOfElt);
private:
size_t updateBufferSize(size_t nrows, size_t ncols, size_t sizeOfElt);
int compressionLevel;
size_t compressedIndex;
char* outBuf;
size_t outBufLen;
fstream* dataStream;
ofstream* metaStream;
};
//Auto-chunking version of BinaryFile
class BinaryFileAutoChunking {
public:
BinaryFileAutoChunking(const string& dataFile, const string& metaFile, int compressionLevel, size_t nrows, size_t ncols, size_t sizeOfElt);
~BinaryFileAutoChunking();
//No copy assignment or constructor
BinaryFileAutoChunking(const BinaryFileAutoChunking&) = delete;
BinaryFileAutoChunking& operator=(const BinaryFileAutoChunking&) = delete;
void writeRow(const void* row, size_t ncols, size_t sizeOfElt);
private:
size_t nrows;
size_t ncols;
size_t sizeOfElt;
size_t rowLen;
char* buf;
size_t curNumRows;
BinaryFile* binaryFile;
};
#endif