-
Notifications
You must be signed in to change notification settings - Fork 18
Expand file tree
/
Copy pathdataset.py
More file actions
29 lines (25 loc) · 895 Bytes
/
dataset.py
File metadata and controls
29 lines (25 loc) · 895 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
import torch
from torch.utils.data import Dataset
import gzip
import json
class HumanEvalDataset(Dataset):
def __init__(self, file_name):
self.file_name = file_name
self.data = []
self.load_data(file_name)
def load_data(self, filename):
if filename.endswith(".gz"):
with open(filename, "rb") as gzfp:
with gzip.open(gzfp, 'rt') as fp:
for line in fp:
if any(not x.isspace() for x in line):
self.data.append(json.loads(line))
else:
with open(filename, "r") as fp:
for line in fp:
if any(not x.isspace() for x in line):
self.data.append(json.loads(line))
def __len__(self):
return len(self.data)
def __getitem__(self, idx):
return self.data[idx]