-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathembed_users.py
More file actions
65 lines (53 loc) · 2.13 KB
/
Copy pathembed_users.py
File metadata and controls
65 lines (53 loc) · 2.13 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
from tqdm import tqdm
import numpy as np
from SimuLine.Preprocessing.UnbiasedEmbedding.src.preprocess_datasets import main as preprocess_datasets
from SimuLine.Preprocessing.UnbiasedEmbedding.src.main import main as training
DATA_DIR = './Data/Adressa'
SAVE_DIR = './SimuLine/Preprocessing/UnbiasedEmbedding/data/raw'
EMBEDDING_DIR = './SimuLine/Preprocessing/Data'
def main():
build_train_test()
preprocess_datasets()
training()
scale()
def scale():
article_embedding = np.load(EMBEDDING_DIR + '/article_embedding.npy')
user_embedding = np.load(EMBEDDING_DIR + '/user_embedding.npy')
article_norm = np.sqrt((article_embedding ** 2).sum(1)).mean(0)
user_norm = np.sqrt((user_embedding ** 2).sum(1)).mean(0)
user_embedding = user_embedding / user_norm * article_norm
np.save(EMBEDDING_DIR + '/user_embedding.npy', user_embedding)
def save(table, path):
table_string = '\n'.join(['{}\t{}'.format(int(record[0]), int(record[1])) for record in table])
with open(path, 'w') as f:
f.write(table_string)
def build_train_test():
with open(DATA_DIR + '/user_article_5_hot.txt', 'r') as f:
user_article_str = f.read()
user_article = user_article_str.split('\n')
user_article = [record.split('\t') for record in user_article]
user_article = [[int(record[0]), int(record[1])] for record in user_article]
train_set = []
test_set = []
tmp = []
cur_user = -1
for record in tqdm(user_article):
if record[0] == cur_user:
tmp.append(record)
else:
# deal with the last user
for i, r in enumerate(tmp):
if i % 5 == 0:
test_set.append(r)
else:
train_set.append(r)
# new user start
tmp = [record]
cur_user = record[0]
for i, r in enumerate(tmp):
if i % 5 == 0:
test_set.append(r)
else:
train_set.append(r)
save(train_set, '{}/train.txt'.format(SAVE_DIR))
save(test_set, '{}/test.txt'.format(SAVE_DIR))