Skip to content

Commit b5d8165

Browse files
henonhenon
committed
TextClassification: reverted to simpler shuffling and created dbpedia_subset.zip for debugging
1 parent ff9d875 commit b5d8165

4 files changed

Lines changed: 79 additions & 91 deletions

File tree

‎data/dbpedia_subset.zip‎

770 KB
Binary file not shown.

‎test/TensorFlowNET.Examples/TextProcess/DataHelpers.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.Linq;
66
using System.Text;
77
using System.Text.RegularExpressions;
8+
using TensorFlowNET.Examples.Utility;
89

910
namespace TensorFlowNET.Examples
1011
{
@@ -25,7 +26,8 @@ public static (int[][], int[], int) build_char_dataset(string step, string model
2526
char_dict["<unk>"] = 1;
2627
foreach (char c in alphabet)
2728
char_dict[c.ToString()] = char_dict.Count;
28-
var contents = File.ReadAllLines(TRAIN_PATH);
29+
var contents = new Random(17).Shuffle( File.ReadAllLines(TRAIN_PATH));
30+
//File.WriteAllLines("text_classification/dbpedia_csv/train_6400.csv", contents.Take(6400));
2931
var size = limit == null ? contents.Length : limit.Value;
3032

3133
var x = new int[size][];

‎test/TensorFlowNET.Examples/TextProcess/TextClassificationTrain.cs‎

Lines changed: 54 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,8 @@ protected virtual bool RunWithImportedGraph(Session sess, Graph graph)
5959
Console.WriteLine("\tDONE ");
6060

6161
var (train_x, valid_x, train_y, valid_y) = train_test_split(x, y, test_size: 0.15f);
62-
Console.WriteLine("Training set size: " + train_x.Length);
63-
Console.WriteLine("Test set size: " + valid_x.Length);
62+
Console.WriteLine("Training set size: " + train_x.len);
63+
Console.WriteLine("Test set size: " + valid_x.len);
6464

6565
Console.WriteLine("Import graph...");
6666
var meta_file = model_name + ".meta";
@@ -164,106 +164,70 @@ protected virtual bool RunWithBuiltGraph(Session session, Graph graph)
164164
}
165165

166166
// TODO: this originally is an SKLearn utility function. it randomizes train and test which we don't do here
167-
//private (NDArray, NDArray, NDArray, NDArray) train_test_split(NDArray x, NDArray y, float test_size = 0.3f)
168-
//{
169-
// Console.WriteLine("Splitting in Training and Testing data...");
170-
// int len = x.shape[0];
171-
// //int classes = y.Data<int>().Distinct().Count();
172-
// //int samples = len / classes;
173-
// int train_size = (int)Math.Round(len * (1 - test_size));
174-
// var train_x = x[new Slice(stop: train_size), new Slice()];
175-
// var valid_x = x[new Slice(start: train_size + 1), new Slice()];
176-
// var train_y = y[new Slice(stop: train_size)];
177-
// var valid_y = y[new Slice(start: train_size + 1)];
178-
// Console.WriteLine("\tDONE");
179-
// return (train_x, valid_x, train_y, valid_y);
180-
//}
181-
182-
private (int[][], int[][], int[], int[]) train_test_split(int[][] x, int[] y, float test_size = 0.3f)
183-
{
184-
Console.WriteLine("Splitting in Training and Testing data...");
185-
var stopwatch = Stopwatch.StartNew();
186-
int len = x.Length;
187-
//int classes = y.Distinct().Count();
188-
//int samples = len / classes;
189-
int train_size = int.Parse((len * (1 - test_size)).ToString());
190-
191-
//var train_x = new List<int[]>();
192-
//var valid_x = new List<int[]>();
193-
//var train_y = new List<int>();
194-
//var valid_y = new List<int>();
167+
private (NDArray, NDArray, NDArray, NDArray) train_test_split(NDArray x, NDArray y, float test_size = 0.3f)
168+
{
169+
Console.WriteLine("Splitting in Training and Testing data...");
170+
int len = x.shape[0];
171+
//int classes = y.Data<int>().Distinct().Count();
172+
//int samples = len / classes;
173+
int train_size = (int)Math.Round(len * (1 - test_size));
174+
var train_x = x[new Slice(stop: train_size), new Slice()];
175+
var valid_x = x[new Slice(start: train_size + 1), new Slice()];
176+
var train_y = y[new Slice(stop: train_size)];
177+
var valid_y = y[new Slice(start: train_size + 1)];
178+
Console.WriteLine("\tDONE");
179+
return (train_x, valid_x, train_y, valid_y);
180+
}
195181

196-
//for (int i = 0; i < classes; i++)
197-
//{
198-
// for (int j = 0; j < samples; j++)
199-
// {
200-
// int idx = i * samples + j;
201-
// if (idx < train_size + samples * i)
202-
// {
203-
// train_x.Add(x[idx]);
204-
// train_y.Add(y[idx]);
205-
// }
206-
// else
207-
// {
208-
// valid_x.Add(x[idx]);
209-
// valid_y.Add(y[idx]);
210-
// }
211-
// }
212-
//}
213-
var random = new Random(17);
182+
//private (int[][], int[][], int[], int[]) train_test_split(int[][] x, int[] y, float test_size = 0.3f)
183+
//{
184+
// Console.WriteLine("Splitting in Training and Testing data...");
185+
// var stopwatch = Stopwatch.StartNew();
186+
// int len = x.Length;
187+
// int train_size = int.Parse((len * (1 - test_size)).ToString());
188+
// var random = new Random(17);
214189

215-
// we collect indices of labels
216-
var labels = new Dictionary<int, HashSet<int>>();
217-
var shuffled_indices = Shuffle<int>(random, range(len).ToArray());
218-
foreach (var i in shuffled_indices)
219-
{
220-
var label = y[i];
221-
if (!labels.ContainsKey(i))
222-
labels[label] = new HashSet<int>();
223-
labels[label].Add(i);
224-
}
190+
// // we collect indices of labels
191+
// var labels = new Dictionary<int, HashSet<int>>();
192+
// var shuffled_indices = random.Shuffle<int>(range(len).ToArray());
193+
// foreach (var i in shuffled_indices)
194+
// {
195+
// var label = y[i];
196+
// if (!labels.ContainsKey(i))
197+
// labels[label] = new HashSet<int>();
198+
// labels[label].Add(i);
199+
// }
225200

226-
var train_x = new int[train_size][];
227-
var valid_x = new int[len - train_size][];
228-
var train_y = new int[train_size];
229-
var valid_y = new int[len - train_size];
201+
// var train_x = new int[train_size][];
202+
// var valid_x = new int[len - train_size][];
203+
// var train_y = new int[train_size];
204+
// var valid_y = new int[len - train_size];
230205

231-
FillWithShuffledLabels(x, y, train_x, train_y, random, labels);
232-
FillWithShuffledLabels(x, y, valid_x, valid_y, random, labels);
206+
// FillWithShuffledLabels(x, y, train_x, train_y, random, labels);
207+
// FillWithShuffledLabels(x, y, valid_x, valid_y, random, labels);
233208

234-
Console.WriteLine("\tDONE " + stopwatch.Elapsed);
235-
return (train_x, valid_x, train_y, valid_y);
236-
}
209+
// Console.WriteLine("\tDONE " + stopwatch.Elapsed);
210+
// return (train_x, valid_x, train_y, valid_y);
211+
//}
237212

238213
private static void FillWithShuffledLabels(int[][] x, int[] y, int[][] shuffled_x, int[] shuffled_y, Random random, Dictionary<int, HashSet<int>> labels)
239214
{
240215
int i = 0;
216+
var label_keys = labels.Keys.ToArray();
241217
while (i < shuffled_x.Length)
242218
{
243-
foreach (var key in Shuffle<int>(random, labels.Keys.ToArray()))
244-
{
245-
var set = labels[key];
246-
var index = set.First();
247-
if (set.Count == 0)
248-
labels.Remove(key); // remove the set as it is empty
249-
shuffled_x[i] = x[index];
250-
shuffled_y[i] = y[index];
251-
i++;
252-
}
253-
}
254-
}
255-
256-
public static T[] Shuffle<T>(Random rng, T[] array)
257-
{
258-
int n = array.Length;
259-
while (n > 1)
260-
{
261-
int k = rng.Next(n--);
262-
T temp = array[n];
263-
array[n] = array[k];
264-
array[k] = temp;
219+
var key = label_keys[random.Next(label_keys.Length)];
220+
var set = labels[key];
221+
var index = set.First();
222+
if (set.Count == 0)
223+
{
224+
labels.Remove(key); // remove the set as it is empty
225+
label_keys = labels.Keys.ToArray();
226+
}
227+
shuffled_x[i] = x[index];
228+
shuffled_y[i] = y[index];
229+
i++;
265230
}
266-
return array;
267231
}
268232

269233
private IEnumerable<(NDArray, NDArray, int)> batch_iter(NDArray inputs, NDArray outputs, int batch_size, int num_epochs)

‎test/TensorFlowNET.Examples/Utility/ArrayShuffling.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Text;
4+
5+
namespace TensorFlowNET.Examples.Utility
6+
{
7+
public static class ArrayShuffling
8+
{
9+
public static T[] Shuffle<T>(this Random rng, T[] array)
10+
{
11+
int n = array.Length;
12+
while (n > 1)
13+
{
14+
int k = rng.Next(n--);
15+
T temp = array[n];
16+
array[n] = array[k];
17+
array[k] = temp;
18+
}
19+
return array;
20+
}
21+
}
22+
}

0 commit comments

Comments
 (0)