update imdb data.

Oceania2018 · Oceania2018 · commit 87f667c67b62 · 2019-02-19T17:24:36.000-06:00
diff --git a/data/imdb.zip b/data/imdb.zip
diff --git a/src/TensorFlowNET.Utility/Compress.cs b/src/TensorFlowNET.Utility/Compress.cs
@@ -3,6 +3,7 @@
 using System;
 using System.IO;
 using System.IO.Compression;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -12,6 +13,9 @@ public class Compress
     {
         public static void UnZip(String gzArchiveName, String destFolder)
         {
+            var flag = gzArchiveName.Split(Path.DirectorySeparatorChar).Last().Split('.').First() + ".bin";
+            if (File.Exists(Path.Combine(destFolder, flag))) return;
+
             Console.WriteLine($"Extracting.");
             var task = Task.Run(() =>
             {
@@ -24,12 +28,16 @@ public static void UnZip(String gzArchiveName, String destFolder)
                 Console.Write(".");
             }
 
+            File.Create(Path.Combine(destFolder, flag));
             Console.WriteLine("");
             Console.WriteLine("Extracting is completed.");
         }
 
         public static void ExtractTGZ(String gzArchiveName, String destFolder)
         {
+            var flag = gzArchiveName.Split(Path.DirectorySeparatorChar).Last().Split('.').First() + ".bin";
+            if (File.Exists(Path.Combine(destFolder, flag))) return;
+
             Console.WriteLine($"Extracting.");
             var task = Task.Run(() =>
             {
@@ -49,6 +57,7 @@ public static void ExtractTGZ(String gzArchiveName, String destFolder)
                 Console.Write(".");
             }
 
+            File.Create(Path.Combine(destFolder, flag));
             Console.WriteLine("");
             Console.WriteLine("Extracting is completed.");
         }
diff --git a/test/TensorFlowNET.Examples/ImageRecognition.cs b/test/TensorFlowNET.Examples/ImageRecognition.cs
@@ -88,8 +88,7 @@ private void PrepareData()
             string zipFile = Path.Join(dir, "inception5h.zip");
             Utility.Web.Download(url, zipFile);
 
-            if (!File.Exists(Path.Join(dir, pbFile)))
-                Utility.Compress.UnZip(zipFile, dir);
+            Utility.Compress.UnZip(zipFile, dir);
 
             // download sample picture
             string pic = Path.Join(dir, "img", "grace_hopper.jpg");
diff --git a/test/TensorFlowNET.Examples/LabelImage.cs b/test/TensorFlowNET.Examples/LabelImage.cs
@@ -106,8 +106,7 @@ private void PrepareData()
             string zipFile = Path.Join(dir, $"{pbFile}.tar.gz");
             Utility.Web.Download(url, zipFile);
 
-            if (!File.Exists(Path.Join(dir, pbFile)))
-                Utility.Compress.ExtractTGZ(zipFile, dir);
+            Utility.Compress.ExtractTGZ(zipFile, dir);
 
             // download sample picture
             string pic = "grace_hopper.jpg";
diff --git a/test/TensorFlowNET.Examples/TextClassificationWithMovieReviews.cs b/test/TensorFlowNET.Examples/TextClassificationWithMovieReviews.cs
@@ -3,12 +3,14 @@
 using System.IO;
 using System.Text;
 using Tensorflow;
+using NumSharp.Core;
 
 namespace TensorFlowNET.Examples
 {
     public class TextClassificationWithMovieReviews : Python, IExample
     {
         string dir = "text_classification_with_movie_reviews";
+        string dataFile = "imdb.zip";
 
         public void Run()
         {
@@ -17,17 +19,27 @@ public void Run()
 
         private void PrepareData()
         {
-
             Directory.CreateDirectory(dir);
 
             // get model file
-            string url = "https://storage.googleapis.com/download.tensorflow.org/models/inception_v3_2016_08_28_frozen.pb.tar.gz";
+            string url = $"https://github.com/SciSharp/TensorFlow.NET/raw/master/data/{dataFile}";
 
             string zipFile = Path.Join(dir, $"imdb.zip");
             Utility.Web.Download(url, zipFile);
+            Utility.Compress.UnZip(zipFile, dir);
+
+            // prepare training dataset
+            NDArray x_train = File.ReadAllLines(Path.Join(dir, "x_train.txt"));
+            NDArray labels_train = File.ReadAllLines(Path.Join(dir, "y_train.txt"));
+            NDArray indices_train = File.ReadAllLines(Path.Join(dir, "indices_train.txt"));
+            x_train = x_train[indices_train];
+            labels_train = labels_train[indices_train];
 
-            if (!File.Exists(Path.Join(dir, zipFile)))
-                Utility.Compress.ExtractTGZ(zipFile, dir);
+            NDArray x_test = File.ReadAllLines(Path.Join(dir, "x_test.txt"));
+            NDArray labels_test = File.ReadAllLines(Path.Join(dir, "y_test.txt"));
+            NDArray indices_test = File.ReadAllLines(Path.Join(dir, "indices_test.txt"));
+            x_test = x_test[indices_test];
+            labels_test = labels_test[indices_test];
         }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -3,6 +3,7 @@`
`3`	`3`	`using System;`
`4`	`4`	`using System.IO;`
`5`	`5`	`using System.IO.Compression;`
	`6`	`+using System.Linq;`
`6`	`7`	`using System.Threading;`
`7`	`8`	`using System.Threading.Tasks;`
`8`	`9`
`@@ -12,6 +13,9 @@ public class Compress`
`12`	`13`	`{`
`13`	`14`	`public static void UnZip(String gzArchiveName, String destFolder)`
`14`	`15`	`{`
	`16`	`+ var flag = gzArchiveName.Split(Path.DirectorySeparatorChar).Last().Split('.').First() + ".bin";`
	`17`	`+ if (File.Exists(Path.Combine(destFolder, flag))) return;`
	`18`	`+`
`15`	`19`	`Console.WriteLine($"Extracting.");`
`16`	`20`	`var task = Task.Run(() =>`
`17`	`21`	`{`
`@@ -24,12 +28,16 @@ public static void UnZip(String gzArchiveName, String destFolder)`
`24`	`28`	`Console.Write(".");`
`25`	`29`	`}`
`26`	`30`
	`31`	`+ File.Create(Path.Combine(destFolder, flag));`
`27`	`32`	`Console.WriteLine("");`
`28`	`33`	`Console.WriteLine("Extracting is completed.");`
`29`	`34`	`}`
`30`	`35`
`31`	`36`	`public static void ExtractTGZ(String gzArchiveName, String destFolder)`
`32`	`37`	`{`
	`38`	`+ var flag = gzArchiveName.Split(Path.DirectorySeparatorChar).Last().Split('.').First() + ".bin";`
	`39`	`+ if (File.Exists(Path.Combine(destFolder, flag))) return;`
	`40`	`+`
`33`	`41`	`Console.WriteLine($"Extracting.");`
`34`	`42`	`var task = Task.Run(() =>`
`35`	`43`	`{`
`@@ -49,6 +57,7 @@ public static void ExtractTGZ(String gzArchiveName, String destFolder)`
`49`	`57`	`Console.Write(".");`
`50`	`58`	`}`
`51`	`59`
	`60`	`+ File.Create(Path.Combine(destFolder, flag));`
`52`	`61`	`Console.WriteLine("");`
`53`	`62`	`Console.WriteLine("Extracting is completed.");`
`54`	`63`	`}`