Skip to content

Commit 773a3ba

Browse files
Oceania2018Oceania2018
committed
Add Preprocessing.timeseries_dataset_from_array
1 parent 1b7e25c commit 773a3ba

7 files changed

Lines changed: 122 additions & 11 deletions

File tree

‎src/TensorFlowNET.Core/Binding.Util.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,8 @@ public static int len(object a)
155155
{
156156
switch (a)
157157
{
158+
case Tensor tensor:
159+
return tensor.shape[0];
158160
case Tensors arr:
159161
return arr.Length;
160162
case Array arr:

‎src/TensorFlowNET.Core/Tensorflow.Binding.csproj‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<AssemblyName>TensorFlow.NET</AssemblyName>
66
<RootNamespace>Tensorflow</RootNamespace>
77
<TargetTensorFlow>2.2.0</TargetTensorFlow>
8-
<Version>0.40.1</Version>
8+
<Version>0.50.0</Version>
99
<LangVersion>8.0</LangVersion>
1010
<Authors>Haiping Chen, Meinrad Recheis, Eli Belash</Authors>
1111
<Company>SciSharp STACK</Company>
@@ -19,7 +19,7 @@
1919
<Description>Google's TensorFlow full binding in .NET Standard.
2020
Building, training and infering deep learning models.
2121
https://tensorflownet.readthedocs.io</Description>
22-
<AssemblyVersion>0.40.1.0</AssemblyVersion>
22+
<AssemblyVersion>0.50.0.0</AssemblyVersion>
2323
<PackageReleaseNotes>tf.net 0.20.x and above are based on tensorflow native 2.x.
2424

2525
* Eager Mode is added finally.
@@ -31,8 +31,9 @@ https://tensorflownet.readthedocs.io</Description>
3131
TensorFlow .NET v0.3x is focused on making more Keras API works.
3232
Keras API is a separate package released as TensorFlow.Keras.
3333

34-
tf.net 0.4x.x aligns with TensorFlow v2.4.1 native library.</PackageReleaseNotes>
35-
<FileVersion>0.40.1.0</FileVersion>
34+
tf.net 0.4x.x aligns with TensorFlow v2.4.1 native library.
35+
tf.net 0.5x.x aligns with TensorFlow v2.5.x native library.</PackageReleaseNotes>
36+
<FileVersion>0.50.0.0</FileVersion>
3637
<PackageLicenseFile>LICENSE</PackageLicenseFile>
3738
<PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
3839
<SignAssembly>true</SignAssembly>

‎src/TensorFlowNET.Core/Tensors/Tensor.Index.cs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,35 @@ public Tensor slice(Slice slice)
120120
});
121121
}
122122

123+
public Tensor this[params Tensor[] slice]
124+
{
125+
get
126+
{
127+
var args = tensor_util.ParseSlices(slice);
128+
129+
return tf_with(ops.name_scope(null, "strided_slice", args), scope =>
130+
{
131+
string name = scope;
132+
133+
var tensor = gen_array_ops.strided_slice(
134+
this,
135+
args.PackedBegin,
136+
args.PackedEnd,
137+
args.PackedStrides,
138+
begin_mask: args.BeginMask,
139+
end_mask: args.EndMask,
140+
shrink_axis_mask: args.ShrinkAxisMask,
141+
new_axis_mask: args.NewAxisMask,
142+
ellipsis_mask: args.EllipsisMask,
143+
name: name);
144+
145+
tensor.OriginalVarSlice = args;
146+
147+
return tensor;
148+
});
149+
}
150+
}
151+
123152
public Tensor slice(int start)
124153
{
125154
var slice_spec = new int[] { start };

‎src/TensorFlowNET.Core/Tensors/constant_op.cs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -146,9 +146,7 @@ private static EagerTensor convert_to_eager_tensor(object value, Context ctx, TF
146146
}
147147

148148
if (dtype == TF_DataType.TF_STRING && value is byte[] bytes)
149-
{
150149
return new EagerTensor(bytes, ctx.DeviceName, TF_DataType.TF_STRING);
151-
}
152150

153151
switch (value)
154152
{

‎src/TensorFlowNET.Core/Tensors/tensor_util.cs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -673,5 +673,38 @@ public static ParsedSliceArgs ParseSlices(Slice[] slices)
673673
NewAxisMask = new_axis_mask
674674
};
675675
}
676+
677+
public static ParsedSliceArgs ParseSlices(Tensor[] slices)
678+
{
679+
var begin = new List<Tensor>();
680+
var end = new List<Tensor>();
681+
var strides = new List<Tensor>();
682+
683+
var index = 0;
684+
var (new_axis_mask, shrink_axis_mask) = (0, 0);
685+
var (begin_mask, end_mask) = (0, 0);
686+
var ellipsis_mask = 0;
687+
688+
foreach (var s in slices)
689+
{
690+
begin.Add(s);
691+
end.Add(s + 1);
692+
shrink_axis_mask |= (1 << index);
693+
strides.Add(tf.constant(1, dtype: s.dtype));
694+
index += 1;
695+
}
696+
697+
return new ParsedSliceArgs
698+
{
699+
PackedBegin = array_ops.stack(begin),
700+
PackedEnd = array_ops.stack(end),
701+
PackedStrides = array_ops.stack(strides),
702+
BeginMask = begin_mask,
703+
EndMask = end_mask,
704+
EllipsisMask = ellipsis_mask,
705+
ShrinkAxisMask = shrink_axis_mask,
706+
NewAxisMask = new_axis_mask
707+
};
708+
}
676709
}
677710
}

‎src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
using static Tensorflow.KerasApi;
2+
using static Tensorflow.Binding;
3+
using NumSharp;
24

35
namespace Tensorflow.Keras
46
{
@@ -41,7 +43,7 @@ public IDatasetV2 image_dataset_from_directory(string directory,
4143
int num_channels = 0;
4244
if (color_mode == "rgb")
4345
num_channels = 3;
44-
46+
4547
var (image_paths, label_list, class_name_list) = keras.preprocessing.dataset_utils.index_directory(directory,
4648
labels,
4749
formats: WHITELIST_FORMATS,
@@ -90,5 +92,50 @@ public IDatasetV2 text_dataset_from_directory(string directory,
9092
dataset.class_names = class_name_list;
9193
return dataset;
9294
}
95+
96+
/// <summary>
97+
/// Creates a dataset of sliding windows over a timeseries provided as array.
98+
/// </summary>
99+
/// <param name="data"></param>
100+
/// <param name="sequence_length"></param>
101+
/// <param name="sequence_stride"></param>
102+
/// <param name="sampling_rate"></param>
103+
/// <param name="batch_size"></param>
104+
/// <param name="shuffle"></param>
105+
/// <returns></returns>
106+
public IDatasetV2 timeseries_dataset_from_array(Tensor data, int sequence_length,
107+
int sequence_stride = 1, int sampling_rate = 1, int batch_size = 128,
108+
bool shuffle = false, int seed = (int)1e6, int start_index = 0, int? end_index = null)
109+
{
110+
if (!end_index.HasValue)
111+
end_index = len(data);
112+
113+
var num_seqs = end_index.Value - start_index - (sequence_length * sampling_rate) + 1;
114+
var index_dtype = num_seqs < 2147483647 ? tf.int32 : tf.int64;
115+
var start_positions = np.arange(0, num_seqs, sequence_stride);
116+
if (shuffle)
117+
{
118+
var rng = np.random.RandomState(seed);
119+
rng.shuffle(start_positions);
120+
}
121+
122+
var sequence_length_tensor = math_ops.cast(sequence_length, dtype: index_dtype);
123+
var sampling_rate_tensor = math_ops.cast(sampling_rate, dtype: index_dtype);
124+
125+
var start_positions_tensor = tf.constant(start_positions);
126+
var positions_ds = tf.data.Dataset.from_tensor(start_positions_tensor).repeat();
127+
var z = tf.data.Dataset.zip(tf.data.Dataset.range(len(start_positions)), positions_ds);
128+
var indices = z.map(m =>
129+
{
130+
var (i, positions) = (m[0], m[1]);
131+
return tf.range(positions[i], positions[i] + sequence_length_tensor * sampling_rate_tensor, sampling_rate_tensor);
132+
}, num_parallel_calls: -1);
133+
return null;
134+
}
135+
136+
IDatasetV2 sequences_from_indices(Tensor array, Tensor indices_ds, Tensor start_index, Tensor end_index)
137+
{
138+
return null;
139+
}
93140
}
94141
}

‎src/TensorFlowNET.Keras/Tensorflow.Keras.csproj‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<LangVersion>8.0</LangVersion>
77
<RootNamespace>Tensorflow.Keras</RootNamespace>
88
<Platforms>AnyCPU;x64</Platforms>
9-
<Version>0.5.1</Version>
9+
<Version>0.5.2</Version>
1010
<Authors>Haiping Chen</Authors>
1111
<Product>Keras for .NET</Product>
1212
<Copyright>Apache 2.0, Haiping Chen 2021</Copyright>
@@ -24,7 +24,8 @@
2424
* Support model.load_weights.
2525
* Add Subtract layer
2626
* Support YOLOv3 model.
27-
* Text preprocessing</PackageReleaseNotes>
27+
* Text preprocessing
28+
* Preprocessing.timeseries_dataset_from_array</PackageReleaseNotes>
2829
<Description>Keras for .NET
2930

3031
Keras is an API designed for human beings, not machines. Keras follows best practices for reducing cognitive load: it offers consistent &amp; simple APIs, it minimizes the number of user actions required for common use cases, and it provides clear &amp; actionable error messages.</Description>
@@ -35,8 +36,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac
3536
<RepositoryType>Git</RepositoryType>
3637
<SignAssembly>true</SignAssembly>
3738
<AssemblyOriginatorKeyFile>Open.snk</AssemblyOriginatorKeyFile>
38-
<AssemblyVersion>0.5.1.0</AssemblyVersion>
39-
<FileVersion>0.5.1.0</FileVersion>
39+
<AssemblyVersion>0.5.2.0</AssemblyVersion>
40+
<FileVersion>0.5.2.0</FileVersion>
4041
<PackageLicenseFile>LICENSE</PackageLicenseFile>
4142
</PropertyGroup>
4243

0 commit comments

Comments
 (0)