Skip to content

Commit cc9f062

Browse files
Oceania2018Oceania2018
committed
Fix SparseCategoricalCrossentropy.
1 parent 4945c66 commit cc9f062

20 files changed

Lines changed: 276 additions & 22 deletions

‎src/TensorFlowNET.Core/Keras/Engine/DataAdapters/DataHandler.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ public class DataHandler
1313
{
1414
DataHandlerArgs args;
1515
IDataAdapter _adapter;
16+
public IDataAdapter DataAdapter => _adapter;
1617
IDatasetV2 _dataset;
1718
int _inferred_steps;
1819
int _current_step;

‎src/TensorFlowNET.Core/Keras/Engine/DataAdapters/IDataAdapter.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,5 +20,6 @@ public interface IDataAdapter
2020
bool CanHandle(Tensor x, Tensor y = null);
2121
IDatasetV2 GetDataset();
2222
int GetSize();
23+
(Tensor, Tensor) Expand1d(Tensor x, Tensor y);
2324
}
2425
}

‎src/TensorFlowNET.Core/Keras/Engine/DataAdapters/TensorLikeDataAdapter.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,5 +89,12 @@ public IDatasetV2 GetDataset()
8989

9090
public int GetSize()
9191
=> _size;
92+
93+
public (Tensor, Tensor) Expand1d(Tensor x, Tensor y)
94+
{
95+
if (y.TensorShape.ndim == 1)
96+
y = array_ops.expand_dims(y, axis: -1);
97+
return (x, y);
98+
}
9299
}
93100
}

‎src/TensorFlowNET.Core/Keras/Engine/LossesContainer.cs‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
using System;
22
using System.Collections.Generic;
3+
using System.Linq;
34
using System.Text;
45
using Tensorflow.Keras.ArgsDefinition;
56
using Tensorflow.Keras.Losses;
67
using Tensorflow.Keras.Metrics;
8+
using Tensorflow.Keras.Utils;
79

810
namespace Tensorflow.Keras.Engine
911
{
@@ -12,13 +14,19 @@ public class LossesContainer : Container
1214
ILossFunc _user_losses;
1315
ILossFunc _losses;
1416
Mean _loss_metric;
17+
bool _built;
18+
Tensor[] _per_output_metrics;
19+
List<Tensor> loss_values;
20+
List<Tensor> loss_metric_values;
1521

1622
public LossesContainer(ILossFunc losses, string[] output_names = null)
1723
: base(output_names)
1824
{
1925
_user_losses = losses;
2026
_losses = losses;
2127
_loss_metric = new Mean(name: "loss");
28+
loss_values = new List<Tensor>();
29+
loss_metric_values = new List<Tensor>();
2230
_built = false;
2331
}
2432

@@ -27,14 +35,44 @@ public LossesContainer(ILossFunc losses, string[] output_names = null)
2735
/// </summary>
2836
/// <param name="y_true"></param>
2937
/// <param name="y_pred"></param>
30-
public void Apply(Tensor y_true, Tensor y_pred)
38+
public Tensor Call(Tensor y_true, Tensor y_pred)
3139
{
40+
Build(y_pred);
41+
var loss_value = _losses.Call(y_true, y_pred);
42+
var loss_metric_value = loss_value;
43+
var batch_dim = array_ops.shape(y_true)[0];
3244

45+
/*if (_losses.Reduction == ReductionV2.SUM_OVER_BATCH_SIZE
46+
|| _losses.Reduction == ReductionV2.AUTO)
47+
loss_value = losses_utils.scale_loss_for_distribution(loss_value);*/
48+
49+
loss_values.append(loss_value);
50+
loss_metric_values.append(loss_metric_value);
51+
52+
if(loss_values.Count > 0)
53+
{
54+
var total_loss_metric_value = math_ops.add_n(loss_metric_values.ToArray());
55+
_loss_metric.update_state(total_loss_metric_value, batch_dim);
56+
// loss_values = losses_utils.cast_losses_to_common_dtype(loss_values);
57+
var total_loss = math_ops.add_n(loss_values.ToArray());
58+
return total_loss;
59+
}
60+
else
61+
{
62+
// Ok for a model to have no compiled loss.
63+
return array_ops.zeros(new TensorShape());
64+
}
3365
}
3466

35-
public void Build()
67+
public void Build(Tensor y_pred)
3668
{
69+
_create_metrics();
70+
_built = true;
71+
}
3772

73+
void _create_metrics()
74+
{
75+
// _per_output_metrics = _output_names.Select(x => null);
3876
}
3977
}
4078
}

‎src/TensorFlowNET.Core/Keras/Engine/MetricsContainer.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,18 @@ public MetricsContainer(string[] metrics, string[] output_names = null)
1616
_metrics = metrics;
1717
_built = false;
1818
}
19+
20+
public void update_state(Tensor y_true, Tensor y_pred, Tensor sample_weight = null)
21+
{
22+
if (!_built)
23+
Build();
24+
25+
_built = true;
26+
}
27+
28+
void Build()
29+
{
30+
31+
}
1932
}
2033
}

‎src/TensorFlowNET.Core/Keras/Engine/Model.Compile.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ namespace Tensorflow.Keras.Engine
1010
{
1111
public partial class Model
1212
{
13+
LossesContainer compiled_loss;
14+
MetricsContainer compiled_metrics;
1315
public void compile(string optimizerName, ILossFunc lossName)
1416
{
1517
throw new NotImplementedException("");
@@ -18,8 +20,8 @@ public void compile(string optimizerName, ILossFunc lossName)
1820
public void compile(ILossFunc loss, OptimizerV2 optimizer, string[] metrics)
1921
{
2022
this.optimizer = optimizer;
21-
var compiled_loss = new LossesContainer(loss, output_names: output_names);
22-
var compiled_metrics = new MetricsContainer(metrics, output_names: output_names);
23+
compiled_loss = new LossesContainer(loss, output_names: output_names);
24+
compiled_metrics = new MetricsContainer(metrics, output_names: output_names);
2325

2426
int experimental_steps_per_execution = 1;
2527
_configure_steps_per_execution(experimental_steps_per_execution);

‎src/TensorFlowNET.Core/Keras/Engine/Model.Fit.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ public void fit(NDArray x, NDArray y,
3737
var val_x = x[new Slice(train_count)];
3838
var val_y = y[new Slice(train_count)];
3939

40-
var data_handler = new DataHandler(new DataHandlerArgs
40+
data_handler = new DataHandler(new DataHandlerArgs
4141
{
4242
X = train_x,
4343
Y = train_y,

‎src/TensorFlowNET.Core/Keras/Engine/Model.Train.cs‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
using NumSharp;
22
using System;
33
using System.Collections.Generic;
4+
using System.Linq;
45
using System.Text;
6+
using Tensorflow.Gradients;
7+
using Tensorflow.Keras.Optimizers;
58
using static Tensorflow.Binding;
69

710
namespace Tensorflow.Keras.Engine
@@ -11,7 +14,8 @@ public partial class Model
1114
Tensor step_function(OwnedIterator iterator)
1215
{
1316
var data = iterator.next();
14-
train_step(data[0], data[1]);
17+
var outputs = train_step(data[0], data[1]);
18+
tf_with(ops.control_dependencies(new object[0]), ctl => _train_counter.assign_add(1));
1519
throw new NotImplementedException("");
1620
}
1721

@@ -20,11 +24,33 @@ Tensor step_function(OwnedIterator iterator)
2024
/// </summary>
2125
/// <param name="data"></param>
2226
/// <returns></returns>
23-
Tensor train_step(Tensor x, Tensor y)
27+
IEnumerable<(string, Tensor)> train_step(Tensor x, Tensor y)
2428
{
29+
(x, y) = data_handler.DataAdapter.Expand1d(x, y);
2530
using var tape = tf.GradientTape();
2631
var y_pred = Apply(x, is_training: true);
27-
throw new NotImplementedException("");
32+
var loss = compiled_loss.Call(y, y_pred);
33+
34+
// For custom training steps, users can just write:
35+
// trainable_variables = self.trainable_variables
36+
// gradients = tape.gradient(loss, trainable_variables)
37+
// self.optimizer.apply_gradients(zip(gradients, trainable_variables))
38+
// The _minimize call does a few extra steps unnecessary in most cases,
39+
// such as loss scaling and gradient clipping.
40+
_minimize(tape, optimizer, loss, trainable_variables);
41+
42+
compiled_metrics.update_state(y, y_pred);
43+
return new[] { ("loss", loss) };
44+
}
45+
46+
void _minimize(GradientTape tape, OptimizerV2 optimizer, Tensor loss, List<IVariableV1> trainable_variables)
47+
{
48+
var gradients = tape.gradient(loss, trainable_variables);
49+
gradients = optimizer._aggregate_gradients(zip(gradients, trainable_variables));
50+
gradients = optimizer._clip_gradients(gradients);
51+
52+
optimizer.apply_gradients(zip(gradients, trainable_variables.Select(x => x as ResourceVariable)),
53+
experimental_aggregate_gradients: false);
2854
}
2955
}
3056
}

‎src/TensorFlowNET.Core/Keras/Engine/Model.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using Tensorflow.Keras.Optimizers;
77
using NumSharp;
88
using System.Collections.Generic;
9+
using System.Data.Common;
910

1011
namespace Tensorflow.Keras.Engine
1112
{
@@ -23,7 +24,7 @@ public partial class Model : Layer
2324
#pragma warning restore CS0414 // The field 'Model._is_compiled' is assigned but its value is never used
2425
#pragma warning restore CS0108 // Member hides inherited member; missing new keyword
2526
ILossFunc loss;
26-
IOptimizer optimizer;
27+
OptimizerV2 optimizer;
2728
IVariableV1 _steps_per_execution;
2829
protected bool _is_graph_network;
2930
protected Tensors inputs;
@@ -34,6 +35,7 @@ public partial class Model : Layer
3435
IVariableV1 _predict_counter;
3536
bool _base_model_initialized;
3637
bool stop_training;
38+
DataHandler data_handler;
3739

3840
public Model(ModelArgs args)
3941
: base(args)

‎src/TensorFlowNET.Core/Keras/Losses/ILossFunc.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,7 @@ namespace Tensorflow.Keras.Losses
66
{
77
public interface ILossFunc
88
{
9+
string Reduction { get; }
10+
Tensor Call(Tensor y_true, Tensor y_pred);
911
}
1012
}

0 commit comments

Comments
 (0)