remove generic in OpTape.

Oceania2018 · Oceania2018 · commit aee6cd69d07b · 2021-11-14T15:24:56.000-06:00
diff --git a/src/TensorFlowNET.Core/Eager/EagerRunner.RecordGradient.cs b/src/TensorFlowNET.Core/Eager/EagerRunner.RecordGradient.cs
@@ -11,7 +11,7 @@ public bool RecordGradient(string op_name,
             Tensor[] inputs,
             object[] attrs,
             Tensor[] results,
-            Func<BackwardFunction> getBackwardFunction = null)
+            BackwardFunction backwardFunction = null)
         {
             bool should_record = ShouldRecord(inputs);
 
@@ -28,9 +28,9 @@ public bool RecordGradient(string op_name,
             }
             
             if (!should_record) return should_record;
-            tf.Logger.Debug($"RecordGradient: op_name={op_name}");
+            // tf.Logger.Debug($"RecordGradient: op_name={op_name}");
 
-            Tensor[] op_outputs;
+            /*Tensor[] op_outputs = null;
             var unused_output_indices = gradient_exclustions.OpGradientUnusedOutputIndices(op_name);
             if (unused_output_indices != null)
             {
@@ -44,7 +44,7 @@ public bool RecordGradient(string op_name,
             else
                 op_outputs = results;
 
-            Tensor[] op_inputs;
+            Tensor[] op_inputs = null;
             var unused_input_indices = gradient_exclustions.OpGradientUnusedInputIndices(op_name);
             if (unused_input_indices != null)
             {
@@ -56,22 +56,14 @@ public bool RecordGradient(string op_name,
                 }
             }
             else
-                op_inputs = inputs;
+                op_inputs = inputs;*/
 
-            TapeSetRecordOperation(op_name, inputs, results,
-                getBackwardFunction ?? GetBackwradFunction(op_name, inputs, attrs, results));
+            backwardFunction = backwardFunction ?? GetGradientFunction(op_name, inputs, attrs, results);
+            TapeSetRecordOperation(op_name, inputs, results, backwardFunction);
 
             return true;
         }
 
-        Func<BackwardFunction> GetBackwradFunction(string op_name,
-                     Tensor[] op_inputs,
-                     object[] attrs,
-                     Tensor[] op_outputs)
-        {
-            return () => GetGradientFunction(op_name, op_inputs, attrs, op_outputs);
-        }
-
         BackwardFunction GetGradientFunction(string op_name,
                      Tensor[] op_inputs,
                      object[] attrs,
diff --git a/src/TensorFlowNET.Core/Eager/EagerRunner.TapeSetRecordBackprop.cs b/src/TensorFlowNET.Core/Eager/EagerRunner.TapeSetRecordBackprop.cs
@@ -1,7 +1,6 @@
 &#65279;using System;
 using Tensorflow.Gradients;
 using static Tensorflow.Binding;
-using static Tensorflow.tensorflow;
 
 namespace Tensorflow.Eager
 {
@@ -10,7 +9,7 @@ public partial class EagerRunner
         void TapeSetRecordBackprop(string op_type,
             Tensor[] input_tensors,
             TapeTensor[] output_tensors,
-            Func<BackwardFunction> backward_function_getter)
+            BackwardFunction backward_function)
         {
             if (!CouldBackprop())
             {
@@ -19,8 +18,7 @@ void TapeSetRecordBackprop(string op_type,
 
             foreach (var tape in tf.GetTapeSet())
             {
-                tape.RecordOperation(op_type, input_tensors, output_tensors,
-                    backward_function_getter);
+                tape.RecordOperation(op_type, input_tensors, output_tensors, backward_function);
             }
         }
     }
diff --git a/src/TensorFlowNET.Core/Eager/EagerRunner.TapeSetRecordForwardprop.cs b/src/TensorFlowNET.Core/Eager/EagerRunner.TapeSetRecordForwardprop.cs
@@ -9,7 +9,7 @@ public partial class EagerRunner
         bool TapeSetRecordForwardprop(string op_type,
             Tensor[] input_tensors,
             TapeTensor[] output_tensors,
-            Func<BackwardFunction> backward_function_getter)
+            BackwardFunction backward_function_getter)
         {
             if (!CouldForwardprop())
             {
diff --git a/src/TensorFlowNET.Core/Eager/EagerRunner.TapeSetRecordOperation.cs b/src/TensorFlowNET.Core/Eager/EagerRunner.TapeSetRecordOperation.cs
@@ -10,16 +10,16 @@ public partial class EagerRunner
         public bool TapeSetRecordOperation(string op_type,
             Tensor[] input_tensors,
             Tensor[] output_tensors,
-            Func<BackwardFunction> backward_function_getter)
+            BackwardFunction backward_function)
         {
             var output_info = output_tensors.Select(x => new TapeTensor(x)).ToArray();
 
             if (!TapeSetRecordForwardprop(op_type, input_tensors, output_info,
-                    backward_function_getter))
+                    backward_function))
                 return false;
 
             TapeSetRecordBackprop(op_type, input_tensors, output_info,
-                backward_function_getter);
+                backward_function);
 
             return true;
         }
diff --git a/src/TensorFlowNET.Core/Eager/IEagerRunner.cs b/src/TensorFlowNET.Core/Eager/IEagerRunner.cs
@@ -9,7 +9,8 @@ public interface IEagerRunner
     {
         Tensor[] Execute(Context ctx, string op_name,
             int num_outputs,
-            Tensor[] inputs, object[] attrs,
+            Tensor[] inputs, 
+            object[] attrs,
             string name = null);
 
         (TF_DataType, Tensor[]) ArgsToMatchingEager(Context ctx,
@@ -34,7 +35,7 @@ bool RecordGradient(string op_name,
             Tensor[] inputs,
             object[] attrs,
             Tensor[] results,
-            Func<BackwardFunction> getBackwardFunction = null);
+            BackwardFunction getBackwardFunction = null);
 
         bool MustRecordGradient();
 
diff --git a/src/TensorFlowNET.Core/Functions/TapeGradientFunctions.cs b/src/TensorFlowNET.Core/Functions/TapeGradientFunctions.cs
@@ -47,7 +47,7 @@ public void Record(Tensors flat_outputs, Tensors inference_args)
         {
             var (backward_function, to_record) = _wrap_backward_function(_forward_graph, _backward, flat_outputs);
             tf.Runner.RecordGradient(_forward.Name, inference_args, new object[0], to_record,
-                getBackwardFunction: () => backward_function);
+                getBackwardFunction: backward_function);
         }
 
         /// <summary>
diff --git a/src/TensorFlowNET.Core/Gradients/BackpropInitialState.cs b/src/TensorFlowNET.Core/Gradients/BackpropInitialState.cs
@@ -5,7 +5,7 @@ namespace Tensorflow.Gradients
 {
     public class BackpropInitialState
     {
-        public OpTape<BackwardFunction, TapeTensor> op_tape { get; set; }
+        public OpTape op_tape { get; set; }
         /// <summary>
         /// Map from tensor to how many references still exist for this tensor in
         /// the tape.
@@ -19,7 +19,7 @@ public class BackpropInitialState
 
         public BackpropInitialState()
         {
-            op_tape = new OpTape<BackwardFunction, TapeTensor>();
+            op_tape = new OpTape();
             tensor_usage_counts = new UnorderedMap<Tensor, long>();
             op_missing_tensor = new UnorderedMap<Tensor, long>();
         }
diff --git a/src/TensorFlowNET.Core/Gradients/ITape.cs b/src/TensorFlowNET.Core/Gradients/ITape.cs
@@ -13,7 +13,7 @@ public interface ITape
         void RecordOperation(string op_type,
             Tensor[] input_tensors,
             TapeTensor[] output_tensors,
-            Func<BackwardFunction> backward_function_getter);
+            BackwardFunction backward_function);
 
         void VariableAccessed(ResourceVariable variable);
 
diff --git a/src/TensorFlowNET.Core/Gradients/OpTape.cs b/src/TensorFlowNET.Core/Gradients/OpTape.cs
@@ -5,10 +5,7 @@ namespace Tensorflow.Gradients
     /// <summary>
     /// Map from operation-id to tape entry.
     /// </summary>
-    /// <typeparam name="BackwardFunction"></typeparam>
-    /// <typeparam name="TapeTensor"></typeparam>
-    public class OpTape<BackwardFunction, TapeTensor> :
-        UnorderedMap<Tensor, OpTapeEntry<BackwardFunction, TapeTensor>>
+    public class OpTape : UnorderedMap<Tensor, OpTapeEntry>
     {
 
     }
diff --git a/src/TensorFlowNET.Core/Gradients/OpTapeEntry.cs b/src/TensorFlowNET.Core/Gradients/OpTapeEntry.cs
@@ -5,9 +5,7 @@ namespace Tensorflow.Gradients
     /// <summary>
     /// Represents an entry in the tape.
     /// </summary>
-    /// <typeparam name="BackwardFunction"></typeparam>
-    /// <typeparam name="TapeTensor"></typeparam>
-    public class OpTapeEntry<BackwardFunction, TapeTensor>
+    public class OpTapeEntry
     {
         public string op_type { get; set; }
         public TapeTensor[] output_tensor_info { get; set; }
diff --git a/src/TensorFlowNET.Core/Gradients/Tape.CallBackwardFunction.cs b/src/TensorFlowNET.Core/Gradients/Tape.CallBackwardFunction.cs
@@ -1,5 +1,4 @@
 &#65279;using System.Collections.Generic;
-using static Tensorflow.tensorflow;
 
 namespace Tensorflow.Gradients
 {
@@ -9,7 +8,7 @@ public Tensor[] CallBackwardFunction(BackwardFunction backward_function,
             List<long> unneeded_gradients,
             List<Tensor> output_gradients)
         {
-            var grads = new Tensor[output_gradients.Count];
+            // var grads = new Tensor[output_gradients.Count];
             var result = backward_function(output_gradients.ToArray(),
                 unneeded_gradients.ToArray());
 
diff --git a/src/TensorFlowNET.Core/Gradients/Tape.ComputeGradient.cs b/src/TensorFlowNET.Core/Gradients/Tape.ComputeGradient.cs
@@ -2,24 +2,22 @@
 using System.Collections.Generic;
 using System.Linq;
 using Tensorflow.Util;
-using static Tensorflow.tensorflow;
 
 namespace Tensorflow.Gradients
 {
     public partial class Tape
     {
-        int kMinAggregateCount = 4;
-        int kMinAggregateBytes = 128 * 1024 * 1024;
+        // int kMinAggregateCount = 4;
+        // int kMinAggregateBytes = 128 * 1024 * 1024;
 
         public Tensor[] ComputeGradient(Tensor[] target_tensor_ids,
             Tensor[] source_tensor_ids,
             UnorderedMap<Tensor, TapeTensor> sources_that_are_targets,
             Tensor[] output_gradients)
         {
-            var result = new List<Tensor>(source_tensor_ids.Length);
             var sources_set = new UnorderedSet<Tensor>(source_tensor_ids);
-            var gradients_size = new UnorderedMap<Tensor, long>();
-
+            // var gradients_size = new UnorderedMap<Tensor, long>();
+            var functionsAcceptingNoneForIndicesMap = FunctionsAcceptingNoneForIndicesMap();
             var state = PrepareBackprop(
                 target_tensor_ids, tensor_tape_, op_tape_, sources_set, _persistent);
             var op_stack = InitialStack(state.op_tape, state.op_missing_tensor);
@@ -54,7 +52,7 @@ public Tensor[] ComputeGradient(Tensor[] target_tensor_ids,
                     var id = trace.output_tensor_info[i].GetTensor();
                     if (!gradients.find(id, out var grad_it))
                     {
-                        if (FunctionsAcceptingNoneForIndicesMap().find(trace.op_type, out var func_name_it) &&
+                        if (functionsAcceptingNoneForIndicesMap.find(trace.op_type, out var func_name_it) &&
                             func_name_it.find(i))
                         {
                             out_gradients.Add(null);
@@ -76,8 +74,8 @@ public Tensor[] ComputeGradient(Tensor[] target_tensor_ids,
                             gradients.Remove(id);
                         else
                         {
-                            grad_it.Clear();
-                            grad_it.Add(new_gradients);
+                            // grad_it.Clear();
+                            // grad_it.Add(new_gradients);
                             // vspace.MarkAsResult(new_gradients);
                         }
                         out_gradients.Add(new_gradients);
@@ -87,18 +85,17 @@ public Tensor[] ComputeGradient(Tensor[] target_tensor_ids,
                 Tensor[] in_gradients;
                 if (any_gradient_nonzero)
                 {
-                    foreach (var i in zero_indices)
-                        out_gradients[i] = trace.output_tensor_info[i].ZerosLike();
+                    // foreach (var i in zero_indices)
+                    //     out_gradients[i] = trace.output_tensor_info[i].ZerosLike();
 
-                    in_gradients = CallBackwardFunction(trace.backward_function,
-                        unneeded_gradients,
-                        out_gradients);
+                    in_gradients = trace.backward_function(out_gradients.ToArray(), unneeded_gradients.ToArray());
 
                     if (in_gradients.Count() != trace.input_tensor_id.Count())
                         throw new RuntimeError($"Recorded operation '{trace.op_type}' returned too few gradients. Expected {trace.input_tensor_id.Length} but received {in_gradients.Count()}");
                     if (!_persistent)
                     {
                         // trace.backward_function_deleter(trace.backward_function);
+                        trace.backward_function = null;
                     }
                 }
                 else
@@ -113,7 +110,7 @@ public Tensor[] ComputeGradient(Tensor[] target_tensor_ids,
                     {
                         var unaggregated_grads = gradients[id];
                         unaggregated_grads.Add(in_gradients[i]);
-                        if (unaggregated_grads.Count > kMinAggregateCount)
+                        /*if (unaggregated_grads.Count > kMinAggregateCount)
                         {
                             if (!gradients_size.find(id, out var size))
                             {
@@ -125,7 +122,7 @@ public Tensor[] ComputeGradient(Tensor[] target_tensor_ids,
                             {
                                 throw new NotImplementedException("");
                             }
-                        }
+                        }*/
                     }
 
                     if (!state.tensor_usage_counts.find(id))
@@ -162,36 +159,21 @@ public Tensor[] ComputeGradient(Tensor[] target_tensor_ids,
             if (state.op_tape.Count > 0)
                 throw new RuntimeError("Invalid tape state.");
 
-            var used_gradient_ids = new List<Tensor>(source_tensor_ids.Length);
+            var result = new Tensor[source_tensor_ids.Length];
+            var j = 0;
             foreach (var id in source_tensor_ids)
             {
-                if (!gradients.find(id, out var grad_it))
-                    result.Add(null);
-                else
+                if (gradients.find(id, out var grad_it))
                 {
                     if (grad_it.Count > 1)
-                    {
-                        var grad = gen_math_ops.add_n(grad_it.ToArray());
-                        grad_it.Clear();
-                        grad_it.Add(grad);
-                    }
-                    result.Add(grad_it[0]);
-                    used_gradient_ids.Add(id);
+                        result[j] = gen_math_ops.add_n(grad_it.ToArray());
+                    else
+                        result[j] = grad_it[0];
                 }
+                j++;
             }
 
-            /*foreach(var grad_pair in gradients)
-            {
-                if(!used_gradient_ids.Contains(grad_pair.Key))
-                {
-                    foreach(var g in grad_pair.Value)
-                    {
-                        vspace.DeleteGradient(g);
-                    }
-                }
-            }*/
-
-            return result.ToArray();
+            return result;
         }
 
         UnorderedMap<string, UnorderedSet<int>> FunctionsAcceptingNoneForIndicesMap()
@@ -207,7 +189,7 @@ UnorderedMapEnumerable<Tensor, List<Tensor>> InitialGradients(Tensor[] target_te
             UnorderedMap<Tensor, TapeTensor> sources_that_are_targets,
             Tensor[] output_gradients,
             TensorTape tensor_tape,
-            OpTape<BackwardFunction, TapeTensor> op_tape)
+            OpTape op_tape)
         {
             var result = new UnorderedMapEnumerable<Tensor, List<Tensor>>();
             for (int i = 0; i < target_tensor_ids.Length; ++i)
@@ -253,7 +235,7 @@ UnorderedMapEnumerable<Tensor, List<Tensor>> InitialGradients(Tensor[] target_te
             return result;
         }
 
-        Queue<Tensor> InitialStack(OpTape<BackwardFunction, TapeTensor> op_tape,
+        Queue<Tensor> InitialStack(OpTape op_tape,
             UnorderedMap<Tensor, long> op_missing_tensor)
         {
             var result = new Queue<Tensor>();
diff --git a/src/TensorFlowNET.Core/Gradients/Tape.PrepareBackprop.cs b/src/TensorFlowNET.Core/Gradients/Tape.PrepareBackprop.cs
@@ -8,7 +8,7 @@ public partial class Tape
     {
         public BackpropInitialState PrepareBackprop(Tensor[] target,
             TensorTape tensor_tape,
-            OpTape<BackwardFunction, TapeTensor> op_tape,
+            OpTape op_tape,
             UnorderedSet<Tensor> sources_set,
             bool persistent_tape)
         {
diff --git a/src/TensorFlowNET.Core/Gradients/Tape.RecordOperation.cs b/src/TensorFlowNET.Core/Gradients/Tape.RecordOperation.cs
@@ -16,7 +16,7 @@ public partial class Tape
         public void RecordOperation(string op_type,
             Tensor[] input_tensors,
             TapeTensor[] output_tensors,
-            Func<BackwardFunction> backward_function_getter)
+            BackwardFunction backward_function)
         {
             if (!ShouldRecord(input_tensors))
                 return;
@@ -32,12 +32,12 @@ public void RecordOperation(string op_type,
                 tensor_usage_[o.GetTensor()] = 1;
             }
 
-            op_tape_[op_id] = new OpTapeEntry<BackwardFunction, TapeTensor>
+            op_tape_[op_id] = new OpTapeEntry
             {
                 op_type = op_type,
                 output_tensor_info = output_tensors,
                 input_tensor_id = input_tensors,
-                backward_function = backward_function_getter()
+                backward_function = backward_function
             };
         }
     }
diff --git a/src/TensorFlowNET.Core/Gradients/Tape.cs b/src/TensorFlowNET.Core/Gradients/Tape.cs
diff --git a/src/TensorFlowNET.Core/Graphs/FuncGraph.cs b/src/TensorFlowNET.Core/Graphs/FuncGraph.cs
diff --git a/src/TensorFlowNET.Core/Operations/gen_math_ops.cs b/src/TensorFlowNET.Core/Operations/gen_math_ops.cs
diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.cs b/src/TensorFlowNET.Core/Tensors/Tensor.cs
diff --git a/src/TensorFlowNET.Core/tensorflow.cs b/src/TensorFlowNET.Core/tensorflow.cs

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,6 @@`
`1`	`1`	`using System;`
`2`	`2`	`using Tensorflow.Gradients;`
`3`	`3`	`using static Tensorflow.Binding;`
`4`		`-using static Tensorflow.tensorflow;`
`5`	`4`
`6`	`5`	`namespace Tensorflow.Eager`
`7`	`6`	`{`
`@@ -10,7 +9,7 @@ public partial class EagerRunner`
`10`	`9`	`void TapeSetRecordBackprop(string op_type,`
`11`	`10`	`Tensor[] input_tensors,`
`12`	`11`	`TapeTensor[] output_tensors,`
`13`		`- Func<BackwardFunction> backward_function_getter)`
	`12`	`+ BackwardFunction backward_function)`
`14`	`13`	`{`
`15`	`14`	`if (!CouldBackprop())`
`16`	`15`	`{`
`@@ -19,8 +18,7 @@ void TapeSetRecordBackprop(string op_type,`
`19`	`18`
`20`	`19`	`foreach (var tape in tf.GetTapeSet())`
`21`	`20`	`{`
`22`		`- tape.RecordOperation(op_type, input_tensors, output_tensors,`
`23`		`- backward_function_getter);`
	`21`	`+ tape.RecordOperation(op_type, input_tensors, output_tensors, backward_function);`
`24`	`22`	`}`
`25`	`23`	`}`
`26`	`24`	`}`
Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@ public partial class EagerRunner`
`9`	`9`	`bool TapeSetRecordForwardprop(string op_type,`
`10`	`10`	`Tensor[] input_tensors,`
`11`	`11`	`TapeTensor[] output_tensors,`
`12`		`- Func<BackwardFunction> backward_function_getter)`
	`12`	`+ BackwardFunction backward_function_getter)`
`13`	`13`	`{`
`14`	`14`	`if (!CouldForwardprop())`
`15`	`15`	`{`
Original file line number	Diff line number	Diff line change
`@@ -47,7 +47,7 @@ public void Record(Tensors flat_outputs, Tensors inference_args)`
`47`	`47`	`{`
`48`	`48`	`var (backward_function, to_record) = _wrap_backward_function(_forward_graph, _backward, flat_outputs);`
`49`	`49`	`tf.Runner.RecordGradient(_forward.Name, inference_args, new object[0], to_record,`
`50`		`- getBackwardFunction: () => backward_function);`
	`50`	`+ getBackwardFunction: backward_function);`
`51`	`51`	`}`
`52`	`52`
`53`	`53`	`/// <summary>`
Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,7 @@ namespace Tensorflow.Gradients`
`5`	`5`	`{`
`6`	`6`	`public class BackpropInitialState`
`7`	`7`	`{`
`8`		`- public OpTape<BackwardFunction, TapeTensor> op_tape { get; set; }`
	`8`	`+ public OpTape op_tape { get; set; }`
`9`	`9`	`/// <summary>`
`10`	`10`	`/// Map from tensor to how many references still exist for this tensor in`
`11`	`11`	`/// the tape.`
`@@ -19,7 +19,7 @@ public class BackpropInitialState`
`19`	`19`
`20`	`20`	`public BackpropInitialState()`
`21`	`21`	`{`
`22`		`- op_tape = new OpTape<BackwardFunction, TapeTensor>();`
	`22`	`+ op_tape = new OpTape();`
`23`	`23`	`tensor_usage_counts = new UnorderedMap<Tensor, long>();`
`24`	`24`	`op_missing_tensor = new UnorderedMap<Tensor, long>();`
`25`	`25`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,10 +5,7 @@ namespace Tensorflow.Gradients`
`5`	`5`	`/// <summary>`
`6`	`6`	`/// Map from operation-id to tape entry.`
`7`	`7`	`/// </summary>`
`8`		`- /// <typeparam name="BackwardFunction"></typeparam>`
`9`		`- /// <typeparam name="TapeTensor"></typeparam>`
`10`		`- public class OpTape<BackwardFunction, TapeTensor> :`
`11`		`- UnorderedMap<Tensor, OpTapeEntry<BackwardFunction, TapeTensor>>`
	`8`	`+ public class OpTape : UnorderedMap<Tensor, OpTapeEntry>`
`12`	`9`	`{`
`13`	`10`
`14`	`11`	`}`