Skip to content

Commit dded90a

Browse files
Oceania2018Oceania2018
committed
fix string tensor for non ascii char.
1 parent 3c7207c commit dded90a

9 files changed

Lines changed: 102 additions & 62 deletions

File tree

‎src/TensorFlowNET.Core/APIs/tf.linalg.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public class LinalgApi
2828
public Tensor eye(int num_rows,
2929
int num_columns = -1,
3030
TensorShape batch_shape = null,
31-
TF_DataType dtype = TF_DataType.TF_FLOAT,
31+
TF_DataType dtype = TF_DataType.TF_DOUBLE,
3232
string name = null)
3333
=> ops.eye(num_rows, num_columns: num_columns, batch_shape: batch_shape, dtype: dtype, name: name);
3434

‎src/TensorFlowNET.Core/Eager/EagerTensor.Creation.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,10 @@ public EagerTensor(object value, Shape? shape = null, string device_name = null,
5050
public EagerTensor(Shape shape, TF_DataType dtype) : base(shape, dtype)
5151
=> NewEagerTensorHandle(_handle);
5252

53-
internal unsafe EagerTensor(Array array, Shape shape) : base(array, shape)
53+
public EagerTensor(Array array, Shape shape) : base(array, shape)
54+
=> NewEagerTensorHandle(_handle);
55+
56+
public EagerTensor(byte[] bytes, TF_DataType dtype) : base(bytes, dtype)
5457
=> NewEagerTensorHandle(_handle);
5558

5659
void NewEagerTensorHandle(IntPtr h)

‎src/TensorFlowNET.Core/Numpy/NDArray.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ public NDIterator<T> AsIterator<T>(bool autoreset = false) where T : unmanaged
3939
public bool HasNext() => throw new NotImplementedException("");
4040
public T MoveNext<T>() => throw new NotImplementedException("");
4141
public NDArray reshape(Shape newshape) => new NDArray(_tensor, newshape);
42-
public NDArray astype(Type type) => throw new NotImplementedException("");
43-
public NDArray astype(TF_DataType type) => throw new NotImplementedException("");
42+
public NDArray astype(Type type) => new NDArray(math_ops.cast(_tensor, type.as_tf_dtype()));
43+
public NDArray astype(TF_DataType dtype) => new NDArray(math_ops.cast(_tensor, dtype));
4444
public NDArray ravel() => throw new NotImplementedException("");
4545
public void shuffle(NDArray nd) => throw new NotImplementedException("");
4646
public Array ToMuliDimArray<T>() => throw new NotImplementedException("");

‎src/TensorFlowNET.Core/Numpy/Shape.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,11 +62,17 @@ public ulong size
6262
{
6363
get
6464
{
65+
// scalar
66+
if (ndim == 0)
67+
return 1;
68+
6569
var computed = 1L;
6670
for (int i = 0; i < _dims.Length; i++)
6771
{
6872
var val = _dims[i];
69-
if (val <= 0)
73+
if (val == 0)
74+
return 0;
75+
else if (val < 0)
7076
continue;
7177
computed *= val;
7278
}

‎src/TensorFlowNET.Core/Operations/linalg_ops.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ public class linalg_ops
88
public Tensor eye(int num_rows,
99
int num_columns = -1,
1010
TensorShape batch_shape = null,
11-
TF_DataType dtype = TF_DataType.TF_FLOAT,
11+
TF_DataType dtype = TF_DataType.TF_DOUBLE,
1212
string name = null)
1313
{
1414
return tf_with(ops.name_scope(name, default_name: "eye", new { num_rows, num_columns, batch_shape }), scope =>

‎src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs‎

Lines changed: 79 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -67,16 +67,17 @@ public unsafe Tensor(NDArray nd)
6767
}
6868

6969
#region scala
70-
public Tensor(bool value) => InitTensor(value);
71-
public Tensor(byte value) => InitTensor(value);
72-
public Tensor(sbyte value) => InitTensor(value);
73-
public Tensor(short value) => InitTensor(value);
74-
public Tensor(int value) => InitTensor(value);
75-
public Tensor(uint value) => InitTensor(value);
76-
public Tensor(long value) => InitTensor(value);
77-
public Tensor(ulong value) => InitTensor(value);
78-
public Tensor(float value) => InitTensor(value);
79-
public Tensor(double value) => InitTensor(value);
70+
public Tensor(bool value) => InitTensor(new[] { value }, Shape.Scalar);
71+
public Tensor(byte value) => InitTensor(new[] { value }, Shape.Scalar);
72+
public Tensor(sbyte value) => InitTensor(new[] { value }, Shape.Scalar);
73+
public Tensor(short value) => InitTensor(new[] { value }, Shape.Scalar);
74+
public Tensor(int value) => InitTensor(new[] { value }, Shape.Scalar);
75+
public Tensor(uint value) => InitTensor(new[] { value }, Shape.Scalar);
76+
public Tensor(long value) => InitTensor(new[] { value }, Shape.Scalar);
77+
public Tensor(ulong value) => InitTensor(new[] { value }, Shape.Scalar);
78+
public Tensor(float value) => InitTensor(new[] { value }, Shape.Scalar);
79+
public Tensor(double value) => InitTensor(new[] { value }, Shape.Scalar);
80+
public Tensor(string value) => InitTensor(new[] { value }, TensorShape.Scalar);
8081
#endregion
8182

8283
#region 1d array
@@ -94,6 +95,10 @@ public unsafe Tensor(NDArray nd)
9495
public Tensor(Complex[] data, Shape? shape = null) => InitTensor(data, shape);
9596
#endregion
9697

98+
public Tensor(Shape shape, TF_DataType dtype) => InitTensor(shape, dtype);
99+
public Tensor(Array array, Shape? shape = null) => InitTensor(array, shape);
100+
public Tensor(byte[] bytes, TF_DataType dtype) => InitTensor(bytes, dtype);
101+
97102
public Tensor(Operation op, int value_index, TF_DataType dtype)
98103
{
99104
_op = op;
@@ -103,65 +108,87 @@ public Tensor(Operation op, int value_index, TF_DataType dtype)
103108
isCreatedInGraphMode = !tf.executing_eagerly();
104109
}
105110

106-
internal Tensor(Shape shape, TF_DataType dtype) => InitTensor(shape, dtype);
107-
internal Tensor(Array array, Shape? shape = null) => InitTensor(array, shape);
108-
internal Tensor(string value) => InitTensor(value);
109-
110-
protected unsafe void InitTensor<T>(T data) where T : unmanaged
111-
{
112-
_handle = TF_NewTensor(data);
113-
isCreatedInGraphMode = !tf.executing_eagerly();
114-
}
115-
116111
protected unsafe void InitTensor(Shape shape, TF_DataType dtype)
117112
{
118113
_handle = TF_NewTensor(shape, dtype, null);
119114
isCreatedInGraphMode = !tf.executing_eagerly();
120115
}
121116

122-
protected void InitTensor(string value)
117+
protected unsafe void InitTensor(byte[] bytes, TF_DataType dtype)
123118
{
124-
_handle = StringTensor(new[] { value }, TensorShape.Scalar);
119+
if (dtype == TF_DataType.TF_STRING)
120+
_handle = StringTensor(new byte[][] { bytes }, TensorShape.Scalar);
121+
else
122+
throw new NotImplementedException("");
125123
isCreatedInGraphMode = !tf.executing_eagerly();
126124
}
127125

128126
protected unsafe void InitTensor(Array array, Shape? shape = null)
129127
{
128+
isCreatedInGraphMode = !tf.executing_eagerly();
129+
130130
shape = shape ?? array.GetShape();
131-
var dtype = array.GetType().GetElementType().as_tf_dtype();
131+
var dtype = array.GetDataType();
132132

133-
switch (array)
133+
if (shape.size == 0)
134134
{
135-
case bool[] val: fixed (void* addr = &val[0]) _handle = TF_NewTensor(shape, dtype, addr); break;
136-
case bool[,] val: fixed (void* addr = &val[0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
137-
case bool[,,] val: fixed (void* addr = &val[0, 0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
138-
case bool[,,,] val: fixed (void* addr = &val[0, 0, 0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
139-
case byte[] val: fixed (void* addr = &val[0]) _handle = TF_NewTensor(shape, dtype, addr); break;
140-
case byte[,] val: fixed (void* addr = &val[0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
141-
case byte[,,] val: fixed (void* addr = &val[0, 0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
142-
case byte[,,,] val: fixed (void* addr = &val[0, 0, 0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
143-
case int[] val: fixed (void* addr = &val[0]) _handle = TF_NewTensor(shape, dtype, addr); break;
144-
case int[,] val: fixed (void* addr = &val[0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
145-
case int[,,] val: fixed (void* addr = &val[0, 0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
146-
case int[,,,] val: fixed (void* addr = &val[0, 0, 0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
147-
case long[] val: fixed (void* addr = &val[0]) _handle = TF_NewTensor(shape, dtype, addr); break;
148-
case long[,] val: fixed (void* addr = &val[0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
149-
case long[,,] val: fixed (void* addr = &val[0, 0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
150-
case long[,,,] val: fixed (void* addr = &val[0, 0, 0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
151-
case float[] val: fixed (void* addr = &val[0]) _handle = TF_NewTensor(shape, dtype, addr); break;
152-
case float[,] val: fixed (void* addr = &val[0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
153-
case float[,,] val: fixed (void* addr = &val[0, 0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
154-
case float[,,,] val: fixed (void* addr = &val[0, 0, 0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
155-
case double[] val: fixed (void* addr = &val[0]) _handle = TF_NewTensor(shape, dtype, addr); break;
156-
case double[,] val: fixed (void* addr = &val[0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
157-
case double[,,] val: fixed (void* addr = &val[0, 0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
158-
case double[,,,] val: fixed (void* addr = &val[0, 0, 0, 0]) _handle = TF_NewTensor(shape, dtype, addr); break;
159-
case string[] val: _handle = StringTensor(val, shape); break;
160-
default:
161-
throw new NotImplementedException("");
135+
_handle = TF_NewTensor(shape, dtype, null);
136+
return;
162137
}
163138

164-
isCreatedInGraphMode = !tf.executing_eagerly();
139+
_handle = array switch
140+
{
141+
bool[] val => InitTensor(val, shape, dtype),
142+
bool[,] val => InitTensor(val, shape, dtype),
143+
bool[,,] val => InitTensor(val, shape, dtype),
144+
bool[,,,] val => InitTensor(val, shape, dtype),
145+
byte[] val => InitTensor(val, shape, dtype),
146+
byte[,] val => InitTensor(val, shape, dtype),
147+
byte[,,] val => InitTensor(val, shape, dtype),
148+
byte[,,,] val => InitTensor(val, shape, dtype),
149+
int[] val => InitTensor(val, shape, dtype),
150+
int[,] val => InitTensor(val, shape, dtype),
151+
int[,,] val => InitTensor(val, shape, dtype),
152+
int[,,,] val => InitTensor(val, shape, dtype),
153+
long[] val => InitTensor(val, shape, dtype),
154+
long[,] val => InitTensor(val, shape, dtype),
155+
long[,,] val => InitTensor(val, shape, dtype),
156+
long[,,,] val => InitTensor(val, shape, dtype),
157+
float[] val => InitTensor(val, shape, dtype),
158+
float[,] val => InitTensor(val, shape, dtype),
159+
float[,,] val => InitTensor(val, shape, dtype),
160+
float[,,,] val => InitTensor(val, shape, dtype),
161+
double[] val => InitTensor(val, shape, dtype),
162+
double[,] val => InitTensor(val, shape, dtype),
163+
double[,,] val => InitTensor(val, shape, dtype),
164+
double[,,,] val => InitTensor(val, shape, dtype),
165+
string[] val => StringTensor(val, shape),
166+
_ => throw new NotImplementedException("")
167+
};
168+
}
169+
170+
unsafe IntPtr InitTensor<T>(T[] array, Shape shape, TF_DataType dtype) where T : unmanaged
171+
{
172+
fixed (T* addr = &array[0])
173+
return TF_NewTensor(shape, dtype, addr);
174+
}
175+
176+
unsafe IntPtr InitTensor<T>(T[,] array, Shape shape, TF_DataType dtype) where T : unmanaged
177+
{
178+
fixed (T* addr = &array[0, 0])
179+
return TF_NewTensor(shape, dtype, addr);
180+
}
181+
182+
unsafe IntPtr InitTensor<T>(T[,,] array, Shape shape, TF_DataType dtype) where T : unmanaged
183+
{
184+
fixed (T* addr = &array[0, 0, 0])
185+
return TF_NewTensor(shape, dtype, addr);
186+
}
187+
188+
unsafe IntPtr InitTensor<T>(T[,,,] array, Shape shape, TF_DataType dtype) where T : unmanaged
189+
{
190+
fixed (T* addr = &array[0, 0, 0, 0])
191+
return TF_NewTensor(shape, dtype, addr);
165192
}
166193
}
167194
}

‎src/TensorFlowNET.Core/Tensors/TensorShape.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,9 @@ public bool is_compatible_with(TensorShape shape2)
160160
{
161161
if (dims != null && shape2.dims != null)
162162
{
163+
if (dims.Contains(-1) || shape2.dims.Contains(-1))
164+
return true;
165+
163166
if (shape.size != (ulong)shape2.size)
164167
return false;
165168
}

‎src/TensorFlowNET.Core/Tensors/constant_op.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,8 +152,9 @@ value is NDArray nd &&
152152
value = nd.astype(dtype.as_system_dtype());
153153
}
154154

155+
// non ascii char
155156
if (dtype == TF_DataType.TF_STRING && value is byte[] bytes)
156-
return new EagerTensor(bytes, ctx.DeviceName, TF_DataType.TF_STRING);
157+
return new EagerTensor(bytes, TF_DataType.TF_STRING);
157158

158159
switch (value)
159160
{

‎test/TensorFlowNET.UnitTest/ManagedAPI/LinalgTest.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@ public void EyeTest()
1313

1414
Assert.AreEqual((3, 3), tensor.TensorShape);
1515

16-
Assert.AreEqual(0.0f, (float)tensor[2, 0]);
17-
Assert.AreEqual(0.0f, (float)tensor[2, 1]);
18-
Assert.AreEqual(1.0f, (float)tensor[2, 2]);
16+
Assert.AreEqual(0.0f, (double)tensor[2, 0]);
17+
Assert.AreEqual(0.0f, (double)tensor[2, 1]);
18+
Assert.AreEqual(1.0f, (double)tensor[2, 2]);
1919
}
2020
}
2121
}

0 commit comments

Comments
 (0)