-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathmemory.h
More file actions
1097 lines (960 loc) · 39.6 KB
/
Copy pathmemory.h
File metadata and controls
1097 lines (960 loc) · 39.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#ifndef KERNEL_FLOAT_MEMORY_H
#define KERNEL_FLOAT_MEMORY_H
#include "binops.h"
#include "conversion.h"
#include "iterate.h"
namespace kernel_float {
namespace detail {
template<typename T, size_t N, typename Is = make_index_sequence<N>>
struct copy_impl;
template<typename T, size_t N, size_t... Is>
struct copy_impl<T, N, index_sequence<Is...>> {
KERNEL_FLOAT_INLINE
static vector_storage<T, N> load(const T* input, const size_t* offsets, const bool* mask) {
return {(mask[Is] ? input[offsets[Is]] : T {})...};
}
KERNEL_FLOAT_INLINE
static void store(T* outputs, const T* inputs, const size_t* offsets, const bool* mask) {
((mask[Is] ? outputs[offsets[Is]] = inputs[Is] : T {}), ...);
}
};
} // namespace detail
/**
* Load the elements from the buffer ``ptr`` at the locations specified by ``indices``.
*
* The ``mask`` should be a vector of booleans where ``true`` indicates that the value should
* be loaded and ``false`` indicates that the value should be skipped. This can be used
* to prevent reading out of bounds.
*
* ```
* // Load 2 elements at data[0] and data[8], skip data[2] and data[4]
* vec<T, 4> values = read(data, make_vec(0, 2, 4, 8), make_vec(true, false, false, true));
* ```
*/
template<typename T, typename I, typename M = bool, typename E = broadcast_vector_extent_type<I, M>>
KERNEL_FLOAT_INLINE vector<T, E> read(const T* ptr, const I& indices, const M& mask = true) {
return detail::copy_impl<T, extent_size<E>>::load(
ptr,
convert_storage<size_t>(indices, E()).data(),
convert_storage<bool>(mask, E()).data());
}
/**
* Store the elements from the vector `values` in the buffer ``ptr`` at the locations specified by ``indices``.
*
* The ``mask`` should be a vector of booleans where ``true`` indicates that the value should
* be store and ``false`` indicates that the value should be skipped. This can be used
* to prevent writing out of bounds.
*
* ```
* // Store 2 elements at data[0] and data[8], skip data[2] and data[4]
* auto values = make_vec(42, 13, 87, 12);
* auto mask = make_vec(true, false, false, true);
* write(data, make_vec(0, 2, 4, 8), values, mask);
* ```
*/
template<
typename T,
typename V,
typename I,
typename M = bool,
typename E = broadcast_vector_extent_type<V, I, M>>
KERNEL_FLOAT_INLINE void write(T* ptr, const I& indices, const V& values, const M& mask = true) {
return detail::copy_impl<T, extent_size<E>>::store(
ptr,
convert_storage<T>(values, E()).data(),
convert_storage<size_t>(indices, E()).data(),
convert_storage<bool>(mask, E()).data());
}
/**
* Load ``N`` elements at the location ``ptr[0], ptr[1], ptr[2], ...``.
*
* ```
* // Load 4 elements at locations data[0], data[1], data[2], data[3]
* vec<T, 4> values = read<4>(data);
*
* // Load 4 elements at locations data[10], data[11], data[12], data[13]
* vec<T, 4> values = read<4>(data + 10);
* ```
*/
template<size_t N, typename T>
KERNEL_FLOAT_INLINE vector<T, extent<N>> read(const T* ptr) {
return read(ptr, range<size_t, N>());
}
/**
* Store ``N`` elements at the location ``ptr[0], ptr[1], ptr[2], ...``.
*
* ```
* // Store 4 elements at locations data[0], data[1], data[2], data[3]
* vec<float, 4> values = {1.0f, 2.0f, 3.0f, 4.0f};
* write(data, values);
*
* // Store 4 elements at locations data[10], data[11], data[12], data[13]
* write(data + 10, values);
* ```
*/
template<typename V, typename T>
KERNEL_FLOAT_INLINE void write(T* ptr, const V& values) {
static constexpr size_t N = vector_size<V>;
write(ptr, range<size_t, N>(), values);
}
/**
* Cache-eviction hint for `ld.global`/`st.global` operations, mirroring CUDA's `__ldca`/`__ldcg`/`__ldcs`/
* `__ldlu`/`__ldcv` load intrinsics and `__stwb`/`__stcg`/`__stcs`/`__stwt` store intrinsics. Used as a
* template parameter of `read_aligned`/`write_aligned` (and `copy_aligned_impl::load`/`::store`) to control
* caching behavior.
*
* Modifiers that are semantically equivalent between loads and stores share the same underlying value: `ca`
* (load) and `wb` (store) both mean "cache at all levels", and `cv` (load) and `wt` (store) both mean
* "bypass caching". Each PTX-derived short name also has a more descriptive human-readable alias with the
* same value (e.g. `cache_all` for `ca`/`wb`).
*/
enum struct cache_modifier {
normal, // default behavior, caching behavior is left to the compiler
cache_all, // Cache at all levels
ca = cache_all, // Load: cache at all levels, likely to be accessed again (`__ldca`).
wb = cache_all, // Store: write-back, cache at all levels (`__stwb`); equivalent to `ca`.
cache_global, // cache at global level only, bypassing L1
cg = cache_global, // Load/store: cache at global level only, bypassing L1 (`__ldcg`/`__stcg`).
streaming, // streaming, likely to be accessed only once.
cs = streaming, // Load/store: streaming, likely to be accessed only once (`__ldcs`/`__stcs`).
uncached, // don't cache, already read or write directly from memory
cv = uncached, // Load: don't cache, always re-fetch from memory (`__ldcv`).
wt = uncached, // Store: write-through to system memory (`__stwt`); equivalent to `cv`.
last_use, // last use, the cache line will not be re-used afterwards (loads only, not store equivalent).
lu = last_use, // Load only: last use, the cache line will not be re-used (`__ldlu`)
read_only, // load through the read-only/non-coherent cache (loads only, not store equivalent).
nc = read_only // Load only: non-coherent, read-only cache load (`ld.global.nc`, i.e. `__ldg`).
};
namespace detail {
/**
* Returns the greatest common divisor of `a` and `b`.
*/
KERNEL_FLOAT_INLINE
constexpr size_t gcd(size_t a, size_t b) {
return b == 0 ? a : gcd(b, a % b);
}
/**
* Maps a size in bytes to the built-in type that a value of that size can be reinterpreted as for the
* `__ldXX`/`__stXX` cache-modifier intrinsics. Only the sizes of `char`, `short`, `int`/`float`,
* `long long`/`double` and `int4`/`double2`/`ulonglong2` are supported; every other size maps to `void`.
*/
template<size_t Bytes>
struct cache_intrinsic_type {
using type = void;
};
template<>
struct cache_intrinsic_type<1> {
using type = unsigned char;
};
template<>
struct cache_intrinsic_type<2> {
using type = unsigned short;
};
template<>
struct cache_intrinsic_type<4> {
using type = unsigned int;
};
template<>
struct cache_intrinsic_type<8> {
using type = unsigned long long;
};
template<>
struct cache_intrinsic_type<16> {
using type = ulonglong2;
};
/**
* Returns true if a value of `Bytes` bytes can be reinterpreted as one of the built-in types supported by
* the cache-modifier intrinsics.
*/
template<size_t Bytes>
static constexpr bool cache_intrinsic_supported =
!is_same_type<typename cache_intrinsic_type<Bytes>::type, void>;
/**
* Loads a value of type `S` from `ptr`, applying the given cache-eviction hint. Falls back to a plain load
* if `Modifier` is `normal`, if `S` does not match one of the sizes supported by the cache intrinsics, or if
* this code is not compiled for a CUDA device (e.g., host compilation or HIP).
*/
template<typename S, cache_modifier Modifier>
KERNEL_FLOAT_INLINE S cache_load(const S* ptr) {
#if KERNEL_FLOAT_IS_CUDA && KERNEL_FLOAT_IS_DEVICE
if constexpr (!cache_intrinsic_supported<sizeof(S)> || Modifier == cache_modifier::normal) {
return *ptr;
} else {
using R = typename cache_intrinsic_type<sizeof(S)>::type;
const R* raw_ptr = reinterpret_cast<const R*>(ptr);
R result;
if constexpr (Modifier == cache_modifier::ca) {
result = __ldca(raw_ptr);
} else if constexpr (Modifier == cache_modifier::cg) {
result = __ldcg(raw_ptr);
} else if constexpr (Modifier == cache_modifier::cs) {
result = __ldcs(raw_ptr);
} else if constexpr (Modifier == cache_modifier::lu) {
result = __ldlu(raw_ptr);
} else if constexpr (Modifier == cache_modifier::cv) {
result = __ldcv(raw_ptr);
} else if constexpr (Modifier == cache_modifier::nc) {
result = __ldg(raw_ptr);
} else {
return *ptr;
}
return *reinterpret_cast<const S*>(&result);
}
#else
return *ptr;
#endif
}
/**
* Stores `value` of type `S` to `ptr`, applying the given cache-eviction hint. Falls back to a plain store
* if `Modifier` is `normal` or `lu` (which has no store equivalent), if `S` does not match one of the sizes
* supported by the cache intrinsics, or if this code is not compiled for a CUDA device (e.g., host
* compilation or HIP).
*/
template<typename S, cache_modifier Modifier>
KERNEL_FLOAT_INLINE void cache_store(S* ptr, const S& value) {
#if KERNEL_FLOAT_IS_CUDA && KERNEL_FLOAT_IS_DEVICE
if constexpr (!cache_intrinsic_supported<sizeof(S)> || Modifier == cache_modifier::normal) {
*ptr = value;
} else {
using R = typename cache_intrinsic_type<sizeof(S)>::type;
R* raw_ptr = reinterpret_cast<R*>(ptr);
R raw_value = *reinterpret_cast<const R*>(&value);
if constexpr (Modifier == cache_modifier::ca) {
__stwb(raw_ptr, raw_value);
} else if constexpr (Modifier == cache_modifier::cg) {
__stcg(raw_ptr, raw_value);
} else if constexpr (Modifier == cache_modifier::cs) {
__stcs(raw_ptr, raw_value);
} else if constexpr (Modifier == cache_modifier::cv) {
__stwt(raw_ptr, raw_value);
} else {
// Modifier == cache_modifier::normal or cache_modifier::lu (no store equivalent)
*ptr = value;
}
}
#else
*ptr = value;
#endif
}
template<typename T, size_t N, size_t Alignment, typename = void>
struct copy_aligned_impl {
static constexpr size_t K = N > 8 ? 8 : (N > 4 ? 4 : (N > 2 ? 2 : 1));
static constexpr size_t Alignment_K = gcd(Alignment, sizeof(T) * K);
template<cache_modifier LoadModifier = cache_modifier::normal>
KERNEL_FLOAT_INLINE static void load(T* output, const T* input) {
copy_aligned_impl<T, K, Alignment>::template load<LoadModifier>(output, input);
copy_aligned_impl<T, N - K, Alignment_K>::template load<LoadModifier>(
output + K,
input + K);
}
template<cache_modifier StoreModifier = cache_modifier::normal>
KERNEL_FLOAT_INLINE static void store(T* output, const T* input) {
copy_aligned_impl<T, K, Alignment>::template store<StoreModifier>(output, input);
copy_aligned_impl<T, N - K, Alignment_K>::template store<StoreModifier>(
output + K,
input + K);
}
};
template<typename T, size_t Alignment>
struct copy_aligned_impl<T, 0, Alignment> {
template<cache_modifier LoadModifier = cache_modifier::normal>
KERNEL_FLOAT_INLINE static void load(T* output, const T* input) {}
template<cache_modifier StoreModifier = cache_modifier::normal>
KERNEL_FLOAT_INLINE static void store(T* output, const T* input) {}
};
template<typename T, size_t Alignment>
struct copy_aligned_impl<T, 1, Alignment> {
using storage_type = T;
template<cache_modifier LoadModifier = cache_modifier::normal>
KERNEL_FLOAT_INLINE static void load(T* output, const T* input) {
output[0] = cache_load<T, LoadModifier>(&input[0]);
}
template<cache_modifier StoreModifier = cache_modifier::normal>
KERNEL_FLOAT_INLINE static void store(T* output, const T* input) {
cache_store<T, StoreModifier>(&output[0], input[0]);
}
};
template<typename T, size_t Alignment>
struct copy_aligned_impl<T, 2, Alignment, enable_if_t<(Alignment > sizeof(T))>> {
static constexpr size_t storage_alignment = gcd(Alignment, 2 * sizeof(T));
struct alignas(storage_alignment) storage_type {
T v0, v1;
};
template<cache_modifier LoadModifier = cache_modifier::normal>
KERNEL_FLOAT_INLINE static void load(T* output, const T* input) {
storage_type storage =
cache_load<storage_type, LoadModifier>(reinterpret_cast<const storage_type*>(input));
output[0] = storage.v0;
output[1] = storage.v1;
}
template<cache_modifier StoreModifier = cache_modifier::normal>
KERNEL_FLOAT_INLINE static void store(T* output, const T* input) {
cache_store<storage_type, StoreModifier>(
reinterpret_cast<storage_type*>(output),
storage_type {input[0], input[1]});
}
};
template<typename T, size_t Alignment>
struct copy_aligned_impl<T, 4, Alignment, enable_if_t<(Alignment > 2 * sizeof(T))>> {
static constexpr size_t storage_alignment = gcd(Alignment, 4 * sizeof(T));
struct alignas(storage_alignment) storage_type {
T v0, v1, v2, v3;
};
template<cache_modifier LoadModifier = cache_modifier::normal>
KERNEL_FLOAT_INLINE static void load(T* output, const T* input) {
storage_type storage =
cache_load<storage_type, LoadModifier>(reinterpret_cast<const storage_type*>(input));
output[0] = storage.v0;
output[1] = storage.v1;
output[2] = storage.v2;
output[3] = storage.v3;
}
template<cache_modifier StoreModifier = cache_modifier::normal>
KERNEL_FLOAT_INLINE static void store(T* output, const T* input) {
cache_store<storage_type, StoreModifier>(
reinterpret_cast<storage_type*>(output),
storage_type {
input[0], //
input[1],
input[2],
input[3]});
}
};
template<typename T, size_t Alignment>
struct copy_aligned_impl<T, 8, Alignment, enable_if_t<(Alignment > 4 * sizeof(T))>> {
static constexpr size_t storage_alignment = gcd(Alignment, 8 * sizeof(T));
struct alignas(storage_alignment) storage_type {
T v0, v1, v2, v3, v4, v5, v6, v7;
};
template<cache_modifier LoadModifier = cache_modifier::normal>
KERNEL_FLOAT_INLINE static void load(T* output, const T* input) {
storage_type storage =
cache_load<storage_type, LoadModifier>(reinterpret_cast<const storage_type*>(input));
output[0] = storage.v0;
output[1] = storage.v1;
output[2] = storage.v2;
output[3] = storage.v3;
output[4] = storage.v4;
output[5] = storage.v5;
output[6] = storage.v6;
output[7] = storage.v7;
}
template<cache_modifier StoreModifier = cache_modifier::normal>
KERNEL_FLOAT_INLINE static void store(T* output, const T* input) {
cache_store<storage_type, StoreModifier>(
reinterpret_cast<storage_type*>(output),
storage_type {
input[0], //
input[1],
input[2],
input[3],
input[4],
input[5],
input[6],
input[7]});
}
};
} // namespace detail
/**
* Load ``N`` elements at the locations ``ptr[0], ptr[1], ptr[2], ...``.
*
* It is assumed that ``ptr`` is maximum aligned such that all ``N`` elements can be loaded at once using a vector
* operation. If the pointer is not aligned, undefined behavior will occur.
*
* ```
* // Load 4 elements at locations data[0], data[1], data[2], data[3]
* vec<T, 4> values = read_aligned<4>(data);
*
* // Load 4 elements at locations data[12], data[13], data[14], data[15]
* vec<T, 4> values2 = read_aligned<4>(data + 12);
* ```
*/
template<
size_t Align,
cache_modifier LoadModifier = cache_modifier::normal,
size_t N = Align,
typename T>
KERNEL_FLOAT_INLINE vector<T, extent<N>> read_aligned(const T* ptr) {
static constexpr size_t alignment = detail::gcd(Align * sizeof(T), KERNEL_FLOAT_MAX_ALIGNMENT);
vector_storage<T, N> result;
detail::copy_aligned_impl<T, N, alignment>::template load<LoadModifier>(
result.data(),
KERNEL_FLOAT_ASSUME_ALIGNED(const T, ptr, alignment));
return result;
}
/**
* Store ``N`` elements at the locations ``ptr[0], ptr[1], ptr[2], ...``.
*
* It is assumed that ``ptr`` is maximum aligned such that all ``N`` elements can be loaded at once using a vector
* operation. If the pointer is not aligned, undefined behavior will occur.
*
* ```
* // Store 4 elements at locations data[0], data[1], data[2], data[3]
* vec<float, 4> values = {1.0f, 2.0f, 3.0f, 4.0f};
* write_aligned(data, values);
*
* // Load 4 elements at locations data[10], data[11], data[12], data[13]
* write_aligned(data + 10, values);
* ```
*/
template<
size_t Align,
cache_modifier StoreModifier = cache_modifier::normal,
typename V,
typename T>
KERNEL_FLOAT_INLINE void write_aligned(T* ptr, const V& values) {
static constexpr size_t N = vector_size<V>;
static constexpr size_t alignment = detail::gcd(Align * sizeof(T), KERNEL_FLOAT_MAX_ALIGNMENT);
return detail::copy_aligned_impl<T, N, alignment>::template store<StoreModifier>(
KERNEL_FLOAT_ASSUME_ALIGNED(T, ptr, alignment),
convert_storage<T, N>(values).data());
}
enum struct access_mode { read_only, read_write };
/**
* An access policy describes how a `vector_ref`/`vector_ptr` reads from and writes to its underlying
* storage. The read/write entry points are non-static member functions so that a policy can carry
* runtime state. Policies that are stateless (such as the default `access_policy` below) remain empty
* classes and therefore add no size to `vector_ref`/`vector_ptr` thanks to empty-base-class optimization.
*/
template<
typename U,
size_t Alignment = alignof(U),
cache_modifier ReadPolicy = cache_modifier::normal,
cache_modifier WritePolicy = ReadPolicy>
struct access_policy {
using storage_type = U;
static constexpr access_mode mode = access_mode::read_write;
static constexpr size_t alignment = Alignment;
static constexpr size_t access_alignment = detail::gcd(alignment, KERNEL_FLOAT_MAX_ALIGNMENT);
static_assert(access_alignment >= alignof(storage_type), "invalid alignment for pointer type");
template<size_t N>
using with_offset = access_policy<
storage_type,
detail::gcd(alignment, N * sizeof(storage_type)),
ReadPolicy,
WritePolicy>;
template<size_t M, cache_modifier RP, cache_modifier WP>
KERNEL_FLOAT_INLINE access_policy(access_policy<U, M, RP, WP>) {}
access_policy() = default;
template<size_t N>
KERNEL_FLOAT_INLINE with_offset<N> offset_impl(size_t index) const {
return with_offset<N> {};
}
template<typename T, size_t N>
KERNEL_FLOAT_INLINE void
read_impl(const storage_type* input, vector_storage<T, N>& output) const {
access_policy<const U, Alignment, ReadPolicy, WritePolicy>().read_impl(input, output);
}
template<typename T, size_t N>
KERNEL_FLOAT_INLINE void
write_impl(storage_type* output, const vector_storage<T, N>& input) const {
detail::copy_aligned_impl<storage_type, N, access_alignment>::template store<WritePolicy>(
KERNEL_FLOAT_ASSUME_ALIGNED(storage_type, output, access_alignment),
convert_storage<storage_type, N>(input).data());
}
};
template<typename U, size_t Alignment, cache_modifier ReadPolicy, cache_modifier WritePolicy>
struct access_policy<const U, Alignment, ReadPolicy, WritePolicy> {
using storage_type = const U;
static constexpr access_mode mode = access_mode::read_only;
static constexpr size_t alignment = Alignment;
static constexpr size_t access_alignment = detail::gcd(alignment, KERNEL_FLOAT_MAX_ALIGNMENT);
static_assert(access_alignment >= alignof(storage_type), "invalid alignment for pointer type");
template<size_t N>
using with_offset = access_policy<
storage_type,
detail::gcd(alignment, N * sizeof(storage_type)),
ReadPolicy,
WritePolicy>;
template<size_t M, cache_modifier RP, cache_modifier WP>
KERNEL_FLOAT_INLINE access_policy(access_policy<const U, M, RP, WP>) {}
template<size_t M, cache_modifier RP, cache_modifier WP>
KERNEL_FLOAT_INLINE access_policy(access_policy<U, M, RP, WP>) {}
access_policy() = default;
template<size_t N>
KERNEL_FLOAT_INLINE with_offset<N> offset_impl(size_t index) const {
return with_offset<N> {};
}
template<typename T, size_t N>
KERNEL_FLOAT_INLINE void
read_impl(const storage_type* input, vector_storage<T, N>& output) const {
vector_storage<U, N> result;
detail::copy_aligned_impl<U, N, access_alignment>::template load<ReadPolicy>(
result.data(),
KERNEL_FLOAT_ASSUME_ALIGNED(storage_type, input, access_alignment));
output = convert<T, N>(result);
}
};
namespace detail {
/**
* Returns true if a pointer having alignment of `a` bytes also has an alignment of `b` bytes. Returns false otherwise.
*/
KERNEL_FLOAT_INLINE
constexpr bool is_alignment_divisible(size_t a, size_t b) {
return gcd(a, KERNEL_FLOAT_MAX_ALIGNMENT) % gcd(b, KERNEL_FLOAT_MAX_ALIGNMENT) == 0;
}
template<typename DstPolicy, typename SrcPolicy>
struct is_policy_convertible {
static constexpr bool value = is_same_type<DstPolicy, SrcPolicy>;
};
template<
typename U,
size_t A1,
size_t A2,
cache_modifier ReadPolicy1,
cache_modifier WritePolicy1,
cache_modifier ReadPolicy2,
cache_modifier WritePolicy2>
struct is_policy_convertible<
access_policy<U, A1, ReadPolicy1, WritePolicy1>,
access_policy<U, A2, ReadPolicy2, WritePolicy2>> {
static constexpr bool value = is_alignment_divisible(A2, A1);
};
template<
typename U,
size_t A1,
size_t A2,
cache_modifier ReadPolicy1,
cache_modifier WritePolicy1,
cache_modifier ReadPolicy2,
cache_modifier WritePolicy2>
struct is_policy_convertible<
access_policy<const U, A1, ReadPolicy1, WritePolicy1>,
access_policy<U, A2, ReadPolicy2, WritePolicy2>>:
is_policy_convertible<
access_policy<const U, A1, ReadPolicy1, WritePolicy1>,
access_policy<const U, A2, ReadPolicy2, WritePolicy2>> {};
} // namespace detail
template<typename T, size_t N, typename Policy = access_policy<T>, access_mode = Policy::mode>
struct vector_ref;
/**
* A wrapper for a pointer that enables vectorized access and supports type conversions..
*
* The `vector_ptr<T, N, U>` type is designed to function as if its a `vec<T, N>*` pointer, allowing of reading and
* writing `vec<T, N>` elements. However, the actual type of underlying storage is a pointer of type `U*`, where
* automatic conversion is performed between `T` and `U` when reading/writing items.
*
* For example, a `vector_ptr<double, N, half>` is useful where the data is stored in low precision (here 16 bit)
* but it should be accessed as if it was in a higher precision format (here 64 bit).
*
* The access policy is stored as a (privately inherited) subobject. For the stateless default policy this base is
* empty, so `sizeof(vector_ptr)` equals `sizeof(pointer)` (EBCO). The stored policy is forwarded to the
* `vector_ref` returned by `operator*`.
*
* @tparam T The type of the elements as viewed by the user.
* @tparam N The alignment of T in number of elements.
* @tparam Policy The access policy, which also determines the underlying storage type and alignment.
*/
template<
typename T,
size_t N,
typename Policy = access_policy<T, sizeof(T) * N>,
access_mode = Policy::mode>
struct vector_ptr: private Policy {
using policy_type = Policy;
using storage_type = typename policy_type::storage_type;
using pointer_type = storage_type*;
using value_type = decay_t<T>;
static constexpr size_t alignment = policy_type::alignment;
static constexpr size_t offset_alignment = detail::gcd(alignment, sizeof(storage_type) * N);
using offset_policy_type = typename Policy::template with_offset<N>;
/**
* Default constructor sets the pointer to `NULL`.
*/
KERNEL_FLOAT_INLINE vector_ptr() {}
KERNEL_FLOAT_INLINE vector_ptr(decltype(nullptr)) {}
/**
* Constructor from a given pointer. It is up to the user to assert that the pointer is aligned to `Alignment`.
*/
template<typename V = storage_type, enable_if_t<alignment != alignof(V), int> = 0>
KERNEL_FLOAT_INLINE explicit vector_ptr(pointer_type p, policy_type policy = {}) :
policy_type(policy),
data_(p) {}
/**
* Constructor from a given pointer. This assumes that the alignment of the pointer equals `Alignment`.
*/
template<typename V = storage_type, enable_if_t<alignment == alignof(V), int> = 0>
KERNEL_FLOAT_INLINE vector_ptr(pointer_type p, policy_type policy = {}) :
policy_type(policy),
data_(p) {}
/**
* Constructs a vector_ptr from another vector_ptr with potentially different alignment and type. This constructor
* only allows conversion if the alignment of the source is greater than or equal to the alignment of the target.
* The target policy is default-constructed.
*/
template<
typename T2,
size_t N2,
typename P2,
enable_if_t<detail::is_policy_convertible<policy_type, P2>::value, int> = 0>
KERNEL_FLOAT_INLINE vector_ptr(vector_ptr<T2, N2, P2> p) :
policy_type(p.policy()),
data_(p.get()) {}
/**
* Returns a `vector_ptr` where the pointer has been offset by `index * N` elements.
*/
KERNEL_FLOAT_INLINE vector_ptr<value_type, N, offset_policy_type> offset(size_t index) const {
return vector_ptr<value_type, N, offset_policy_type> {
data_ + index * N,
policy().template offset_impl<N>(index)};
}
/**
* Shorthand for `at(0)`. The stored policy is forwarded to the resulting reference.
*/
KERNEL_FLOAT_INLINE const vector_ref<value_type, N, policy_type> operator*() const {
return vector_ref<value_type, N, policy_type> {data_, policy()};
}
/**
* Accesses a reference to a vector at a specific index with optional alignment considerations.
*
* @tparam K The number of elements in the vector to access, defaults to N.
* @param index The index at which to access the vector.
*/
template<size_t K = N>
KERNEL_FLOAT_INLINE vector_ref<value_type, K, offset_policy_type> at(size_t index) const {
return vector_ref<value_type, K, offset_policy_type> {
data_ + index * N,
policy().template offset_impl<N>(index)};
}
/**
* Shorthand for `at(index)`.
*/
KERNEL_FLOAT_INLINE vector_ref<value_type, N, offset_policy_type>
operator[](size_t index) const {
return at(index);
}
/**
* Accesses a vector at a specific index.
*
* @tparam K The number of elements to read, defaults to `N`.
* @param index The index from which to read the data.
*/
template<size_t K = N>
KERNEL_FLOAT_INLINE vector<value_type, extent<K>> read(size_t index = 0) const {
return this->template at<K>(index).read();
}
/**
* @brief Writes data to a specific index.
*
* @tparam K The number of elements to write, defaults to `N`.
* @tparam V The type of the values being written.
* @param index The index at which to write the data.
* @param values The vector of values to write.
*/
template<size_t K = N, typename V>
KERNEL_FLOAT_INLINE void write(size_t index, const V& values) const {
this->template at<K>(index).write(values);
}
/**
* Gets the raw data pointer managed by this `vector_ptr`.
*/
KERNEL_FLOAT_INLINE pointer_type get() const {
return data_;
}
/**
* Returns a reference to the access policy stored within this vector_ptr.
*/
KERNEL_FLOAT_INLINE const policy_type& policy() const {
return static_cast<const policy_type&>(*this);
}
private:
pointer_type data_ = nullptr;
};
/**
* Specialization for `vector_ptr` if the backing storage is const.
*/
template<typename T, size_t N, typename Policy>
struct vector_ptr<T, N, Policy, access_mode::read_only>: private Policy {
using policy_type = Policy;
using storage_type = typename policy_type::storage_type;
using pointer_type = storage_type*;
using value_type = decay_t<T>;
static constexpr size_t alignment = policy_type::alignment;
static constexpr size_t offset_alignment = detail::gcd(alignment, sizeof(storage_type) * N);
using offset_policy_type = typename Policy::template with_offset<N>;
KERNEL_FLOAT_INLINE vector_ptr() {}
KERNEL_FLOAT_INLINE vector_ptr(decltype(nullptr)) {}
template<typename V = storage_type, enable_if_t<alignment != alignof(V), int> = 0>
KERNEL_FLOAT_INLINE explicit vector_ptr(pointer_type p, policy_type policy = {}) :
policy_type(policy),
data_(p) {}
template<typename V = storage_type, enable_if_t<alignment == alignof(V), int> = 0>
KERNEL_FLOAT_INLINE vector_ptr(pointer_type p, policy_type policy = {}) :
policy_type(policy),
data_(p) {}
template<
typename T2,
size_t N2,
typename P2,
enable_if_t<detail::is_policy_convertible<policy_type, P2>::value, int> = 0>
KERNEL_FLOAT_INLINE vector_ptr(vector_ptr<T2, N2, P2> p) :
policy_type(p.policy()),
data_(p.get()) {}
KERNEL_FLOAT_INLINE vector_ptr<value_type, N, offset_policy_type> offset(size_t index) const {
return vector_ptr<value_type, N, offset_policy_type> {
data_ + index * N,
policy().template offset_impl<N>(index)};
}
KERNEL_FLOAT_INLINE vector_ref<value_type, N, policy_type> operator*() const {
return vector_ref<value_type, N, policy_type> {data_, policy()};
}
template<size_t K = N>
KERNEL_FLOAT_INLINE vector_ref<value_type, K, offset_policy_type> at(size_t index) const {
return vector_ref<value_type, K, offset_policy_type> {
data_ + index * N,
policy().template offset_impl<N>(index)};
}
template<size_t K = N>
KERNEL_FLOAT_INLINE vector<value_type, extent<K>> read(size_t index = 0) const {
return this->template at<K>(index).read();
}
KERNEL_FLOAT_INLINE const vector<value_type, extent<N>> operator[](size_t index) const {
return read(index);
}
KERNEL_FLOAT_INLINE pointer_type get() const {
return data_;
}
KERNEL_FLOAT_INLINE const policy_type& policy() const {
return static_cast<const policy_type&>(*this);
}
private:
pointer_type data_ = nullptr;
};
template<typename T, size_t N, typename P>
KERNEL_FLOAT_INLINE vector_ptr<T, N, typename P::template with_offset<N>>
operator+(vector_ptr<T, N, P> p, size_t i) {
return p.offset(i);
}
template<typename T, size_t N, typename P>
KERNEL_FLOAT_INLINE vector_ptr<T, N, typename P::template with_offset<N>>
operator+(size_t i, vector_ptr<T, N, P> p) {
return p.offset(i);
}
/**
* Offsets `p` in place by `i * N` elements.
*
* This is only available when offsetting the pointer does not lose alignment, i.e. when the policy after
* offsetting is convertible back to the original policy. Otherwise `p + i` has a different (less aligned)
* type than `p` and cannot be assigned back; capture it in a new variable instead (e.g. `auto q = p + i`).
*/
template<
typename T,
size_t N,
typename P,
typename =
enable_if_t<detail::is_policy_convertible<P, typename P::template with_offset<N>>::value>>
KERNEL_FLOAT_INLINE vector_ptr<T, N, P>& operator+=(vector_ptr<T, N, P>& p, size_t i) {
return p = p + i;
}
/**
* @brief A reference wrapper that allows reading/writing a vector of type `T`and length `N` with optional data
* conversion.
*
* The access policy is stored as a (privately inherited) subobject so that stateful policies are possible. For the
* stateless default policy this base is empty and contributes nothing to the size of `vector_ref` (EBCO).
*
* @tparam T The type of the elements as seen from the user's perspective.
* @tparam N The number of elements in the vector.
* @tparam Policy The access policy controlling reads/writes and the underlying storage type.
*/
template<typename T, size_t N, typename Policy, access_mode>
struct vector_ref: private Policy {
using policy_type = Policy;
using storage_type = typename Policy::storage_type;
using pointer_type = storage_type*;
using value_type = T;
using vector_type = vector<value_type, extent<N>>;
static constexpr size_t alignment = Policy::alignment;
/**
* Constructs a vector_ref to manage access to a raw data pointer.
*
* @param data Pointer to the raw data this vector_ref will manage.
* @param policy The access policy instance (defaulted; empty for the stateless default policy).
*/
KERNEL_FLOAT_INLINE explicit vector_ref(pointer_type data, policy_type policy = {}) :
policy_type(policy),
data_(data) {}
/**
* Return this `vector_ref` as a `vector_ptr`.
*/
KERNEL_FLOAT_INLINE vector_ptr<value_type, N, policy_type> operator&() const {
return vector_ptr<value_type, N, policy_type> {data_, policy()};
}
/**
* Reads data from the underlying raw pointer, converting it to type `T`.
*
* @return vector_type A vector of type vector_type containing the read and converted data.
*/
KERNEL_FLOAT_INLINE vector_type read() const {
vector_type result;
policy().read_impl(data_, result);
return result;
}
/**
* Writes data to the underlying raw pointer, converting it from the input vector if necessary.
*
* @param values The values to be written.
*/
KERNEL_FLOAT_INLINE void write(const vector_type& values) const {
policy().write_impl(data_, values);
}
/**
* Conversion operator that is shorthand for `read()`.
*/
KERNEL_FLOAT_INLINE operator vector_type() const {
return read();
}
/**
* Assignment operator that is shorthand for `write(values)`.
*/
template<typename V>
KERNEL_FLOAT_INLINE vector_ref operator=(const V& values) const {
write(values);
return *this;
}
/**
* Gets the raw data pointer managed by this vector_ref
*/
KERNEL_FLOAT_INLINE pointer_type get() const {
return data_;
}
/**
* Returns a reference to the access policy stored within this vector_ref.
*/
KERNEL_FLOAT_INLINE const policy_type& policy() const {
return static_cast<const policy_type&>(*this);
}
private:
pointer_type data_ = nullptr;
};
/**
* Specialization for `vector_ref` if the backing storage is const.
*/
template<typename T, size_t N, typename Policy>
struct vector_ref<T, N, Policy, access_mode::read_only>: private Policy {
using policy_type = Policy;
using storage_type = typename policy_type::storage_type;
using pointer_type = storage_type*;
using value_type = T;
using vector_type = vector<value_type, extent<N>>;
static constexpr size_t alignment = policy_type::alignment;
static constexpr size_t access_alignment = detail::gcd(alignment, KERNEL_FLOAT_MAX_ALIGNMENT);
static_assert(access_alignment >= alignof(storage_type), "invalid alignment for pointer type");
KERNEL_FLOAT_INLINE explicit vector_ref(pointer_type data, policy_type policy = {}) :
policy_type(policy),
data_(data) {}
KERNEL_FLOAT_INLINE vector_ptr<value_type, N, policy_type> operator&() const {
return vector_ptr<value_type, N, policy_type> {data_, policy()};
}
KERNEL_FLOAT_INLINE vector_type read() const {
vector_type result;
policy().read_impl(data_, result);
return result;
}
KERNEL_FLOAT_INLINE operator vector_type() const {
return read();
}
KERNEL_FLOAT_INLINE pointer_type get() const {
return data_;
}
KERNEL_FLOAT_INLINE const policy_type& policy() const {
return static_cast<const policy_type&>(*this);
}
private:
pointer_type data_ = nullptr;
};
#define KERNEL_FLOAT_VECTOR_REF_ASSIGN_OP(OP, OP_ASSIGN) \
template<typename T, size_t N, typename Policy, typename V> \
KERNEL_FLOAT_INLINE vector_ref<T, N, Policy> operator OP_ASSIGN( \
vector_ref<T, N, Policy> ptr, \
const V \
& value) { \
ptr.write(ptr.read() OP value); \
return ptr; \
}
KERNEL_FLOAT_VECTOR_REF_ASSIGN_OP(+, +=)
KERNEL_FLOAT_VECTOR_REF_ASSIGN_OP(-, -=)
KERNEL_FLOAT_VECTOR_REF_ASSIGN_OP(*, *=)