Line data Source code
1 : //@HEADER
2 : // ************************************************************************
3 : //
4 : // Kokkos v. 4.0
5 : // Copyright (2022) National Technology & Engineering
6 : // Solutions of Sandia, LLC (NTESS).
7 : //
8 : // Under the terms of Contract DE-NA0003525 with NTESS,
9 : // the U.S. Government retains certain rights in this software.
10 : //
11 : // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 : // See https://kokkos.org/LICENSE for license information.
13 : // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 : //
15 : //@HEADER
16 :
17 : #ifndef KOKKOS_EXPERIMENTAL_VIEW_MAPPING_HPP
18 : #define KOKKOS_EXPERIMENTAL_VIEW_MAPPING_HPP
19 :
20 : #include <cstring>
21 : #include <type_traits>
22 : #include <initializer_list>
23 :
24 : #include <Kokkos_Core_fwd.hpp>
25 : #include <Kokkos_DetectionIdiom.hpp>
26 : #include <Kokkos_Pair.hpp>
27 : #include <Kokkos_Layout.hpp>
28 : #include <Kokkos_Extents.hpp>
29 : #include <impl/Kokkos_Error.hpp>
30 : #include <impl/Kokkos_Traits.hpp>
31 : #include <View/Kokkos_ViewTracker.hpp>
32 : #include <View/Kokkos_ViewTraits.hpp>
33 : #include <View/Kokkos_ViewCtor.hpp>
34 : #include <View/Kokkos_ViewAtomic.hpp>
35 : #include <impl/Kokkos_Tools.hpp>
36 : #include <impl/Kokkos_StringManipulation.hpp>
37 : #include <impl/Kokkos_ZeroMemset_fwd.hpp>
38 : #include <View/Kokkos_ViewDataAnalysis.hpp>
39 : #include <View/Kokkos_ViewAlloc.hpp>
40 :
41 : //----------------------------------------------------------------------------
42 : //----------------------------------------------------------------------------
43 :
44 : namespace Kokkos {
45 : namespace Impl {
46 :
47 : template <class T>
48 : struct is_integral_extent_type {
49 : enum : bool { value = std::is_same_v<T, Kokkos::ALL_t> ? 1 : 0 };
50 : };
51 :
52 : template <class iType>
53 : struct is_integral_extent_type<std::pair<iType, iType>> {
54 : enum : bool { value = std::is_integral_v<iType> ? 1 : 0 };
55 : };
56 :
57 : template <class iType>
58 : struct is_integral_extent_type<Kokkos::pair<iType, iType>> {
59 : enum : bool { value = std::is_integral_v<iType> ? 1 : 0 };
60 : };
61 :
62 : // Assuming '2 == initializer_list<iType>::size()'
63 : template <class iType>
64 : struct is_integral_extent_type<std::initializer_list<iType>> {
65 : enum : bool { value = std::is_integral_v<iType> ? 1 : 0 };
66 : };
67 :
68 : template <unsigned I, class... Args>
69 : struct is_integral_extent {
70 : // get_type is void when sizeof...(Args) <= I
71 : using type = std::remove_cv_t<std::remove_reference_t<
72 : typename Kokkos::Impl::get_type<I, Args...>::type>>;
73 :
74 : enum : bool { value = is_integral_extent_type<type>::value };
75 :
76 : static_assert(value || std::is_integral_v<type> || std::is_void_v<type>,
77 : "subview argument must be either integral or integral extent");
78 : };
79 :
80 : // Rules for subview arguments and layouts matching
81 :
82 : template <class LayoutDest, class LayoutSrc, int RankDest, int RankSrc,
83 : int CurrentArg, class... SubViewArgs>
84 : struct SubviewLegalArgsCompileTime;
85 :
86 : // Rules which allow LayoutLeft to LayoutLeft assignment
87 :
88 : template <int RankDest, int RankSrc, int CurrentArg, class Arg,
89 : class... SubViewArgs>
90 : struct SubviewLegalArgsCompileTime<Kokkos::LayoutLeft, Kokkos::LayoutLeft,
91 : RankDest, RankSrc, CurrentArg, Arg,
92 : SubViewArgs...> {
93 : enum {
94 : value =
95 : (((CurrentArg == RankDest - 1) &&
96 : (Kokkos::Impl::is_integral_extent_type<Arg>::value)) ||
97 : ((CurrentArg >= RankDest) && (std::is_integral_v<Arg>)) ||
98 : ((CurrentArg < RankDest) && (std::is_same_v<Arg, Kokkos::ALL_t>)) ||
99 : ((CurrentArg == 0) &&
100 : (Kokkos::Impl::is_integral_extent_type<Arg>::value))) &&
101 : (SubviewLegalArgsCompileTime<Kokkos::LayoutLeft, Kokkos::LayoutLeft,
102 : RankDest, RankSrc, CurrentArg + 1,
103 : SubViewArgs...>::value)
104 : };
105 : };
106 :
107 : template <int RankDest, int RankSrc, int CurrentArg, class Arg>
108 : struct SubviewLegalArgsCompileTime<Kokkos::LayoutLeft, Kokkos::LayoutLeft,
109 : RankDest, RankSrc, CurrentArg, Arg> {
110 : enum {
111 : value = ((CurrentArg == RankDest - 1) || (std::is_integral_v<Arg>)) &&
112 : (CurrentArg == RankSrc - 1)
113 : };
114 : };
115 :
116 : // Rules which allow LayoutRight to LayoutRight assignment
117 :
118 : template <int RankDest, int RankSrc, int CurrentArg, class Arg,
119 : class... SubViewArgs>
120 : struct SubviewLegalArgsCompileTime<Kokkos::LayoutRight, Kokkos::LayoutRight,
121 : RankDest, RankSrc, CurrentArg, Arg,
122 : SubViewArgs...> {
123 : enum {
124 : value = (((CurrentArg == RankSrc - RankDest) &&
125 : (Kokkos::Impl::is_integral_extent_type<Arg>::value)) ||
126 : ((CurrentArg < RankSrc - RankDest) && (std::is_integral_v<Arg>)) ||
127 : ((CurrentArg >= RankSrc - RankDest) &&
128 : (std::is_same_v<Arg, Kokkos::ALL_t>))) &&
129 : (SubviewLegalArgsCompileTime<Kokkos::LayoutRight,
130 : Kokkos::LayoutRight, RankDest, RankSrc,
131 : CurrentArg + 1, SubViewArgs...>::value)
132 : };
133 : };
134 :
135 : template <int RankDest, int RankSrc, int CurrentArg, class Arg>
136 : struct SubviewLegalArgsCompileTime<Kokkos::LayoutRight, Kokkos::LayoutRight,
137 : RankDest, RankSrc, CurrentArg, Arg> {
138 : enum {
139 : value =
140 : ((CurrentArg == RankSrc - 1) && (std::is_same_v<Arg, Kokkos::ALL_t>))
141 : };
142 : };
143 :
144 : // Rules which allow assignment to LayoutStride
145 :
146 : template <int RankDest, int RankSrc, int CurrentArg, class... SubViewArgs>
147 : struct SubviewLegalArgsCompileTime<Kokkos::LayoutStride, Kokkos::LayoutLeft,
148 : RankDest, RankSrc, CurrentArg,
149 : SubViewArgs...> {
150 : enum : bool { value = true };
151 : };
152 :
153 : template <int RankDest, int RankSrc, int CurrentArg, class... SubViewArgs>
154 : struct SubviewLegalArgsCompileTime<Kokkos::LayoutStride, Kokkos::LayoutRight,
155 : RankDest, RankSrc, CurrentArg,
156 : SubViewArgs...> {
157 : enum : bool { value = true };
158 : };
159 :
160 : template <int RankDest, int RankSrc, int CurrentArg, class... SubViewArgs>
161 : struct SubviewLegalArgsCompileTime<Kokkos::LayoutStride, Kokkos::LayoutStride,
162 : RankDest, RankSrc, CurrentArg,
163 : SubViewArgs...> {
164 : enum : bool { value = true };
165 : };
166 :
167 : template <unsigned DomainRank, unsigned RangeRank>
168 : struct SubviewExtents {
169 : private:
170 : // Cannot declare zero-length arrays
171 : // '+' is used to silence GCC 7.2.0 -Wduplicated-branches warning when
172 : // RangeRank=1
173 : enum { InternalRangeRank = RangeRank ? RangeRank : +1u };
174 :
175 : size_t m_begin[DomainRank];
176 : size_t m_length[InternalRangeRank];
177 : unsigned m_index[InternalRangeRank];
178 :
179 : template <size_t... DimArgs>
180 : KOKKOS_FORCEINLINE_FUNCTION bool set(unsigned, unsigned,
181 : const ViewDimension<DimArgs...>&) {
182 : return true;
183 : }
184 :
185 : template <class T, size_t... DimArgs, class... Args>
186 : KOKKOS_FORCEINLINE_FUNCTION bool set(unsigned domain_rank,
187 : unsigned range_rank,
188 : const ViewDimension<DimArgs...>& dim,
189 : const T& val, Args... args) {
190 : const size_t v = static_cast<size_t>(val);
191 :
192 : m_begin[domain_rank] = v;
193 :
194 : return set(domain_rank + 1, range_rank, dim, args...)
195 : #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK)
196 : && (v < dim.extent(domain_rank))
197 : #endif
198 : ;
199 : }
200 :
201 : // ALL_t
202 : template <size_t... DimArgs, class... Args>
203 : KOKKOS_FORCEINLINE_FUNCTION bool set(unsigned domain_rank,
204 : unsigned range_rank,
205 : const ViewDimension<DimArgs...>& dim,
206 : Kokkos::ALL_t, Args... args) {
207 : m_begin[domain_rank] = 0;
208 : m_length[range_rank] = dim.extent(domain_rank);
209 : m_index[range_rank] = domain_rank;
210 :
211 : return set(domain_rank + 1, range_rank + 1, dim, args...);
212 : }
213 :
214 : // std::pair range
215 : template <class T, size_t... DimArgs, class... Args>
216 : KOKKOS_FORCEINLINE_FUNCTION bool set(unsigned domain_rank,
217 : unsigned range_rank,
218 : const ViewDimension<DimArgs...>& dim,
219 : const std::pair<T, T>& val,
220 : Args... args) {
221 : const size_t b = static_cast<size_t>(val.first);
222 : const size_t e = static_cast<size_t>(val.second);
223 :
224 : m_begin[domain_rank] = b;
225 : m_length[range_rank] = e - b;
226 : m_index[range_rank] = domain_rank;
227 :
228 : return set(domain_rank + 1, range_rank + 1, dim, args...)
229 : #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK)
230 : && (e <= b + dim.extent(domain_rank))
231 : #endif
232 : ;
233 : }
234 :
235 : // Kokkos::pair range
236 : template <class T, size_t... DimArgs, class... Args>
237 : KOKKOS_FORCEINLINE_FUNCTION bool set(unsigned domain_rank,
238 : unsigned range_rank,
239 : const ViewDimension<DimArgs...>& dim,
240 : const Kokkos::pair<T, T>& val,
241 : Args... args) {
242 : const size_t b = static_cast<size_t>(val.first);
243 : const size_t e = static_cast<size_t>(val.second);
244 :
245 : m_begin[domain_rank] = b;
246 : m_length[range_rank] = e - b;
247 : m_index[range_rank] = domain_rank;
248 :
249 : return set(domain_rank + 1, range_rank + 1, dim, args...)
250 : #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK)
251 : && (e <= b + dim.extent(domain_rank))
252 : #endif
253 : ;
254 : }
255 :
256 : // { begin , end } range
257 : template <class T, size_t... DimArgs, class... Args>
258 : KOKKOS_FORCEINLINE_FUNCTION bool set(unsigned domain_rank,
259 : unsigned range_rank,
260 : const ViewDimension<DimArgs...>& dim,
261 : const std::initializer_list<T>& val,
262 : Args... args) {
263 : const size_t b = static_cast<size_t>(val.begin()[0]);
264 : const size_t e = static_cast<size_t>(val.begin()[1]);
265 :
266 : m_begin[domain_rank] = b;
267 : m_length[range_rank] = e - b;
268 : m_index[range_rank] = domain_rank;
269 :
270 : return set(domain_rank + 1, range_rank + 1, dim, args...)
271 : #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK)
272 : && (val.size() == 2) && (e <= b + dim.extent(domain_rank))
273 : #endif
274 : ;
275 : }
276 :
277 : //------------------------------
278 :
279 : #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK)
280 :
281 : template <size_t... DimArgs>
282 : void error(char*, int, unsigned, unsigned,
283 : const ViewDimension<DimArgs...>&) const {}
284 :
285 : template <class T, size_t... DimArgs, class... Args>
286 : void error(char* buf, int buf_len, unsigned domain_rank, unsigned range_rank,
287 : const ViewDimension<DimArgs...>& dim, const T& val,
288 : Args... args) const {
289 : const int n = std::min(
290 : buf_len,
291 : snprintf(buf, buf_len, " %lu < %lu %c", static_cast<unsigned long>(val),
292 : static_cast<unsigned long>(dim.extent(domain_rank)),
293 : int(sizeof...(Args) ? ',' : ')')));
294 :
295 : error(buf + n, buf_len - n, domain_rank + 1, range_rank, dim, args...);
296 : }
297 :
298 : // std::pair range
299 : template <size_t... DimArgs, class... Args>
300 : void error(char* buf, int buf_len, unsigned domain_rank, unsigned range_rank,
301 : const ViewDimension<DimArgs...>& dim, Kokkos::ALL_t,
302 : Args... args) const {
303 : const int n = std::min(buf_len, snprintf(buf, buf_len, " Kokkos::ALL %c",
304 : int(sizeof...(Args) ? ',' : ')')));
305 :
306 : error(buf + n, buf_len - n, domain_rank + 1, range_rank + 1, dim, args...);
307 : }
308 :
309 : // std::pair range
310 : template <class T, size_t... DimArgs, class... Args>
311 : void error(char* buf, int buf_len, unsigned domain_rank, unsigned range_rank,
312 : const ViewDimension<DimArgs...>& dim, const std::pair<T, T>& val,
313 : Args... args) const {
314 : // d <= e - b
315 : const int n = std::min(
316 : buf_len, snprintf(buf, buf_len, " %lu <= %lu - %lu %c",
317 : static_cast<unsigned long>(dim.extent(domain_rank)),
318 : static_cast<unsigned long>(val.second),
319 : static_cast<unsigned long>(val.first),
320 : int(sizeof...(Args) ? ',' : ')')));
321 :
322 : error(buf + n, buf_len - n, domain_rank + 1, range_rank + 1, dim, args...);
323 : }
324 :
325 : // Kokkos::pair range
326 : template <class T, size_t... DimArgs, class... Args>
327 : void error(char* buf, int buf_len, unsigned domain_rank, unsigned range_rank,
328 : const ViewDimension<DimArgs...>& dim,
329 : const Kokkos::pair<T, T>& val, Args... args) const {
330 : // d <= e - b
331 : const int n = std::min(
332 : buf_len, snprintf(buf, buf_len, " %lu <= %lu - %lu %c",
333 : static_cast<unsigned long>(dim.extent(domain_rank)),
334 : static_cast<unsigned long>(val.second),
335 : static_cast<unsigned long>(val.first),
336 : int(sizeof...(Args) ? ',' : ')')));
337 :
338 : error(buf + n, buf_len - n, domain_rank + 1, range_rank + 1, dim, args...);
339 : }
340 :
341 : // { begin , end } range
342 : template <class T, size_t... DimArgs, class... Args>
343 : void error(char* buf, int buf_len, unsigned domain_rank, unsigned range_rank,
344 : const ViewDimension<DimArgs...>& dim,
345 : const std::initializer_list<T>& val, Args... args) const {
346 : // d <= e - b
347 : int n = 0;
348 : if (val.size() == 2) {
349 : n = std::min(buf_len,
350 : snprintf(buf, buf_len, " %lu <= %lu - %lu %c",
351 : static_cast<unsigned long>(dim.extent(domain_rank)),
352 : static_cast<unsigned long>(val.begin()[0]),
353 : static_cast<unsigned long>(val.begin()[1]),
354 : int(sizeof...(Args) ? ',' : ')')));
355 : } else {
356 : n = std::min(buf_len, snprintf(buf, buf_len, " { ... }.size() == %u %c",
357 : unsigned(val.size()),
358 : int(sizeof...(Args) ? ',' : ')')));
359 : }
360 :
361 : error(buf + n, buf_len - n, domain_rank + 1, range_rank + 1, dim, args...);
362 : }
363 :
364 : template <size_t... DimArgs, class... Args>
365 : KOKKOS_FORCEINLINE_FUNCTION void error(const ViewDimension<DimArgs...>& dim,
366 : Args... args) const {
367 : KOKKOS_IF_ON_HOST(
368 : (enum {LEN = 1024}; char buffer[LEN];
369 :
370 : const int n = snprintf(buffer, LEN, "Kokkos::subview bounds error (");
371 : error(buffer + n, LEN - n, 0, 0, dim, args...);
372 :
373 : Kokkos::abort(buffer);))
374 :
375 : KOKKOS_IF_ON_DEVICE(((void)dim;
376 : Kokkos::abort("Kokkos::subview bounds error");
377 : [](Args...) {}(args...);))
378 : }
379 :
380 : #else
381 :
382 : template <size_t... DimArgs, class... Args>
383 : KOKKOS_FORCEINLINE_FUNCTION void error(const ViewDimension<DimArgs...>&,
384 : Args...) const {}
385 :
386 : #endif
387 :
388 : public:
389 : template <size_t... DimArgs, class... Args>
390 : KOKKOS_INLINE_FUNCTION SubviewExtents(const ViewDimension<DimArgs...>& dim,
391 : Args... args) {
392 : static_assert(DomainRank == sizeof...(DimArgs));
393 : static_assert(DomainRank == sizeof...(Args));
394 :
395 : // Verifies that all arguments, up to 8, are integral types,
396 : // integral extents, or don't exist.
397 : static_assert(RangeRank ==
398 : unsigned(is_integral_extent<0, Args...>::value) +
399 : unsigned(is_integral_extent<1, Args...>::value) +
400 : unsigned(is_integral_extent<2, Args...>::value) +
401 : unsigned(is_integral_extent<3, Args...>::value) +
402 : unsigned(is_integral_extent<4, Args...>::value) +
403 : unsigned(is_integral_extent<5, Args...>::value) +
404 : unsigned(is_integral_extent<6, Args...>::value) +
405 : unsigned(is_integral_extent<7, Args...>::value));
406 :
407 : if (RangeRank == 0) {
408 : m_length[0] = 0;
409 : m_index[0] = ~0u;
410 : }
411 :
412 : if (!set(0, 0, dim, args...)) error(dim, args...);
413 : }
414 :
415 : template <typename iType>
416 : KOKKOS_FORCEINLINE_FUNCTION constexpr size_t domain_offset(
417 : const iType i) const {
418 : return unsigned(i) < DomainRank ? m_begin[i] : 0;
419 : }
420 :
421 : template <typename iType>
422 : KOKKOS_FORCEINLINE_FUNCTION constexpr size_t range_extent(
423 : const iType i) const {
424 : return unsigned(i) < InternalRangeRank ? m_length[i] : 0;
425 : }
426 :
427 : template <typename iType>
428 : KOKKOS_FORCEINLINE_FUNCTION constexpr unsigned range_index(
429 : const iType i) const {
430 : return unsigned(i) < InternalRangeRank ? m_index[i] : ~0u;
431 : }
432 : };
433 :
434 : } // namespace Impl
435 : } // namespace Kokkos
436 :
437 : //----------------------------------------------------------------------------
438 : //----------------------------------------------------------------------------
439 :
440 : namespace Kokkos {
441 : namespace Impl {
442 : //----------------------------------------------------------------------------
443 : // LayoutLeft AND ( 1 >= rank OR 0 == rank_dynamic ) : no padding / striding
444 : template <class Dimension>
445 : struct ViewOffset<
446 : Dimension, Kokkos::LayoutLeft,
447 : std::enable_if_t<(1 >= Dimension::rank || 0 == Dimension::rank_dynamic)>> {
448 : using is_mapping_plugin = std::true_type;
449 : using is_regular = std::true_type;
450 :
451 : using size_type = size_t;
452 : using dimension_type = Dimension;
453 : using array_layout = Kokkos::LayoutLeft;
454 :
455 : dimension_type m_dim;
456 :
457 : //----------------------------------------
458 :
459 : // rank 1
460 : template <typename I0>
461 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0) const {
462 : return i0;
463 : }
464 :
465 : // rank 2
466 : template <typename I0, typename I1>
467 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
468 : I1 const& i1) const {
469 : return i0 + m_dim.N0 * i1;
470 : }
471 :
472 : // rank 3
473 : template <typename I0, typename I1, typename I2>
474 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
475 : I1 const& i1,
476 : I2 const& i2) const {
477 : return i0 + m_dim.N0 * (i1 + m_dim.N1 * i2);
478 : }
479 :
480 : // rank 4
481 : template <typename I0, typename I1, typename I2, typename I3>
482 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
483 : I1 const& i1,
484 : I2 const& i2,
485 : I3 const& i3) const {
486 : return i0 + m_dim.N0 * (i1 + m_dim.N1 * (i2 + m_dim.N2 * i3));
487 : }
488 :
489 : // rank 5
490 : template <typename I0, typename I1, typename I2, typename I3, typename I4>
491 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
492 : I1 const& i1,
493 : I2 const& i2,
494 : I3 const& i3,
495 : I4 const& i4) const {
496 : return i0 +
497 : m_dim.N0 * (i1 + m_dim.N1 * (i2 + m_dim.N2 * (i3 + m_dim.N3 * i4)));
498 : }
499 :
500 : // rank 6
501 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
502 : typename I5>
503 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
504 : I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
505 : I5 const& i5) const {
506 : return i0 +
507 : m_dim.N0 *
508 : (i1 +
509 : m_dim.N1 *
510 : (i2 + m_dim.N2 * (i3 + m_dim.N3 * (i4 + m_dim.N4 * i5))));
511 : }
512 :
513 : // rank 7
514 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
515 : typename I5, typename I6>
516 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
517 : I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
518 : I5 const& i5, I6 const& i6) const {
519 : return i0 +
520 : m_dim.N0 *
521 : (i1 + m_dim.N1 *
522 : (i2 + m_dim.N2 *
523 : (i3 + m_dim.N3 *
524 : (i4 + m_dim.N4 *
525 : (i5 + m_dim.N5 * i6)))));
526 : }
527 :
528 : // rank 8
529 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
530 : typename I5, typename I6, typename I7>
531 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
532 : I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
533 : I5 const& i5, I6 const& i6, I7 const& i7) const {
534 : return i0 +
535 : m_dim.N0 *
536 : (i1 +
537 : m_dim.N1 *
538 : (i2 + m_dim.N2 *
539 : (i3 + m_dim.N3 *
540 : (i4 + m_dim.N4 *
541 : (i5 + m_dim.N5 *
542 : (i6 + m_dim.N6 *
543 : i7))))));
544 : }
545 :
546 : //----------------------------------------
547 :
548 : KOKKOS_INLINE_FUNCTION
549 : constexpr array_layout layout() const {
550 : constexpr auto r = dimension_type::rank;
551 : return array_layout((r > 0 ? m_dim.N0 : KOKKOS_INVALID_INDEX),
552 : (r > 1 ? m_dim.N1 : KOKKOS_INVALID_INDEX),
553 : (r > 2 ? m_dim.N2 : KOKKOS_INVALID_INDEX),
554 : (r > 3 ? m_dim.N3 : KOKKOS_INVALID_INDEX),
555 : (r > 4 ? m_dim.N4 : KOKKOS_INVALID_INDEX),
556 : (r > 5 ? m_dim.N5 : KOKKOS_INVALID_INDEX),
557 : (r > 6 ? m_dim.N6 : KOKKOS_INVALID_INDEX),
558 : (r > 7 ? m_dim.N7 : KOKKOS_INVALID_INDEX));
559 : }
560 :
561 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_0() const {
562 : return m_dim.N0;
563 : }
564 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_1() const {
565 : return m_dim.N1;
566 : }
567 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_2() const {
568 : return m_dim.N2;
569 : }
570 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_3() const {
571 : return m_dim.N3;
572 : }
573 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_4() const {
574 : return m_dim.N4;
575 : }
576 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_5() const {
577 : return m_dim.N5;
578 : }
579 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_6() const {
580 : return m_dim.N6;
581 : }
582 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_7() const {
583 : return m_dim.N7;
584 : }
585 :
586 : /* Cardinality of the domain index space */
587 : KOKKOS_INLINE_FUNCTION
588 : constexpr size_type size() const {
589 : return size_type(m_dim.N0) * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 *
590 : m_dim.N5 * m_dim.N6 * m_dim.N7;
591 : }
592 :
593 : /* Span of the range space */
594 : KOKKOS_INLINE_FUNCTION
595 : constexpr size_type span() const {
596 : return size_type(m_dim.N0) * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 *
597 : m_dim.N5 * m_dim.N6 * m_dim.N7;
598 : }
599 :
600 : KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const {
601 : return true;
602 : }
603 :
604 : /* Strides of dimensions */
605 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_0() const { return 1; }
606 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_1() const {
607 : return m_dim.N0;
608 : }
609 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_2() const {
610 : return size_type(m_dim.N0) * m_dim.N1;
611 : }
612 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_3() const {
613 : return size_type(m_dim.N0) * m_dim.N1 * m_dim.N2;
614 : }
615 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_4() const {
616 : return size_type(m_dim.N0) * m_dim.N1 * m_dim.N2 * m_dim.N3;
617 : }
618 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_5() const {
619 : return size_type(m_dim.N0) * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4;
620 : }
621 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_6() const {
622 : return size_type(m_dim.N0) * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 *
623 : m_dim.N5;
624 : }
625 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_7() const {
626 : return size_type(m_dim.N0) * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 *
627 : m_dim.N5 * m_dim.N6;
628 : }
629 :
630 : // Fill the target unbounded array s with the stride.
631 : // This method differs from stride() in that it does not write the total
632 : // length to the last index of the array. Preconditions: s must be an array of
633 : // dimension_type::rank elements
634 : // FIXME: The version of clang-format in CI fails from maybe_unused
635 : // clang-format off
636 : template <typename iType>
637 : KOKKOS_INLINE_FUNCTION iType
638 : stride_fill([[maybe_unused]] iType* const s) const {
639 : iType n = 1;
640 : if constexpr (0 < dimension_type::rank) {
641 : s[0] = n;
642 : n *= m_dim.N0;
643 : }
644 : if constexpr (1 < dimension_type::rank) {
645 : s[1] = n;
646 : n *= m_dim.N1;
647 : }
648 : if constexpr (2 < dimension_type::rank) {
649 : s[2] = n;
650 : n *= m_dim.N2;
651 : }
652 : if constexpr (3 < dimension_type::rank) {
653 : s[3] = n;
654 : n *= m_dim.N3;
655 : }
656 : if constexpr (4 < dimension_type::rank) {
657 : s[4] = n;
658 : n *= m_dim.N4;
659 : }
660 : if constexpr (5 < dimension_type::rank) {
661 : s[5] = n;
662 : n *= m_dim.N5;
663 : }
664 : if constexpr (6 < dimension_type::rank) {
665 : s[6] = n;
666 : n *= m_dim.N6;
667 : }
668 : if constexpr (7 < dimension_type::rank) {
669 : s[7] = n;
670 : n *= m_dim.N7;
671 : }
672 : return n;
673 : }
674 : // clang-format on
675 :
676 : // Fill the target unbounded array s with the stride and the total spanned
677 : // size. This method differs from stride_fill() in that it writes the total
678 : // spanned size to the last index of the array. Preconditions: s must be an
679 : // array of dimension_type::rank + 1 elements Stride with [ rank ] value is
680 : // the total length
681 : template <typename iType>
682 : KOKKOS_INLINE_FUNCTION void stride(iType* const s) const {
683 : s[dimension_type::rank] = stride_fill(s);
684 : }
685 :
686 : //----------------------------------------
687 :
688 : // MSVC (16.5.5) + CUDA (10.2) did not generate the defaulted functions
689 : // correct and errors out during compilation. Same for the other places where
690 : // I changed this.
691 : #ifdef KOKKOS_IMPL_WINDOWS_CUDA
692 : KOKKOS_FUNCTION ViewOffset() : m_dim(dimension_type()) {}
693 : KOKKOS_FUNCTION ViewOffset(const ViewOffset& src) { m_dim = src.m_dim; }
694 : KOKKOS_FUNCTION ViewOffset& operator=(const ViewOffset& src) {
695 : m_dim = src.m_dim;
696 : return *this;
697 : }
698 : #else
699 : ViewOffset() = default;
700 : ViewOffset(const ViewOffset&) = default;
701 : ViewOffset& operator=(const ViewOffset&) = default;
702 : #endif
703 :
704 : template <unsigned TrivialScalarSize>
705 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
706 : std::integral_constant<unsigned, TrivialScalarSize> const&,
707 : Kokkos::LayoutLeft const& arg_layout)
708 : : m_dim(arg_layout.dimension[0], 0, 0, 0, 0, 0, 0, 0) {}
709 :
710 : template <class DimRHS>
711 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
712 : const ViewOffset<DimRHS, Kokkos::LayoutLeft, void>& rhs)
713 : : m_dim(rhs.m_dim.N0, rhs.m_dim.N1, rhs.m_dim.N2, rhs.m_dim.N3,
714 : rhs.m_dim.N4, rhs.m_dim.N5, rhs.m_dim.N6, rhs.m_dim.N7) {
715 : static_assert(int(DimRHS::rank) == int(dimension_type::rank),
716 : "ViewOffset assignment requires equal rank");
717 : // Also requires equal static dimensions ...
718 : }
719 :
720 : template <class DimRHS>
721 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
722 : const ViewOffset<DimRHS, Kokkos::LayoutRight, void>& rhs)
723 : : m_dim(rhs.m_dim.N0, 0, 0, 0, 0, 0, 0, 0) {
724 : static_assert(((DimRHS::rank == 0 && dimension_type::rank == 0) ||
725 : (DimRHS::rank == 1 && dimension_type::rank == 1)),
726 : "ViewOffset LayoutLeft and LayoutRight are only compatible "
727 : "when rank <= 1");
728 : }
729 :
730 : template <class DimRHS>
731 : KOKKOS_INLINE_FUNCTION ViewOffset(
732 : const ViewOffset<DimRHS, Kokkos::LayoutStride, void>& rhs)
733 : : m_dim(rhs.m_dim.N0, 0, 0, 0, 0, 0, 0, 0) {
734 : if (rhs.m_stride.S0 != 1) {
735 : Kokkos::abort(
736 : "Kokkos::Impl::ViewOffset assignment of LayoutLeft from LayoutStride "
737 : " requires stride == 1");
738 : }
739 : }
740 :
741 : //----------------------------------------
742 : // Subview construction
743 :
744 : template <class DimRHS>
745 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
746 : const ViewOffset<DimRHS, Kokkos::LayoutLeft, void>&,
747 : const SubviewExtents<DimRHS::rank, dimension_type::rank>& sub)
748 : : m_dim(sub.range_extent(0), 0, 0, 0, 0, 0, 0, 0) {
749 : static_assert((0 == dimension_type::rank_dynamic) ||
750 : (1 == dimension_type::rank &&
751 : 1 == dimension_type::rank_dynamic && 1 <= DimRHS::rank),
752 : "ViewOffset subview construction requires compatible rank");
753 : }
754 : };
755 :
756 : //----------------------------------------------------------------------------
757 : // LayoutLeft AND ( 1 < rank AND 0 < rank_dynamic ) : has padding / striding
758 : template <class Dimension>
759 : struct ViewOffset<
760 : Dimension, Kokkos::LayoutLeft,
761 : std::enable_if_t<(1 < Dimension::rank && 0 < Dimension::rank_dynamic)>> {
762 : using is_mapping_plugin = std::true_type;
763 : using is_regular = std::true_type;
764 :
765 : using size_type = size_t;
766 : using dimension_type = Dimension;
767 : using array_layout = Kokkos::LayoutLeft;
768 :
769 : dimension_type m_dim;
770 : size_type m_stride;
771 :
772 : //----------------------------------------
773 :
774 : // rank 1
775 : template <typename I0>
776 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0) const {
777 : return i0;
778 : }
779 :
780 : // rank 2
781 : template <typename I0, typename I1>
782 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
783 : I1 const& i1) const {
784 : return i0 + m_stride * i1;
785 : }
786 :
787 : // rank 3
788 : template <typename I0, typename I1, typename I2>
789 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
790 : I1 const& i1,
791 : I2 const& i2) const {
792 : return i0 + m_stride * (i1 + m_dim.N1 * i2);
793 : }
794 :
795 : // rank 4
796 : template <typename I0, typename I1, typename I2, typename I3>
797 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
798 : I1 const& i1,
799 : I2 const& i2,
800 : I3 const& i3) const {
801 : return i0 + m_stride * (i1 + m_dim.N1 * (i2 + m_dim.N2 * i3));
802 : }
803 :
804 : // rank 5
805 : template <typename I0, typename I1, typename I2, typename I3, typename I4>
806 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
807 : I1 const& i1,
808 : I2 const& i2,
809 : I3 const& i3,
810 : I4 const& i4) const {
811 : return i0 +
812 : m_stride * (i1 + m_dim.N1 * (i2 + m_dim.N2 * (i3 + m_dim.N3 * i4)));
813 : }
814 :
815 : // rank 6
816 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
817 : typename I5>
818 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
819 : I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
820 : I5 const& i5) const {
821 : return i0 +
822 : m_stride *
823 : (i1 +
824 : m_dim.N1 *
825 : (i2 + m_dim.N2 * (i3 + m_dim.N3 * (i4 + m_dim.N4 * i5))));
826 : }
827 :
828 : // rank 7
829 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
830 : typename I5, typename I6>
831 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
832 : I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
833 : I5 const& i5, I6 const& i6) const {
834 : return i0 +
835 : m_stride *
836 : (i1 + m_dim.N1 *
837 : (i2 + m_dim.N2 *
838 : (i3 + m_dim.N3 *
839 : (i4 + m_dim.N4 *
840 : (i5 + m_dim.N5 * i6)))));
841 : }
842 :
843 : // rank 8
844 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
845 : typename I5, typename I6, typename I7>
846 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
847 : I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
848 : I5 const& i5, I6 const& i6, I7 const& i7) const {
849 : return i0 +
850 : m_stride *
851 : (i1 +
852 : m_dim.N1 *
853 : (i2 + m_dim.N2 *
854 : (i3 + m_dim.N3 *
855 : (i4 + m_dim.N4 *
856 : (i5 + m_dim.N5 *
857 : (i6 + m_dim.N6 *
858 : i7))))));
859 : }
860 :
861 : //----------------------------------------
862 :
863 : KOKKOS_INLINE_FUNCTION
864 : constexpr array_layout layout() const {
865 : constexpr auto r = dimension_type::rank;
866 : array_layout l((r > 0 ? m_dim.N0 : KOKKOS_INVALID_INDEX),
867 : (r > 1 ? m_dim.N1 : KOKKOS_INVALID_INDEX),
868 : (r > 2 ? m_dim.N2 : KOKKOS_INVALID_INDEX),
869 : (r > 3 ? m_dim.N3 : KOKKOS_INVALID_INDEX),
870 : (r > 4 ? m_dim.N4 : KOKKOS_INVALID_INDEX),
871 : (r > 5 ? m_dim.N5 : KOKKOS_INVALID_INDEX),
872 : (r > 6 ? m_dim.N6 : KOKKOS_INVALID_INDEX),
873 : (r > 7 ? m_dim.N7 : KOKKOS_INVALID_INDEX));
874 : // Without span_is_contiguous Sacado hidden dimensions get messed up
875 : l.stride = span_is_contiguous() ? KOKKOS_IMPL_CTOR_DEFAULT_ARG : m_stride;
876 : return l;
877 : }
878 :
879 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_0() const {
880 : return m_dim.N0;
881 : }
882 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_1() const {
883 : return m_dim.N1;
884 : }
885 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_2() const {
886 : return m_dim.N2;
887 : }
888 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_3() const {
889 : return m_dim.N3;
890 : }
891 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_4() const {
892 : return m_dim.N4;
893 : }
894 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_5() const {
895 : return m_dim.N5;
896 : }
897 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_6() const {
898 : return m_dim.N6;
899 : }
900 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_7() const {
901 : return m_dim.N7;
902 : }
903 :
904 : /* Cardinality of the domain index space */
905 : KOKKOS_INLINE_FUNCTION
906 : constexpr size_type size() const {
907 : return size_type(m_dim.N0) * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 *
908 : m_dim.N5 * m_dim.N6 * m_dim.N7;
909 : }
910 :
911 : /* Span of the range space */
912 : KOKKOS_INLINE_FUNCTION
913 : constexpr size_type span() const {
914 : return (m_dim.N0 > size_type(0) ? m_stride : size_type(0)) * m_dim.N1 *
915 : m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5 * m_dim.N6 * m_dim.N7;
916 : }
917 :
918 : KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const {
919 : return m_stride == m_dim.N0;
920 : }
921 :
922 : /* Strides of dimensions */
923 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_0() const { return 1; }
924 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_1() const {
925 : return m_stride;
926 : }
927 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_2() const {
928 : return m_stride * m_dim.N1;
929 : }
930 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_3() const {
931 : return m_stride * m_dim.N1 * m_dim.N2;
932 : }
933 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_4() const {
934 : return m_stride * m_dim.N1 * m_dim.N2 * m_dim.N3;
935 : }
936 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_5() const {
937 : return m_stride * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4;
938 : }
939 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_6() const {
940 : return m_stride * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5;
941 : }
942 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_7() const {
943 : return m_stride * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5 *
944 : m_dim.N6;
945 : }
946 :
947 : // Fill the target unbounded array s with the stride.
948 : // This method differs from stride() in that it does not write the total
949 : // length to the last index of the array. Preconditions: s must be an array of
950 : // dimension_type::rank elements
951 : // The version of clang-format in CI fails from maybe_unused
952 : // clang-format off
953 : template <typename iType>
954 : KOKKOS_INLINE_FUNCTION iType
955 : stride_fill([[maybe_unused]] iType* const s) const {
956 : iType n = 1;
957 : if constexpr (0 < dimension_type::rank) {
958 : s[0] = n;
959 : n *= m_stride;
960 : }
961 : if constexpr (1 < dimension_type::rank) {
962 : s[1] = n;
963 : n *= m_dim.N1;
964 : }
965 : if constexpr (2 < dimension_type::rank) {
966 : s[2] = n;
967 : n *= m_dim.N2;
968 : }
969 : if constexpr (3 < dimension_type::rank) {
970 : s[3] = n;
971 : n *= m_dim.N3;
972 : }
973 : if constexpr (4 < dimension_type::rank) {
974 : s[4] = n;
975 : n *= m_dim.N4;
976 : }
977 : if constexpr (5 < dimension_type::rank) {
978 : s[5] = n;
979 : n *= m_dim.N5;
980 : }
981 : if constexpr (6 < dimension_type::rank) {
982 : s[6] = n;
983 : n *= m_dim.N6;
984 : }
985 : if constexpr (7 < dimension_type::rank) {
986 : s[7] = n;
987 : n *= m_dim.N7;
988 : }
989 : return n;
990 : }
991 : // clang-format on
992 :
993 : // Fill the target unbounded array s with the stride and the total spanned
994 : // size. This method differs from stride_fill() in that it writes the total
995 : // spanned size to the last index of the array. Preconditions: s must be an
996 : // array of dimension_type::rank + 1 elements
997 : template <typename iType>
998 : KOKKOS_INLINE_FUNCTION void stride(iType* const s) const {
999 : s[dimension_type::rank] = stride_fill(s);
1000 : }
1001 :
1002 : //----------------------------------------
1003 :
1004 : private:
1005 : template <unsigned TrivialScalarSize>
1006 : struct Padding {
1007 : enum {
1008 : div = TrivialScalarSize == 0
1009 : ? 0
1010 : : Kokkos::Impl::MEMORY_ALIGNMENT /
1011 : (TrivialScalarSize ? TrivialScalarSize : 1)
1012 : };
1013 : enum {
1014 : mod = TrivialScalarSize == 0
1015 : ? 0
1016 : : Kokkos::Impl::MEMORY_ALIGNMENT %
1017 : (TrivialScalarSize ? TrivialScalarSize : 1)
1018 : };
1019 :
1020 : // If memory alignment is a multiple of the trivial scalar size then attempt
1021 : // to align.
1022 : enum { align = 0 != TrivialScalarSize && 0 == mod ? div : 0 };
1023 : enum {
1024 : div_ok = (div != 0) ? div : 1
1025 : }; // To valid modulo zero in constexpr
1026 :
1027 : KOKKOS_INLINE_FUNCTION
1028 : static constexpr size_t stride(size_t const N) {
1029 : return ((align != 0) &&
1030 : ((static_cast<size_t>(Kokkos::Impl::MEMORY_ALIGNMENT_THRESHOLD) *
1031 : align) < N) &&
1032 : ((N % div_ok) != 0))
1033 : ? N + align - (N % div_ok)
1034 : : N;
1035 : }
1036 : };
1037 :
1038 : public:
1039 : // MSVC (16.5.5) + CUDA (10.2) did not generate the defaulted functions
1040 : // correct and errors out during compilation. Same for the other places where
1041 : // I changed this.
1042 : #ifdef KOKKOS_IMPL_WINDOWS_CUDA
1043 : KOKKOS_FUNCTION ViewOffset() : m_dim(dimension_type()), m_stride(0) {}
1044 : KOKKOS_FUNCTION ViewOffset(const ViewOffset& src) {
1045 : m_dim = src.m_dim;
1046 : m_stride = src.m_stride;
1047 : }
1048 : KOKKOS_FUNCTION ViewOffset& operator=(const ViewOffset& src) {
1049 : m_dim = src.m_dim;
1050 : m_stride = src.m_stride;
1051 : return *this;
1052 : }
1053 : #else
1054 :
1055 : ViewOffset() = default;
1056 : ViewOffset(const ViewOffset&) = default;
1057 : ViewOffset& operator=(const ViewOffset&) = default;
1058 : #endif
1059 :
1060 : /* Enable padding for trivial scalar types with non-zero trivial scalar size
1061 : */
1062 : template <unsigned TrivialScalarSize>
1063 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1064 : std::integral_constant<unsigned, TrivialScalarSize> const&,
1065 : Kokkos::LayoutLeft const& arg_layout)
1066 : : m_dim(arg_layout.dimension[0], arg_layout.dimension[1],
1067 : arg_layout.dimension[2], arg_layout.dimension[3],
1068 : arg_layout.dimension[4], arg_layout.dimension[5],
1069 : arg_layout.dimension[6], arg_layout.dimension[7]),
1070 : m_stride(
1071 : arg_layout.stride != KOKKOS_IMPL_CTOR_DEFAULT_ARG
1072 : ? arg_layout.stride
1073 : : Padding<TrivialScalarSize>::stride(arg_layout.dimension[0])) {
1074 : }
1075 :
1076 : template <class DimRHS>
1077 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1078 : const ViewOffset<DimRHS, Kokkos::LayoutLeft, void>& rhs)
1079 : : m_dim(rhs.m_dim.N0, rhs.m_dim.N1, rhs.m_dim.N2, rhs.m_dim.N3,
1080 : rhs.m_dim.N4, rhs.m_dim.N5, rhs.m_dim.N6, rhs.m_dim.N7),
1081 : m_stride(rhs.stride_1()) {
1082 : static_assert(int(DimRHS::rank) == int(dimension_type::rank),
1083 : "ViewOffset assignment requires equal rank");
1084 : // Also requires equal static dimensions ...
1085 : }
1086 :
1087 : template <class DimRHS>
1088 : KOKKOS_INLINE_FUNCTION ViewOffset(
1089 : const ViewOffset<DimRHS, Kokkos::LayoutStride, void>& rhs)
1090 : : m_dim(rhs.m_dim.N0, rhs.m_dim.N1, rhs.m_dim.N2, rhs.m_dim.N3,
1091 : rhs.m_dim.N4, rhs.m_dim.N5, rhs.m_dim.N6, rhs.m_dim.N7),
1092 : m_stride(rhs.stride_1()) {
1093 : if (rhs.m_stride.S0 != 1) {
1094 : Kokkos::abort(
1095 : "Kokkos::Impl::ViewOffset assignment of LayoutLeft from LayoutStride "
1096 : "requires stride == 1");
1097 : }
1098 : }
1099 :
1100 : //----------------------------------------
1101 : // Subview construction
1102 : // This subview must be 2 == rank and 2 == rank_dynamic
1103 : // due to only having stride #0.
1104 : // The source dimension #0 must be non-zero for stride-one leading dimension.
1105 : // At most subsequent dimension can be non-zero.
1106 :
1107 : template <class DimRHS>
1108 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1109 : const ViewOffset<DimRHS, Kokkos::LayoutLeft, void>& rhs,
1110 : const SubviewExtents<DimRHS::rank, dimension_type::rank>& sub)
1111 : : m_dim(sub.range_extent(0), sub.range_extent(1), sub.range_extent(2),
1112 : sub.range_extent(3), sub.range_extent(4), sub.range_extent(5),
1113 : sub.range_extent(6), sub.range_extent(7)),
1114 : m_stride(
1115 : (1 == sub.range_index(1)
1116 : ? rhs.stride_1()
1117 : : (2 == sub.range_index(1)
1118 : ? rhs.stride_2()
1119 : : (3 == sub.range_index(1)
1120 : ? rhs.stride_3()
1121 : : (4 == sub.range_index(1)
1122 : ? rhs.stride_4()
1123 : : (5 == sub.range_index(1)
1124 : ? rhs.stride_5()
1125 : : (6 == sub.range_index(1)
1126 : ? rhs.stride_6()
1127 : : (7 == sub.range_index(1)
1128 : ? rhs.stride_7()
1129 : : 0)))))))) {
1130 : // static_assert( ( 2 == dimension_type::rank ) &&
1131 : // ( 2 == dimension_type::rank_dynamic ) &&
1132 : // ( 2 <= DimRHS::rank )
1133 : // , "ViewOffset subview construction requires compatible rank"
1134 : // );
1135 : }
1136 : };
1137 :
1138 : //----------------------------------------------------------------------------
1139 : // LayoutRight AND ( 1 >= rank OR 0 == rank_dynamic ) : no padding / striding
1140 : template <class Dimension>
1141 : struct ViewOffset<
1142 : Dimension, Kokkos::LayoutRight,
1143 : std::enable_if_t<(1 >= Dimension::rank || 0 == Dimension::rank_dynamic)>> {
1144 : using is_mapping_plugin = std::true_type;
1145 : using is_regular = std::true_type;
1146 :
1147 : using size_type = size_t;
1148 : using dimension_type = Dimension;
1149 : using array_layout = Kokkos::LayoutRight;
1150 :
1151 : dimension_type m_dim;
1152 :
1153 : //----------------------------------------
1154 :
1155 : // rank 1
1156 : template <typename I0>
1157 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0) const {
1158 : return i0;
1159 : }
1160 :
1161 : // rank 2
1162 : template <typename I0, typename I1>
1163 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1164 : I1 const& i1) const {
1165 : return i1 + m_dim.N1 * i0;
1166 : }
1167 :
1168 : // rank 3
1169 : template <typename I0, typename I1, typename I2>
1170 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1171 : I1 const& i1,
1172 : I2 const& i2) const {
1173 : return i2 + m_dim.N2 * (i1 + m_dim.N1 * (i0));
1174 : }
1175 :
1176 : // rank 4
1177 : template <typename I0, typename I1, typename I2, typename I3>
1178 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1179 : I1 const& i1,
1180 : I2 const& i2,
1181 : I3 const& i3) const {
1182 : return i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1 + m_dim.N1 * (i0)));
1183 : }
1184 :
1185 : // rank 5
1186 : template <typename I0, typename I1, typename I2, typename I3, typename I4>
1187 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1188 : I1 const& i1,
1189 : I2 const& i2,
1190 : I3 const& i3,
1191 : I4 const& i4) const {
1192 : return i4 + m_dim.N4 *
1193 : (i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1 + m_dim.N1 * (i0))));
1194 : }
1195 :
1196 : // rank 6
1197 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
1198 : typename I5>
1199 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
1200 : I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
1201 : I5 const& i5) const {
1202 : return i5 +
1203 : m_dim.N5 *
1204 : (i4 +
1205 : m_dim.N4 *
1206 : (i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1 + m_dim.N1 * (i0)))));
1207 : }
1208 :
1209 : // rank 7
1210 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
1211 : typename I5, typename I6>
1212 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
1213 : I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
1214 : I5 const& i5, I6 const& i6) const {
1215 : return i6 +
1216 : m_dim.N6 *
1217 : (i5 +
1218 : m_dim.N5 *
1219 : (i4 +
1220 : m_dim.N4 *
1221 : (i3 + m_dim.N3 *
1222 : (i2 + m_dim.N2 * (i1 + m_dim.N1 * (i0))))));
1223 : }
1224 :
1225 : // rank 8
1226 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
1227 : typename I5, typename I6, typename I7>
1228 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
1229 : I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
1230 : I5 const& i5, I6 const& i6, I7 const& i7) const {
1231 : return i7 +
1232 : m_dim.N7 *
1233 : (i6 +
1234 : m_dim.N6 *
1235 : (i5 +
1236 : m_dim.N5 *
1237 : (i4 +
1238 : m_dim.N4 *
1239 : (i3 +
1240 : m_dim.N3 *
1241 : (i2 + m_dim.N2 * (i1 + m_dim.N1 * (i0)))))));
1242 : }
1243 :
1244 : //----------------------------------------
1245 :
1246 : KOKKOS_INLINE_FUNCTION
1247 : constexpr array_layout layout() const {
1248 : constexpr auto r = dimension_type::rank;
1249 : return array_layout((r > 0 ? m_dim.N0 : KOKKOS_INVALID_INDEX),
1250 : (r > 1 ? m_dim.N1 : KOKKOS_INVALID_INDEX),
1251 : (r > 2 ? m_dim.N2 : KOKKOS_INVALID_INDEX),
1252 : (r > 3 ? m_dim.N3 : KOKKOS_INVALID_INDEX),
1253 : (r > 4 ? m_dim.N4 : KOKKOS_INVALID_INDEX),
1254 : (r > 5 ? m_dim.N5 : KOKKOS_INVALID_INDEX),
1255 : (r > 6 ? m_dim.N6 : KOKKOS_INVALID_INDEX),
1256 : (r > 7 ? m_dim.N7 : KOKKOS_INVALID_INDEX));
1257 : }
1258 :
1259 1378872 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_0() const {
1260 1378872 : return m_dim.N0;
1261 : }
1262 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_1() const {
1263 : return m_dim.N1;
1264 : }
1265 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_2() const {
1266 : return m_dim.N2;
1267 : }
1268 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_3() const {
1269 : return m_dim.N3;
1270 : }
1271 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_4() const {
1272 : return m_dim.N4;
1273 : }
1274 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_5() const {
1275 : return m_dim.N5;
1276 : }
1277 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_6() const {
1278 : return m_dim.N6;
1279 : }
1280 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_7() const {
1281 : return m_dim.N7;
1282 : }
1283 :
1284 : /* Cardinality of the domain index space */
1285 : KOKKOS_INLINE_FUNCTION
1286 : constexpr size_type size() const {
1287 : return size_type(m_dim.N0) * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 *
1288 : m_dim.N5 * m_dim.N6 * m_dim.N7;
1289 : }
1290 :
1291 : /* Span of the range space */
1292 : KOKKOS_INLINE_FUNCTION
1293 : constexpr size_type span() const {
1294 : return size_type(m_dim.N0) * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 *
1295 : m_dim.N5 * m_dim.N6 * m_dim.N7;
1296 : }
1297 :
1298 : KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const {
1299 : return true;
1300 : }
1301 :
1302 : /* Strides of dimensions */
1303 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_7() const { return 1; }
1304 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_6() const {
1305 : return m_dim.N7;
1306 : }
1307 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_5() const {
1308 : return m_dim.N7 * m_dim.N6;
1309 : }
1310 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_4() const {
1311 : return m_dim.N7 * m_dim.N6 * m_dim.N5;
1312 : }
1313 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_3() const {
1314 : return m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4;
1315 : }
1316 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_2() const {
1317 : return m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4 * m_dim.N3;
1318 : }
1319 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_1() const {
1320 : return m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4 * m_dim.N3 * m_dim.N2;
1321 : }
1322 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_0() const {
1323 : return m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4 * m_dim.N3 * m_dim.N2 *
1324 : m_dim.N1;
1325 : }
1326 :
1327 : // Fill the target unbounded array s with the stride.
1328 : // This method differs from stride() in that it does not write the total
1329 : // length to the last index of the array. Preconditions: s must be an array of
1330 : // dimension_type::rank elements
1331 : // The version of clang-format in CI fails from maybe_unused
1332 : // clang-format off
1333 : template <typename iType>
1334 : KOKKOS_INLINE_FUNCTION iType
1335 : stride_fill([[maybe_unused]] iType* const s) const {
1336 : size_type n = 1;
1337 : if constexpr (7 < dimension_type::rank) {
1338 : s[7] = n;
1339 : n *= m_dim.N7;
1340 : }
1341 : if constexpr (6 < dimension_type::rank) {
1342 : s[6] = n;
1343 : n *= m_dim.N6;
1344 : }
1345 : if constexpr (5 < dimension_type::rank) {
1346 : s[5] = n;
1347 : n *= m_dim.N5;
1348 : }
1349 : if constexpr (4 < dimension_type::rank) {
1350 : s[4] = n;
1351 : n *= m_dim.N4;
1352 : }
1353 : if constexpr (3 < dimension_type::rank) {
1354 : s[3] = n;
1355 : n *= m_dim.N3;
1356 : }
1357 : if constexpr (2 < dimension_type::rank) {
1358 : s[2] = n;
1359 : n *= m_dim.N2;
1360 : }
1361 : if constexpr (1 < dimension_type::rank) {
1362 : s[1] = n;
1363 : n *= m_dim.N1;
1364 : }
1365 : if constexpr (0 < dimension_type::rank) {
1366 : s[0] = n;
1367 : }
1368 : return n * m_dim.N0;
1369 : }
1370 : // clang-format on
1371 :
1372 : // Fill the target unbounded array s with the stride and the total spanned
1373 : // size. This method differs from stride_fill() in that it writes the total
1374 : // spanned size to the last index of the array. Preconditions: s must be an
1375 : // array of dimension_type::rank + 1 elements
1376 : template <typename iType>
1377 : KOKKOS_INLINE_FUNCTION void stride(iType* const s) const {
1378 : s[dimension_type::rank] = stride_fill(s);
1379 : }
1380 :
1381 : //----------------------------------------
1382 : // MSVC (16.5.5) + CUDA (10.2) did not generate the defaulted functions
1383 : // correct and errors out during compilation. Same for the other places where
1384 : // I changed this.
1385 :
1386 : #ifdef KOKKOS_IMPL_WINDOWS_CUDA
1387 : KOKKOS_FUNCTION ViewOffset() : m_dim(dimension_type()) {}
1388 : KOKKOS_FUNCTION ViewOffset(const ViewOffset& src) { m_dim = src.m_dim; }
1389 : KOKKOS_FUNCTION ViewOffset& operator=(const ViewOffset& src) {
1390 : m_dim = src.m_dim;
1391 : return *this;
1392 : }
1393 : #else
1394 :
1395 : ViewOffset() = default;
1396 : ViewOffset(const ViewOffset&) = default;
1397 : ViewOffset& operator=(const ViewOffset&) = default;
1398 : #endif
1399 :
1400 : template <unsigned TrivialScalarSize>
1401 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1402 : std::integral_constant<unsigned, TrivialScalarSize> const&,
1403 : Kokkos::LayoutRight const& arg_layout)
1404 : : m_dim(arg_layout.dimension[0], 0, 0, 0, 0, 0, 0, 0) {}
1405 :
1406 : template <class DimRHS>
1407 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1408 : const ViewOffset<DimRHS, Kokkos::LayoutRight, void>& rhs)
1409 : : m_dim(rhs.m_dim.N0, rhs.m_dim.N1, rhs.m_dim.N2, rhs.m_dim.N3,
1410 : rhs.m_dim.N4, rhs.m_dim.N5, rhs.m_dim.N6, rhs.m_dim.N7) {
1411 : static_assert(int(DimRHS::rank) == int(dimension_type::rank),
1412 : "ViewOffset assignment requires equal rank");
1413 : // Also requires equal static dimensions ...
1414 : }
1415 :
1416 : template <class DimRHS>
1417 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1418 : const ViewOffset<DimRHS, Kokkos::LayoutLeft, void>& rhs)
1419 : : m_dim(rhs.m_dim.N0, 0, 0, 0, 0, 0, 0, 0) {
1420 : static_assert((DimRHS::rank == 0 && dimension_type::rank == 0) ||
1421 : (DimRHS::rank == 1 && dimension_type::rank == 1),
1422 : "ViewOffset LayoutRight and LayoutLeft are only compatible "
1423 : "when rank <= 1");
1424 : }
1425 :
1426 : template <class DimRHS>
1427 : KOKKOS_INLINE_FUNCTION ViewOffset(
1428 : const ViewOffset<DimRHS, Kokkos::LayoutStride, void>& rhs)
1429 : : m_dim(rhs.m_dim.N0, 0, 0, 0, 0, 0, 0, 0) {}
1430 :
1431 : //----------------------------------------
1432 : // Subview construction
1433 :
1434 : template <class DimRHS>
1435 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1436 : const ViewOffset<DimRHS, Kokkos::LayoutRight, void>&,
1437 : const SubviewExtents<DimRHS::rank, dimension_type::rank>& sub)
1438 : : m_dim(sub.range_extent(0), 0, 0, 0, 0, 0, 0, 0) {
1439 : static_assert((0 == dimension_type::rank_dynamic) ||
1440 : (1 == dimension_type::rank &&
1441 : 1 == dimension_type::rank_dynamic && 1 <= DimRHS::rank),
1442 : "ViewOffset subview construction requires compatible rank");
1443 : }
1444 : };
1445 :
1446 : //----------------------------------------------------------------------------
1447 : // LayoutRight AND ( 1 < rank AND 0 < rank_dynamic ) : has padding / striding
1448 : template <class Dimension>
1449 : struct ViewOffset<
1450 : Dimension, Kokkos::LayoutRight,
1451 : std::enable_if_t<(1 < Dimension::rank && 0 < Dimension::rank_dynamic)>> {
1452 : using is_mapping_plugin = std::true_type;
1453 : using is_regular = std::true_type;
1454 :
1455 : using size_type = size_t;
1456 : using dimension_type = Dimension;
1457 : using array_layout = Kokkos::LayoutRight;
1458 :
1459 : dimension_type m_dim;
1460 : size_type m_stride;
1461 :
1462 : //----------------------------------------
1463 :
1464 : // rank 1
1465 : template <typename I0>
1466 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0) const {
1467 : return i0;
1468 : }
1469 :
1470 : // rank 2
1471 : template <typename I0, typename I1>
1472 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1473 : I1 const& i1) const {
1474 : return i1 + i0 * m_stride;
1475 : }
1476 :
1477 : // rank 3
1478 : template <typename I0, typename I1, typename I2>
1479 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1480 : I1 const& i1,
1481 : I2 const& i2) const {
1482 : return i2 + m_dim.N2 * (i1) + i0 * m_stride;
1483 : }
1484 :
1485 : // rank 4
1486 : template <typename I0, typename I1, typename I2, typename I3>
1487 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1488 : I1 const& i1,
1489 : I2 const& i2,
1490 : I3 const& i3) const {
1491 : return i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1)) + i0 * m_stride;
1492 : }
1493 :
1494 : // rank 5
1495 : template <typename I0, typename I1, typename I2, typename I3, typename I4>
1496 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1497 : I1 const& i1,
1498 : I2 const& i2,
1499 : I3 const& i3,
1500 : I4 const& i4) const {
1501 : return i4 + m_dim.N4 * (i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1))) +
1502 : i0 * m_stride;
1503 : }
1504 :
1505 : // rank 6
1506 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
1507 : typename I5>
1508 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
1509 : I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
1510 : I5 const& i5) const {
1511 : return i5 +
1512 : m_dim.N5 *
1513 : (i4 + m_dim.N4 * (i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1)))) +
1514 : i0 * m_stride;
1515 : }
1516 :
1517 : // rank 7
1518 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
1519 : typename I5, typename I6>
1520 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
1521 : I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
1522 : I5 const& i5, I6 const& i6) const {
1523 : return i6 +
1524 : m_dim.N6 *
1525 : (i5 + m_dim.N5 *
1526 : (i4 + m_dim.N4 *
1527 : (i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1))))) +
1528 : i0 * m_stride;
1529 : }
1530 :
1531 : // rank 8
1532 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
1533 : typename I5, typename I6, typename I7>
1534 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
1535 : I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
1536 : I5 const& i5, I6 const& i6, I7 const& i7) const {
1537 : return i7 +
1538 : m_dim.N7 *
1539 : (i6 +
1540 : m_dim.N6 *
1541 : (i5 +
1542 : m_dim.N5 *
1543 : (i4 + m_dim.N4 *
1544 : (i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1)))))) +
1545 : i0 * m_stride;
1546 : }
1547 :
1548 : //----------------------------------------
1549 :
1550 : KOKKOS_INLINE_FUNCTION
1551 : constexpr array_layout layout() const {
1552 : constexpr auto r = dimension_type::rank;
1553 : array_layout l((r > 0 ? m_dim.N0 : KOKKOS_INVALID_INDEX),
1554 : (r > 1 ? m_dim.N1 : KOKKOS_INVALID_INDEX),
1555 : (r > 2 ? m_dim.N2 : KOKKOS_INVALID_INDEX),
1556 : (r > 3 ? m_dim.N3 : KOKKOS_INVALID_INDEX),
1557 : (r > 4 ? m_dim.N4 : KOKKOS_INVALID_INDEX),
1558 : (r > 5 ? m_dim.N5 : KOKKOS_INVALID_INDEX),
1559 : (r > 6 ? m_dim.N6 : KOKKOS_INVALID_INDEX),
1560 : (r > 7 ? m_dim.N7 : KOKKOS_INVALID_INDEX));
1561 : // Without span_is_contiguous Sacado hidden dimensions get messed up
1562 : l.stride = span_is_contiguous() ? KOKKOS_IMPL_CTOR_DEFAULT_ARG : m_stride;
1563 : return l;
1564 : }
1565 :
1566 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_0() const {
1567 : return m_dim.N0;
1568 : }
1569 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_1() const {
1570 : return m_dim.N1;
1571 : }
1572 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_2() const {
1573 : return m_dim.N2;
1574 : }
1575 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_3() const {
1576 : return m_dim.N3;
1577 : }
1578 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_4() const {
1579 : return m_dim.N4;
1580 : }
1581 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_5() const {
1582 : return m_dim.N5;
1583 : }
1584 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_6() const {
1585 : return m_dim.N6;
1586 : }
1587 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_7() const {
1588 : return m_dim.N7;
1589 : }
1590 :
1591 : /* Cardinality of the domain index space */
1592 : KOKKOS_INLINE_FUNCTION
1593 : constexpr size_type size() const {
1594 : return size_type(m_dim.N0) * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 *
1595 : m_dim.N5 * m_dim.N6 * m_dim.N7;
1596 : }
1597 :
1598 : /* Span of the range space */
1599 : KOKKOS_INLINE_FUNCTION
1600 : constexpr size_type span() const {
1601 : return size() > 0 ? size_type(m_dim.N0) * m_stride : 0;
1602 : }
1603 :
1604 : KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const {
1605 : return m_stride == static_cast<size_type>(m_dim.N7) * m_dim.N6 * m_dim.N5 *
1606 : m_dim.N4 * m_dim.N3 * m_dim.N2 * m_dim.N1;
1607 : }
1608 :
1609 : /* Strides of dimensions */
1610 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_7() const { return 1; }
1611 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_6() const {
1612 : return m_dim.N7;
1613 : }
1614 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_5() const {
1615 : return static_cast<size_type>(m_dim.N7) * m_dim.N6;
1616 : }
1617 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_4() const {
1618 : return static_cast<size_type>(m_dim.N7) * m_dim.N6 * m_dim.N5;
1619 : }
1620 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_3() const {
1621 : return static_cast<size_type>(m_dim.N7) * m_dim.N6 * m_dim.N5 * m_dim.N4;
1622 : }
1623 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_2() const {
1624 : return static_cast<size_type>(m_dim.N7) * m_dim.N6 * m_dim.N5 * m_dim.N4 *
1625 : m_dim.N3;
1626 : }
1627 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_1() const {
1628 : return static_cast<size_type>(m_dim.N7) * m_dim.N6 * m_dim.N5 * m_dim.N4 *
1629 : m_dim.N3 * m_dim.N2;
1630 : }
1631 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_0() const {
1632 : return m_stride;
1633 : }
1634 :
1635 : // Fill the target unbounded array s with the stride.
1636 : // This method differs from stride() in that it does not write the total
1637 : // length to the last index of the array. Preconditions: s must be an array of
1638 : // dimension_type::rank elements
1639 : // The version of clang-format in CI fails from maybe_unused
1640 : // clang-format off
1641 : template <typename iType>
1642 : KOKKOS_INLINE_FUNCTION iType
1643 : stride_fill([[maybe_unused]] iType* const s) const {
1644 : size_type n = 1;
1645 : if constexpr (7 < dimension_type::rank) {
1646 : s[7] = n;
1647 : n *= m_dim.N7;
1648 : }
1649 : if constexpr (6 < dimension_type::rank) {
1650 : s[6] = n;
1651 : n *= m_dim.N6;
1652 : }
1653 : if constexpr (5 < dimension_type::rank) {
1654 : s[5] = n;
1655 : n *= m_dim.N5;
1656 : }
1657 : if constexpr (4 < dimension_type::rank) {
1658 : s[4] = n;
1659 : n *= m_dim.N4;
1660 : }
1661 : if constexpr (3 < dimension_type::rank) {
1662 : s[3] = n;
1663 : n *= m_dim.N3;
1664 : }
1665 : if constexpr (2 < dimension_type::rank) {
1666 : s[2] = n;
1667 : n *= m_dim.N2;
1668 : }
1669 : if constexpr (1 < dimension_type::rank) {
1670 : s[1] = n;
1671 : }
1672 : if constexpr (0 < dimension_type::rank) {
1673 : s[0] = m_stride;
1674 : }
1675 : return m_stride * m_dim.N0;
1676 : }
1677 : // clang-format on
1678 :
1679 : // Fill the target unbounded array s with the stride and the total spanned
1680 : // size. This method differs from stride_fill() in that it writes the total
1681 : // spanned size to the last index of the array. Preconditions: s must be an
1682 : // array of dimension_type::rank + 1 elements
1683 : template <typename iType>
1684 : KOKKOS_INLINE_FUNCTION void stride(iType* const s) const {
1685 : s[dimension_type::rank] = stride_fill(s);
1686 : }
1687 :
1688 : //----------------------------------------
1689 :
1690 : private:
1691 : template <unsigned TrivialScalarSize>
1692 : struct Padding {
1693 : enum {
1694 : div = TrivialScalarSize == 0
1695 : ? 0
1696 : : Kokkos::Impl::MEMORY_ALIGNMENT /
1697 : (TrivialScalarSize ? TrivialScalarSize : 1)
1698 : };
1699 : enum {
1700 : mod = TrivialScalarSize == 0
1701 : ? 0
1702 : : Kokkos::Impl::MEMORY_ALIGNMENT %
1703 : (TrivialScalarSize ? TrivialScalarSize : 1)
1704 : };
1705 :
1706 : // If memory alignment is a multiple of the trivial scalar size then attempt
1707 : // to align.
1708 : enum { align = 0 != TrivialScalarSize && 0 == mod ? div : 0 };
1709 : enum {
1710 : div_ok = (div != 0) ? div : 1
1711 : }; // To valid modulo zero in constexpr
1712 :
1713 : KOKKOS_INLINE_FUNCTION
1714 : static constexpr size_t stride(size_t const N) {
1715 : return ((align != 0) &&
1716 : ((static_cast<size_t>(Kokkos::Impl::MEMORY_ALIGNMENT_THRESHOLD) *
1717 : align) < N) &&
1718 : ((N % div_ok) != 0))
1719 : ? N + align - (N % div_ok)
1720 : : N;
1721 : }
1722 : };
1723 :
1724 : public:
1725 : // MSVC (16.5.5) + CUDA (10.2) did not generate the defaulted functions
1726 : // correct and errors out during compilation. Same for the other places where
1727 : // I changed this.
1728 :
1729 : #ifdef KOKKOS_IMPL_WINDOWS_CUDA
1730 : KOKKOS_FUNCTION ViewOffset() : m_dim(dimension_type()), m_stride(0) {}
1731 : KOKKOS_FUNCTION ViewOffset(const ViewOffset& src) {
1732 : m_dim = src.m_dim;
1733 : m_stride = src.m_stride;
1734 : }
1735 : KOKKOS_FUNCTION ViewOffset& operator=(const ViewOffset& src) {
1736 : m_dim = src.m_dim;
1737 : m_stride = src.m_stride;
1738 : return *this;
1739 : }
1740 : #else
1741 :
1742 : ViewOffset() = default;
1743 : ViewOffset(const ViewOffset&) = default;
1744 : ViewOffset& operator=(const ViewOffset&) = default;
1745 : #endif
1746 :
1747 : /* Enable padding for trivial scalar types with non-zero trivial scalar size.
1748 : */
1749 :
1750 : private:
1751 : template <unsigned TrivialScalarSize>
1752 : KOKKOS_FUNCTION constexpr size_type compute_stride(
1753 : const Kokkos::LayoutRight& arg_layout) {
1754 : if (arg_layout.stride != KOKKOS_IMPL_CTOR_DEFAULT_ARG)
1755 : return arg_layout.stride;
1756 : size_type value = m_dim.N1;
1757 : if constexpr (dimension_type::rank > 2) value *= m_dim.N2;
1758 : if constexpr (dimension_type::rank > 3) value *= m_dim.N3;
1759 : if constexpr (dimension_type::rank > 4) value *= m_dim.N4;
1760 : if constexpr (dimension_type::rank > 5) value *= m_dim.N5;
1761 : if constexpr (dimension_type::rank > 6) value *= m_dim.N6;
1762 : if constexpr (dimension_type::rank > 7) value *= m_dim.N7;
1763 : return Padding<TrivialScalarSize>::stride(value);
1764 : }
1765 :
1766 : public:
1767 : template <unsigned TrivialScalarSize>
1768 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1769 : std::integral_constant<unsigned, TrivialScalarSize> const&,
1770 : Kokkos::LayoutRight const& arg_layout)
1771 : : m_dim(arg_layout.dimension[0], arg_layout.dimension[1],
1772 : arg_layout.dimension[2], arg_layout.dimension[3],
1773 : arg_layout.dimension[4], arg_layout.dimension[5],
1774 : arg_layout.dimension[6], arg_layout.dimension[7]),
1775 : m_stride(compute_stride<TrivialScalarSize>(arg_layout)) {}
1776 :
1777 : template <class DimRHS>
1778 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1779 : const ViewOffset<DimRHS, Kokkos::LayoutRight, void>& rhs)
1780 : : m_dim(rhs.m_dim.N0, rhs.m_dim.N1, rhs.m_dim.N2, rhs.m_dim.N3,
1781 : rhs.m_dim.N4, rhs.m_dim.N5, rhs.m_dim.N6, rhs.m_dim.N7),
1782 : m_stride(rhs.stride_0()) {
1783 : static_assert(int(DimRHS::rank) == int(dimension_type::rank),
1784 : "ViewOffset assignment requires equal rank");
1785 : // Also requires equal static dimensions ...
1786 : }
1787 :
1788 : template <class DimRHS>
1789 : KOKKOS_INLINE_FUNCTION ViewOffset(
1790 : const ViewOffset<DimRHS, Kokkos::LayoutStride, void>& rhs)
1791 : : m_dim(rhs.m_dim.N0, rhs.m_dim.N1, rhs.m_dim.N2, rhs.m_dim.N3,
1792 : rhs.m_dim.N4, rhs.m_dim.N5, rhs.m_dim.N6, rhs.m_dim.N7),
1793 : m_stride(rhs.stride_0()) {
1794 : if (((dimension_type::rank == 2)
1795 : ? rhs.m_stride.S1
1796 : : ((dimension_type::rank == 3)
1797 : ? rhs.m_stride.S2
1798 : : ((dimension_type::rank == 4)
1799 : ? rhs.m_stride.S3
1800 : : ((dimension_type::rank == 5)
1801 : ? rhs.m_stride.S4
1802 : : ((dimension_type::rank == 6)
1803 : ? rhs.m_stride.S5
1804 : : ((dimension_type::rank == 7)
1805 : ? rhs.m_stride.S6
1806 : : rhs.m_stride.S7)))))) != 1) {
1807 : Kokkos::abort(
1808 : "Kokkos::Impl::ViewOffset assignment of LayoutRight from "
1809 : "LayoutStride requires right-most stride == 1");
1810 : }
1811 : }
1812 :
1813 : //----------------------------------------
1814 : // Subview construction
1815 : // Last dimension must be non-zero
1816 :
1817 : template <class DimRHS>
1818 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1819 : const ViewOffset<DimRHS, Kokkos::LayoutRight, void>& rhs,
1820 : const SubviewExtents<DimRHS::rank, dimension_type::rank>& sub)
1821 : : m_dim(sub.range_extent(0), sub.range_extent(1), sub.range_extent(2),
1822 : sub.range_extent(3), sub.range_extent(4), sub.range_extent(5),
1823 : sub.range_extent(6), sub.range_extent(7)),
1824 : m_stride(
1825 : 0 == sub.range_index(0)
1826 : ? rhs.stride_0()
1827 : : (1 == sub.range_index(0)
1828 : ? rhs.stride_1()
1829 : : (2 == sub.range_index(0)
1830 : ? rhs.stride_2()
1831 : : (3 == sub.range_index(0)
1832 : ? rhs.stride_3()
1833 : : (4 == sub.range_index(0)
1834 : ? rhs.stride_4()
1835 : : (5 == sub.range_index(0)
1836 : ? rhs.stride_5()
1837 : : (6 == sub.range_index(0)
1838 : ? rhs.stride_6()
1839 : : 0))))))) {
1840 : /* // This subview must be 2 == rank and 2 == rank_dynamic
1841 : // due to only having stride #0.
1842 : // The source dimension #0 must be non-zero for stride-one leading
1843 : dimension.
1844 : // At most subsequent dimension can be non-zero.
1845 :
1846 : static_assert( (( 2 == dimension_type::rank ) &&
1847 : ( 2 <= DimRHS::rank )) ||
1848 : ()
1849 : , "ViewOffset subview construction requires compatible
1850 : rank" );
1851 : */
1852 : }
1853 : };
1854 :
1855 : //----------------------------------------------------------------------------
1856 : /* Strided array layout only makes sense for 0 < rank */
1857 : /* rank = 0 included for DynRankView case */
1858 :
1859 : template <unsigned Rank>
1860 : struct ViewStride;
1861 :
1862 : template <>
1863 : struct ViewStride<0> {
1864 : static constexpr size_t S0 = 0, S1 = 0, S2 = 0, S3 = 0, S4 = 0, S5 = 0,
1865 : S6 = 0, S7 = 0;
1866 :
1867 : ViewStride() = default;
1868 : ViewStride(const ViewStride&) = default;
1869 : ViewStride& operator=(const ViewStride&) = default;
1870 :
1871 : KOKKOS_INLINE_FUNCTION
1872 : constexpr ViewStride(size_t, size_t, size_t, size_t, size_t, size_t, size_t,
1873 : size_t) {}
1874 : };
1875 :
1876 : template <>
1877 : struct ViewStride<1> {
1878 : size_t S0;
1879 : static constexpr size_t S1 = 0, S2 = 0, S3 = 0, S4 = 0, S5 = 0, S6 = 0,
1880 : S7 = 0;
1881 :
1882 : ViewStride() = default;
1883 : ViewStride(const ViewStride&) = default;
1884 : ViewStride& operator=(const ViewStride&) = default;
1885 :
1886 : KOKKOS_INLINE_FUNCTION
1887 : constexpr ViewStride(size_t aS0, size_t, size_t, size_t, size_t, size_t,
1888 : size_t, size_t)
1889 : : S0(aS0) {}
1890 : };
1891 :
1892 : template <>
1893 : struct ViewStride<2> {
1894 : size_t S0, S1;
1895 : static constexpr size_t S2 = 0, S3 = 0, S4 = 0, S5 = 0, S6 = 0, S7 = 0;
1896 :
1897 : ViewStride() = default;
1898 : ViewStride(const ViewStride&) = default;
1899 : ViewStride& operator=(const ViewStride&) = default;
1900 :
1901 : KOKKOS_INLINE_FUNCTION
1902 : constexpr ViewStride(size_t aS0, size_t aS1, size_t, size_t, size_t, size_t,
1903 : size_t, size_t)
1904 : : S0(aS0), S1(aS1) {}
1905 : };
1906 :
1907 : template <>
1908 : struct ViewStride<3> {
1909 : size_t S0, S1, S2;
1910 : static constexpr size_t S3 = 0, S4 = 0, S5 = 0, S6 = 0, S7 = 0;
1911 :
1912 : ViewStride() = default;
1913 : ViewStride(const ViewStride&) = default;
1914 : ViewStride& operator=(const ViewStride&) = default;
1915 :
1916 : KOKKOS_INLINE_FUNCTION
1917 : constexpr ViewStride(size_t aS0, size_t aS1, size_t aS2, size_t, size_t,
1918 : size_t, size_t, size_t)
1919 : : S0(aS0), S1(aS1), S2(aS2) {}
1920 : };
1921 :
1922 : template <>
1923 : struct ViewStride<4> {
1924 : size_t S0, S1, S2, S3;
1925 : static constexpr size_t S4 = 0, S5 = 0, S6 = 0, S7 = 0;
1926 :
1927 : ViewStride() = default;
1928 : ViewStride(const ViewStride&) = default;
1929 : ViewStride& operator=(const ViewStride&) = default;
1930 :
1931 : KOKKOS_INLINE_FUNCTION
1932 : constexpr ViewStride(size_t aS0, size_t aS1, size_t aS2, size_t aS3, size_t,
1933 : size_t, size_t, size_t)
1934 : : S0(aS0), S1(aS1), S2(aS2), S3(aS3) {}
1935 : };
1936 :
1937 : template <>
1938 : struct ViewStride<5> {
1939 : size_t S0, S1, S2, S3, S4;
1940 : static constexpr size_t S5 = 0, S6 = 0, S7 = 0;
1941 :
1942 : ViewStride() = default;
1943 : ViewStride(const ViewStride&) = default;
1944 : ViewStride& operator=(const ViewStride&) = default;
1945 :
1946 : KOKKOS_INLINE_FUNCTION
1947 : constexpr ViewStride(size_t aS0, size_t aS1, size_t aS2, size_t aS3,
1948 : size_t aS4, size_t, size_t, size_t)
1949 : : S0(aS0), S1(aS1), S2(aS2), S3(aS3), S4(aS4) {}
1950 : };
1951 :
1952 : template <>
1953 : struct ViewStride<6> {
1954 : size_t S0, S1, S2, S3, S4, S5;
1955 : static constexpr size_t S6 = 0, S7 = 0;
1956 :
1957 : ViewStride() = default;
1958 : ViewStride(const ViewStride&) = default;
1959 : ViewStride& operator=(const ViewStride&) = default;
1960 :
1961 : KOKKOS_INLINE_FUNCTION
1962 : constexpr ViewStride(size_t aS0, size_t aS1, size_t aS2, size_t aS3,
1963 : size_t aS4, size_t aS5, size_t, size_t)
1964 : : S0(aS0), S1(aS1), S2(aS2), S3(aS3), S4(aS4), S5(aS5) {}
1965 : };
1966 :
1967 : template <>
1968 : struct ViewStride<7> {
1969 : size_t S0, S1, S2, S3, S4, S5, S6;
1970 : static constexpr size_t S7 = 0;
1971 :
1972 : ViewStride() = default;
1973 : ViewStride(const ViewStride&) = default;
1974 : ViewStride& operator=(const ViewStride&) = default;
1975 :
1976 : KOKKOS_INLINE_FUNCTION
1977 : constexpr ViewStride(size_t aS0, size_t aS1, size_t aS2, size_t aS3,
1978 : size_t aS4, size_t aS5, size_t aS6, size_t)
1979 : : S0(aS0), S1(aS1), S2(aS2), S3(aS3), S4(aS4), S5(aS5), S6(aS6) {}
1980 : };
1981 :
1982 : template <>
1983 : struct ViewStride<8> {
1984 : size_t S0, S1, S2, S3, S4, S5, S6, S7;
1985 :
1986 : ViewStride() = default;
1987 : ViewStride(const ViewStride&) = default;
1988 : ViewStride& operator=(const ViewStride&) = default;
1989 :
1990 : KOKKOS_INLINE_FUNCTION
1991 : constexpr ViewStride(size_t aS0, size_t aS1, size_t aS2, size_t aS3,
1992 : size_t aS4, size_t aS5, size_t aS6, size_t aS7)
1993 : : S0(aS0),
1994 : S1(aS1),
1995 : S2(aS2),
1996 : S3(aS3),
1997 : S4(aS4),
1998 : S5(aS5),
1999 : S6(aS6),
2000 : S7(aS7) {}
2001 : };
2002 :
2003 : template <class Dimension>
2004 : struct ViewOffset<Dimension, Kokkos::LayoutStride, void> {
2005 : private:
2006 : using stride_type = ViewStride<Dimension::rank>;
2007 :
2008 : public:
2009 : using is_mapping_plugin = std::true_type;
2010 : using is_regular = std::true_type;
2011 :
2012 : using size_type = size_t;
2013 : using dimension_type = Dimension;
2014 : using array_layout = Kokkos::LayoutStride;
2015 :
2016 : dimension_type m_dim;
2017 : stride_type m_stride;
2018 :
2019 : //----------------------------------------
2020 :
2021 : // rank 1
2022 : template <typename I0>
2023 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0) const {
2024 : return i0 * m_stride.S0;
2025 : }
2026 :
2027 : // rank 2
2028 : template <typename I0, typename I1>
2029 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
2030 : I1 const& i1) const {
2031 : return i0 * m_stride.S0 + i1 * m_stride.S1;
2032 : }
2033 :
2034 : // rank 3
2035 : template <typename I0, typename I1, typename I2>
2036 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
2037 : I1 const& i1,
2038 : I2 const& i2) const {
2039 : return i0 * m_stride.S0 + i1 * m_stride.S1 + i2 * m_stride.S2;
2040 : }
2041 :
2042 : // rank 4
2043 : template <typename I0, typename I1, typename I2, typename I3>
2044 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
2045 : I1 const& i1,
2046 : I2 const& i2,
2047 : I3 const& i3) const {
2048 : return i0 * m_stride.S0 + i1 * m_stride.S1 + i2 * m_stride.S2 +
2049 : i3 * m_stride.S3;
2050 : }
2051 :
2052 : // rank 5
2053 : template <typename I0, typename I1, typename I2, typename I3, typename I4>
2054 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
2055 : I1 const& i1,
2056 : I2 const& i2,
2057 : I3 const& i3,
2058 : I4 const& i4) const {
2059 : return i0 * m_stride.S0 + i1 * m_stride.S1 + i2 * m_stride.S2 +
2060 : i3 * m_stride.S3 + i4 * m_stride.S4;
2061 : }
2062 :
2063 : // rank 6
2064 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
2065 : typename I5>
2066 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
2067 : I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
2068 : I5 const& i5) const {
2069 : return i0 * m_stride.S0 + i1 * m_stride.S1 + i2 * m_stride.S2 +
2070 : i3 * m_stride.S3 + i4 * m_stride.S4 + i5 * m_stride.S5;
2071 : }
2072 :
2073 : // rank 7
2074 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
2075 : typename I5, typename I6>
2076 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
2077 : I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
2078 : I5 const& i5, I6 const& i6) const {
2079 : return i0 * m_stride.S0 + i1 * m_stride.S1 + i2 * m_stride.S2 +
2080 : i3 * m_stride.S3 + i4 * m_stride.S4 + i5 * m_stride.S5 +
2081 : i6 * m_stride.S6;
2082 : }
2083 :
2084 : // rank 8
2085 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
2086 : typename I5, typename I6, typename I7>
2087 : KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
2088 : I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
2089 : I5 const& i5, I6 const& i6, I7 const& i7) const {
2090 : return i0 * m_stride.S0 + i1 * m_stride.S1 + i2 * m_stride.S2 +
2091 : i3 * m_stride.S3 + i4 * m_stride.S4 + i5 * m_stride.S5 +
2092 : i6 * m_stride.S6 + i7 * m_stride.S7;
2093 : }
2094 :
2095 : //----------------------------------------
2096 :
2097 : KOKKOS_INLINE_FUNCTION
2098 : constexpr array_layout layout() const {
2099 : constexpr auto r = dimension_type::rank;
2100 : return array_layout((r > 0 ? m_dim.N0 : KOKKOS_INVALID_INDEX), m_stride.S0,
2101 : (r > 1 ? m_dim.N1 : KOKKOS_INVALID_INDEX), m_stride.S1,
2102 : (r > 2 ? m_dim.N2 : KOKKOS_INVALID_INDEX), m_stride.S2,
2103 : (r > 3 ? m_dim.N3 : KOKKOS_INVALID_INDEX), m_stride.S3,
2104 : (r > 4 ? m_dim.N4 : KOKKOS_INVALID_INDEX), m_stride.S4,
2105 : (r > 5 ? m_dim.N5 : KOKKOS_INVALID_INDEX), m_stride.S5,
2106 : (r > 6 ? m_dim.N6 : KOKKOS_INVALID_INDEX), m_stride.S6,
2107 : (r > 7 ? m_dim.N7 : KOKKOS_INVALID_INDEX), m_stride.S7);
2108 : }
2109 :
2110 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_0() const {
2111 : return m_dim.N0;
2112 : }
2113 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_1() const {
2114 : return m_dim.N1;
2115 : }
2116 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_2() const {
2117 : return m_dim.N2;
2118 : }
2119 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_3() const {
2120 : return m_dim.N3;
2121 : }
2122 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_4() const {
2123 : return m_dim.N4;
2124 : }
2125 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_5() const {
2126 : return m_dim.N5;
2127 : }
2128 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_6() const {
2129 : return m_dim.N6;
2130 : }
2131 : KOKKOS_INLINE_FUNCTION constexpr size_type dimension_7() const {
2132 : return m_dim.N7;
2133 : }
2134 :
2135 : /* Cardinality of the domain index space */
2136 : KOKKOS_INLINE_FUNCTION
2137 : constexpr size_type size() const {
2138 : return dimension_type::rank == 0
2139 : ? 1
2140 : : size_type(m_dim.N0) * m_dim.N1 * m_dim.N2 * m_dim.N3 *
2141 : m_dim.N4 * m_dim.N5 * m_dim.N6 * m_dim.N7;
2142 : }
2143 :
2144 : private:
2145 : KOKKOS_INLINE_FUNCTION
2146 : static constexpr size_type Max(size_type lhs, size_type rhs) {
2147 : return lhs < rhs ? rhs : lhs;
2148 : }
2149 :
2150 : public:
2151 : /* Span of the range space, largest stride * dimension */
2152 : KOKKOS_INLINE_FUNCTION
2153 : constexpr size_type span() const {
2154 : return dimension_type::rank == 0
2155 : ? 1
2156 : : (size() == size_type(0)
2157 : ? size_type(0)
2158 : : Max(m_dim.N0 * m_stride.S0,
2159 : Max(m_dim.N1 * m_stride.S1,
2160 : Max(m_dim.N2 * m_stride.S2,
2161 : Max(m_dim.N3 * m_stride.S3,
2162 : Max(m_dim.N4 * m_stride.S4,
2163 : Max(m_dim.N5 * m_stride.S5,
2164 : Max(m_dim.N6 * m_stride.S6,
2165 : m_dim.N7 *
2166 : m_stride.S7))))))));
2167 : }
2168 :
2169 : KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const {
2170 : return span() == size();
2171 : }
2172 :
2173 : /* Strides of dimensions */
2174 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_0() const {
2175 : return m_stride.S0;
2176 : }
2177 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_1() const {
2178 : return m_stride.S1;
2179 : }
2180 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_2() const {
2181 : return m_stride.S2;
2182 : }
2183 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_3() const {
2184 : return m_stride.S3;
2185 : }
2186 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_4() const {
2187 : return m_stride.S4;
2188 : }
2189 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_5() const {
2190 : return m_stride.S5;
2191 : }
2192 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_6() const {
2193 : return m_stride.S6;
2194 : }
2195 : KOKKOS_INLINE_FUNCTION constexpr size_type stride_7() const {
2196 : return m_stride.S7;
2197 : }
2198 :
2199 : // Fill the target unbounded array s with the stride.
2200 : // This method differs from stride() in that it does not write the total
2201 : // length to the last index of the array. Preconditions: s must be an array of
2202 : // dimension_type::rank elements
2203 : // The version of clang-format in CI fails from maybe_unused
2204 : // clang-format off
2205 : template <typename iType>
2206 : KOKKOS_INLINE_FUNCTION iType
2207 : stride_fill([[maybe_unused]] iType* const s) const {
2208 : if constexpr (0 < dimension_type::rank) {
2209 : s[0] = m_stride.S0;
2210 : }
2211 : if constexpr (1 < dimension_type::rank) {
2212 : s[1] = m_stride.S1;
2213 : }
2214 : if constexpr (2 < dimension_type::rank) {
2215 : s[2] = m_stride.S2;
2216 : }
2217 : if constexpr (3 < dimension_type::rank) {
2218 : s[3] = m_stride.S3;
2219 : }
2220 : if constexpr (4 < dimension_type::rank) {
2221 : s[4] = m_stride.S4;
2222 : }
2223 : if constexpr (5 < dimension_type::rank) {
2224 : s[5] = m_stride.S5;
2225 : }
2226 : if constexpr (6 < dimension_type::rank) {
2227 : s[6] = m_stride.S6;
2228 : }
2229 : if constexpr (7 < dimension_type::rank) {
2230 : s[7] = m_stride.S7;
2231 : }
2232 : return span();
2233 : }
2234 : // clang-format on
2235 :
2236 : // Fill the target unbounded array s with the stride and the total spanned
2237 : // size. This method differs from stride_fill() in that it writes the total
2238 : // spanned size to the last index of the array. Preconditions: s must be an
2239 : // array of dimension_type::rank + 1 elements
2240 : template <typename iType>
2241 : KOKKOS_INLINE_FUNCTION void stride(iType* const s) const {
2242 : s[dimension_type::rank] = stride_fill(s);
2243 : }
2244 :
2245 : //----------------------------------------
2246 : // MSVC (16.5.5) + CUDA (10.2) did not generate the defaulted functions
2247 : // correct and errors out during compilation. Same for the other places where
2248 : // I changed this.
2249 :
2250 : #ifdef KOKKOS_IMPL_WINDOWS_CUDA
2251 : KOKKOS_FUNCTION ViewOffset()
2252 : : m_dim(dimension_type()), m_stride(stride_type()) {}
2253 : KOKKOS_FUNCTION ViewOffset(const ViewOffset& src) {
2254 : m_dim = src.m_dim;
2255 : m_stride = src.m_stride;
2256 : }
2257 : KOKKOS_FUNCTION ViewOffset& operator=(const ViewOffset& src) {
2258 : m_dim = src.m_dim;
2259 : m_stride = src.m_stride;
2260 : return *this;
2261 : }
2262 : #else
2263 :
2264 : ViewOffset() = default;
2265 : ViewOffset(const ViewOffset&) = default;
2266 : ViewOffset& operator=(const ViewOffset&) = default;
2267 : #endif
2268 :
2269 : KOKKOS_INLINE_FUNCTION
2270 : constexpr ViewOffset(std::integral_constant<unsigned, 0> const&,
2271 : Kokkos::LayoutStride const& rhs)
2272 : : m_dim(rhs.dimension[0], rhs.dimension[1], rhs.dimension[2],
2273 : rhs.dimension[3], rhs.dimension[4], rhs.dimension[5],
2274 : rhs.dimension[6], rhs.dimension[7]),
2275 : m_stride(rhs.stride[0], rhs.stride[1], rhs.stride[2], rhs.stride[3],
2276 : rhs.stride[4], rhs.stride[5], rhs.stride[6], rhs.stride[7]) {}
2277 :
2278 : template <class DimRHS, class LayoutRHS>
2279 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
2280 : const ViewOffset<DimRHS, LayoutRHS, void>& rhs)
2281 : : m_dim(rhs.m_dim.N0, rhs.m_dim.N1, rhs.m_dim.N2, rhs.m_dim.N3,
2282 : rhs.m_dim.N4, rhs.m_dim.N5, rhs.m_dim.N6, rhs.m_dim.N7),
2283 : m_stride(rhs.stride_0(), rhs.stride_1(), rhs.stride_2(), rhs.stride_3(),
2284 : rhs.stride_4(), rhs.stride_5(), rhs.stride_6(),
2285 : rhs.stride_7()) {
2286 : static_assert(int(DimRHS::rank) == int(dimension_type::rank),
2287 : "ViewOffset assignment requires equal rank");
2288 : // Also requires equal static dimensions ...
2289 : }
2290 :
2291 : //----------------------------------------
2292 : // Subview construction
2293 :
2294 : private:
2295 : template <class DimRHS, class LayoutRHS>
2296 : KOKKOS_INLINE_FUNCTION static constexpr size_t stride(
2297 : unsigned r, const ViewOffset<DimRHS, LayoutRHS, void>& rhs) {
2298 : return r > 7
2299 : ? 0
2300 : : (r == 0
2301 : ? rhs.stride_0()
2302 : : (r == 1
2303 : ? rhs.stride_1()
2304 : : (r == 2
2305 : ? rhs.stride_2()
2306 : : (r == 3
2307 : ? rhs.stride_3()
2308 : : (r == 4
2309 : ? rhs.stride_4()
2310 : : (r == 5
2311 : ? rhs.stride_5()
2312 : : (r == 6
2313 : ? rhs.stride_6()
2314 : : rhs.stride_7())))))));
2315 : }
2316 :
2317 : public:
2318 : template <class DimRHS, class LayoutRHS>
2319 : KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
2320 : const ViewOffset<DimRHS, LayoutRHS, void>& rhs,
2321 : const SubviewExtents<DimRHS::rank, dimension_type::rank>& sub)
2322 : // range_extent(r) returns 0 when dimension_type::rank <= r
2323 : : m_dim(sub.range_extent(0), sub.range_extent(1), sub.range_extent(2),
2324 : sub.range_extent(3), sub.range_extent(4), sub.range_extent(5),
2325 : sub.range_extent(6), sub.range_extent(7))
2326 : // range_index(r) returns ~0u when dimension_type::rank <= r
2327 : ,
2328 : m_stride(
2329 : stride(sub.range_index(0), rhs), stride(sub.range_index(1), rhs),
2330 : stride(sub.range_index(2), rhs), stride(sub.range_index(3), rhs),
2331 : stride(sub.range_index(4), rhs), stride(sub.range_index(5), rhs),
2332 : stride(sub.range_index(6), rhs), stride(sub.range_index(7), rhs)) {}
2333 : };
2334 :
2335 : } // namespace Impl
2336 : } // namespace Kokkos
2337 :
2338 : //----------------------------------------------------------------------------
2339 : //----------------------------------------------------------------------------
2340 :
2341 : namespace Kokkos {
2342 : namespace Impl {
2343 :
2344 : /** \brief ViewDataHandle provides the type of the 'data handle' which the view
2345 : * uses to access data with the [] operator. It also provides
2346 : * an allocate function and a function to extract a raw ptr from the
2347 : * data handle. ViewDataHandle also defines an enum ReferenceAble which
2348 : * specifies whether references/pointers to elements can be taken and a
2349 : * 'return_type' which is what the view operators will give back.
2350 : * Specialisation of this object allows three things depending
2351 : * on ViewTraits and compiler options:
2352 : * (i) Use special allocator (e.g. huge pages/small pages and pinned
2353 : * memory) (ii) Use special data handle type (e.g. add Cuda Texture Object)
2354 : * (iii) Use special access intrinsics (e.g. texture fetch and
2355 : * non-caching loads)
2356 : */
2357 : template <class Traits, class Enable = void>
2358 : struct ViewDataHandle {
2359 : using value_type = typename Traits::value_type;
2360 : using handle_type = typename Traits::value_type*;
2361 : using return_type = typename Traits::value_type&;
2362 : using track_type = Kokkos::Impl::SharedAllocationTracker;
2363 :
2364 : KOKKOS_INLINE_FUNCTION
2365 : static handle_type assign(value_type* arg_data_ptr,
2366 : track_type const& /*arg_tracker*/) {
2367 : return handle_type(arg_data_ptr);
2368 : }
2369 :
2370 : KOKKOS_INLINE_FUNCTION
2371 : static handle_type assign(handle_type const arg_data_ptr, size_t offset) {
2372 : return handle_type(arg_data_ptr + offset);
2373 : }
2374 : };
2375 :
2376 : template <class Traits>
2377 : struct ViewDataHandle<
2378 : Traits,
2379 : std::enable_if_t<(std::is_same_v<typename Traits::non_const_value_type,
2380 : typename Traits::value_type> &&
2381 : std::is_void_v<typename Traits::specialize> &&
2382 : Traits::memory_traits::is_atomic)>> {
2383 : using value_type = typename Traits::value_type;
2384 : using handle_type = typename Kokkos::Impl::AtomicViewDataHandle<Traits>;
2385 : using return_type = typename Kokkos::Impl::AtomicDataElement<Traits>;
2386 : using track_type = Kokkos::Impl::SharedAllocationTracker;
2387 :
2388 : KOKKOS_INLINE_FUNCTION
2389 : static handle_type assign(value_type* arg_data_ptr,
2390 : track_type const& /*arg_tracker*/) {
2391 : return handle_type(arg_data_ptr);
2392 : }
2393 :
2394 : template <class SrcHandleType>
2395 : KOKKOS_INLINE_FUNCTION static handle_type assign(
2396 : const SrcHandleType& arg_handle, size_t offset) {
2397 : return handle_type(arg_handle + offset);
2398 : }
2399 : };
2400 :
2401 : template <class Traits>
2402 : struct ViewDataHandle<
2403 : Traits, std::enable_if_t<(std::is_void_v<typename Traits::specialize> &&
2404 : (!Traits::memory_traits::is_aligned) &&
2405 : Traits::memory_traits::is_restrict &&
2406 : (!Traits::memory_traits::is_atomic))>> {
2407 : using value_type = typename Traits::value_type;
2408 : using handle_type = typename Traits::value_type* KOKKOS_RESTRICT;
2409 : using return_type = typename Traits::value_type& KOKKOS_RESTRICT;
2410 : using track_type = Kokkos::Impl::SharedAllocationTracker;
2411 :
2412 : KOKKOS_INLINE_FUNCTION
2413 : static value_type* assign(value_type* arg_data_ptr,
2414 : track_type const& /*arg_tracker*/) {
2415 : return (value_type*)(arg_data_ptr);
2416 : }
2417 :
2418 : KOKKOS_INLINE_FUNCTION
2419 : static value_type* assign(handle_type const arg_data_ptr, size_t offset) {
2420 : return (value_type*)(arg_data_ptr + offset);
2421 : }
2422 : };
2423 :
2424 : template <class Traits>
2425 : struct ViewDataHandle<
2426 : Traits, std::enable_if_t<(std::is_void_v<typename Traits::specialize> &&
2427 : Traits::memory_traits::is_aligned &&
2428 : (!Traits::memory_traits::is_restrict) &&
2429 : (!Traits::memory_traits::is_atomic))>> {
2430 : using value_type = typename Traits::value_type;
2431 : // typedef work-around for intel compilers error #3186: expected typedef
2432 : // declaration
2433 : // NOLINTNEXTLINE(modernize-use-using)
2434 : typedef value_type* KOKKOS_IMPL_ALIGN_PTR(KOKKOS_MEMORY_ALIGNMENT)
2435 : handle_type;
2436 : using return_type = typename Traits::value_type&;
2437 : using track_type = Kokkos::Impl::SharedAllocationTracker;
2438 :
2439 : KOKKOS_INLINE_FUNCTION
2440 : static handle_type assign(value_type* arg_data_ptr,
2441 : track_type const& /*arg_tracker*/) {
2442 : if (reinterpret_cast<uintptr_t>(arg_data_ptr) % Impl::MEMORY_ALIGNMENT) {
2443 : Kokkos::abort(
2444 : "Assigning NonAligned View or Pointer to Kokkos::View with Aligned "
2445 : "attribute");
2446 : }
2447 : return handle_type(arg_data_ptr);
2448 : }
2449 :
2450 : KOKKOS_INLINE_FUNCTION
2451 : static handle_type assign(handle_type const arg_data_ptr, size_t offset) {
2452 : if (reinterpret_cast<uintptr_t>(arg_data_ptr + offset) %
2453 : Impl::MEMORY_ALIGNMENT) {
2454 : Kokkos::abort(
2455 : "Assigning NonAligned View or Pointer to Kokkos::View with Aligned "
2456 : "attribute");
2457 : }
2458 : return handle_type(arg_data_ptr + offset);
2459 : }
2460 : };
2461 :
2462 : template <class Traits>
2463 : struct ViewDataHandle<
2464 : Traits, std::enable_if_t<(std::is_void_v<typename Traits::specialize> &&
2465 : Traits::memory_traits::is_aligned &&
2466 : Traits::memory_traits::is_restrict &&
2467 : (!Traits::memory_traits::is_atomic))>> {
2468 : using value_type = typename Traits::value_type;
2469 : // typedef work-around for intel compilers error #3186: expected typedef
2470 : // declaration
2471 : // NOLINTNEXTLINE(modernize-use-using)
2472 : typedef value_type* KOKKOS_IMPL_ALIGN_PTR(KOKKOS_MEMORY_ALIGNMENT)
2473 : handle_type;
2474 : using return_type = typename Traits::value_type& KOKKOS_RESTRICT;
2475 : using track_type = Kokkos::Impl::SharedAllocationTracker;
2476 :
2477 : KOKKOS_INLINE_FUNCTION
2478 : static value_type* assign(value_type* arg_data_ptr,
2479 : track_type const& /*arg_tracker*/) {
2480 : if (reinterpret_cast<uintptr_t>(arg_data_ptr) % Impl::MEMORY_ALIGNMENT) {
2481 : Kokkos::abort(
2482 : "Assigning NonAligned View or Pointer to Kokkos::View with Aligned "
2483 : "attribute");
2484 : }
2485 : return (value_type*)(arg_data_ptr);
2486 : }
2487 :
2488 : KOKKOS_INLINE_FUNCTION
2489 : static value_type* assign(handle_type const arg_data_ptr, size_t offset) {
2490 : if (reinterpret_cast<uintptr_t>(arg_data_ptr + offset) %
2491 : Impl::MEMORY_ALIGNMENT) {
2492 : Kokkos::abort(
2493 : "Assigning NonAligned View or Pointer to Kokkos::View with Aligned "
2494 : "attribute");
2495 : }
2496 : return (value_type*)(arg_data_ptr + offset);
2497 : }
2498 : };
2499 : } // namespace Impl
2500 : } // namespace Kokkos
2501 :
2502 : //----------------------------------------------------------------------------
2503 : //----------------------------------------------------------------------------
2504 :
2505 : namespace Kokkos {
2506 : namespace Impl {
2507 : //----------------------------------------------------------------------------
2508 : /** \brief View mapping for non-specialized data type and standard layout */
2509 : template <class Traits>
2510 : class ViewMapping<
2511 : Traits, std::enable_if_t<(std::is_void_v<typename Traits::specialize> &&
2512 : ViewOffset<typename Traits::dimension,
2513 : typename Traits::array_layout,
2514 : void>::is_mapping_plugin::value)>> {
2515 : public:
2516 : using offset_type = ViewOffset<typename Traits::dimension,
2517 : typename Traits::array_layout, void>;
2518 :
2519 : using handle_type = typename ViewDataHandle<Traits>::handle_type;
2520 :
2521 : handle_type m_impl_handle;
2522 : offset_type m_impl_offset;
2523 :
2524 : private:
2525 : template <class, class...>
2526 : friend class ViewMapping;
2527 :
2528 : KOKKOS_INLINE_FUNCTION
2529 : ViewMapping(const handle_type& arg_handle, const offset_type& arg_offset)
2530 : : m_impl_handle(arg_handle), m_impl_offset(arg_offset) {}
2531 :
2532 : public:
2533 : using printable_label_typedef = void;
2534 : enum { is_managed = Traits::is_managed };
2535 :
2536 : //----------------------------------------
2537 : // Domain dimensions
2538 :
2539 : static constexpr unsigned Rank = Traits::dimension::rank;
2540 :
2541 : template <typename iType>
2542 106864929 : KOKKOS_INLINE_FUNCTION constexpr size_t extent(const iType& r) const {
2543 106864929 : return m_impl_offset.m_dim.extent(r);
2544 : }
2545 :
2546 : static KOKKOS_INLINE_FUNCTION constexpr size_t static_extent(
2547 : const unsigned r) noexcept {
2548 : using dim_type = typename offset_type::dimension_type;
2549 : return dim_type::static_extent(r);
2550 : }
2551 :
2552 : KOKKOS_INLINE_FUNCTION constexpr typename Traits::array_layout layout()
2553 : const {
2554 : return m_impl_offset.layout();
2555 : }
2556 :
2557 1378872 : KOKKOS_INLINE_FUNCTION constexpr size_t dimension_0() const {
2558 1378872 : return m_impl_offset.dimension_0();
2559 : }
2560 : KOKKOS_INLINE_FUNCTION constexpr size_t dimension_1() const {
2561 : return m_impl_offset.dimension_1();
2562 : }
2563 : KOKKOS_INLINE_FUNCTION constexpr size_t dimension_2() const {
2564 : return m_impl_offset.dimension_2();
2565 : }
2566 : KOKKOS_INLINE_FUNCTION constexpr size_t dimension_3() const {
2567 : return m_impl_offset.dimension_3();
2568 : }
2569 : KOKKOS_INLINE_FUNCTION constexpr size_t dimension_4() const {
2570 : return m_impl_offset.dimension_4();
2571 : }
2572 : KOKKOS_INLINE_FUNCTION constexpr size_t dimension_5() const {
2573 : return m_impl_offset.dimension_5();
2574 : }
2575 : KOKKOS_INLINE_FUNCTION constexpr size_t dimension_6() const {
2576 : return m_impl_offset.dimension_6();
2577 : }
2578 : KOKKOS_INLINE_FUNCTION constexpr size_t dimension_7() const {
2579 : return m_impl_offset.dimension_7();
2580 : }
2581 :
2582 : // Is a regular layout with uniform striding for each index.
2583 : using is_regular = typename offset_type::is_regular;
2584 :
2585 : KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const {
2586 : return m_impl_offset.stride_0();
2587 : }
2588 : KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const {
2589 : return m_impl_offset.stride_1();
2590 : }
2591 : KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const {
2592 : return m_impl_offset.stride_2();
2593 : }
2594 : KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const {
2595 : return m_impl_offset.stride_3();
2596 : }
2597 : KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const {
2598 : return m_impl_offset.stride_4();
2599 : }
2600 : KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const {
2601 : return m_impl_offset.stride_5();
2602 : }
2603 : KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const {
2604 : return m_impl_offset.stride_6();
2605 : }
2606 : KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const {
2607 : return m_impl_offset.stride_7();
2608 : }
2609 :
2610 : // Fill the target unbounded array s with the stride and the total spanned
2611 : // size. This method differs from stride_fill() in that it writes the total
2612 : // spanned size to the last index of the array. Preconditions: s must be an
2613 : // array of dimension_type::rank + 1 elements
2614 : template <typename iType>
2615 : KOKKOS_INLINE_FUNCTION void stride(iType* const s) const {
2616 : m_impl_offset.stride(s);
2617 : }
2618 :
2619 : // Fill the target unbounded array s with the stride.
2620 : // This method differs from stride() in that it does not write the total
2621 : // length to the last index of the array. Preconditions: s must be an array of
2622 : // dimension_type::rank elements
2623 : template <typename iType>
2624 : KOKKOS_INLINE_FUNCTION iType stride_fill(iType* const s) const {
2625 : return m_impl_offset.stride_fill(s);
2626 : }
2627 :
2628 : //----------------------------------------
2629 : // Range span
2630 :
2631 : /** \brief Span of the mapped range */
2632 : KOKKOS_INLINE_FUNCTION constexpr size_t span() const {
2633 : return m_impl_offset.span();
2634 : }
2635 :
2636 : /** \brief Is the mapped range span contiguous */
2637 : KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const {
2638 : return m_impl_offset.span_is_contiguous();
2639 : }
2640 :
2641 : using reference_type = typename ViewDataHandle<Traits>::return_type;
2642 : using pointer_type = typename Traits::value_type*;
2643 :
2644 : /** \brief Query raw pointer to memory */
2645 : KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const {
2646 : return m_impl_handle;
2647 : }
2648 :
2649 : //----------------------------------------
2650 : // The View class performs all rank and bounds checking before
2651 : // calling these element reference methods.
2652 :
2653 : KOKKOS_FORCEINLINE_FUNCTION
2654 : reference_type reference() const { return m_impl_handle[0]; }
2655 :
2656 : template <typename I0>
2657 : KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<
2658 : (std::is_integral_v<I0> &&
2659 : // if layout is neither stride nor irregular,
2660 : // then just use the handle directly
2661 : !(std::is_same_v<typename Traits::array_layout, Kokkos::LayoutStride> ||
2662 : !is_regular::value)),
2663 : reference_type>
2664 : reference(const I0& i0) const {
2665 : return m_impl_handle[i0];
2666 : }
2667 :
2668 : template <typename I0>
2669 : KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t<
2670 : (std::is_integral_v<I0> &&
2671 : // if the layout is strided or irregular, then
2672 : // we have to use the offset
2673 : (std::is_same_v<typename Traits::array_layout, Kokkos::LayoutStride> ||
2674 : !is_regular::value)),
2675 : reference_type>
2676 : reference(const I0& i0) const {
2677 : return m_impl_handle[m_impl_offset(i0)];
2678 : }
2679 :
2680 : template <typename I0, typename I1>
2681 : KOKKOS_FORCEINLINE_FUNCTION reference_type reference(const I0& i0,
2682 : const I1& i1) const {
2683 : return m_impl_handle[m_impl_offset(i0, i1)];
2684 : }
2685 :
2686 : template <typename I0, typename I1, typename I2>
2687 : KOKKOS_FORCEINLINE_FUNCTION reference_type reference(const I0& i0,
2688 : const I1& i1,
2689 : const I2& i2) const {
2690 : return m_impl_handle[m_impl_offset(i0, i1, i2)];
2691 : }
2692 :
2693 : template <typename I0, typename I1, typename I2, typename I3>
2694 : KOKKOS_FORCEINLINE_FUNCTION reference_type
2695 : reference(const I0& i0, const I1& i1, const I2& i2, const I3& i3) const {
2696 : return m_impl_handle[m_impl_offset(i0, i1, i2, i3)];
2697 : }
2698 :
2699 : template <typename I0, typename I1, typename I2, typename I3, typename I4>
2700 : KOKKOS_FORCEINLINE_FUNCTION reference_type reference(const I0& i0,
2701 : const I1& i1,
2702 : const I2& i2,
2703 : const I3& i3,
2704 : const I4& i4) const {
2705 : return m_impl_handle[m_impl_offset(i0, i1, i2, i3, i4)];
2706 : }
2707 :
2708 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
2709 : typename I5>
2710 : KOKKOS_FORCEINLINE_FUNCTION reference_type
2711 : reference(const I0& i0, const I1& i1, const I2& i2, const I3& i3,
2712 : const I4& i4, const I5& i5) const {
2713 : return m_impl_handle[m_impl_offset(i0, i1, i2, i3, i4, i5)];
2714 : }
2715 :
2716 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
2717 : typename I5, typename I6>
2718 : KOKKOS_FORCEINLINE_FUNCTION reference_type
2719 : reference(const I0& i0, const I1& i1, const I2& i2, const I3& i3,
2720 : const I4& i4, const I5& i5, const I6& i6) const {
2721 : return m_impl_handle[m_impl_offset(i0, i1, i2, i3, i4, i5, i6)];
2722 : }
2723 :
2724 : template <typename I0, typename I1, typename I2, typename I3, typename I4,
2725 : typename I5, typename I6, typename I7>
2726 : KOKKOS_FORCEINLINE_FUNCTION reference_type
2727 : reference(const I0& i0, const I1& i1, const I2& i2, const I3& i3,
2728 : const I4& i4, const I5& i5, const I6& i6, const I7& i7) const {
2729 : return m_impl_handle[m_impl_offset(i0, i1, i2, i3, i4, i5, i6, i7)];
2730 : }
2731 :
2732 : //----------------------------------------
2733 :
2734 : private:
2735 : enum { MemorySpanMask = 8 - 1 /* Force alignment on 8 byte boundary */ };
2736 : enum { MemorySpanSize = sizeof(typename Traits::value_type) };
2737 :
2738 : public:
2739 : /** \brief Span, in bytes, of the referenced memory */
2740 : KOKKOS_INLINE_FUNCTION constexpr size_t memory_span() const {
2741 : return (m_impl_offset.span() * sizeof(typename Traits::value_type) +
2742 : MemorySpanMask) &
2743 : ~size_t(MemorySpanMask);
2744 : }
2745 :
2746 : //----------------------------------------
2747 :
2748 : KOKKOS_DEFAULTED_FUNCTION ~ViewMapping() = default;
2749 : KOKKOS_INLINE_FUNCTION ViewMapping() : m_impl_handle(), m_impl_offset() {}
2750 :
2751 : KOKKOS_DEFAULTED_FUNCTION ViewMapping(const ViewMapping&) = default;
2752 : KOKKOS_DEFAULTED_FUNCTION ViewMapping& operator=(const ViewMapping&) =
2753 : default;
2754 :
2755 : KOKKOS_DEFAULTED_FUNCTION ViewMapping(ViewMapping&&) = default;
2756 : KOKKOS_DEFAULTED_FUNCTION ViewMapping& operator=(ViewMapping&&) = default;
2757 :
2758 : //----------------------------------------
2759 :
2760 : /**\brief Span, in bytes, of the required memory */
2761 : KOKKOS_INLINE_FUNCTION
2762 : static constexpr size_t memory_span(
2763 : typename Traits::array_layout const& arg_layout) {
2764 : using padding = std::integral_constant<unsigned int, 0>;
2765 : return (offset_type(padding(), arg_layout).span() * MemorySpanSize +
2766 : MemorySpanMask) &
2767 : ~size_t(MemorySpanMask);
2768 : }
2769 :
2770 : /**\brief Wrap a span of memory */
2771 : template <class... P>
2772 : KOKKOS_INLINE_FUNCTION ViewMapping(
2773 : Kokkos::Impl::ViewCtorProp<P...> const& arg_prop,
2774 : typename Traits::array_layout const& arg_layout)
2775 : : m_impl_handle(Impl::get_property<Impl::PointerTag>(arg_prop)),
2776 : m_impl_offset(std::integral_constant<unsigned, 0>(), arg_layout) {}
2777 :
2778 : /**\brief Assign data */
2779 : KOKKOS_INLINE_FUNCTION
2780 : void assign_data(pointer_type arg_ptr) {
2781 : m_impl_handle = handle_type(arg_ptr);
2782 : }
2783 :
2784 : //----------------------------------------
2785 : /* Allocate and construct mapped array.
2786 : * Allocate via shared allocation record and
2787 : * return that record for allocation tracking.
2788 : */
2789 : template <class... P>
2790 : Kokkos::Impl::SharedAllocationRecord<>* allocate_shared(
2791 : Kokkos::Impl::ViewCtorProp<P...> const& arg_prop,
2792 : typename Traits::array_layout const& arg_layout,
2793 : bool execution_space_specified) {
2794 : using alloc_prop = Kokkos::Impl::ViewCtorProp<P...>;
2795 :
2796 : using execution_space = typename alloc_prop::execution_space;
2797 : using memory_space = typename Traits::memory_space;
2798 : static_assert(
2799 : SpaceAccessibility<execution_space, memory_space>::accessible);
2800 : using device_type = Kokkos::Device<execution_space, memory_space>;
2801 : using value_type = typename Traits::value_type;
2802 : using functor_type = std::conditional_t<
2803 : alloc_prop::sequential_host_init,
2804 : ViewValueFunctorSequentialHostInit<device_type, value_type>,
2805 : ViewValueFunctor<device_type, value_type>>;
2806 : using record_type =
2807 : Kokkos::Impl::SharedAllocationRecord<memory_space, functor_type>;
2808 :
2809 : // Query the mapping for byte-size of allocation.
2810 : // If padding is allowed then pass in sizeof value type
2811 : // for padding computation.
2812 : using padding = std::integral_constant<
2813 : unsigned int, alloc_prop::allow_padding ? sizeof(value_type) : 0>;
2814 :
2815 : m_impl_offset = offset_type(padding(), arg_layout);
2816 :
2817 : const size_t alloc_size =
2818 : (m_impl_offset.span() * MemorySpanSize + MemorySpanMask) &
2819 : ~size_t(MemorySpanMask);
2820 : const std::string& alloc_name =
2821 : Impl::get_property<Impl::LabelTag>(arg_prop);
2822 : const execution_space& exec_space =
2823 : Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop);
2824 : const memory_space& mem_space =
2825 : Impl::get_property<Impl::MemorySpaceTag>(arg_prop);
2826 :
2827 : // Create shared memory tracking record with allocate memory from the memory
2828 : // space
2829 : record_type* const record =
2830 : execution_space_specified
2831 : ? record_type::allocate(exec_space, mem_space, alloc_name,
2832 : alloc_size)
2833 : : record_type::allocate(mem_space, alloc_name, alloc_size);
2834 :
2835 : m_impl_handle = handle_type(reinterpret_cast<pointer_type>(record->data()));
2836 :
2837 : functor_type functor =
2838 : execution_space_specified
2839 : ? functor_type(exec_space, (value_type*)m_impl_handle,
2840 : m_impl_offset.span(), alloc_name)
2841 : : functor_type((value_type*)m_impl_handle, m_impl_offset.span(),
2842 : alloc_name);
2843 :
2844 : // Only initialize if the allocation is non-zero.
2845 : // May be zero if one of the dimensions is zero.
2846 : if constexpr (alloc_prop::initialize)
2847 : if (alloc_size) {
2848 : // Assume destruction is only required when construction is requested.
2849 : // The ViewValueFunctor has both value construction and destruction
2850 : // operators.
2851 : record->m_destroy = std::move(functor);
2852 :
2853 : // Construct values
2854 : record->m_destroy.construct_shared_allocation();
2855 : }
2856 :
2857 : return record;
2858 : }
2859 : };
2860 :
2861 : //----------------------------------------------------------------------------
2862 : //----------------------------------------------------------------------------
2863 : /** \brief Assign compatible default mappings */
2864 :
2865 : template <class DstTraits, class SrcTraits>
2866 : class ViewMapping<
2867 : DstTraits, SrcTraits,
2868 : std::enable_if_t<(
2869 : !(std::is_same_v<typename SrcTraits::array_layout,
2870 : LayoutStride>)&& // Added to have a new
2871 : // specialization for
2872 : // SrcType of
2873 : // LayoutStride
2874 : // default mappings
2875 : std::is_void_v<typename DstTraits::specialize> &&
2876 : std::is_void_v<typename SrcTraits::specialize> &&
2877 : (
2878 : // same layout
2879 : std::is_same_v<typename DstTraits::array_layout,
2880 : typename SrcTraits::array_layout> ||
2881 : // known layout
2882 : ((std::is_same_v<typename DstTraits::array_layout,
2883 : Kokkos::LayoutLeft> ||
2884 : std::is_same_v<typename DstTraits::array_layout,
2885 : Kokkos::LayoutRight> ||
2886 : std::is_same_v<
2887 : typename DstTraits::array_layout,
2888 : Kokkos::LayoutStride>)&&(std::is_same_v<typename SrcTraits::
2889 : array_layout,
2890 : Kokkos::LayoutLeft> ||
2891 : std::is_same_v<
2892 : typename SrcTraits::array_layout,
2893 : Kokkos::LayoutRight> ||
2894 : std::is_same_v<
2895 : typename SrcTraits::array_layout,
2896 : Kokkos::LayoutStride>))))>> {
2897 : private:
2898 : enum {
2899 : is_assignable_space = Kokkos::Impl::MemorySpaceAccess<
2900 : typename DstTraits::memory_space,
2901 : typename SrcTraits::memory_space>::assignable
2902 : };
2903 :
2904 : enum {
2905 : is_assignable_value_type =
2906 : std::is_same_v<typename DstTraits::value_type,
2907 : typename SrcTraits::value_type> ||
2908 : std::is_same_v<typename DstTraits::value_type,
2909 : typename SrcTraits::const_value_type>
2910 : };
2911 :
2912 : enum {
2913 : is_assignable_dimension =
2914 : ViewDimensionAssignable<typename DstTraits::dimension,
2915 : typename SrcTraits::dimension>::value
2916 : };
2917 :
2918 : enum {
2919 : is_assignable_layout = std::is_same_v<typename DstTraits::array_layout,
2920 : typename SrcTraits::array_layout> ||
2921 : std::is_same_v<typename DstTraits::array_layout,
2922 : Kokkos::LayoutStride> ||
2923 : (DstTraits::dimension::rank == 0) ||
2924 : (DstTraits::dimension::rank == 1)
2925 : };
2926 :
2927 : public:
2928 : enum {
2929 : is_assignable_data_type =
2930 : is_assignable_value_type && is_assignable_dimension
2931 : };
2932 : enum {
2933 : is_assignable = is_assignable_space && is_assignable_value_type &&
2934 : is_assignable_dimension && is_assignable_layout
2935 : };
2936 :
2937 : using TrackType = Kokkos::Impl::SharedAllocationTracker;
2938 : using DstType = ViewMapping<DstTraits, void>;
2939 : using SrcType = ViewMapping<SrcTraits, void>;
2940 :
2941 : KOKKOS_INLINE_FUNCTION
2942 : static void assign(DstType& dst, const SrcType& src,
2943 : const TrackType& src_track) {
2944 : static_assert(is_assignable_space,
2945 : "View assignment must have compatible spaces");
2946 :
2947 : static_assert(
2948 : is_assignable_value_type,
2949 : "View assignment must have same value type or const = non-const");
2950 :
2951 : static_assert(is_assignable_dimension,
2952 : "View assignment must have compatible dimensions");
2953 :
2954 : static_assert(
2955 : is_assignable_layout,
2956 : "View assignment must have compatible layout or have rank <= 1");
2957 :
2958 : using dst_offset_type = typename DstType::offset_type;
2959 :
2960 : if (size_t(DstTraits::dimension::rank_dynamic) <
2961 : size_t(SrcTraits::dimension::rank_dynamic)) {
2962 : using dst_dim = typename DstTraits::dimension;
2963 : bool assignable = ((1 > DstTraits::dimension::rank_dynamic &&
2964 : 1 <= SrcTraits::dimension::rank_dynamic)
2965 : ? dst_dim::ArgN0 == src.dimension_0()
2966 : : true) &&
2967 : ((2 > DstTraits::dimension::rank_dynamic &&
2968 : 2 <= SrcTraits::dimension::rank_dynamic)
2969 : ? dst_dim::ArgN1 == src.dimension_1()
2970 : : true) &&
2971 : ((3 > DstTraits::dimension::rank_dynamic &&
2972 : 3 <= SrcTraits::dimension::rank_dynamic)
2973 : ? dst_dim::ArgN2 == src.dimension_2()
2974 : : true) &&
2975 : ((4 > DstTraits::dimension::rank_dynamic &&
2976 : 4 <= SrcTraits::dimension::rank_dynamic)
2977 : ? dst_dim::ArgN3 == src.dimension_3()
2978 : : true) &&
2979 : ((5 > DstTraits::dimension::rank_dynamic &&
2980 : 5 <= SrcTraits::dimension::rank_dynamic)
2981 : ? dst_dim::ArgN4 == src.dimension_4()
2982 : : true) &&
2983 : ((6 > DstTraits::dimension::rank_dynamic &&
2984 : 6 <= SrcTraits::dimension::rank_dynamic)
2985 : ? dst_dim::ArgN5 == src.dimension_5()
2986 : : true) &&
2987 : ((7 > DstTraits::dimension::rank_dynamic &&
2988 : 7 <= SrcTraits::dimension::rank_dynamic)
2989 : ? dst_dim::ArgN6 == src.dimension_6()
2990 : : true) &&
2991 : ((8 > DstTraits::dimension::rank_dynamic &&
2992 : 8 <= SrcTraits::dimension::rank_dynamic)
2993 : ? dst_dim::ArgN7 == src.dimension_7()
2994 : : true);
2995 : if (!assignable)
2996 : Kokkos::abort(
2997 : "View Assignment: trying to assign runtime dimension to non "
2998 : "matching compile time dimension.");
2999 : }
3000 : dst.m_impl_offset = dst_offset_type(src.m_impl_offset);
3001 : dst.m_impl_handle = Kokkos::Impl::ViewDataHandle<DstTraits>::assign(
3002 : src.m_impl_handle, src_track);
3003 : }
3004 : };
3005 :
3006 : //----------------------------------------------------------------------------
3007 : // Create new specialization for SrcType of LayoutStride. Runtime check for
3008 : // compatible layout
3009 : template <class DstTraits, class SrcTraits>
3010 : class ViewMapping<
3011 : DstTraits, SrcTraits,
3012 : std::enable_if_t<(std::is_same_v<typename SrcTraits::array_layout,
3013 : Kokkos::LayoutStride> &&
3014 : std::is_void_v<typename DstTraits::specialize> &&
3015 : std::is_void_v<typename SrcTraits::specialize> &&
3016 : (
3017 : // same layout
3018 : std::is_same_v<typename DstTraits::array_layout,
3019 : typename SrcTraits::array_layout> ||
3020 : // known layout
3021 : (std::is_same_v<typename DstTraits::array_layout,
3022 : Kokkos::LayoutLeft> ||
3023 : std::is_same_v<typename DstTraits::array_layout,
3024 : Kokkos::LayoutRight> ||
3025 : std::is_same_v<typename DstTraits::array_layout,
3026 : Kokkos::LayoutStride>)))>> {
3027 : private:
3028 : enum {
3029 : is_assignable_space = Kokkos::Impl::MemorySpaceAccess<
3030 : typename DstTraits::memory_space,
3031 : typename SrcTraits::memory_space>::assignable
3032 : };
3033 :
3034 : enum {
3035 : is_assignable_value_type =
3036 : std::is_same_v<typename DstTraits::value_type,
3037 : typename SrcTraits::value_type> ||
3038 : std::is_same_v<typename DstTraits::value_type,
3039 : typename SrcTraits::const_value_type>
3040 : };
3041 :
3042 : enum {
3043 : is_assignable_dimension =
3044 : ViewDimensionAssignable<typename DstTraits::dimension,
3045 : typename SrcTraits::dimension>::value
3046 : };
3047 :
3048 : public:
3049 : enum {
3050 : is_assignable_data_type =
3051 : is_assignable_value_type && is_assignable_dimension
3052 : };
3053 : enum {
3054 : is_assignable = is_assignable_space && is_assignable_value_type &&
3055 : is_assignable_dimension
3056 : };
3057 :
3058 : using TrackType = Kokkos::Impl::SharedAllocationTracker;
3059 : using DstType = ViewMapping<DstTraits, void>;
3060 : using SrcType = ViewMapping<SrcTraits, void>;
3061 :
3062 : KOKKOS_INLINE_FUNCTION
3063 : static bool assignable_layout_check(DstType&,
3064 : const SrcType& src) // Runtime check
3065 : {
3066 : size_t strides[9];
3067 : bool assignable = true;
3068 : src.stride(strides);
3069 : size_t exp_stride = 1;
3070 : if (std::is_same_v<typename DstTraits::array_layout, Kokkos::LayoutLeft>) {
3071 : for (int i = 0; i < (int)src.Rank; i++) {
3072 : if (i > 0) exp_stride *= src.extent(i - 1);
3073 : if (strides[i] != exp_stride) {
3074 : assignable = false;
3075 : break;
3076 : }
3077 : }
3078 : } else if (std::is_same_v<typename DstTraits::array_layout,
3079 : Kokkos::LayoutRight>) {
3080 : for (int i = 0; i < (int)src.Rank; i++) {
3081 : if (i > 0) exp_stride *= src.extent(src.Rank - i);
3082 : if (strides[src.Rank - 1 - i] != exp_stride) {
3083 : assignable = false;
3084 : break;
3085 : }
3086 : }
3087 : }
3088 : return assignable;
3089 : }
3090 :
3091 : KOKKOS_INLINE_FUNCTION
3092 : static void assign(DstType& dst, const SrcType& src,
3093 : const TrackType& src_track) {
3094 : static_assert(is_assignable_space,
3095 : "View assignment must have compatible spaces");
3096 :
3097 : static_assert(
3098 : is_assignable_value_type,
3099 : "View assignment must have same value type or const = non-const");
3100 :
3101 : static_assert(is_assignable_dimension,
3102 : "View assignment must have compatible dimensions");
3103 :
3104 : bool assignable_layout = assignable_layout_check(dst, src); // Runtime
3105 : // check
3106 : if (!assignable_layout)
3107 : Kokkos::abort("View assignment must have compatible layouts\n");
3108 :
3109 : using dst_offset_type = typename DstType::offset_type;
3110 :
3111 : if (size_t(DstTraits::dimension::rank_dynamic) <
3112 : size_t(SrcTraits::dimension::rank_dynamic)) {
3113 : using dst_dim = typename DstTraits::dimension;
3114 : bool assignable = ((1 > DstTraits::dimension::rank_dynamic &&
3115 : 1 <= SrcTraits::dimension::rank_dynamic)
3116 : ? dst_dim::ArgN0 == src.dimension_0()
3117 : : true) &&
3118 : ((2 > DstTraits::dimension::rank_dynamic &&
3119 : 2 <= SrcTraits::dimension::rank_dynamic)
3120 : ? dst_dim::ArgN1 == src.dimension_1()
3121 : : true) &&
3122 : ((3 > DstTraits::dimension::rank_dynamic &&
3123 : 3 <= SrcTraits::dimension::rank_dynamic)
3124 : ? dst_dim::ArgN2 == src.dimension_2()
3125 : : true) &&
3126 : ((4 > DstTraits::dimension::rank_dynamic &&
3127 : 4 <= SrcTraits::dimension::rank_dynamic)
3128 : ? dst_dim::ArgN3 == src.dimension_3()
3129 : : true) &&
3130 : ((5 > DstTraits::dimension::rank_dynamic &&
3131 : 5 <= SrcTraits::dimension::rank_dynamic)
3132 : ? dst_dim::ArgN4 == src.dimension_4()
3133 : : true) &&
3134 : ((6 > DstTraits::dimension::rank_dynamic &&
3135 : 6 <= SrcTraits::dimension::rank_dynamic)
3136 : ? dst_dim::ArgN5 == src.dimension_5()
3137 : : true) &&
3138 : ((7 > DstTraits::dimension::rank_dynamic &&
3139 : 7 <= SrcTraits::dimension::rank_dynamic)
3140 : ? dst_dim::ArgN6 == src.dimension_6()
3141 : : true) &&
3142 : ((8 > DstTraits::dimension::rank_dynamic &&
3143 : 8 <= SrcTraits::dimension::rank_dynamic)
3144 : ? dst_dim::ArgN7 == src.dimension_7()
3145 : : true);
3146 : if (!assignable)
3147 : Kokkos::abort(
3148 : "View Assignment: trying to assign runtime dimension to non "
3149 : "matching compile time dimension.");
3150 : }
3151 : dst.m_impl_offset = dst_offset_type(src.m_impl_offset);
3152 : dst.m_impl_handle = Kokkos::Impl::ViewDataHandle<DstTraits>::assign(
3153 : src.m_impl_handle, src_track);
3154 : }
3155 : };
3156 :
3157 : //----------------------------------------------------------------------------
3158 : //----------------------------------------------------------------------------
3159 : // Subview mapping.
3160 : // Deduce destination view type from source view traits and subview arguments
3161 :
3162 : template <class, class ValueType, class Exts, class... Args>
3163 : struct SubViewDataTypeImpl;
3164 :
3165 : /* base case */
3166 : template <class ValueType>
3167 : struct SubViewDataTypeImpl<void, ValueType, Kokkos::Experimental::Extents<>> {
3168 : using type = ValueType;
3169 : };
3170 :
3171 : /* for integral args, subview doesn't have that dimension */
3172 : template <class ValueType, size_t Ext, size_t... Exts, class Integral,
3173 : class... Args>
3174 : struct SubViewDataTypeImpl<
3175 : std::enable_if_t<std::is_integral_v<std::decay_t<Integral>>>, ValueType,
3176 : Kokkos::Experimental::Extents<Ext, Exts...>, Integral, Args...>
3177 : : SubViewDataTypeImpl<void, ValueType,
3178 : Kokkos::Experimental::Extents<Exts...>, Args...> {};
3179 :
3180 : /* for ALL slice, subview has the same dimension */
3181 : template <class ValueType, size_t Ext, size_t... Exts, class... Args>
3182 : struct SubViewDataTypeImpl<void, ValueType,
3183 : Kokkos::Experimental::Extents<Ext, Exts...>,
3184 : Kokkos::ALL_t, Args...>
3185 : : SubViewDataTypeImpl<void, typename ApplyExtent<ValueType, Ext>::type,
3186 : Kokkos::Experimental::Extents<Exts...>, Args...> {};
3187 :
3188 : /* for pair-style slice, subview has dynamic dimension, since pair doesn't give
3189 : * static sizes */
3190 : /* Since we don't allow interleaving of dynamic and static extents, make all of
3191 : * the dimensions to the left dynamic */
3192 : template <class ValueType, size_t Ext, size_t... Exts, class PairLike,
3193 : class... Args>
3194 : struct SubViewDataTypeImpl<
3195 : std::enable_if_t<is_pair_like<PairLike>::value>, ValueType,
3196 : Kokkos::Experimental::Extents<Ext, Exts...>, PairLike, Args...>
3197 : : SubViewDataTypeImpl<
3198 : void, typename make_all_extents_into_pointers<ValueType>::type*,
3199 : Kokkos::Experimental::Extents<Exts...>, Args...> {};
3200 :
3201 : template <class ValueType, class Exts, class... Args>
3202 : struct SubViewDataType : SubViewDataTypeImpl<void, ValueType, Exts, Args...> {};
3203 :
3204 : //----------------------------------------------------------------------------
3205 :
3206 : template <class SrcTraits, class... Args>
3207 : class ViewMapping<
3208 : std::enable_if_t<(
3209 : std::is_void_v<typename SrcTraits::specialize> &&
3210 : (std::is_same_v<typename SrcTraits::array_layout, Kokkos::LayoutLeft> ||
3211 : std::is_same_v<typename SrcTraits::array_layout,
3212 : Kokkos::LayoutRight> ||
3213 : std::is_same_v<typename SrcTraits::array_layout,
3214 : Kokkos::LayoutStride>))>,
3215 : SrcTraits, Args...> {
3216 : private:
3217 : static_assert(SrcTraits::rank == sizeof...(Args),
3218 : "Subview mapping requires one argument for each dimension of "
3219 : "source View");
3220 :
3221 : enum {
3222 : RZ = false,
3223 : R0 = bool(is_integral_extent<0, Args...>::value),
3224 : R1 = bool(is_integral_extent<1, Args...>::value),
3225 : R2 = bool(is_integral_extent<2, Args...>::value),
3226 : R3 = bool(is_integral_extent<3, Args...>::value),
3227 : R4 = bool(is_integral_extent<4, Args...>::value),
3228 : R5 = bool(is_integral_extent<5, Args...>::value),
3229 : R6 = bool(is_integral_extent<6, Args...>::value),
3230 : R7 = bool(is_integral_extent<7, Args...>::value)
3231 : };
3232 :
3233 : enum {
3234 : rank = unsigned(R0) + unsigned(R1) + unsigned(R2) + unsigned(R3) +
3235 : unsigned(R4) + unsigned(R5) + unsigned(R6) + unsigned(R7)
3236 : };
3237 :
3238 : // Whether right-most rank is a range.
3239 : enum {
3240 : R0_rev =
3241 : (0 == SrcTraits::rank
3242 : ? RZ
3243 : : (1 == SrcTraits::rank
3244 : ? R0
3245 : : (2 == SrcTraits::rank
3246 : ? R1
3247 : : (3 == SrcTraits::rank
3248 : ? R2
3249 : : (4 == SrcTraits::rank
3250 : ? R3
3251 : : (5 == SrcTraits::rank
3252 : ? R4
3253 : : (6 == SrcTraits::rank
3254 : ? R5
3255 : : (7 == SrcTraits::rank
3256 : ? R6
3257 : : R7))))))))
3258 : };
3259 :
3260 : // Subview's layout
3261 : using array_layout = std::conditional_t<
3262 : ( /* Same array layout IF */
3263 : (rank == 0) /* output rank zero */
3264 : || SubviewLegalArgsCompileTime<typename SrcTraits::array_layout,
3265 : typename SrcTraits::array_layout, rank,
3266 : SrcTraits::rank, 0, Args...>::value ||
3267 : // OutputRank 1 or 2, InputLayout Left, Interval 0
3268 : // because single stride one or second index has a stride.
3269 : (rank <= 2 && R0 &&
3270 : std::is_same_v<typename SrcTraits::array_layout,
3271 : Kokkos::LayoutLeft>) // replace with input rank
3272 : ||
3273 : // OutputRank 1 or 2, InputLayout Right, Interval [InputRank-1]
3274 : // because single stride one or second index has a stride.
3275 : (rank <= 2 && R0_rev &&
3276 : std::is_same_v<typename SrcTraits::array_layout,
3277 : Kokkos::LayoutRight>) // replace input rank
3278 : ),
3279 : typename SrcTraits::array_layout, Kokkos::LayoutStride>;
3280 :
3281 : using value_type = typename SrcTraits::value_type;
3282 :
3283 : using data_type =
3284 : typename SubViewDataType<value_type,
3285 : typename Kokkos::Impl::ParseViewExtents<
3286 : typename SrcTraits::data_type>::type,
3287 : Args...>::type;
3288 :
3289 : public:
3290 : using traits_type = Kokkos::ViewTraits<data_type, array_layout,
3291 : typename SrcTraits::device_type,
3292 : typename SrcTraits::memory_traits>;
3293 :
3294 : using type =
3295 : Kokkos::View<data_type, array_layout, typename SrcTraits::device_type,
3296 : typename SrcTraits::memory_traits>;
3297 :
3298 : template <class MemoryTraits>
3299 : struct apply {
3300 : static_assert(Kokkos::is_memory_traits<MemoryTraits>::value);
3301 :
3302 : using traits_type =
3303 : Kokkos::ViewTraits<data_type, array_layout,
3304 : typename SrcTraits::device_type, MemoryTraits>;
3305 :
3306 : using type = Kokkos::View<data_type, array_layout,
3307 : typename SrcTraits::device_type, MemoryTraits>;
3308 : };
3309 :
3310 : // The presumed type is 'ViewMapping< traits_type , void >'
3311 : // However, a compatible ViewMapping is acceptable.
3312 : template <class DstTraits>
3313 : KOKKOS_INLINE_FUNCTION static void assign(
3314 : ViewMapping<DstTraits, void>& dst,
3315 : ViewMapping<SrcTraits, void> const& src, Args... args) {
3316 : static_assert(ViewMapping<DstTraits, traits_type, void>::is_assignable,
3317 : "Subview destination type must be compatible with subview "
3318 : "derived type");
3319 :
3320 : using DstType = ViewMapping<DstTraits, void>;
3321 :
3322 : using dst_offset_type = typename DstType::offset_type;
3323 :
3324 : const SubviewExtents<SrcTraits::rank, rank> extents(src.m_impl_offset.m_dim,
3325 : args...);
3326 :
3327 : dst.m_impl_offset = dst_offset_type(src.m_impl_offset, extents);
3328 :
3329 : dst.m_impl_handle = ViewDataHandle<DstTraits>::assign(
3330 : src.m_impl_handle,
3331 : src.m_impl_offset(extents.domain_offset(0), extents.domain_offset(1),
3332 : extents.domain_offset(2), extents.domain_offset(3),
3333 : extents.domain_offset(4), extents.domain_offset(5),
3334 : extents.domain_offset(6), extents.domain_offset(7)));
3335 : }
3336 : };
3337 :
3338 : //----------------------------------------------------------------------------
3339 :
3340 : } // namespace Impl
3341 : } // namespace Kokkos
3342 :
3343 : //----------------------------------------------------------------------------
3344 : //----------------------------------------------------------------------------
3345 :
3346 : namespace Kokkos {
3347 : namespace Impl {
3348 :
3349 : template <class Map, class... Indices, std::size_t... Enumerate>
3350 73224123 : KOKKOS_FUNCTION bool within_range(Map const& map,
3351 : std::index_sequence<Enumerate...>,
3352 : Indices... indices) {
3353 73224123 : return (((std::size_t)indices < map.extent(Enumerate)) && ...);
3354 : }
3355 :
3356 : // Disabled when using MDSpan because the MDSpan implementation has its own
3357 : // version
3358 : #ifndef KOKKOS_ENABLE_IMPL_MDSPAN
3359 : template <class... Indices>
3360 : KOKKOS_FUNCTION constexpr char* append_formatted_multidimensional_index(
3361 : char* dest, Indices... indices) {
3362 : char* d = dest;
3363 : strcat(d, "[");
3364 : (
3365 : [&] {
3366 : d += strlen(d);
3367 : to_chars_i(d,
3368 : d + 20, // 20 digits ought to be enough
3369 : indices);
3370 : strcat(d, ",");
3371 : }(),
3372 : ...);
3373 : d[strlen(d) - 1] = ']'; // overwrite trailing comma
3374 : return dest;
3375 : }
3376 : #endif
3377 :
3378 : template <class Map, class... Indices, std::size_t... Enumerate>
3379 0 : KOKKOS_FUNCTION void print_extents(char* dest, Map const& map,
3380 : std::index_sequence<Enumerate...>) {
3381 0 : append_formatted_multidimensional_index(dest, map.extent(Enumerate)...);
3382 0 : }
3383 :
3384 : template <class T>
3385 : using printable_label_typedef_t = typename T::printable_label_typedef;
3386 :
3387 : template <class MemorySpace, class ViewType, class MapType, class... Args>
3388 39447979 : KOKKOS_INLINE_FUNCTION void view_verify_operator_bounds(
3389 : Kokkos::Impl::ViewTracker<ViewType> const& tracker, const MapType& map,
3390 : Args... args) {
3391 39447979 : if (!within_range(map, std::make_index_sequence<sizeof...(Args)>(),
3392 : args...)) {
3393 0 : char err[256] = "";
3394 0 : strcat(err, "Kokkos::View ERROR: out of bounds access");
3395 0 : strcat(err, " label=(\"");
3396 0 : KOKKOS_IF_ON_HOST(
3397 : if (tracker.m_tracker.has_record()) {
3398 : strncat(err, tracker.m_tracker.template get_label<void>().c_str(),
3399 : 128);
3400 : } else { strcat(err, "**UNMANAGED**"); })
3401 : KOKKOS_IF_ON_DEVICE([&] {
3402 : // Check #1: is there a SharedAllocationRecord? (we won't use it, but
3403 : // if its not there then there isn't a corresponding
3404 : // SharedAllocationHeader containing a label). This check should cover
3405 : // the case of Views that don't have the Unmanaged trait but were
3406 : // initialized by pointer.
3407 : if (!tracker.m_tracker.has_record()) {
3408 : strcat(err, "**UNMANAGED**");
3409 : return;
3410 : }
3411 : // Check #2: does the ViewMapping have the printable_label_typedef
3412 : // defined? See above that only the non-specialized standard-layout
3413 : // ViewMapping has this defined by default. The existence of this
3414 : // alias indicates the existence of MapType::is_managed
3415 : if constexpr (is_detected_v<printable_label_typedef_t, MapType>) {
3416 : // Check #3: is the View managed as determined by the MemoryTraits?
3417 : if constexpr (MapType::is_managed != 0) {
3418 : SharedAllocationHeader const* const header =
3419 : SharedAllocationHeader::get_header(
3420 : static_cast<void const*>(map.data()));
3421 : char const* const label = header->label();
3422 : strcat(err, label);
3423 : return;
3424 : }
3425 : strcat(err, "**UNAVAILABLE**");
3426 : }
3427 : }();)
3428 0 : strcat(err, "\") with indices ");
3429 0 : append_formatted_multidimensional_index(err, args...);
3430 0 : strcat(err, " but extents ");
3431 0 : print_extents(err, map, std::make_index_sequence<sizeof...(Args)>());
3432 : Kokkos::abort(err);
3433 : }
3434 39447979 : }
3435 :
3436 : // primary template: memory space is accessible, do nothing.
3437 : template <class MemorySpace, class AccessSpace,
3438 : bool = SpaceAccessibility<AccessSpace, MemorySpace>::accessible>
3439 : struct RuntimeCheckViewMemoryAccessViolation {
3440 : template <class Track, class Map>
3441 38778703 : KOKKOS_FUNCTION RuntimeCheckViewMemoryAccessViolation(char const* const,
3442 : Track const&,
3443 : Map const&) {}
3444 : };
3445 :
3446 : // explicit specialization: memory access violation will occur, call abort with
3447 : // the specified error message.
3448 : template <class MemorySpace, class AccessSpace>
3449 : struct RuntimeCheckViewMemoryAccessViolation<MemorySpace, AccessSpace, false> {
3450 : template <class Track, class Map>
3451 : KOKKOS_FUNCTION RuntimeCheckViewMemoryAccessViolation(char const* const msg,
3452 : Track const& track,
3453 : Map const&) {
3454 : char err[256] = "";
3455 : strncat(err, msg, 64);
3456 : strcat(err, " (label=\"");
3457 :
3458 : KOKKOS_IF_ON_HOST(({
3459 : auto const tracker = track.m_tracker;
3460 :
3461 : if (tracker.has_record()) {
3462 : strncat(err, tracker.template get_label<void>().c_str(), 128);
3463 : } else {
3464 : strcat(err, "**UNMANAGED**");
3465 : }
3466 : }))
3467 :
3468 : KOKKOS_IF_ON_DEVICE(({
3469 : strcat(err, "**UNAVAILABLE**");
3470 : (void)track;
3471 : }))
3472 :
3473 : strcat(err, "\")");
3474 :
3475 : Kokkos::abort(err);
3476 : }
3477 : };
3478 :
3479 : template <class MemorySpace, class Track, class Map, class... Ignore>
3480 38778703 : KOKKOS_FUNCTION void runtime_check_memory_access_violation(
3481 : char const* const msg, Track const& track, Map const& map, Ignore...) {
3482 38778703 : KOKKOS_IF_ON_HOST(
3483 : ((void)RuntimeCheckViewMemoryAccessViolation<MemorySpace,
3484 : DefaultHostExecutionSpace>(
3485 : msg, track, map);))
3486 : KOKKOS_IF_ON_DEVICE(
3487 : ((void)RuntimeCheckViewMemoryAccessViolation<MemorySpace,
3488 : DefaultExecutionSpace>(
3489 : msg, track, map);))
3490 38778703 : }
3491 :
3492 : } /* namespace Impl */
3493 : } /* namespace Kokkos */
3494 :
3495 : //----------------------------------------------------------------------------
3496 : //----------------------------------------------------------------------------
3497 :
3498 : #endif /* #ifndef KOKKOS_EXPERIMENTAL_VIEW_MAPPING_HPP */
|
