Line data    Source code

       1             : //@HEADER
       2             : // ************************************************************************
       3             : //
       4             : //                        Kokkos v. 4.0
       5             : //       Copyright (2022) National Technology & Engineering
       6             : //               Solutions of Sandia, LLC (NTESS).
       7             : //
       8             : // Under the terms of Contract DE-NA0003525 with NTESS,
       9             : // the U.S. Government retains certain rights in this software.
      10             : //
      11             : // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
      12             : // See https://kokkos.org/LICENSE for license information.
      13             : // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
      14             : //
      15             : //@HEADER
      16             : 
      17             : #ifndef KOKKOS_CORE_HPP
      18             : #define KOKKOS_CORE_HPP
      19             : #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
      20             : #define KOKKOS_IMPL_PUBLIC_INCLUDE
      21             : #define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_CORE
      22             : #endif
      23             : 
      24             : //----------------------------------------------------------------------------
      25             : // In the case windows.h is included before Kokkos_Core.hpp there might be
      26             : // errors due to the potentially defined macros with name "min" and "max" in
      27             : // windows.h. These collide with the use of "min" and "max" in names inside
      28             : // Kokkos. The macros will be redefined at the end of Kokkos_Core.hpp
      29             : #if defined(min)
      30             : #pragma push_macro("min")
      31             : #undef min
      32             : #define KOKKOS_IMPL_PUSH_MACRO_MIN
      33             : #endif
      34             : #if defined(max)
      35             : #pragma push_macro("max")
      36             : #undef max
      37             : #define KOKKOS_IMPL_PUSH_MACRO_MAX
      38             : #endif
      39             : 
      40             : //----------------------------------------------------------------------------
      41             : // Include the execution space header files for the enabled execution spaces.
      42             : 
      43             : #include <Kokkos_Core_fwd.hpp>
      44             : 
      45             : #include <KokkosCore_Config_DeclareBackend.hpp>
      46             : 
      47             : #include <Kokkos_Half.hpp>
      48             : #include <Kokkos_AnonymousSpace.hpp>
      49             : #include <Kokkos_Pair.hpp>
      50             : #include <Kokkos_Clamp.hpp>
      51             : #include <Kokkos_MinMax.hpp>
      52             : #include <Kokkos_MathematicalConstants.hpp>
      53             : #include <Kokkos_MathematicalFunctions.hpp>
      54             : #include <Kokkos_MathematicalSpecialFunctions.hpp>
      55             : #include <Kokkos_NumericTraits.hpp>
      56             : #include <Kokkos_BitManipulation.hpp>
      57             : #include <Kokkos_Swap.hpp>
      58             : #include <Kokkos_MemoryPool.hpp>
      59             : #include <Kokkos_Array.hpp>
      60             : #include <Kokkos_View.hpp>
      61             : #include <Kokkos_Vectorization.hpp>
      62             : #include <Kokkos_Atomic.hpp>
      63             : #include <Kokkos_hwloc.hpp>
      64             : #include <Kokkos_Timer.hpp>
      65             : #include <Kokkos_Tuners.hpp>
      66             : #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4
      67             : #include <Kokkos_TaskScheduler.hpp>
      68             : #endif
      69             : #include <Kokkos_Complex.hpp>
      70             : #include <Kokkos_CopyViews.hpp>
      71             : #include <impl/Kokkos_TeamMDPolicy.hpp>
      72             : #include <impl/Kokkos_InitializationSettings.hpp>
      73             : #include <functional>
      74             : #include <iosfwd>
      75             : #include <memory>
      76             : #include <vector>
      77             : 
      78             : //----------------------------------------------------------------------------
      79             : 
      80             : namespace Kokkos {
      81             : 
      82             : void initialize(int& argc, char* argv[]);
      83             : 
      84             : void initialize(
      85             :     InitializationSettings const& settings = InitializationSettings());
      86             : 
      87             : namespace Impl {
      88             : 
      89             : void pre_initialize(const InitializationSettings& settings);
      90             : 
      91             : void post_initialize(const InitializationSettings& settings);
      92             : 
      93             : void pre_finalize();
      94             : 
      95             : void post_finalize();
      96             : 
      97             : void declare_configuration_metadata(const std::string& category,
      98             :                                     const std::string& key,
      99             :                                     const std::string& value);
     100             : 
     101             : }  // namespace Impl
     102             : 
     103             : [[nodiscard]] bool is_initialized() noexcept;
     104             : [[nodiscard]] bool is_finalized() noexcept;
     105             : 
     106             : [[nodiscard]] int device_id() noexcept;
     107             : [[nodiscard]] int num_devices() noexcept;
     108             : [[nodiscard]] int num_threads() noexcept;
     109             : 
     110             : bool show_warnings() noexcept;
     111             : bool tune_internals() noexcept;
     112             : 
     113             : /** \brief  Finalize the spaces that were initialized via Kokkos::initialize */
     114             : void finalize();
     115             : 
     116             : /**
     117             :  * \brief Push a user-defined function to be called in
     118             :  *   Kokkos::finalize, before any Kokkos state is finalized.
     119             :  *
     120             :  * \warning Only call this after Kokkos::initialize, but before
     121             :  *   Kokkos::finalize.
     122             :  *
     123             :  * This function is the Kokkos analog to std::atexit.  If you call
     124             :  * this with a function f, then your function will get called when
     125             :  * Kokkos::finalize is called.  Specifically, it will be called BEFORE
     126             :  * Kokkos does any finalization.  This means that all execution
     127             :  * spaces, memory spaces, etc. that were initialized will still be
     128             :  * initialized when your function is called.
     129             :  *
     130             :  * Just like std::atexit, if you call push_finalize_hook in sequence
     131             :  * with multiple functions (f, g, h), Kokkos::finalize will call them
     132             :  * in reverse order (h, g, f), as if popping a stack.  Furthermore,
     133             :  * just like std::atexit, if any of your functions throws but does not
     134             :  * catch an exception, Kokkos::finalize will call std::terminate.
     135             :  */
     136             : void push_finalize_hook(std::function<void()> f);
     137             : 
     138             : void fence(const std::string& name /*= "Kokkos::fence: Unnamed Global Fence"*/);
     139             : 
     140             : /** \brief Print "Bill of Materials" */
     141             : void print_configuration(std::ostream& os, bool verbose = false);
     142             : 
     143             : }  // namespace Kokkos
     144             : 
     145             : //----------------------------------------------------------------------------
     146             : //----------------------------------------------------------------------------
     147             : 
     148             : namespace Kokkos {
     149             : 
     150             : /* Allocate memory from a memory space.
     151             :  * The allocation is tracked in Kokkos memory tracking system, so
     152             :  * leaked memory can be identified.
     153             :  */
     154             : template <class Space = Kokkos::DefaultExecutionSpace::memory_space>
     155             : inline void* kokkos_malloc(const std::string& arg_alloc_label,
     156             :                            const size_t arg_alloc_size) {
     157             :   using MemorySpace = typename Space::memory_space;
     158             :   return Impl::SharedAllocationRecord<MemorySpace>::allocate_tracked(
     159             :       MemorySpace(), arg_alloc_label, arg_alloc_size);
     160             : }
     161             : 
     162             : template <class Space = Kokkos::DefaultExecutionSpace::memory_space>
     163             : inline void* kokkos_malloc(const size_t arg_alloc_size) {
     164             :   using MemorySpace = typename Space::memory_space;
     165             :   return Impl::SharedAllocationRecord<MemorySpace>::allocate_tracked(
     166             :       MemorySpace(), "no-label", arg_alloc_size);
     167             : }
     168             : 
     169             : template <class Space = Kokkos::DefaultExecutionSpace::memory_space>
     170             : inline void kokkos_free(void* arg_alloc) {
     171             :   using MemorySpace = typename Space::memory_space;
     172             :   return Impl::SharedAllocationRecord<MemorySpace>::deallocate_tracked(
     173             :       arg_alloc);
     174             : }
     175             : 
     176             : template <class Space = Kokkos::DefaultExecutionSpace::memory_space>
     177             : inline void* kokkos_realloc(void* arg_alloc, const size_t arg_alloc_size) {
     178             :   using MemorySpace = typename Space::memory_space;
     179             :   return Impl::SharedAllocationRecord<MemorySpace>::reallocate_tracked(
     180             :       arg_alloc, arg_alloc_size);
     181             : }
     182             : 
     183             : }  // namespace Kokkos
     184             : 
     185             : namespace Kokkos {
     186             : 
     187             : /** \brief  ScopeGuard
     188             :  *  Some user scope issues have been identified with some Kokkos::finalize
     189             :  * calls; ScopeGuard aims to correct these issues.
     190             :  *
     191             :  *  Two requirements for ScopeGuard:
     192             :  *     if Kokkos::is_initialized() in the constructor, don't call
     193             :  * Kokkos::initialize or Kokkos::finalize it is not copyable or assignable
     194             :  */
     195             : namespace Impl {
     196             : 
     197           0 : inline std::string scopeguard_correct_usage() {
     198           0 :   return std::string(
     199             :       "Do instead:\n"
     200             :       "  std::unique_ptr<Kokkos::ScopeGuard> guard =\n"
     201             :       "    !Kokkos::is_initialized() && !Kokkos::is_finalized()?\n"
     202           0 :       "    new ScopeGuard(argc,argv) : nullptr;\n");
     203             : }
     204             : 
     205           0 : inline std::string scopeguard_create_while_initialized_warning() {
     206           0 :   return std::string(
     207             :              "Kokkos Error: Creating a ScopeGuard while Kokkos is initialized "
     208             :              "is illegal.\n")
     209           0 :       .append(scopeguard_correct_usage());
     210             : }
     211             : 
     212           0 : inline std::string scopeguard_create_after_finalize_warning() {
     213           0 :   return std::string(
     214             :              "Kokkos Error: Creating a ScopeGuard after Kokkos was finalized "
     215             :              "is illegal.\n")
     216           0 :       .append(scopeguard_correct_usage());
     217             : }
     218             : 
     219           0 : inline std::string scopeguard_destruct_after_finalize_warning() {
     220           0 :   return std::string(
     221             :              "Kokkos Error: Destroying a ScopeGuard after Kokkos was finalized "
     222             :              "is illegal.\n")
     223           0 :       .append(scopeguard_correct_usage());
     224             : }
     225             : 
     226             : }  // namespace Impl
     227             : 
     228             : class KOKKOS_ATTRIBUTE_NODISCARD ScopeGuard {
     229             :  public:
     230             :   template <class... Args>
     231             : #if defined(__has_cpp_attribute) && __has_cpp_attribute(nodiscard) >= 201907
     232             :   [[nodiscard]]
     233             : #endif
     234         400 :   ScopeGuard(Args&&... args) {
     235         400 :     if (is_initialized()) {
     236           0 :       Kokkos::abort(
     237             :           Impl::scopeguard_create_while_initialized_warning().c_str());
     238             :     }
     239         400 :     if (is_finalized()) {
     240           0 :       Kokkos::abort(Impl::scopeguard_create_after_finalize_warning().c_str());
     241             :     }
     242         400 :     initialize(static_cast<Args&&>(args)...);
     243         400 :   }
     244             : 
     245         400 :   ~ScopeGuard() {
     246         400 :     if (is_finalized()) {
     247           0 :       Kokkos::abort(Impl::scopeguard_destruct_after_finalize_warning().c_str());
     248             :     }
     249         400 :     finalize();
     250         400 :   }
     251             : 
     252             :   ScopeGuard& operator=(const ScopeGuard&) = delete;
     253             :   ScopeGuard& operator=(ScopeGuard&&)      = delete;
     254             :   ScopeGuard(const ScopeGuard&)            = delete;
     255             :   ScopeGuard(ScopeGuard&&)                 = delete;
     256             : };
     257             : 
     258             : }  // namespace Kokkos
     259             : 
     260             : namespace Kokkos {
     261             : namespace Experimental {
     262             : // Partitioning an Execution Space: expects space and integer arguments for
     263             : // relative weight
     264             : //   Customization point for backends
     265             : //   Default behavior is to return the passed in instance
     266             : template <class ExecSpace, class... Args>
     267             : std::vector<ExecSpace> partition_space(ExecSpace const& space, Args...) {
     268             :   static_assert(is_execution_space<ExecSpace>::value,
     269             :                 "Kokkos Error: partition_space expects an Execution Space as "
     270             :                 "first argument");
     271             :   static_assert(
     272             :       (... && std::is_arithmetic_v<Args>),
     273             :       "Kokkos Error: partitioning arguments must be integers or floats");
     274             :   std::vector<ExecSpace> instances(sizeof...(Args));
     275             :   for (int s = 0; s < int(sizeof...(Args)); s++) instances[s] = space;
     276             :   return instances;
     277             : }
     278             : 
     279             : template <class ExecSpace, class T>
     280             : std::vector<ExecSpace> partition_space(ExecSpace const& space,
     281             :                                        std::vector<T> const& weights) {
     282             :   static_assert(is_execution_space<ExecSpace>::value,
     283             :                 "Kokkos Error: partition_space expects an Execution Space as "
     284             :                 "first argument");
     285             :   static_assert(
     286             :       std::is_arithmetic_v<T>,
     287             :       "Kokkos Error: partitioning arguments must be integers or floats");
     288             : 
     289             :   std::vector<ExecSpace> instances(weights.size());
     290             :   for (int s = 0; s < int(weights.size()); s++) instances[s] = space;
     291             :   return instances;
     292             : }
     293             : }  // namespace Experimental
     294             : }  // namespace Kokkos
     295             : 
     296             : #include <Kokkos_Crs.hpp>
     297             : #include <Kokkos_WorkGraphPolicy.hpp>
     298             : // Including this in Kokkos_Parallel_Reduce.hpp led to a circular dependency
     299             : // because Kokkos::Sum is used in Kokkos_Combined_Reducer.hpp and the default.
     300             : // The real answer is to finally break up Kokkos_Parallel_Reduce.hpp into
     301             : // smaller parts...
     302             : #include <impl/Kokkos_Combined_Reducer.hpp>
     303             : // Yet another workaround to deal with circular dependency issues because the
     304             : // implementation of the RAII wrapper is using Kokkos::single.
     305             : #include <Kokkos_AcquireUniqueTokenImpl.hpp>
     306             : 
     307             : //----------------------------------------------------------------------------
     308             : // Redefinition of the macros min and max if we pushed them at entry of
     309             : // Kokkos_Core.hpp
     310             : #if defined(KOKKOS_IMPL_PUSH_MACRO_MIN)
     311             : #pragma pop_macro("min")
     312             : #undef KOKKOS_IMPL_PUSH_MACRO_MIN
     313             : #endif
     314             : #if defined(KOKKOS_IMPL_PUSH_MACRO_MAX)
     315             : #pragma pop_macro("max")
     316             : #undef KOKKOS_IMPL_PUSH_MACRO_MAX
     317             : #endif
     318             : 
     319             : //----------------------------------------------------------------------------
     320             : //----------------------------------------------------------------------------
     321             : 
     322             : #ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_CORE
     323             : #undef KOKKOS_IMPL_PUBLIC_INCLUDE
     324             : #undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_CORE
     325             : #endif
     326             : #endif