1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008 Gael Guennebaud <[email protected]>
5// Copyright (C) 2007-2011 Benoit Jacob <[email protected]>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10
11#ifndef EIGEN_CORE_H
12#define EIGEN_CORE_H
13
14// first thing Eigen does: stop the compiler from committing suicide
15#include "src/Core/util/DisableStupidWarnings.h"
16
17#if defined(__CUDACC__) && !defined(EIGEN_NO_CUDA)
18 #define EIGEN_CUDACC __CUDACC__
19#endif
20
21#if defined(__CUDA_ARCH__) && !defined(EIGEN_NO_CUDA)
22 #define EIGEN_CUDA_ARCH __CUDA_ARCH__
23#endif
24
25#if defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 9)
26#define EIGEN_CUDACC_VER ((__CUDACC_VER_MAJOR__ * 10000) + (__CUDACC_VER_MINOR__ * 100))
27#elif defined(__CUDACC_VER__)
28#define EIGEN_CUDACC_VER __CUDACC_VER__
29#else
30#define EIGEN_CUDACC_VER 0
31#endif
32
33// Handle NVCC/CUDA/SYCL
34#if defined(__CUDACC__) || defined(__SYCL_DEVICE_ONLY__)
35 // Do not try asserts on CUDA and SYCL!
36 #ifndef EIGEN_NO_DEBUG
37 #define EIGEN_NO_DEBUG
38 #endif
39
40 #ifdef EIGEN_INTERNAL_DEBUGGING
41 #undef EIGEN_INTERNAL_DEBUGGING
42 #endif
43
44 #ifdef EIGEN_EXCEPTIONS
45 #undef EIGEN_EXCEPTIONS
46 #endif
47
48 // All functions callable from CUDA code must be qualified with __device__
49 #ifdef __CUDACC__
50 // Do not try to vectorize on CUDA and SYCL!
51 #ifndef EIGEN_DONT_VECTORIZE
52 #define EIGEN_DONT_VECTORIZE
53 #endif
54
55 #define EIGEN_DEVICE_FUNC __host__ __device__
56 // We need cuda_runtime.h to ensure that that EIGEN_USING_STD_MATH macro
57 // works properly on the device side
58 #include <cuda_runtime.h>
59 #else
60 #define EIGEN_DEVICE_FUNC
61 #endif
62
63#else
64 #define EIGEN_DEVICE_FUNC
65
66#endif
67
68// When compiling CUDA device code with NVCC, pull in math functions from the
69// global namespace. In host mode, and when device doee with clang, use the
70// std versions.
71#if defined(__CUDA_ARCH__) && defined(__NVCC__)
72 #define EIGEN_USING_STD_MATH(FUNC) using ::FUNC;
73#else
74 #define EIGEN_USING_STD_MATH(FUNC) using std::FUNC;
75#endif
76
77#if (defined(_CPPUNWIND) || defined(__EXCEPTIONS)) && !defined(__CUDA_ARCH__) && !defined(EIGEN_EXCEPTIONS) && !defined(EIGEN_USE_SYCL)
78 #define EIGEN_EXCEPTIONS
79#endif
80
81#ifdef EIGEN_EXCEPTIONS
82 #include <new>
83#endif
84
85// then include this file where all our macros are defined. It's really important to do it first because
86// it's where we do all the alignment settings (platform detection and honoring the user's will if he
87// defined e.g. EIGEN_DONT_ALIGN) so it needs to be done before we do anything with vectorization.
88#include "src/Core/util/Macros.h"
89
90// Disable the ipa-cp-clone optimization flag with MinGW 6.x or newer (enabled by default with -O3)
91// See http://eigen.tuxfamily.org/bz/show_bug.cgi?id=556 for details.
92#if EIGEN_COMP_MINGW && EIGEN_GNUC_AT_LEAST(4,6)
93 #pragma GCC optimize ("-fno-ipa-cp-clone")
94#endif
95
96#include <complex>
97
98// this include file manages BLAS and MKL related macros
99// and inclusion of their respective header files
100#include "src/Core/util/MKL_support.h"
101
102// if alignment is disabled, then disable vectorization. Note: EIGEN_MAX_ALIGN_BYTES is the proper check, it takes into
103// account both the user's will (EIGEN_MAX_ALIGN_BYTES,EIGEN_DONT_ALIGN) and our own platform checks
104#if EIGEN_MAX_ALIGN_BYTES==0
105 #ifndef EIGEN_DONT_VECTORIZE
106 #define EIGEN_DONT_VECTORIZE
107 #endif
108#endif
109
110#if EIGEN_COMP_MSVC
111 #include <malloc.h> // for _aligned_malloc -- need it regardless of whether vectorization is enabled
112 #if (EIGEN_COMP_MSVC >= 1500) // 2008 or later
113 // Remember that usage of defined() in a #define is undefined by the standard.
114 // a user reported that in 64-bit mode, MSVC doesn't care to define _M_IX86_FP.
115 #if (defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) || EIGEN_ARCH_x86_64
116 #define EIGEN_SSE2_ON_MSVC_2008_OR_LATER
117 #endif
118 #endif
119#else
120 // Remember that usage of defined() in a #define is undefined by the standard
121 #if (defined __SSE2__) && ( (!EIGEN_COMP_GNUC) || EIGEN_COMP_ICC || EIGEN_GNUC_AT_LEAST(4,2) )
122 #define EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC
123 #endif
124#endif
125
126#ifndef EIGEN_DONT_VECTORIZE
127
128 #if defined (EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC) || defined(EIGEN_SSE2_ON_MSVC_2008_OR_LATER)
129
130 // Defines symbols for compile-time detection of which instructions are
131 // used.
132 // EIGEN_VECTORIZE_YY is defined if and only if the instruction set YY is used
133 #define EIGEN_VECTORIZE
134 #define EIGEN_VECTORIZE_SSE
135 #define EIGEN_VECTORIZE_SSE2
136
137 // Detect sse3/ssse3/sse4:
138 // gcc and icc defines __SSE3__, ...
139 // there is no way to know about this on msvc. You can define EIGEN_VECTORIZE_SSE* if you
140 // want to force the use of those instructions with msvc.
141 #ifdef __SSE3__
142 #define EIGEN_VECTORIZE_SSE3
143 #endif
144 #ifdef __SSSE3__
145 #define EIGEN_VECTORIZE_SSSE3
146 #endif
147 #ifdef __SSE4_1__
148 #define EIGEN_VECTORIZE_SSE4_1
149 #endif
150 #ifdef __SSE4_2__
151 #define EIGEN_VECTORIZE_SSE4_2
152 #endif
153 #ifdef __AVX__
154 #define EIGEN_VECTORIZE_AVX
155 #define EIGEN_VECTORIZE_SSE3
156 #define EIGEN_VECTORIZE_SSSE3
157 #define EIGEN_VECTORIZE_SSE4_1
158 #define EIGEN_VECTORIZE_SSE4_2
159 #endif
160 #ifdef __AVX2__
161 #define EIGEN_VECTORIZE_AVX2
162 #endif
163 #ifdef __FMA__
164 #define EIGEN_VECTORIZE_FMA
165 #endif
166 #if defined(__AVX512F__) && defined(EIGEN_ENABLE_AVX512)
167 #define EIGEN_VECTORIZE_AVX512
168 #define EIGEN_VECTORIZE_AVX2
169 #define EIGEN_VECTORIZE_AVX
170 #define EIGEN_VECTORIZE_FMA
171 #ifdef __AVX512DQ__
172 #define EIGEN_VECTORIZE_AVX512DQ
173 #endif
174 #ifdef __AVX512ER__
175 #define EIGEN_VECTORIZE_AVX512ER
176 #endif
177 #endif
178
179 // include files
180
181 // This extern "C" works around a MINGW-w64 compilation issue
182 // https://sourceforge.net/tracker/index.php?func=detail&aid=3018394&group_id=202880&atid=983354
183 // In essence, intrin.h is included by windows.h and also declares intrinsics (just as emmintrin.h etc. below do).
184 // However, intrin.h uses an extern "C" declaration, and g++ thus complains of duplicate declarations
185 // with conflicting linkage. The linkage for intrinsics doesn't matter, but at that stage the compiler doesn't know;
186 // so, to avoid compile errors when windows.h is included after Eigen/Core, ensure intrinsics are extern "C" here too.
187 // notice that since these are C headers, the extern "C" is theoretically needed anyways.
188 extern "C" {
189 // In theory we should only include immintrin.h and not the other *mmintrin.h header files directly.
190 // Doing so triggers some issues with ICC. However old gcc versions seems to not have this file, thus:
191 #if EIGEN_COMP_ICC >= 1110
192 #include <immintrin.h>
193 #else
194 #include <mmintrin.h>
195 #include <emmintrin.h>
196 #include <xmmintrin.h>
197 #ifdef EIGEN_VECTORIZE_SSE3
198 #include <pmmintrin.h>
199 #endif
200 #ifdef EIGEN_VECTORIZE_SSSE3
201 #include <tmmintrin.h>
202 #endif
203 #ifdef EIGEN_VECTORIZE_SSE4_1
204 #include <smmintrin.h>
205 #endif
206 #ifdef EIGEN_VECTORIZE_SSE4_2
207 #include <nmmintrin.h>
208 #endif
209 #if defined(EIGEN_VECTORIZE_AVX) || defined(EIGEN_VECTORIZE_AVX512)
210 #include <immintrin.h>
211 #endif
212 #endif
213 } // end extern "C"
214 #elif defined __VSX__
215 #define EIGEN_VECTORIZE
216 #define EIGEN_VECTORIZE_VSX
217 #include <altivec.h>
218 // We need to #undef all these ugly tokens defined in <altivec.h>
219 // => use __vector instead of vector
220 #undef bool
221 #undef vector
222 #undef pixel
223 #elif defined __ALTIVEC__
224 #define EIGEN_VECTORIZE
225 #define EIGEN_VECTORIZE_ALTIVEC
226 #include <altivec.h>
227 // We need to #undef all these ugly tokens defined in <altivec.h>
228 // => use __vector instead of vector
229 #undef bool
230 #undef vector
231 #undef pixel
232 #elif (defined __ARM_NEON) || (defined __ARM_NEON__)
233 #define EIGEN_VECTORIZE
234 #define EIGEN_VECTORIZE_NEON
235 #include <arm_neon.h>
236 #elif (defined __s390x__ && defined __VEC__)
237 #define EIGEN_VECTORIZE
238 #define EIGEN_VECTORIZE_ZVECTOR
239 #include <vecintrin.h>
240 #endif
241#endif
242
243#if defined(__F16C__) && !defined(EIGEN_COMP_CLANG)
244 // We can use the optimized fp16 to float and float to fp16 conversion routines
245 #define EIGEN_HAS_FP16_C
246#endif
247
248#if defined __CUDACC__
249 #define EIGEN_VECTORIZE_CUDA
250 #include <vector_types.h>
251 #if EIGEN_CUDACC_VER >= 70500
252 #define EIGEN_HAS_CUDA_FP16
253 #endif
254#endif
255
256#if defined EIGEN_HAS_CUDA_FP16
257 #include <host_defines.h>
258 #include <cuda_fp16.h>
259#endif
260
261#if (defined _OPENMP) && (!defined EIGEN_DONT_PARALLELIZE)
262 #define EIGEN_HAS_OPENMP
263#endif
264
265#ifdef EIGEN_HAS_OPENMP
266#include <omp.h>
267#endif
268
269// MSVC for windows mobile does not have the errno.h file
270#if !(EIGEN_COMP_MSVC && EIGEN_OS_WINCE) && !EIGEN_COMP_ARM
271#define EIGEN_HAS_ERRNO
272#endif
273
274#ifdef EIGEN_HAS_ERRNO
275#include <cerrno>
276#endif
277#include <cstddef>
278#include <cstdlib>
279#include <cmath>
280#include <cassert>
281#include <functional>
282#include <sstream>
283#ifndef EIGEN_NO_IO
284 #include <iosfwd>
285#endif
286#include <cstring>
287#include <string>
288#include <limits>
289#include <climits> // for CHAR_BIT
290// for min/max:
291#include <algorithm>
292
293// for std::is_nothrow_move_assignable
294#ifdef EIGEN_INCLUDE_TYPE_TRAITS
295#include <type_traits>
296#endif
297
298// for outputting debug info
299#ifdef EIGEN_DEBUG_ASSIGN
300#include <iostream>
301#endif
302
303// required for __cpuid, needs to be included after cmath
304#if EIGEN_COMP_MSVC && EIGEN_ARCH_i386_OR_x86_64 && !EIGEN_OS_WINCE
305 #include <intrin.h>
306#endif
307
308/** \brief Namespace containing all symbols from the %Eigen library. */
309namespace Eigen {
310
311inline static const char *SimdInstructionSetsInUse(void) {
312#if defined(EIGEN_VECTORIZE_AVX512)
313 return "AVX512, FMA, AVX2, AVX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2";
314#elif defined(EIGEN_VECTORIZE_AVX)
315 return "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2";
316#elif defined(EIGEN_VECTORIZE_SSE4_2)
317 return "SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2";
318#elif defined(EIGEN_VECTORIZE_SSE4_1)
319 return "SSE, SSE2, SSE3, SSSE3, SSE4.1";
320#elif defined(EIGEN_VECTORIZE_SSSE3)
321 return "SSE, SSE2, SSE3, SSSE3";
322#elif defined(EIGEN_VECTORIZE_SSE3)
323 return "SSE, SSE2, SSE3";
324#elif defined(EIGEN_VECTORIZE_SSE2)
325 return "SSE, SSE2";
326#elif defined(EIGEN_VECTORIZE_ALTIVEC)
327 return "AltiVec";
328#elif defined(EIGEN_VECTORIZE_VSX)
329 return "VSX";
330#elif defined(EIGEN_VECTORIZE_NEON)
331 return "ARM NEON";
332#elif defined(EIGEN_VECTORIZE_ZVECTOR)
333 return "S390X ZVECTOR";
334#else
335 return "None";
336#endif
337}
338
339} // end namespace Eigen
340
341#if defined EIGEN2_SUPPORT_STAGE40_FULL_EIGEN3_STRICTNESS || defined EIGEN2_SUPPORT_STAGE30_FULL_EIGEN3_API || defined EIGEN2_SUPPORT_STAGE20_RESOLVE_API_CONFLICTS || defined EIGEN2_SUPPORT_STAGE10_FULL_EIGEN2_API || defined EIGEN2_SUPPORT
342// This will generate an error message:
343#error Eigen2-support is only available up to version 3.2. Please go to "http://eigen.tuxfamily.org/index.php?title=Eigen2" for further information
344#endif
345
346namespace Eigen {
347
348// we use size_t frequently and we'll never remember to prepend it with std:: everytime just to
349// ensure QNX/QCC support
350using std::size_t;
351// gcc 4.6.0 wants std:: for ptrdiff_t
352using std::ptrdiff_t;
353
354}
355
356/** \defgroup Core_Module Core module
357 * This is the main module of Eigen providing dense matrix and vector support
358 * (both fixed and dynamic size) with all the features corresponding to a BLAS library
359 * and much more...
360 *
361 * \code
362 * #include <Eigen/Core>
363 * \endcode
364 */
365
366#include "src/Core/util/Constants.h"
367#include "src/Core/util/Meta.h"
368#include "src/Core/util/ForwardDeclarations.h"
369#include "src/Core/util/StaticAssert.h"
370#include "src/Core/util/XprHelper.h"
371#include "src/Core/util/Memory.h"
372
373#include "src/Core/NumTraits.h"
374#include "src/Core/MathFunctions.h"
375#include "src/Core/GenericPacketMath.h"
376#include "src/Core/MathFunctionsImpl.h"
377#include "src/Core/arch/Default/ConjHelper.h"
378
379#if defined EIGEN_VECTORIZE_AVX512
380 #include "src/Core/arch/SSE/PacketMath.h"
381 #include "src/Core/arch/SSE/MathFunctions.h"
382 #include "src/Core/arch/AVX/PacketMath.h"
383 #include "src/Core/arch/AVX/MathFunctions.h"
384 #include "src/Core/arch/AVX512/PacketMath.h"
385 #include "src/Core/arch/AVX512/MathFunctions.h"
386#elif defined EIGEN_VECTORIZE_AVX
387 // Use AVX for floats and doubles, SSE for integers
388 #include "src/Core/arch/SSE/PacketMath.h"
389 #include "src/Core/arch/SSE/Complex.h"
390 #include "src/Core/arch/SSE/MathFunctions.h"
391 #include "src/Core/arch/AVX/PacketMath.h"
392 #include "src/Core/arch/AVX/MathFunctions.h"
393 #include "src/Core/arch/AVX/Complex.h"
394 #include "src/Core/arch/AVX/TypeCasting.h"
395 #include "src/Core/arch/SSE/TypeCasting.h"
396#elif defined EIGEN_VECTORIZE_SSE
397 #include "src/Core/arch/SSE/PacketMath.h"
398 #include "src/Core/arch/SSE/MathFunctions.h"
399 #include "src/Core/arch/SSE/Complex.h"
400 #include "src/Core/arch/SSE/TypeCasting.h"
401#elif defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX)
402 #include "src/Core/arch/AltiVec/PacketMath.h"
403 #include "src/Core/arch/AltiVec/MathFunctions.h"
404 #include "src/Core/arch/AltiVec/Complex.h"
405#elif defined EIGEN_VECTORIZE_NEON
406 #include "src/Core/arch/NEON/PacketMath.h"
407 #include "src/Core/arch/NEON/MathFunctions.h"
408 #include "src/Core/arch/NEON/Complex.h"
409#elif defined EIGEN_VECTORIZE_ZVECTOR
410 #include "src/Core/arch/ZVector/PacketMath.h"
411 #include "src/Core/arch/ZVector/MathFunctions.h"
412 #include "src/Core/arch/ZVector/Complex.h"
413#endif
414
415// Half float support
416#include "src/Core/arch/CUDA/Half.h"
417#include "src/Core/arch/CUDA/PacketMathHalf.h"
418#include "src/Core/arch/CUDA/TypeCasting.h"
419
420#if defined EIGEN_VECTORIZE_CUDA
421 #include "src/Core/arch/CUDA/PacketMath.h"
422 #include "src/Core/arch/CUDA/MathFunctions.h"
423#endif
424
425#include "src/Core/arch/Default/Settings.h"
426
427#include "src/Core/functors/TernaryFunctors.h"
428#include "src/Core/functors/BinaryFunctors.h"
429#include "src/Core/functors/UnaryFunctors.h"
430#include "src/Core/functors/NullaryFunctors.h"
431#include "src/Core/functors/StlFunctors.h"
432#include "src/Core/functors/AssignmentFunctors.h"
433
434// Specialized functors to enable the processing of complex numbers
435// on CUDA devices
436#include "src/Core/arch/CUDA/Complex.h"
437
438#include "src/Core/IO.h"
439#include "src/Core/DenseCoeffsBase.h"
440#include "src/Core/DenseBase.h"
441#include "src/Core/MatrixBase.h"
442#include "src/Core/EigenBase.h"
443
444#include "src/Core/Product.h"
445#include "src/Core/CoreEvaluators.h"
446#include "src/Core/AssignEvaluator.h"
447
448#ifndef EIGEN_PARSED_BY_DOXYGEN // work around Doxygen bug triggered by Assign.h r814874
449 // at least confirmed with Doxygen 1.5.5 and 1.5.6
450 #include "src/Core/Assign.h"
451#endif
452
453#include "src/Core/ArrayBase.h"
454#include "src/Core/util/BlasUtil.h"
455#include "src/Core/DenseStorage.h"
456#include "src/Core/NestByValue.h"
457
458// #include "src/Core/ForceAlignedAccess.h"
459
460#include "src/Core/ReturnByValue.h"
461#include "src/Core/NoAlias.h"
462#include "src/Core/PlainObjectBase.h"
463#include "src/Core/Matrix.h"
464#include "src/Core/Array.h"
465#include "src/Core/CwiseTernaryOp.h"
466#include "src/Core/CwiseBinaryOp.h"
467#include "src/Core/CwiseUnaryOp.h"
468#include "src/Core/CwiseNullaryOp.h"
469#include "src/Core/CwiseUnaryView.h"
470#include "src/Core/SelfCwiseBinaryOp.h"
471#include "src/Core/Dot.h"
472#include "src/Core/StableNorm.h"
473#include "src/Core/Stride.h"
474#include "src/Core/MapBase.h"
475#include "src/Core/Map.h"
476#include "src/Core/Ref.h"
477#include "src/Core/Block.h"
478#include "src/Core/VectorBlock.h"
479#include "src/Core/Transpose.h"
480#include "src/Core/DiagonalMatrix.h"
481#include "src/Core/Diagonal.h"
482#include "src/Core/DiagonalProduct.h"
483#include "src/Core/Redux.h"
484#include "src/Core/Visitor.h"
485#include "src/Core/Fuzzy.h"
486#include "src/Core/Swap.h"
487#include "src/Core/CommaInitializer.h"
488#include "src/Core/GeneralProduct.h"
489#include "src/Core/Solve.h"
490#include "src/Core/Inverse.h"
491#include "src/Core/SolverBase.h"
492#include "src/Core/PermutationMatrix.h"
493#include "src/Core/Transpositions.h"
494#include "src/Core/TriangularMatrix.h"
495#include "src/Core/SelfAdjointView.h"
496#include "src/Core/products/GeneralBlockPanelKernel.h"
497#include "src/Core/products/Parallelizer.h"
498#include "src/Core/ProductEvaluators.h"
499#include "src/Core/products/GeneralMatrixVector.h"
500#include "src/Core/products/GeneralMatrixMatrix.h"
501#include "src/Core/SolveTriangular.h"
502#include "src/Core/products/GeneralMatrixMatrixTriangular.h"
503#include "src/Core/products/SelfadjointMatrixVector.h"
504#include "src/Core/products/SelfadjointMatrixMatrix.h"
505#include "src/Core/products/SelfadjointProduct.h"
506#include "src/Core/products/SelfadjointRank2Update.h"
507#include "src/Core/products/TriangularMatrixVector.h"
508#include "src/Core/products/TriangularMatrixMatrix.h"
509#include "src/Core/products/TriangularSolverMatrix.h"
510#include "src/Core/products/TriangularSolverVector.h"
511#include "src/Core/BandMatrix.h"
512#include "src/Core/CoreIterators.h"
513#include "src/Core/ConditionEstimator.h"
514
515#include "src/Core/BooleanRedux.h"
516#include "src/Core/Select.h"
517#include "src/Core/VectorwiseOp.h"
518#include "src/Core/Random.h"
519#include "src/Core/Replicate.h"
520#include "src/Core/Reverse.h"
521#include "src/Core/ArrayWrapper.h"
522
523#ifdef EIGEN_USE_BLAS
524#include "src/Core/products/GeneralMatrixMatrix_BLAS.h"
525#include "src/Core/products/GeneralMatrixVector_BLAS.h"
526#include "src/Core/products/GeneralMatrixMatrixTriangular_BLAS.h"
527#include "src/Core/products/SelfadjointMatrixMatrix_BLAS.h"
528#include "src/Core/products/SelfadjointMatrixVector_BLAS.h"
529#include "src/Core/products/TriangularMatrixMatrix_BLAS.h"
530#include "src/Core/products/TriangularMatrixVector_BLAS.h"
531#include "src/Core/products/TriangularSolverMatrix_BLAS.h"
532#endif // EIGEN_USE_BLAS
533
534#ifdef EIGEN_USE_MKL_VML
535#include "src/Core/Assign_MKL.h"
536#endif
537
538#include "src/Core/GlobalFunctions.h"
539
540#include "src/Core/util/ReenableStupidWarnings.h"
541
542#endif // EIGEN_CORE_H
543