35#ifndef _BLAZE_MATH_EXPRESSIONS_TDMATTDMATMULTEXPR_H_
36#define _BLAZE_MATH_EXPRESSIONS_TDMATTDMATMULTEXPR_H_
143template<
typename MT1
150 :
public MatMatMultExpr< DenseMatrix< TDMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, true > >
174 static constexpr bool SYM = ( SF && !( HF || LF || UF ) );
175 static constexpr bool HERM = ( HF && !( LF || UF ) );
176 static constexpr bool LOW = ( LF || ( ( SF || HF ) && UF ) );
177 static constexpr bool UPP = ( UF || ( ( SF || HF ) && LF ) );
187 template<
typename T1,
typename T2,
typename T3 >
188 static constexpr bool CanExploitSymmetry_v =
199 template<
typename T1,
typename T2,
typename T3 >
200 static constexpr bool IsEvaluationRequired_v =
210 template<
typename T1,
typename T2,
typename T3 >
211 static constexpr bool UseBlasKernel_v =
218 T1::simdEnabled && T2::simdEnabled && T3::simdEnabled &&
232 template<
typename T1,
typename T2,
typename T3 >
233 static constexpr bool UseVectorizedDefaultKernel_v =
234 ( useOptimizedKernels &&
236 T1::simdEnabled && T2::simdEnabled && T3::simdEnabled &&
309 MT1::simdEnabled && MT2::simdEnabled &&
370 :(
lhs_.columns() ) ) );
394 if( i >=
lhs_.rows() ) {
397 if( j >=
rhs_.columns() ) {
409 inline size_t rows() const noexcept {
420 return rhs_.columns();
450 template<
typename T >
451 inline bool canAlias(
const T* alias )
const noexcept {
452 return (
lhs_.isAliased( alias ) ||
rhs_.isAliased( alias ) );
462 template<
typename T >
463 inline bool isAliased(
const T* alias )
const noexcept {
464 return (
lhs_.isAliased( alias ) ||
rhs_.isAliased( alias ) );
474 return lhs_.isAligned() &&
rhs_.isAligned();
485 !BLAZE_USE_BLAS_MATRIX_MATRIX_MULTIPLICATION ||
487 (
rows() *
columns() < TDMATTDMATMULT_THRESHOLD ) ) &&
488 (
rows() *
columns() >= SMP_TDMATTDMATMULT_THRESHOLD ) &&
512 template<
typename MT
522 if( (*lhs).rows() == 0UL || (*lhs).columns() == 0UL ) {
525 else if( rhs.lhs_.columns() == 0UL ) {
540 TDMatTDMatMultExpr::selectAssignKernel( *lhs, A, B );
556 template<
typename MT3
559 static inline void selectAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
562 ( !BLAZE_DEBUG_MODE && A.rows() <=
SIMDSIZE*10UL ) ||
563 ( C.rows() * C.columns() < TDMATTDMATMULT_THRESHOLD ) )
564 selectSmallAssignKernel( C, A, B );
566 selectBlasAssignKernel( C, A, B );
585 template<
typename MT3
588 static inline auto selectDefaultAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
591 const size_t M( A.rows() );
592 const size_t N( B.columns() );
593 const size_t K( A.columns() );
597 for(
size_t j=0UL; j<N; ++j )
608 for(
size_t i=0UL; i<M; ++i ) {
617 ?(
LOW ?
max(j,kbegin+1UL) : kbegin+1UL )
618 :(
LOW ?
max(j,kbegin) : kbegin ) )
619 :(
LOW ? j : 0UL ) );
622 ?(
UPP ?
min(j+1UL,kbegin) : kbegin )
623 :(
UPP ?
min(j,kbegin)+1UL : kbegin+1UL ) )
624 :(
UPP ? j+1UL : M ) );
627 for(
size_t i=0UL; i<ibegin; ++i ) {
634 for(
size_t i=ibegin; i<iend; ++i ) {
635 C(i,j) = A(i,kbegin) * B(kbegin,j);
638 for(
size_t i=iend; i<M; ++i ) {
647 for(
size_t k=kbegin+1UL; k<kend; ++k )
656 ?(
UPP ?
min(j+1UL,k-1UL) : k-1UL )
657 :(
UPP ?
min(j+1UL,k) : k ) )
658 :(
UPP ? j+1UL : M ) );
660 if( (
SYM ||
HERM ||
LOW ||
UPP ) && ( ibegin > iend ) )
continue;
663 for(
size_t i=ibegin; i<iend; ++i ) {
664 C(i,j) += A(i,k) * B(k,j);
667 C(iend,j) = A(iend,k) * B(k,j);
673 for(
size_t j=1UL; j<N; ++j ) {
674 for(
size_t i=0UL; i<j; ++i ) {
675 C(i,j) =
HERM ?
conj( C(j,i) ) : C(j,i);
697 template<
typename MT3
700 static inline auto selectDefaultAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
705 const size_t M( A.rows() );
706 const size_t N( B.columns() );
708 for(
size_t j=0UL; j<N; ++j )
719 for(
size_t i=0UL; i<ibegin; ++i ) {
723 for(
size_t i=ibegin; i<iend; ++i ) {
724 C(i,j) = A(i,j) * B(j,j);
727 for(
size_t i=iend; i<M; ++i ) {
750 template<
typename MT3
753 static inline auto selectDefaultAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
758 const size_t M( A.rows() );
759 const size_t N( B.columns() );
761 for(
size_t j=0UL; j<N; ++j )
772 for(
size_t i=0UL; i<ibegin; ++i ) {
776 for(
size_t i=ibegin; i<iend; ++i ) {
777 C(i,j) = A(i,i) * B(i,j);
780 for(
size_t i=iend; i<M; ++i ) {
803 template<
typename MT3
806 static inline auto selectDefaultAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
813 for(
size_t i=0UL; i<A.rows(); ++i ) {
814 C(i,i) = A(i,i) * B(i,i);
834 template<
typename MT3
837 static inline auto selectSmallAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
840 selectDefaultAssignKernel( C, A, B );
860 template<
typename MT3
863 static inline auto selectSmallAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
871 const ForwardFunctor fwd;
874 const OppositeType_t<MT5> tmp(
serial( B ) );
875 assign( C, fwd( A * tmp ) );
878 const OppositeType_t<MT4> tmp(
serial( A ) );
879 assign( C, fwd( tmp * B ) );
881 else if( B.rows() * B.columns() <= A.rows() * A.columns() ) {
882 const OppositeType_t<MT5> tmp(
serial( B ) );
883 assign( C, fwd( A * tmp ) );
886 const OppositeType_t<MT4> tmp(
serial( A ) );
887 assign( C, fwd( tmp * B ) );
908 template<
typename MT3
911 static inline auto selectSmallAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
916 const size_t M( A.rows() );
917 const size_t N( B.columns() );
918 const size_t K( A.columns() );
930 for(
size_t j=0UL; j<N; ++j )
957 for( ++k; k<kend; ++k ) {
959 xmm1 += A.load(i ,k) * b1;
961 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
962 xmm4 += A.load(i+
SIMDSIZE*3UL,k) * b1;
963 xmm5 += A.load(i+
SIMDSIZE*4UL,k) * b1;
964 xmm6 += A.load(i+
SIMDSIZE*5UL,k) * b1;
965 xmm7 += A.load(i+
SIMDSIZE*6UL,k) * b1;
966 xmm8 += A.load(i+
SIMDSIZE*7UL,k) * b1;
969 C.store( i , j, xmm1 );
981 C.store( i , j,
zero );
998 for( ; (j+2UL) <= N; j+=2UL )
1033 for( ++k; k<kend; ++k ) {
1039 b1 =
set( B(k,j ) );
1040 b2 =
set( B(k,j+1UL) );
1053 C.store( i , j , xmm1 );
1055 C.store( i+
SIMDSIZE*2UL, j , xmm3 );
1056 C.store( i+
SIMDSIZE*3UL, j , xmm4 );
1057 C.store( i+
SIMDSIZE*4UL, j , xmm5 );
1058 C.store( i , j+1UL, xmm6 );
1059 C.store( i+
SIMDSIZE , j+1UL, xmm7 );
1060 C.store( i+
SIMDSIZE*2UL, j+1UL, xmm8 );
1061 C.store( i+
SIMDSIZE*3UL, j+1UL, xmm9 );
1062 C.store( i+
SIMDSIZE*4UL, j+1UL, xmm10 );
1067 C.store( i , j ,
zero );
1072 C.store( i , j+1UL,
zero );
1094 SIMDType xmm1( A.load(i ,k) * b1 );
1100 for( ++k; k<kend; ++k ) {
1102 xmm1 += A.load(i ,k) * b1;
1103 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
1104 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
1105 xmm4 += A.load(i+
SIMDSIZE*3UL,k) * b1;
1106 xmm5 += A.load(i+
SIMDSIZE*4UL,k) * b1;
1109 C.store( i , j, xmm1 );
1111 C.store( i+
SIMDSIZE*2UL, j, xmm3 );
1112 C.store( i+
SIMDSIZE*3UL, j, xmm4 );
1113 C.store( i+
SIMDSIZE*4UL, j, xmm5 );
1118 C.store( i , j,
zero );
1135 for(
size_t ii=i; ii<iiend; ++ii ) {
1136 C(ii,j) =
HERM ?
conj( C(j,ii) ) : C(j,ii);
1143 for(
size_t ii=i; ii<iiend; ++ii ) {
1149 for( ; (j+2UL) <= jend; j+=2UL )
1181 for( ++k; k<kend; ++k ) {
1186 b1 =
set( B(k,j ) );
1187 b2 =
set( B(k,j+1UL) );
1198 C.store( i , j , xmm1 );
1200 C.store( i+
SIMDSIZE*2UL, j , xmm3 );
1201 C.store( i+
SIMDSIZE*3UL, j , xmm4 );
1202 C.store( i , j+1UL, xmm5 );
1203 C.store( i+
SIMDSIZE , j+1UL, xmm6 );
1204 C.store( i+
SIMDSIZE*2UL, j+1UL, xmm7 );
1205 C.store( i+
SIMDSIZE*3UL, j+1UL, xmm8 );
1210 C.store( i , j ,
zero );
1214 C.store( i , j+1UL,
zero );
1235 SIMDType xmm1( A.load(i ,k) * b1 );
1240 for( ++k; k<kend; ++k ) {
1242 xmm1 += A.load(i ,k) * b1;
1243 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
1244 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
1245 xmm4 += A.load(i+
SIMDSIZE*3UL,k) * b1;
1248 C.store( i , j, xmm1 );
1250 C.store( i+
SIMDSIZE*2UL, j, xmm3 );
1251 C.store( i+
SIMDSIZE*3UL, j, xmm4 );
1256 C.store( i , j,
zero );
1268 for(
size_t ii=i; ii<iiend; ++ii ) {
1283 for(
size_t ii=i; ii<iiend; ++ii ) {
1284 C(ii,j) =
HERM ?
conj( C(j,ii) ) : C(j,ii);
1291 for(
size_t ii=i; ii<iiend; ++ii ) {
1297 for( ; (j+2UL) <= jend; j+=2UL )
1326 for( ++k; k<kend; ++k ) {
1330 b1 =
set( B(k,j ) );
1331 b2 =
set( B(k,j+1UL) );
1340 C.store( i , j , xmm1 );
1342 C.store( i+
SIMDSIZE*2UL, j , xmm3 );
1343 C.store( i , j+1UL, xmm4 );
1344 C.store( i+
SIMDSIZE , j+1UL, xmm5 );
1345 C.store( i+
SIMDSIZE*2UL, j+1UL, xmm6 );
1350 C.store( i , j ,
zero );
1353 C.store( i , j+1UL,
zero );
1373 SIMDType xmm1( A.load(i ,k) * b1 );
1377 for( ++k; k<kend; ++k ) {
1379 xmm1 += A.load(i ,k) * b1;
1380 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
1381 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
1384 C.store( i , j, xmm1 );
1386 C.store( i+
SIMDSIZE*2UL, j, xmm3 );
1391 C.store( i , j,
zero );
1402 for(
size_t ii=i; ii<iiend; ++ii ) {
1417 for(
size_t ii=i; ii<iiend; ++ii ) {
1418 C(ii,j) =
HERM ?
conj( C(j,ii) ) : C(j,ii);
1425 for(
size_t ii=i; ii<iiend; ++ii ) {
1431 for( ; (j+4UL) <= jend; j+=4UL )
1463 for( ++k; k<kend; ++k ) {
1466 b1 =
set( B(k,j ) );
1467 b2 =
set( B(k,j+1UL) );
1468 b3 =
set( B(k,j+2UL) );
1469 b4 =
set( B(k,j+3UL) );
1480 C.store( i , j , xmm1 );
1482 C.store( i , j+1UL, xmm3 );
1483 C.store( i+
SIMDSIZE, j+1UL, xmm4 );
1484 C.store( i , j+2UL, xmm5 );
1485 C.store( i+
SIMDSIZE, j+2UL, xmm6 );
1486 C.store( i , j+3UL, xmm7 );
1487 C.store( i+
SIMDSIZE, j+3UL, xmm8 );
1492 C.store( i , j ,
zero );
1494 C.store( i , j+1UL,
zero );
1496 C.store( i , j+2UL,
zero );
1498 C.store( i , j+3UL,
zero );
1503 for( ; (j+3UL) <= jend; j+=3UL )
1532 for( ++k; k<kend; ++k ) {
1535 b1 =
set( B(k,j ) );
1536 b2 =
set( B(k,j+1UL) );
1537 b3 =
set( B(k,j+2UL) );
1546 C.store( i , j , xmm1 );
1548 C.store( i , j+1UL, xmm3 );
1549 C.store( i+
SIMDSIZE, j+1UL, xmm4 );
1550 C.store( i , j+2UL, xmm5 );
1551 C.store( i+
SIMDSIZE, j+2UL, xmm6 );
1556 C.store( i , j ,
zero );
1558 C.store( i , j+1UL,
zero );
1560 C.store( i , j+2UL,
zero );
1565 for( ; (j+2UL) <= jend; j+=2UL )
1591 for( ++k; k<kend; ++k ) {
1594 b1 =
set( B(k,j ) );
1595 b2 =
set( B(k,j+1UL) );
1602 C.store( i , j , xmm1 );
1604 C.store( i , j+1UL, xmm3 );
1605 C.store( i+
SIMDSIZE, j+1UL, xmm4 );
1610 C.store( i , j ,
zero );
1612 C.store( i , j+1UL,
zero );
1631 SIMDType xmm1( A.load(i ,k) * b1 );
1634 for( ++k; k<kend; ++k ) {
1636 xmm1 += A.load(i ,k) * b1;
1640 C.store( i , j, xmm1 );
1646 C.store( i , j,
zero );
1656 for(
size_t ii=i; ii<iiend; ++ii ) {
1671 for(
size_t ii=i; ii<iiend; ++ii ) {
1672 C(ii,j) =
HERM ?
conj( C(j,ii) ) : C(j,ii);
1679 for(
size_t ii=i; ii<iiend; ++ii ) {
1685 for( ; (j+4UL) <= jend; j+=4UL )
1706 for( ++k; k<kend; ++k ) {
1708 xmm1 += a1 *
set( B(k,j ) );
1709 xmm2 += a1 *
set( B(k,j+1UL) );
1710 xmm3 += a1 *
set( B(k,j+2UL) );
1711 xmm4 += a1 *
set( B(k,j+3UL) );
1714 C.store( i, j , xmm1 );
1715 C.store( i, j+1UL, xmm2 );
1716 C.store( i, j+2UL, xmm3 );
1717 C.store( i, j+3UL, xmm4 );
1722 C.store( i, j ,
zero );
1723 C.store( i, j+1UL,
zero );
1724 C.store( i, j+2UL,
zero );
1725 C.store( i, j+3UL,
zero );
1729 for( ; (j+3UL) <= jend; j+=3UL )
1749 for( ++k; k<kend; ++k ) {
1751 xmm1 += a1 *
set( B(k,j ) );
1752 xmm2 += a1 *
set( B(k,j+1UL) );
1753 xmm3 += a1 *
set( B(k,j+2UL) );
1756 C.store( i, j , xmm1 );
1757 C.store( i, j+1UL, xmm2 );
1758 C.store( i, j+2UL, xmm3 );
1763 C.store( i, j ,
zero );
1764 C.store( i, j+1UL,
zero );
1765 C.store( i, j+2UL,
zero );
1769 for( ; (j+2UL) <= jend; j+=2UL )
1788 for( ++k; k<kend; ++k ) {
1790 xmm1 += a1 *
set( B(k,j ) );
1791 xmm2 += a1 *
set( B(k,j+1UL) );
1794 C.store( i, j , xmm1 );
1795 C.store( i, j+1UL, xmm2 );
1800 C.store( i, j ,
zero );
1801 C.store( i, j+1UL,
zero );
1819 for( ++k; k<K; ++k ) {
1820 xmm1 += A.load(i,k) *
set( B(k,j) );
1823 C.store( i, j, xmm1 );
1828 C.store( i, j,
zero );
1837 for(
size_t ii=i; ii<iiend; ++ii ) {
1844 for( ; remainder && i<M; ++i )
1850 C(i,j) =
HERM ?
conj( C(j,i) ) : C(j,i);
1859 for( ; (j+2UL) <= N; j+=2UL )
1877 for( ++k; k<kend; ++k ) {
1878 value1 += A(i,k) * B(k,j );
1879 value2 += A(i,k) * B(k,j+1UL);
1883 C(i,j+1UL) = value2;
1888 reset( C(i,j+1UL) );
1906 for( ++k; k<K; ++k ) {
1907 value += A(i,k) * B(k,j);
1936 template<
typename MT3
1939 static inline auto selectLargeAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
1942 selectDefaultAssignKernel( C, A, B );
1962 template<
typename MT3
1965 static inline auto selectLargeAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
1996 template<
typename MT3
1999 static inline auto selectBlasAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
2002 selectLargeAssignKernel( C, A, B );
2008#if BLAZE_BLAS_MODE && BLAZE_USE_BLAS_MATRIX_MATRIX_MULTIPLICATION
2022 template<
typename MT3
2025 static inline auto selectBlasAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
2028 using ET = ElementType_t<MT3>;
2032 trmm( C, A, CblasLeft, (
IsLower_v<MT4> )?( CblasLower ):( CblasUpper ), ET(1) );
2036 trmm( C, B, CblasRight, (
IsLower_v<MT5> )?( CblasLower ):( CblasUpper ), ET(1) );
2039 gemm( C, A, B, ET(1), ET(0) );
2059 template<
typename MT
2078 const ForwardFunctor fwd;
2080 const TmpType tmp(
serial( rhs ) );
2081 assign( *lhs, fwd( tmp ) );
2101 template<
typename MT >
2112 const ForwardFunctor fwd;
2117 assign( *lhs, fwd( A * B ) );
2135 template<
typename MT
2137 friend inline auto addAssign( DenseMatrix<MT,SO>& lhs,
const TDMatTDMatMultExpr& rhs )
2145 if( (*lhs).rows() == 0UL || (*lhs).columns() == 0UL || rhs.lhs_.columns() == 0UL ) {
2159 TDMatTDMatMultExpr::selectAddAssignKernel( *lhs, A, B );
2175 template<
typename MT3
2178 static inline void selectAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
2181 ( !BLAZE_DEBUG_MODE && A.rows() <=
SIMDSIZE*10UL ) ||
2182 ( C.rows() * C.columns() < TDMATTDMATMULT_THRESHOLD ) )
2183 selectSmallAddAssignKernel( C, A, B );
2185 selectBlasAddAssignKernel( C, A, B );
2204 template<
typename MT3
2207 static inline auto selectDefaultAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
2210 const size_t M( A.rows() );
2211 const size_t N( B.columns() );
2212 const size_t K( A.columns() );
2216 for(
size_t j=0UL; j<N; ++j )
2226 for(
size_t k=kbegin; k<kend; ++k )
2230 ?(
LOW ?
max(j,k+1UL) : k+1UL )
2231 :(
LOW ?
max(j,k) : k ) )
2232 :(
LOW ? j : 0UL ) );
2235 ?(
UPP ?
min(j+1UL,k) : k )
2236 :(
UPP ?
min(j,k)+1UL : k+1UL ) )
2237 :(
UPP ? j+1UL : M ) );
2239 if( (
LOW ||
UPP ) && ibegin >= iend )
continue;
2242 const size_t inum( iend - ibegin );
2243 const size_t ipos( ibegin +
prevMultiple( inum, 2UL ) );
2246 for(
size_t i=ibegin; i<ipos; i+=2UL ) {
2247 C(i ,j) += A(i ,k) * B(k,j);
2248 C(i+1UL,j) += A(i+1UL,k) * B(k,j);
2251 C(ipos,j) += A(ipos,k) * B(k,j);
2273 template<
typename MT3
2276 static inline auto selectDefaultAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
2281 const size_t M( A.rows() );
2282 const size_t N( B.columns() );
2284 for(
size_t j=0UL; j<N; ++j )
2294 const size_t inum( iend - ibegin );
2295 const size_t ipos( ibegin +
prevMultiple( inum, 2UL ) );
2298 for(
size_t i=ibegin; i<ipos; i+=2UL ) {
2299 C(i ,j) += A(i ,j) * B(j,j);
2300 C(i+1UL,j) += A(i+1UL,j) * B(j,j);
2303 C(ipos,j) += A(ipos,j) * B(j,j);
2324 template<
typename MT3
2327 static inline auto selectDefaultAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
2332 const size_t M( A.rows() );
2333 const size_t N( B.columns() );
2335 for(
size_t j=0UL; j<N; ++j )
2345 const size_t inum( iend - ibegin );
2346 const size_t ipos( ibegin +
prevMultiple( inum, 2UL ) );
2349 for(
size_t i=ibegin; i<ipos; i+=2UL ) {
2350 C(i ,j) += A(i ,i ) * B(i ,j);
2351 C(i+1UL,j) += A(i+1UL,i+1UL) * B(i+1UL,j);
2354 C(ipos,j) += A(ipos,ipos) * B(ipos,j);
2375 template<
typename MT3
2378 static inline auto selectDefaultAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
2383 for(
size_t i=0UL; i<A.rows(); ++i ) {
2384 C(i,i) += A(i,i) * B(i,i);
2404 template<
typename MT3
2407 static inline auto selectSmallAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
2410 selectDefaultAddAssignKernel( C, A, B );
2430 template<
typename MT3
2433 static inline auto selectSmallAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
2441 const ForwardFunctor fwd;
2444 const OppositeType_t<MT5> tmp(
serial( B ) );
2445 addAssign( C, fwd( A * tmp ) );
2448 const OppositeType_t<MT4> tmp(
serial( A ) );
2449 addAssign( C, fwd( tmp * B ) );
2451 else if( B.rows() * B.columns() <= A.rows() * A.columns() ) {
2452 const OppositeType_t<MT5> tmp(
serial( B ) );
2453 addAssign( C, fwd( A * tmp ) );
2456 const OppositeType_t<MT4> tmp(
serial( A ) );
2457 addAssign( C, fwd( tmp * B ) );
2478 template<
typename MT3
2481 static inline auto selectSmallAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
2486 const size_t M( A.rows() );
2487 const size_t N( B.columns() );
2488 const size_t K( A.columns() );
2500 for(
size_t j=0UL; j<N; ++j )
2522 for(
size_t k=kbegin; k<kend; ++k ) {
2524 xmm1 += A.load(i ,k) * b1;
2525 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
2526 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
2527 xmm4 += A.load(i+
SIMDSIZE*3UL,k) * b1;
2528 xmm5 += A.load(i+
SIMDSIZE*4UL,k) * b1;
2529 xmm6 += A.load(i+
SIMDSIZE*5UL,k) * b1;
2530 xmm7 += A.load(i+
SIMDSIZE*6UL,k) * b1;
2531 xmm8 += A.load(i+
SIMDSIZE*7UL,k) * b1;
2534 C.store( i , j, xmm1 );
2536 C.store( i+
SIMDSIZE*2UL, j, xmm3 );
2537 C.store( i+
SIMDSIZE*3UL, j, xmm4 );
2538 C.store( i+
SIMDSIZE*4UL, j, xmm5 );
2539 C.store( i+
SIMDSIZE*5UL, j, xmm6 );
2540 C.store( i+
SIMDSIZE*6UL, j, xmm7 );
2541 C.store( i+
SIMDSIZE*7UL, j, xmm8 );
2550 for( ; (j+2UL) <= N; j+=2UL )
2568 SIMDType xmm6 ( C.load(i ,j+1UL) );
2574 for(
size_t k=kbegin; k<kend; ++k ) {
2594 C.store( i , j , xmm1 );
2596 C.store( i+
SIMDSIZE*2UL, j , xmm3 );
2597 C.store( i+
SIMDSIZE*3UL, j , xmm4 );
2598 C.store( i+
SIMDSIZE*4UL, j , xmm5 );
2599 C.store( i , j+1UL, xmm6 );
2600 C.store( i+
SIMDSIZE , j+1UL, xmm7 );
2601 C.store( i+
SIMDSIZE*2UL, j+1UL, xmm8 );
2602 C.store( i+
SIMDSIZE*3UL, j+1UL, xmm9 );
2603 C.store( i+
SIMDSIZE*4UL, j+1UL, xmm10 );
2621 for(
size_t k=kbegin; k<kend; ++k ) {
2623 xmm1 += A.load(i ,k) * b1;
2624 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
2625 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
2626 xmm4 += A.load(i+
SIMDSIZE*3UL,k) * b1;
2627 xmm5 += A.load(i+
SIMDSIZE*4UL,k) * b1;
2630 C.store( i , j, xmm1 );
2632 C.store( i+
SIMDSIZE*2UL, j, xmm3 );
2633 C.store( i+
SIMDSIZE*3UL, j, xmm4 );
2634 C.store( i+
SIMDSIZE*4UL, j, xmm5 );
2642 for( ; (j+2UL) <= N; j+=2UL )
2664 for(
size_t k=kbegin; k<kend; ++k ) {
2681 C.store( i , j , xmm1 );
2683 C.store( i+
SIMDSIZE*2UL, j , xmm3 );
2684 C.store( i+
SIMDSIZE*3UL, j , xmm4 );
2685 C.store( i , j+1UL, xmm5 );
2686 C.store( i+
SIMDSIZE , j+1UL, xmm6 );
2687 C.store( i+
SIMDSIZE*2UL, j+1UL, xmm7 );
2688 C.store( i+
SIMDSIZE*3UL, j+1UL, xmm8 );
2705 for(
size_t k=kbegin; k<kend; ++k ) {
2707 xmm1 += A.load(i ,k) * b1;
2708 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
2709 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
2710 xmm4 += A.load(i+
SIMDSIZE*3UL,k) * b1;
2713 C.store( i , j, xmm1 );
2715 C.store( i+
SIMDSIZE*2UL, j, xmm3 );
2716 C.store( i+
SIMDSIZE*3UL, j, xmm4 );
2724 for( ; (j+2UL) <= N; j+=2UL )
2744 for(
size_t k=kbegin; k<kend; ++k ) {
2758 C.store( i , j , xmm1 );
2760 C.store( i+
SIMDSIZE*2UL, j , xmm3 );
2761 C.store( i , j+1UL, xmm4 );
2762 C.store( i+
SIMDSIZE , j+1UL, xmm5 );
2763 C.store( i+
SIMDSIZE*2UL, j+1UL, xmm6 );
2779 for(
size_t k=kbegin; k<kend; ++k ) {
2781 xmm1 += A.load(i ,k) * b1;
2782 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
2783 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
2786 C.store( i , j, xmm1 );
2788 C.store( i+
SIMDSIZE*2UL, j, xmm3 );
2795 size_t j(
UPP ? i : 0UL );
2797 for( ; (j+4UL) <= jend; j+=4UL )
2819 for(
size_t k=kbegin; k<kend; ++k ) {
2836 C.store( i , j , xmm1 );
2838 C.store( i , j+1UL, xmm3 );
2839 C.store( i+
SIMDSIZE, j+1UL, xmm4 );
2840 C.store( i , j+2UL, xmm5 );
2841 C.store( i+
SIMDSIZE, j+2UL, xmm6 );
2842 C.store( i , j+3UL, xmm7 );
2843 C.store( i+
SIMDSIZE, j+3UL, xmm8 );
2846 for( ; (j+3UL) <= jend; j+=3UL )
2866 for(
size_t k=kbegin; k<kend; ++k ) {
2880 C.store( i , j , xmm1 );
2882 C.store( i , j+1UL, xmm3 );
2883 C.store( i+
SIMDSIZE, j+1UL, xmm4 );
2884 C.store( i , j+2UL, xmm5 );
2885 C.store( i+
SIMDSIZE, j+2UL, xmm6 );
2888 for( ; (j+2UL) <= jend; j+=2UL )
2906 for(
size_t k=kbegin; k<kend; ++k ) {
2917 C.store( i , j , xmm1 );
2919 C.store( i , j+1UL, xmm3 );
2920 C.store( i+
SIMDSIZE, j+1UL, xmm4 );
2935 for(
size_t k=kbegin; k<kend; ++k ) {
2937 xmm1 += A.load(i ,k) * b1;
2941 C.store( i , j, xmm1 );
2949 size_t j(
UPP ? i : 0UL );
2951 for( ; (j+4UL) <= jend; j+=4UL )
2967 for(
size_t k=kbegin; k<kend; ++k ) {
2969 xmm1 += a1 *
set( B(k,j ) );
2970 xmm2 += a1 *
set( B(k,j+1UL) );
2971 xmm3 += a1 *
set( B(k,j+2UL) );
2972 xmm4 += a1 *
set( B(k,j+3UL) );
2975 C.store( i, j , xmm1 );
2976 C.store( i, j+1UL, xmm2 );
2977 C.store( i, j+2UL, xmm3 );
2978 C.store( i, j+3UL, xmm4 );
2981 for( ; (j+3UL) <= jend; j+=3UL )
2996 for(
size_t k=kbegin; k<kend; ++k ) {
2998 xmm1 += a1 *
set( B(k,j ) );
2999 xmm2 += a1 *
set( B(k,j+1UL) );
3000 xmm3 += a1 *
set( B(k,j+2UL) );
3003 C.store( i, j , xmm1 );
3004 C.store( i, j+1UL, xmm2 );
3005 C.store( i, j+2UL, xmm3 );
3008 for( ; (j+2UL) <= jend; j+=2UL )
3022 for(
size_t k=kbegin; k<kend; ++k ) {
3024 xmm1 += a1 *
set( B(k,j ) );
3025 xmm2 += a1 *
set( B(k,j+1UL) );
3028 C.store( i, j , xmm1 );
3029 C.store( i, j+1UL, xmm2 );
3042 for(
size_t k=kbegin; k<K; ++k ) {
3043 xmm1 += A.load(i,k) *
set( B(k,j) );
3046 C.store( i, j, xmm1 );
3050 for( ; remainder && i<M; ++i )
3052 const size_t jend(
LOW ? i+1UL : N );
3053 size_t j(
UPP ? i : 0UL );
3055 for( ; (j+2UL) <= jend; j+=2UL )
3069 for(
size_t k=kbegin; k<kend; ++k ) {
3070 value1 += A(i,k) * B(k,j );
3071 value2 += A(i,k) * B(k,j+1UL);
3075 C(i,j+1UL) = value2;
3088 for(
size_t k=kbegin; k<K; ++k ) {
3089 value += A(i,k) * B(k,j);
3113 template<
typename MT3
3116 static inline auto selectLargeAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
3119 selectDefaultAddAssignKernel( C, A, B );
3139 template<
typename MT3
3142 static inline auto selectLargeAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
3169 template<
typename MT3
3172 static inline auto selectBlasAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
3175 selectLargeAddAssignKernel( C, A, B );
3181#if BLAZE_BLAS_MODE && BLAZE_USE_BLAS_MATRIX_MATRIX_MULTIPLICATION
3195 template<
typename MT3
3198 static inline auto selectBlasAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
3201 using ET = ElementType_t<MT3>;
3204 ResultType_t<MT3> tmp(
serial( B ) );
3205 trmm( tmp, A, CblasLeft, (
IsLower_v<MT4> )?( CblasLower ):( CblasUpper ), ET(1) );
3206 addAssign( C, tmp );
3209 ResultType_t<MT3> tmp(
serial( A ) );
3210 trmm( tmp, B, CblasRight, (
IsLower_v<MT5> )?( CblasLower ):( CblasUpper ), ET(1) );
3211 addAssign( C, tmp );
3214 gemm( C, A, B, ET(1), ET(1) );
3236 template<
typename MT >
3247 const ForwardFunctor fwd;
3252 addAssign( *lhs, fwd( A * B ) );
3274 template<
typename MT
3276 friend inline auto subAssign( DenseMatrix<MT,SO>& lhs,
const TDMatTDMatMultExpr& rhs )
3284 if( (*lhs).rows() == 0UL || (*lhs).columns() == 0UL || rhs.lhs_.columns() == 0UL ) {
3298 TDMatTDMatMultExpr::selectSubAssignKernel( *lhs, A, B );
3314 template<
typename MT3
3317 static inline void selectSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
3320 ( !BLAZE_DEBUG_MODE && A.rows() <=
SIMDSIZE*10UL ) ||
3321 ( C.rows() * C.columns() < TDMATTDMATMULT_THRESHOLD ) )
3322 selectSmallSubAssignKernel( C, A, B );
3324 selectBlasSubAssignKernel( C, A, B );
3343 template<
typename MT3
3346 static inline auto selectDefaultSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
3349 const size_t M( A.rows() );
3350 const size_t N( B.columns() );
3351 const size_t K( A.columns() );
3355 for(
size_t j=0UL; j<N; ++j )
3365 for(
size_t k=kbegin; k<kend; ++k )
3369 ?(
LOW ?
max(j,k+1UL) : k+1UL )
3370 :(
LOW ?
max(j,k) : k ) )
3371 :(
LOW ? j : 0UL ) );
3374 ?(
UPP ?
min(j+1UL,k) : k )
3375 :(
UPP ?
min(j,k)+1UL : k+1UL ) )
3376 :(
UPP ? j+1UL : M ) );
3378 if( (
LOW ||
UPP ) && ( ibegin >= iend ) )
continue;
3381 const size_t inum( iend - ibegin );
3382 const size_t ipos( ibegin +
prevMultiple( inum, 2UL ) );
3385 for(
size_t i=ibegin; i<ipos; i+=2UL ) {
3386 C(i ,j) -= A(i ,k) * B(k,j);
3387 C(i+1UL,j) -= A(i+1UL,k) * B(k,j);
3390 C(ipos,j) -= A(ipos,k) * B(k,j);
3412 template<
typename MT3
3415 static inline auto selectDefaultSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
3420 const size_t M( A.rows() );
3421 const size_t N( B.columns() );
3423 for(
size_t j=0UL; j<N; ++j )
3433 const size_t inum( iend - ibegin );
3434 const size_t ipos( ibegin +
prevMultiple( inum, 2UL ) );
3437 for(
size_t i=ibegin; i<ipos; i+=2UL ) {
3438 C(i ,j) -= A(i ,j) * B(j,j);
3439 C(i+1UL,j) -= A(i+1UL,j) * B(j,j);
3442 C(ipos,j) -= A(ipos,j) * B(j,j);
3463 template<
typename MT3
3466 static inline auto selectDefaultSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
3471 const size_t M( A.rows() );
3472 const size_t N( B.columns() );
3474 for(
size_t j=0UL; j<N; ++j )
3484 const size_t inum( iend - ibegin );
3485 const size_t ipos( ibegin +
prevMultiple( inum, 2UL ) );
3488 for(
size_t i=ibegin; i<ipos; i+=2UL ) {
3489 C(i ,j) -= A(i ,i ) * B(i ,j);
3490 C(i+1UL,j) -= A(i+1UL,i+1UL) * B(i+1UL,j);
3493 C(ipos,j) -= A(ipos,ipos) * B(ipos,j);
3514 template<
typename MT3
3517 static inline auto selectDefaultSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
3522 for(
size_t i=0UL; i<A.rows(); ++i ) {
3523 C(i,i) -= A(i,i) * B(i,i);
3543 template<
typename MT3
3546 static inline auto selectSmallSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
3549 selectDefaultSubAssignKernel( C, A, B );
3569 template<
typename MT3
3572 static inline auto selectSmallSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
3580 const ForwardFunctor fwd;
3583 const OppositeType_t<MT5> tmp(
serial( B ) );
3584 subAssign( C, fwd( A * tmp ) );
3587 const OppositeType_t<MT4> tmp(
serial( A ) );
3588 subAssign( C, fwd( tmp * B ) );
3590 else if( B.rows() * B.columns() <= A.rows() * A.columns() ) {
3591 const OppositeType_t<MT5> tmp(
serial( B ) );
3592 subAssign( C, fwd( A * tmp ) );
3595 const OppositeType_t<MT4> tmp(
serial( A ) );
3596 subAssign( C, fwd( tmp * B ) );
3617 template<
typename MT3
3620 static inline auto selectSmallSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
3625 const size_t M( A.rows() );
3626 const size_t N( B.columns() );
3627 const size_t K( A.columns() );
3639 for(
size_t j=0UL; j<N; ++j )
3661 for(
size_t k=kbegin; k<kend; ++k ) {
3663 xmm1 -= A.load(i ,k) * b1;
3664 xmm2 -= A.load(i+
SIMDSIZE ,k) * b1;
3665 xmm3 -= A.load(i+
SIMDSIZE*2UL,k) * b1;
3666 xmm4 -= A.load(i+
SIMDSIZE*3UL,k) * b1;
3667 xmm5 -= A.load(i+
SIMDSIZE*4UL,k) * b1;
3668 xmm6 -= A.load(i+
SIMDSIZE*5UL,k) * b1;
3669 xmm7 -= A.load(i+
SIMDSIZE*6UL,k) * b1;
3670 xmm8 -= A.load(i+
SIMDSIZE*7UL,k) * b1;
3673 C.store( i , j, xmm1 );
3675 C.store( i+
SIMDSIZE*2UL, j, xmm3 );
3676 C.store( i+
SIMDSIZE*3UL, j, xmm4 );
3677 C.store( i+
SIMDSIZE*4UL, j, xmm5 );
3678 C.store( i+
SIMDSIZE*5UL, j, xmm6 );
3679 C.store( i+
SIMDSIZE*6UL, j, xmm7 );
3680 C.store( i+
SIMDSIZE*7UL, j, xmm8 );
3689 for( ; (j+2UL) <= N; j+=2UL )
3707 SIMDType xmm6 ( C.load(i ,j+1UL) );
3713 for(
size_t k=kbegin; k<kend; ++k ) {
3733 C.store( i , j , xmm1 );
3735 C.store( i+
SIMDSIZE*2UL, j , xmm3 );
3736 C.store( i+
SIMDSIZE*3UL, j , xmm4 );
3737 C.store( i+
SIMDSIZE*4UL, j , xmm5 );
3738 C.store( i , j+1UL, xmm6 );
3739 C.store( i+
SIMDSIZE , j+1UL, xmm7 );
3740 C.store( i+
SIMDSIZE*2UL, j+1UL, xmm8 );
3741 C.store( i+
SIMDSIZE*3UL, j+1UL, xmm9 );
3742 C.store( i+
SIMDSIZE*4UL, j+1UL, xmm10 );
3760 for(
size_t k=kbegin; k<kend; ++k ) {
3762 xmm1 -= A.load(i ,k) * b1;
3763 xmm2 -= A.load(i+
SIMDSIZE ,k) * b1;
3764 xmm3 -= A.load(i+
SIMDSIZE*2UL,k) * b1;
3765 xmm4 -= A.load(i+
SIMDSIZE*3UL,k) * b1;
3766 xmm5 -= A.load(i+
SIMDSIZE*4UL,k) * b1;
3769 C.store( i , j, xmm1 );
3771 C.store( i+
SIMDSIZE*2UL, j, xmm3 );
3772 C.store( i+
SIMDSIZE*3UL, j, xmm4 );
3773 C.store( i+
SIMDSIZE*4UL, j, xmm5 );
3781 for( ; (j+2UL) <= N; j+=2UL )
3803 for(
size_t k=kbegin; k<kend; ++k ) {
3820 C.store( i , j , xmm1 );
3822 C.store( i+
SIMDSIZE*2UL, j , xmm3 );
3823 C.store( i+
SIMDSIZE*3UL, j , xmm4 );
3824 C.store( i , j+1UL, xmm5 );
3825 C.store( i+
SIMDSIZE , j+1UL, xmm6 );
3826 C.store( i+
SIMDSIZE*2UL, j+1UL, xmm7 );
3827 C.store( i+
SIMDSIZE*3UL, j+1UL, xmm8 );
3844 for(
size_t k=kbegin; k<kend; ++k ) {
3846 xmm1 -= A.load(i ,k) * b1;
3847 xmm2 -= A.load(i+
SIMDSIZE ,k) * b1;
3848 xmm3 -= A.load(i+
SIMDSIZE*2UL,k) * b1;
3849 xmm4 -= A.load(i+
SIMDSIZE*3UL,k) * b1;
3852 C.store( i , j, xmm1 );
3854 C.store( i+
SIMDSIZE*2UL, j, xmm3 );
3855 C.store( i+
SIMDSIZE*3UL, j, xmm4 );
3863 for( ; (j+2UL) <= N; j+=2UL )
3883 for(
size_t k=kbegin; k<kend; ++k ) {
3897 C.store( i , j , xmm1 );
3899 C.store( i+
SIMDSIZE*2UL, j , xmm3 );
3900 C.store( i , j+1UL, xmm4 );
3901 C.store( i+
SIMDSIZE , j+1UL, xmm5 );
3902 C.store( i+
SIMDSIZE*2UL, j+1UL, xmm6 );
3918 for(
size_t k=kbegin; k<kend; ++k ) {
3920 xmm1 -= A.load(i ,k) * b1;
3921 xmm2 -= A.load(i+
SIMDSIZE ,k) * b1;
3922 xmm3 -= A.load(i+
SIMDSIZE*2UL,k) * b1;
3925 C.store( i , j, xmm1 );
3927 C.store( i+
SIMDSIZE*2UL, j, xmm3 );
3934 size_t j(
UPP ? i : 0UL );
3936 for( ; (j+4UL) <= jend; j+=4UL )
3958 for(
size_t k=kbegin; k<kend; ++k ) {
3975 C.store( i , j , xmm1 );
3977 C.store( i , j+1UL, xmm3 );
3978 C.store( i+
SIMDSIZE, j+1UL, xmm4 );
3979 C.store( i , j+2UL, xmm5 );
3980 C.store( i+
SIMDSIZE, j+2UL, xmm6 );
3981 C.store( i , j+3UL, xmm7 );
3982 C.store( i+
SIMDSIZE, j+3UL, xmm8 );
3985 for( ; (j+3UL) <= jend; j+=3UL )
4005 for(
size_t k=kbegin; k<kend; ++k ) {
4019 C.store( i , j , xmm1 );
4021 C.store( i , j+1UL, xmm3 );
4022 C.store( i+
SIMDSIZE, j+1UL, xmm4 );
4023 C.store( i , j+2UL, xmm5 );
4024 C.store( i+
SIMDSIZE, j+2UL, xmm6 );
4027 for( ; (j+2UL) <= jend; j+=2UL )
4045 for(
size_t k=kbegin; k<kend; ++k ) {
4056 C.store( i , j , xmm1 );
4058 C.store( i , j+1UL, xmm3 );
4059 C.store( i+
SIMDSIZE, j+1UL, xmm4 );
4074 for(
size_t k=kbegin; k<kend; ++k ) {
4076 xmm1 -= A.load(i ,k) * b1;
4080 C.store( i , j, xmm1 );
4088 size_t j(
UPP ? i : 0UL );
4090 for( ; (j+4UL) <= jend; j+=4UL )
4106 for(
size_t k=kbegin; k<kend; ++k ) {
4108 xmm1 -= a1 *
set( B(k,j ) );
4109 xmm2 -= a1 *
set( B(k,j+1UL) );
4110 xmm3 -= a1 *
set( B(k,j+2UL) );
4111 xmm4 -= a1 *
set( B(k,j+3UL) );
4114 C.store( i, j , xmm1 );
4115 C.store( i, j+1UL, xmm2 );
4116 C.store( i, j+2UL, xmm3 );
4117 C.store( i, j+3UL, xmm4 );
4120 for( ; (j+3UL) <= jend; j+=3UL )
4135 for(
size_t k=kbegin; k<kend; ++k ) {
4137 xmm1 -= a1 *
set( B(k,j ) );
4138 xmm2 -= a1 *
set( B(k,j+1UL) );
4139 xmm3 -= a1 *
set( B(k,j+2UL) );
4142 C.store( i, j , xmm1 );
4143 C.store( i, j+1UL, xmm2 );
4144 C.store( i, j+2UL, xmm3 );
4147 for( ; (j+2UL) <= jend; j+=2UL )
4161 for(
size_t k=kbegin; k<kend; ++k ) {
4163 xmm1 -= a1 *
set( B(k,j ) );
4164 xmm2 -= a1 *
set( B(k,j+1UL) );
4167 C.store( i, j , xmm1 );
4168 C.store( i, j+1UL, xmm2 );
4181 for(
size_t k=kbegin; k<K; ++k ) {
4182 xmm1 -= A.load(i,k) *
set( B(k,j) );
4185 C.store( i, j, xmm1 );
4189 for( ; remainder && i<M; ++i )
4191 const size_t jend(
LOW ? i+1UL : N );
4192 size_t j(
UPP ? i : 0UL );
4194 for( ; (j+2UL) <= jend; j+=2UL )
4208 for(
size_t k=kbegin; k<kend; ++k ) {
4209 value1 -= A(i,k) * B(k,j );
4210 value2 -= A(i,k) * B(k,j+1UL);
4214 C(i,j+1UL) = value2;
4227 for(
size_t k=kbegin; k<K; ++k ) {
4228 value -= A(i,k) * B(k,j);
4252 template<
typename MT3
4255 static inline auto selectLargeSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
4258 selectDefaultSubAssignKernel( C, A, B );
4278 template<
typename MT3
4281 static inline auto selectLargeSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
4308 template<
typename MT3
4311 static inline auto selectBlasSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
4314 selectLargeSubAssignKernel( C, A, B );
4320#if BLAZE_BLAS_MODE && BLAZE_USE_BLAS_MATRIX_MATRIX_MULTIPLICATION
4334 template<
typename MT3
4337 static inline auto selectBlasSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B )
4340 using ET = ElementType_t<MT3>;
4343 ResultType_t<MT3> tmp(
serial( B ) );
4344 trmm( tmp, A, CblasLeft, (
IsLower_v<MT4> )?( CblasLower ):( CblasUpper ), ET(1) );
4345 subAssign( C, tmp );
4348 ResultType_t<MT3> tmp(
serial( A ) );
4349 trmm( tmp, B, CblasRight, (
IsLower_v<MT5> )?( CblasLower ):( CblasUpper ), ET(1) );
4350 subAssign( C, tmp );
4353 gemm( C, A, B, ET(-1), ET(1) );
4376 template<
typename MT >
4387 const ForwardFunctor fwd;
4392 subAssign( *lhs, fwd( A * B ) );
4414 template<
typename MT
4416 friend inline void schurAssign( DenseMatrix<MT,SO>& lhs,
const TDMatTDMatMultExpr& rhs )
4428 schurAssign( *lhs, tmp );
4457 template<
typename MT
4467 if( (*lhs).rows() == 0UL || (*lhs).columns() == 0UL ) {
4470 else if( rhs.lhs_.columns() == 0UL ) {
4506 template<
typename MT
4525 const ForwardFunctor fwd;
4527 const TmpType tmp( rhs );
4548 template<
typename MT >
4559 const ForwardFunctor fwd;
4585 template<
typename MT
4595 if( (*lhs).rows() == 0UL || (*lhs).columns() == 0UL || rhs.lhs_.columns() == 0UL ) {
4630 template<
typename MT >
4641 const ForwardFunctor fwd;
4671 template<
typename MT
4681 if( (*lhs).rows() == 0UL || (*lhs).columns() == 0UL || rhs.lhs_.columns() == 0UL ) {
4716 template<
typename MT >
4727 const ForwardFunctor fwd;
4755 template<
typename MT
4815template<
typename MT1
4823 :
public MatScalarMultExpr< DenseMatrix< DMatScalarMultExpr< TDMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>, ST, true >, true > >
4829 using MMM = TDMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>;
4831 using RES = ResultType_t<MMM>;
4832 using RT1 = ResultType_t<MT1>;
4833 using RT2 = ResultType_t<MT2>;
4834 using ET1 = ElementType_t<RT1>;
4835 using ET2 = ElementType_t<RT2>;
4836 using CT1 = CompositeType_t<MT1>;
4837 using CT2 = CompositeType_t<MT2>;
4851 static constexpr bool SYM = ( SF && !( HF || LF || UF ) );
4852 static constexpr bool HERM = ( HF && !( LF || UF ) );
4853 static constexpr bool LOW = ( LF || ( ( SF || HF ) && UF ) );
4854 static constexpr bool UPP = ( UF || ( ( SF || HF ) && LF ) );
4863 template<
typename T1,
typename T2,
typename T3 >
4864 static constexpr bool CanExploitSymmetry_v =
4873 template<
typename T1,
typename T2,
typename T3 >
4874 static constexpr bool IsEvaluationRequired_v =
4875 ( ( evaluateLeft || evaluateRight ) && !CanExploitSymmetry_v<T1,T2,T3> );
4882 template<
typename T1,
typename T2,
typename T3,
typename T4 >
4883 static constexpr bool UseBlasKernel_v =
4885 !SYM && !HERM && !LOW && !UPP &&
4890 T1::simdEnabled && T2::simdEnabled && T3::simdEnabled &&
4903 template<
typename T1,
typename T2,
typename T3,
typename T4 >
4904 static constexpr bool UseVectorizedDefaultKernel_v =
4905 ( useOptimizedKernels &&
4907 T1::simdEnabled && T2::simdEnabled && T3::simdEnabled &&
4920 using ForwardFunctor =
If_t< HERM
4939 using BaseType = MatScalarMultExpr< DenseMatrix<This,true> >;
4943 , DeclHermTrait< MultTrait_t<RES,ST> >
4945 , DeclSymTrait< MultTrait_t<RES,ST> >
4948 , DeclDiagTrait< MultTrait_t<RES,ST> >
4949 , DeclLowTrait< MultTrait_t<RES,ST> > >
4951 , DeclUppTrait< MultTrait_t<RES,ST> >
4952 , MultTrait<RES,ST> > > > >::Type;
4962 using LeftOperand =
const TDMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>;
4978 MT1::simdEnabled && MT2::simdEnabled &&
4985 ( !evaluateLeft && MT1::smpAssignable && !evaluateRight && MT2::smpAssignable );
4990 static constexpr size_t SIMDSIZE = SIMDTrait<ElementType>::size;
5031 if( j >=
matrix_.columns() ) {
5034 return (*
this)(i,j);
5043 inline size_t rows()
const {
5053 inline size_t columns()
const {
5084 template<
typename T >
5085 inline bool canAlias(
const T* alias )
const {
5086 return matrix_.canAlias( alias );
5096 template<
typename T >
5097 inline bool isAliased(
const T* alias )
const {
5098 return matrix_.isAliased( alias );
5119 !BLAZE_USE_BLAS_MATRIX_MATRIX_MULTIPLICATION ||
5121 (
rows() *
columns() < TDMATTDMATMULT_THRESHOLD ) ) &&
5122 (
rows() *
columns() >= SMP_TDMATTDMATMULT_THRESHOLD );
5144 template<
typename MT
5154 LeftOperand_t<MMM> left ( rhs.matrix_.leftOperand() );
5155 RightOperand_t<MMM> right( rhs.matrix_.rightOperand() );
5157 if( (*lhs).rows() == 0UL || (*lhs).columns() == 0UL ) {
5160 else if( left.columns() == 0UL ) {
5175 DMatScalarMultExpr::selectAssignKernel( *lhs, A, B, rhs.scalar_ );
5190 template<
typename MT3
5194 static inline void selectAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
5197 ( !BLAZE_DEBUG_MODE && A.rows() <=
SIMDSIZE*10UL ) ||
5198 ( C.rows() * C.columns() < TDMATTDMATMULT_THRESHOLD ) )
5199 selectSmallAssignKernel( C, A, B, scalar );
5201 selectBlasAssignKernel( C, A, B, scalar );
5219 template<
typename MT3
5223 static inline auto selectDefaultAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
5226 const size_t M( A.rows() );
5227 const size_t N( B.columns() );
5228 const size_t K( A.columns() );
5232 for(
size_t j=0UL; j<N; ++j )
5243 for(
size_t i=0UL; i<M; ++i ) {
5252 ?( LOW ?
max(j,kbegin+1UL) : kbegin+1UL )
5253 :( LOW ?
max(j,kbegin) : kbegin ) )
5254 :( LOW ? j : 0UL ) );
5257 ?( UPP ?
min(j+1UL,kbegin) : kbegin )
5258 :( UPP ?
min(j,kbegin)+1UL : kbegin+1UL ) )
5259 :( UPP ? j+1UL : M ) );
5262 for(
size_t i=0UL; i<ibegin; ++i ) {
5269 for(
size_t i=ibegin; i<iend; ++i ) {
5270 C(i,j) = A(i,kbegin) * B(kbegin,j);
5273 for(
size_t i=iend; i<M; ++i ) {
5278 reset( C(M-1UL,j) );
5282 for(
size_t k=kbegin+1UL; k<kend; ++k )
5286 ?( SYM || HERM || LOW ?
max( j, k+1UL ) : k+1UL )
5287 :( SYM || HERM || LOW ?
max( j, k ) : k ) )
5288 :( SYM || HERM || LOW ? j : 0UL ) );
5291 ?( UPP ?
min(j+1UL,k-1UL) : k-1UL )
5292 :( UPP ?
min(j+1UL,k) : k ) )
5293 :( UPP ? j+1UL : M ) );
5295 if( ( SYM || HERM || LOW || UPP ) && ( ibegin > iend ) )
continue;
5298 for(
size_t i=ibegin; i<iend; ++i ) {
5299 C(i,j) += A(i,k) * B(k,j);
5302 C(iend,j) = A(iend,k) * B(k,j);
5309 :( ( SYM || HERM || LOW )?( j ):( 0UL ) ) );
5312 :( UPP ? j+1UL : M ) );
5314 if( ( SYM || HERM || LOW || UPP ) && ( ibegin > iend ) )
continue;
5317 for(
size_t i=ibegin; i<iend; ++i ) {
5324 for(
size_t j=1UL; j<N; ++j ) {
5325 for(
size_t i=0UL; i<j; ++i ) {
5326 C(i,j) = HERM ?
conj( C(j,i) ) : C(j,i);
5347 template<
typename MT3
5351 static inline auto selectDefaultAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
5356 const size_t M( A.rows() );
5357 const size_t N( B.columns() );
5359 for(
size_t j=0UL; j<N; ++j )
5370 for(
size_t i=0UL; i<ibegin; ++i ) {
5374 for(
size_t i=ibegin; i<iend; ++i ) {
5375 C(i,j) = A(i,j) * B(j,j) * scalar;
5378 for(
size_t i=iend; i<M; ++i ) {
5400 template<
typename MT3
5404 static inline auto selectDefaultAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
5409 const size_t M( A.rows() );
5410 const size_t N( B.columns() );
5412 for(
size_t j=0UL; j<N; ++j )
5423 for(
size_t i=0UL; i<ibegin; ++i ) {
5427 for(
size_t i=ibegin; i<iend; ++i ) {
5428 C(i,j) = A(i,i) * B(i,j) * scalar;
5431 for(
size_t i=iend; i<M; ++i ) {
5453 template<
typename MT3
5457 static inline auto selectDefaultAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
5464 for(
size_t i=0UL; i<A.rows(); ++i ) {
5465 C(i,i) = A(i,i) * B(i,i) * scalar;
5484 template<
typename MT3
5488 static inline auto selectSmallAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
5491 selectDefaultAssignKernel( C, A, B, scalar );
5510 template<
typename MT3
5514 static inline auto selectSmallAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
5522 const ForwardFunctor fwd;
5525 const OppositeType_t<MT5> tmp(
serial( B ) );
5526 assign( C, fwd( A * tmp ) * scalar );
5529 const OppositeType_t<MT4> tmp(
serial( A ) );
5530 assign( C, fwd( tmp * B ) * scalar );
5532 else if( B.rows() * B.columns() <= A.rows() * A.columns() ) {
5533 const OppositeType_t<MT5> tmp(
serial( B ) );
5534 assign( C, fwd( A * tmp ) * scalar );
5537 const OppositeType_t<MT4> tmp(
serial( A ) );
5538 assign( C, fwd( tmp * B ) * scalar );
5558 template<
typename MT3
5562 static inline auto selectSmallAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
5567 const size_t M( A.rows() );
5568 const size_t N( B.columns() );
5569 const size_t K( A.columns() );
5576 const SIMDType factor(
set( scalar ) );
5582 for( ; !SYM && !HERM && !LOW && !UPP && (i+
SIMDSIZE*7UL) < ipos; i+=
SIMDSIZE*8UL ) {
5583 for(
size_t j=0UL; j<N; ++j )
5600 SIMDType b1(
set( B(k,j) ) );
5601 SIMDType xmm1( A.load(i ,k) * b1 );
5602 SIMDType xmm2( A.load(i+
SIMDSIZE ,k) * b1 );
5603 SIMDType xmm3( A.load(i+
SIMDSIZE*2UL,k) * b1 );
5604 SIMDType xmm4( A.load(i+
SIMDSIZE*3UL,k) * b1 );
5605 SIMDType xmm5( A.load(i+
SIMDSIZE*4UL,k) * b1 );
5606 SIMDType xmm6( A.load(i+
SIMDSIZE*5UL,k) * b1 );
5607 SIMDType xmm7( A.load(i+
SIMDSIZE*6UL,k) * b1 );
5608 SIMDType xmm8( A.load(i+
SIMDSIZE*7UL,k) * b1 );
5610 for( ++k; k<kend; ++k ) {
5612 xmm1 += A.load(i ,k) * b1;
5613 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
5614 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
5615 xmm4 += A.load(i+
SIMDSIZE*3UL,k) * b1;
5616 xmm5 += A.load(i+
SIMDSIZE*4UL,k) * b1;
5617 xmm6 += A.load(i+
SIMDSIZE*5UL,k) * b1;
5618 xmm7 += A.load(i+
SIMDSIZE*6UL,k) * b1;
5619 xmm8 += A.load(i+
SIMDSIZE*7UL,k) * b1;
5622 C.store( i , j, xmm1 * factor );
5623 C.store( i+
SIMDSIZE , j, xmm2 * factor );
5624 C.store( i+
SIMDSIZE*2UL, j, xmm3 * factor );
5625 C.store( i+
SIMDSIZE*3UL, j, xmm4 * factor );
5626 C.store( i+
SIMDSIZE*4UL, j, xmm5 * factor );
5627 C.store( i+
SIMDSIZE*5UL, j, xmm6 * factor );
5628 C.store( i+
SIMDSIZE*6UL, j, xmm7 * factor );
5629 C.store( i+
SIMDSIZE*7UL, j, xmm8 * factor );
5633 const SIMDType
zero;
5634 C.store( i , j,
zero );
5647 for( ; !SYM && !HERM && !LOW && !UPP && (i+
SIMDSIZE*4UL) < ipos; i+=
SIMDSIZE*5UL )
5651 for( ; (j+2UL) <= N; j+=2UL )
5668 SIMDType a1( A.load(i ,k) );
5669 SIMDType a2( A.load(i+
SIMDSIZE ,k) );
5670 SIMDType a3( A.load(i+
SIMDSIZE*2UL,k) );
5671 SIMDType a4( A.load(i+
SIMDSIZE*3UL,k) );
5672 SIMDType a5( A.load(i+
SIMDSIZE*4UL,k) );
5673 SIMDType b1(
set( B(k,j ) ) );
5674 SIMDType b2(
set( B(k,j+1UL) ) );
5675 SIMDType xmm1 ( a1 * b1 );
5676 SIMDType xmm2 ( a2 * b1 );
5677 SIMDType xmm3 ( a3 * b1 );
5678 SIMDType xmm4 ( a4 * b1 );
5679 SIMDType xmm5 ( a5 * b1 );
5680 SIMDType xmm6 ( a1 * b2 );
5681 SIMDType xmm7 ( a2 * b2 );
5682 SIMDType xmm8 ( a3 * b2 );
5683 SIMDType xmm9 ( a4 * b2 );
5684 SIMDType xmm10( a5 * b2 );
5686 for( ++k; k<kend; ++k ) {
5692 b1 =
set( B(k,j ) );
5693 b2 =
set( B(k,j+1UL) );
5706 C.store( i , j , xmm1 * factor );
5707 C.store( i+
SIMDSIZE , j , xmm2 * factor );
5708 C.store( i+
SIMDSIZE*2UL, j , xmm3 * factor );
5709 C.store( i+
SIMDSIZE*3UL, j , xmm4 * factor );
5710 C.store( i+
SIMDSIZE*4UL, j , xmm5 * factor );
5711 C.store( i , j+1UL, xmm6 * factor );
5712 C.store( i+
SIMDSIZE , j+1UL, xmm7 * factor );
5713 C.store( i+
SIMDSIZE*2UL, j+1UL, xmm8 * factor );
5714 C.store( i+
SIMDSIZE*3UL, j+1UL, xmm9 * factor );
5715 C.store( i+
SIMDSIZE*4UL, j+1UL, xmm10 * factor );
5719 const SIMDType
zero;
5720 C.store( i , j ,
zero );
5725 C.store( i , j+1UL,
zero );
5746 SIMDType b1(
set( B(k,j) ) );
5747 SIMDType xmm1( A.load(i ,k) * b1 );
5748 SIMDType xmm2( A.load(i+
SIMDSIZE ,k) * b1 );
5749 SIMDType xmm3( A.load(i+
SIMDSIZE*2UL,k) * b1 );
5750 SIMDType xmm4( A.load(i+
SIMDSIZE*3UL,k) * b1 );
5751 SIMDType xmm5( A.load(i+
SIMDSIZE*4UL,k) * b1 );
5753 for( ++k; k<kend; ++k ) {
5755 xmm1 += A.load(i ,k) * b1;
5756 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
5757 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
5758 xmm4 += A.load(i+
SIMDSIZE*3UL,k) * b1;
5759 xmm5 += A.load(i+
SIMDSIZE*4UL,k) * b1;
5762 C.store( i , j, xmm1 * factor );
5763 C.store( i+
SIMDSIZE , j, xmm2 * factor );
5764 C.store( i+
SIMDSIZE*2UL, j, xmm3 * factor );
5765 C.store( i+
SIMDSIZE*3UL, j, xmm4 * factor );
5766 C.store( i+
SIMDSIZE*4UL, j, xmm5 * factor );
5770 const SIMDType
zero;
5771 C.store( i , j,
zero );
5782 const size_t jend( LOW ?
min(i+
SIMDSIZE*4UL,N) : N );
5788 for(
size_t ii=i; ii<iiend; ++ii ) {
5789 C(ii,j) = HERM ?
conj( C(j,ii) ) : C(j,ii);
5796 for(
size_t ii=i; ii<iiend; ++ii ) {
5802 for( ; (j+2UL) <= jend; j+=2UL )
5819 SIMDType a1( A.load(i ,k) );
5820 SIMDType a2( A.load(i+
SIMDSIZE ,k) );
5821 SIMDType a3( A.load(i+
SIMDSIZE*2UL,k) );
5822 SIMDType a4( A.load(i+
SIMDSIZE*3UL,k) );
5823 SIMDType b1(
set( B(k,j ) ) );
5824 SIMDType b2(
set( B(k,j+1UL) ) );
5825 SIMDType xmm1( a1 * b1 );
5826 SIMDType xmm2( a2 * b1 );
5827 SIMDType xmm3( a3 * b1 );
5828 SIMDType xmm4( a4 * b1 );
5829 SIMDType xmm5( a1 * b2 );
5830 SIMDType xmm6( a2 * b2 );
5831 SIMDType xmm7( a3 * b2 );
5832 SIMDType xmm8( a4 * b2 );
5834 for( ++k; k<kend; ++k ) {
5839 b1 =
set( B(k,j ) );
5840 b2 =
set( B(k,j+1UL) );
5851 C.store( i , j , xmm1 * factor );
5852 C.store( i+
SIMDSIZE , j , xmm2 * factor );
5853 C.store( i+
SIMDSIZE*2UL, j , xmm3 * factor );
5854 C.store( i+
SIMDSIZE*3UL, j , xmm4 * factor );
5855 C.store( i , j+1UL, xmm5 * factor );
5856 C.store( i+
SIMDSIZE , j+1UL, xmm6 * factor );
5857 C.store( i+
SIMDSIZE*2UL, j+1UL, xmm7 * factor );
5858 C.store( i+
SIMDSIZE*3UL, j+1UL, xmm8 * factor );
5862 const SIMDType
zero;
5863 C.store( i , j ,
zero );
5867 C.store( i , j+1UL,
zero );
5887 SIMDType b1(
set( B(k,j) ) );
5888 SIMDType xmm1( A.load(i ,k) * b1 );
5889 SIMDType xmm2( A.load(i+
SIMDSIZE ,k) * b1 );
5890 SIMDType xmm3( A.load(i+
SIMDSIZE*2UL,k) * b1 );
5891 SIMDType xmm4( A.load(i+
SIMDSIZE*3UL,k) * b1 );
5893 for( ++k; k<kend; ++k ) {
5895 xmm1 += A.load(i ,k) * b1;
5896 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
5897 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
5898 xmm4 += A.load(i+
SIMDSIZE*3UL,k) * b1;
5901 C.store( i , j, xmm1 * factor );
5902 C.store( i+
SIMDSIZE , j, xmm2 * factor );
5903 C.store( i+
SIMDSIZE*2UL, j, xmm3 * factor );
5904 C.store( i+
SIMDSIZE*3UL, j, xmm4 * factor );
5908 const SIMDType
zero;
5909 C.store( i , j,
zero );
5921 for(
size_t ii=i; ii<iiend; ++ii ) {
5930 const size_t jend( LOW ?
min(i+
SIMDSIZE*3UL,N) : N );
5936 for(
size_t ii=i; ii<iiend; ++ii ) {
5937 C(ii,j) = HERM ?
conj( C(j,ii) ) : C(j,ii);
5944 for(
size_t ii=i; ii<iiend; ++ii ) {
5950 for( ; (j+2UL) <= jend; j+=2UL )
5967 SIMDType a1( A.load(i ,k) );
5968 SIMDType a2( A.load(i+
SIMDSIZE ,k) );
5969 SIMDType a3( A.load(i+
SIMDSIZE*2UL,k) );
5970 SIMDType b1(
set( B(k,j ) ) );
5971 SIMDType b2(
set( B(k,j+1UL) ) );
5972 SIMDType xmm1( a1 * b1 );
5973 SIMDType xmm2( a2 * b1 );
5974 SIMDType xmm3( a3 * b1 );
5975 SIMDType xmm4( a1 * b2 );
5976 SIMDType xmm5( a2 * b2 );
5977 SIMDType xmm6( a3 * b2 );
5979 for( ++k; k<kend; ++k ) {
5983 b1 =
set( B(k,j ) );
5984 b2 =
set( B(k,j+1UL) );
5993 C.store( i , j , xmm1 * factor );
5994 C.store( i+
SIMDSIZE , j , xmm2 * factor );
5995 C.store( i+
SIMDSIZE*2UL, j , xmm3 * factor );
5996 C.store( i , j+1UL, xmm4 * factor );
5997 C.store( i+
SIMDSIZE , j+1UL, xmm5 * factor );
5998 C.store( i+
SIMDSIZE*2UL, j+1UL, xmm6 * factor );
6002 const SIMDType
zero;
6003 C.store( i , j ,
zero );
6006 C.store( i , j+1UL,
zero );
6025 SIMDType b1(
set( B(k,j) ) );
6026 SIMDType xmm1( A.load(i ,k) * b1 );
6027 SIMDType xmm2( A.load(i+
SIMDSIZE ,k) * b1 );
6028 SIMDType xmm3( A.load(i+
SIMDSIZE*2UL,k) * b1 );
6030 for( ++k; k<kend; ++k ) {
6032 xmm1 += A.load(i ,k) * b1;
6033 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
6034 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
6037 C.store( i , j, xmm1 * factor );
6038 C.store( i+
SIMDSIZE , j, xmm2 * factor );
6039 C.store( i+
SIMDSIZE*2UL, j, xmm3 * factor );
6043 const SIMDType
zero;
6044 C.store( i , j,
zero );
6055 for(
size_t ii=i; ii<iiend; ++ii ) {
6064 const size_t jend( LOW ?
min(i+
SIMDSIZE*2UL,N) : N );
6070 for(
size_t ii=i; ii<iiend; ++ii ) {
6071 C(ii,j) = HERM ?
conj( C(j,ii) ) : C(j,ii);
6078 for(
size_t ii=i; ii<iiend; ++ii ) {
6084 for( ; (j+4UL) <= jend; j+=4UL )
6101 SIMDType a1( A.load(i ,k) );
6102 SIMDType a2( A.load(i+
SIMDSIZE,k) );
6103 SIMDType b1(
set( B(k,j ) ) );
6104 SIMDType b2(
set( B(k,j+1UL) ) );
6105 SIMDType b3(
set( B(k,j+2UL) ) );
6106 SIMDType b4(
set( B(k,j+3UL) ) );
6107 SIMDType xmm1( a1 * b1 );
6108 SIMDType xmm2( a2 * b1 );
6109 SIMDType xmm3( a1 * b2 );
6110 SIMDType xmm4( a2 * b2 );
6111 SIMDType xmm5( a1 * b3 );
6112 SIMDType xmm6( a2 * b3 );
6113 SIMDType xmm7( a1 * b4 );
6114 SIMDType xmm8( a2 * b4 );
6116 for( ++k; k<kend; ++k ) {
6119 b1 =
set( B(k,j ) );
6120 b2 =
set( B(k,j+1UL) );
6121 b3 =
set( B(k,j+2UL) );
6122 b4 =
set( B(k,j+3UL) );
6133 C.store( i , j , xmm1 * factor );
6134 C.store( i+
SIMDSIZE, j , xmm2 * factor );
6135 C.store( i , j+1UL, xmm3 * factor );
6136 C.store( i+
SIMDSIZE, j+1UL, xmm4 * factor );
6137 C.store( i , j+2UL, xmm5 * factor );
6138 C.store( i+
SIMDSIZE, j+2UL, xmm6 * factor );
6139 C.store( i , j+3UL, xmm7 * factor );
6140 C.store( i+
SIMDSIZE, j+3UL, xmm8 * factor );
6144 const SIMDType
zero;
6145 C.store( i , j ,
zero );
6147 C.store( i , j+1UL,
zero );
6149 C.store( i , j+2UL,
zero );
6151 C.store( i , j+3UL,
zero );
6156 for( ; (j+3UL) <= jend; j+=3UL )
6173 SIMDType a1( A.load(i ,k) );
6174 SIMDType a2( A.load(i+
SIMDSIZE,k) );
6175 SIMDType b1(
set( B(k,j ) ) );
6176 SIMDType b2(
set( B(k,j+1UL) ) );
6177 SIMDType b3(
set( B(k,j+2UL) ) );
6178 SIMDType xmm1( a1 * b1 );
6179 SIMDType xmm2( a2 * b1 );
6180 SIMDType xmm3( a1 * b2 );
6181 SIMDType xmm4( a2 * b2 );
6182 SIMDType xmm5( a1 * b3 );
6183 SIMDType xmm6( a2 * b3 );
6185 for( ++k; k<kend; ++k ) {
6188 b1 =
set( B(k,j ) );
6189 b2 =
set( B(k,j+1UL) );
6190 b3 =
set( B(k,j+2UL) );
6199 C.store( i , j , xmm1 * factor );
6200 C.store( i+
SIMDSIZE, j , xmm2 * factor );
6201 C.store( i , j+1UL, xmm3 * factor );
6202 C.store( i+
SIMDSIZE, j+1UL, xmm4 * factor );
6203 C.store( i , j+2UL, xmm5 * factor );
6204 C.store( i+
SIMDSIZE, j+2UL, xmm6 * factor );
6208 const SIMDType
zero;
6209 C.store( i , j ,
zero );
6211 C.store( i , j+1UL,
zero );
6213 C.store( i , j+2UL,
zero );
6218 for( ; (j+2UL) <= jend; j+=2UL )
6235 SIMDType a1( A.load(i ,k) );
6236 SIMDType a2( A.load(i+
SIMDSIZE,k) );
6237 SIMDType b1(
set( B(k,j ) ) );
6238 SIMDType b2(
set( B(k,j+1UL) ) );
6239 SIMDType xmm1( a1 * b1 );
6240 SIMDType xmm2( a2 * b1 );
6241 SIMDType xmm3( a1 * b2 );
6242 SIMDType xmm4( a2 * b2 );
6244 for( ++k; k<kend; ++k ) {
6247 b1 =
set( B(k,j ) );
6248 b2 =
set( B(k,j+1UL) );
6255 C.store( i , j , xmm1 * factor );
6256 C.store( i+
SIMDSIZE, j , xmm2 * factor );
6257 C.store( i , j+1UL, xmm3 * factor );
6258 C.store( i+
SIMDSIZE, j+1UL, xmm4 * factor );
6262 const SIMDType
zero;
6263 C.store( i , j ,
zero );
6265 C.store( i , j+1UL,
zero );
6283 SIMDType b1(
set( B(k,j) ) );
6284 SIMDType xmm1( A.load(i ,k) * b1 );
6285 SIMDType xmm2( A.load(i+
SIMDSIZE,k) * b1 );
6287 for( ++k; k<kend; ++k ) {
6289 xmm1 += A.load(i ,k) * b1;
6293 C.store( i , j, xmm1 * factor );
6294 C.store( i+
SIMDSIZE, j, xmm2 * factor );
6298 const SIMDType
zero;
6299 C.store( i , j,
zero );
6309 for(
size_t ii=i; ii<iiend; ++ii ) {
6324 for(
size_t ii=i; ii<iiend; ++ii ) {
6325 C(ii,j) = HERM ?
conj( C(j,ii) ) : C(j,ii);
6332 for(
size_t ii=i; ii<iiend; ++ii ) {
6338 for( ; (j+4UL) <= jend; j+=4UL )
6353 SIMDType a1( A.load(i,k) );
6354 SIMDType xmm1( a1 *
set( B(k,j ) ) );
6355 SIMDType xmm2( a1 *
set( B(k,j+1UL) ) );
6356 SIMDType xmm3( a1 *
set( B(k,j+2UL) ) );
6357 SIMDType xmm4( a1 *
set( B(k,j+3UL) ) );
6359 for( ++k; k<kend; ++k ) {
6361 xmm1 += a1 *
set( B(k,j ) );
6362 xmm2 += a1 *
set( B(k,j+1UL) );
6363 xmm3 += a1 *
set( B(k,j+2UL) );
6364 xmm4 += a1 *
set( B(k,j+3UL) );
6367 C.store( i, j , xmm1 * factor );
6368 C.store( i, j+1UL, xmm2 * factor );
6369 C.store( i, j+2UL, xmm3 * factor );
6370 C.store( i, j+3UL, xmm4 * factor );
6374 const SIMDType
zero;
6375 C.store( i, j ,
zero );
6376 C.store( i, j+1UL,
zero );
6377 C.store( i, j+2UL,
zero );
6378 C.store( i, j+3UL,
zero );
6382 for( ; (j+3UL) <= jend; j+=3UL )
6397 SIMDType a1( A.load(i,k) );
6398 SIMDType xmm1( a1 *
set( B(k,j ) ) );
6399 SIMDType xmm2( a1 *
set( B(k,j+1UL) ) );
6400 SIMDType xmm3( a1 *
set( B(k,j+2UL) ) );
6402 for( ++k; k<kend; ++k ) {
6404 xmm1 += a1 *
set( B(k,j ) );
6405 xmm2 += a1 *
set( B(k,j+1UL) );
6406 xmm3 += a1 *
set( B(k,j+2UL) );
6409 C.store( i, j , xmm1 * factor );
6410 C.store( i, j+1UL, xmm2 * factor );
6411 C.store( i, j+2UL, xmm3 * factor );
6415 const SIMDType
zero;
6416 C.store( i, j ,
zero );
6417 C.store( i, j+1UL,
zero );
6418 C.store( i, j+2UL,
zero );
6422 for( ; (j+2UL) <= jend; j+=2UL )
6437 SIMDType a1( A.load(i,k) );
6438 SIMDType xmm1( a1 *
set( B(k,j ) ) );
6439 SIMDType xmm2( a1 *
set( B(k,j+1UL) ) );
6441 for( ++k; k<kend; ++k ) {
6443 xmm1 += a1 *
set( B(k,j ) );
6444 xmm2 += a1 *
set( B(k,j+1UL) );
6447 C.store( i, j , xmm1 * factor );
6448 C.store( i, j+1UL, xmm2 * factor );
6452 const SIMDType
zero;
6453 C.store( i, j ,
zero );
6454 C.store( i, j+1UL,
zero );
6470 SIMDType xmm1( A.load(i,k) *
set( B(k,j) ) );
6472 for( ++k; k<K; ++k ) {
6473 xmm1 += A.load(i,k) *
set( B(k,j) );
6476 C.store( i, j, xmm1 * factor );
6480 const SIMDType
zero;
6481 C.store( i, j,
zero );
6490 for(
size_t ii=i; ii<iiend; ++ii ) {
6497 for( ; remainder && i<M; ++i )
6503 C(i,j) = HERM ?
conj( C(j,i) ) : C(j,i);
6512 for( ; (j+2UL) <= N; j+=2UL )
6530 for( ++k; k<kend; ++k ) {
6531 value1 += A(i,k) * B(k,j );
6532 value2 += A(i,k) * B(k,j+1UL);
6535 C(i,j ) = value1 * scalar;
6536 C(i,j+1UL) = value2 * scalar;
6541 reset( C(i,j+1UL) );
6559 for( ++k; k<K; ++k ) {
6560 value += A(i,k) * B(k,j);
6563 C(i,j) = value * scalar;
6588 template<
typename MT3
6592 static inline auto selectLargeAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
6595 selectDefaultAssignKernel( C, A, B, scalar );
6614 template<
typename MT3
6618 static inline auto selectLargeAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
6622 smmm( C, A, B, scalar );
6624 hmmm( C, A, B, scalar );
6626 lmmm( C, A, B, scalar, ST2(0) );
6628 ummm( C, A, B, scalar, ST2(0) );
6630 mmm( C, A, B, scalar, ST2(0) );
6648 template<
typename MT3
6652 static inline auto selectBlasAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
6655 selectLargeAssignKernel( C, A, B, scalar );
6660#if BLAZE_BLAS_MODE && BLAZE_USE_BLAS_MATRIX_MATRIX_MULTIPLICATION
6674 template<
typename MT3
6678 static inline auto selectBlasAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
6681 using ET = ElementType_t<MT3>;
6685 trmm( C, A, CblasLeft, (
IsLower_v<MT4> )?( CblasLower ):( CblasUpper ),
ET(scalar) );
6689 trmm( C, B, CblasRight, (
IsLower_v<MT5> )?( CblasLower ):( CblasUpper ),
ET(scalar) );
6692 gemm( C, A, B,
ET(scalar),
ET(0) );
6710 template<
typename MT
6729 const ForwardFunctor fwd;
6731 const TmpType tmp(
serial( rhs ) );
6732 assign( *lhs, fwd( tmp ) );
6750 template<
typename MT >
6761 const ForwardFunctor fwd;
6766 assign( *lhs, fwd( A * B ) * rhs.scalar_ );
6782 template<
typename MT
6784 friend inline auto addAssign( DenseMatrix<MT,SO>& lhs,
const DMatScalarMultExpr& rhs )
6792 LeftOperand_t<MMM> left ( rhs.matrix_.leftOperand() );
6793 RightOperand_t<MMM> right( rhs.matrix_.rightOperand() );
6795 if( (*lhs).rows() == 0UL || (*lhs).columns() == 0UL || left.columns() == 0UL ) {
6809 DMatScalarMultExpr::selectAddAssignKernel( *lhs, A, B, rhs.scalar_ );
6824 template<
typename MT3
6828 static inline void selectAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
6831 ( !BLAZE_DEBUG_MODE && A.rows() <=
SIMDSIZE*10UL ) ||
6832 ( C.rows() * C.columns() < TDMATTDMATMULT_THRESHOLD ) )
6833 selectSmallAddAssignKernel( C, A, B, scalar );
6835 selectBlasAddAssignKernel( C, A, B, scalar );
6853 template<
typename MT3
6857 static inline auto selectDefaultAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
6861 addAssign( C, tmp );
6879 template<
typename MT3
6883 static inline auto selectDefaultAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
6888 const size_t M( A.rows() );
6889 const size_t N( B.columns() );
6891 for(
size_t j=0UL; j<N; ++j )
6901 const size_t inum( iend - ibegin );
6902 const size_t ipos( ibegin +
prevMultiple( inum, 2UL ) );
6905 for(
size_t i=ibegin; i<ipos; i+=2UL ) {
6906 C(i ,j) += A(i ,j) * B(j,j) * scalar;
6907 C(i+1UL,j) += A(i+1UL,j) * B(j,j) * scalar;
6910 C(ipos,j) += A(ipos,j) * B(j,j) * scalar;
6930 template<
typename MT3
6934 static inline auto selectDefaultAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
6939 const size_t M( A.rows() );
6940 const size_t N( B.columns() );
6942 for(
size_t j=0UL; j<N; ++j )
6952 const size_t inum( iend - ibegin );
6953 const size_t ipos( ibegin +
prevMultiple( inum, 2UL ) );
6956 for(
size_t i=ibegin; i<ipos; i+=2UL ) {
6957 C(i ,j) += A(i ,i ) * B(i ,j) * scalar;
6958 C(i+1UL,j) += A(i+1UL,i+1UL) * B(i+1UL,j) * scalar;
6961 C(ipos,j) += A(ipos,ipos) * B(ipos,j) * scalar;
6981 template<
typename MT3
6985 static inline auto selectDefaultAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
6990 for(
size_t i=0UL; i<A.rows(); ++i ) {
6991 C(i,i) += A(i,i) * B(i,i) * scalar;
7010 template<
typename MT3
7014 static inline auto selectSmallAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
7017 selectDefaultAddAssignKernel( C, A, B, scalar );
7036 template<
typename MT3
7040 static inline auto selectSmallAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
7048 const ForwardFunctor fwd;
7051 const OppositeType_t<MT5> tmp(
serial( B ) );
7052 addAssign( C, fwd( A * tmp ) * scalar );
7055 const OppositeType_t<MT4> tmp(
serial( A ) );
7056 addAssign( C, fwd( tmp * B ) * scalar );
7058 else if( B.rows() * B.columns() <= A.rows() * A.columns() ) {
7059 const OppositeType_t<MT5> tmp(
serial( B ) );
7060 addAssign( C, fwd( A * tmp ) * scalar );
7063 const OppositeType_t<MT4> tmp(
serial( A ) );
7064 addAssign( C, fwd( tmp * B ) * scalar );
7084 template<
typename MT3
7088 static inline auto selectSmallAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
7093 const size_t M( A.rows() );
7094 const size_t N( B.columns() );
7095 const size_t K( A.columns() );
7102 const SIMDType factor(
set( scalar ) );
7109 for(
size_t j=0UL; j<N; ++j )
7126 SIMDType b1(
set( B(k,j) ) );
7127 SIMDType xmm1( A.load(i ,k) * b1 );
7128 SIMDType xmm2( A.load(i+
SIMDSIZE ,k) * b1 );
7129 SIMDType xmm3( A.load(i+
SIMDSIZE*2UL,k) * b1 );
7130 SIMDType xmm4( A.load(i+
SIMDSIZE*3UL,k) * b1 );
7131 SIMDType xmm5( A.load(i+
SIMDSIZE*4UL,k) * b1 );
7132 SIMDType xmm6( A.load(i+
SIMDSIZE*5UL,k) * b1 );
7133 SIMDType xmm7( A.load(i+
SIMDSIZE*6UL,k) * b1 );
7134 SIMDType xmm8( A.load(i+
SIMDSIZE*7UL,k) * b1 );
7136 for( ++k; k<kend; ++k ) {
7138 xmm1 += A.load(i ,k) * b1;
7139 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
7140 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
7141 xmm4 += A.load(i+
SIMDSIZE*3UL,k) * b1;
7142 xmm5 += A.load(i+
SIMDSIZE*4UL,k) * b1;
7143 xmm6 += A.load(i+
SIMDSIZE*5UL,k) * b1;
7144 xmm7 += A.load(i+
SIMDSIZE*6UL,k) * b1;
7145 xmm8 += A.load(i+
SIMDSIZE*7UL,k) * b1;
7148 C.store( i , j, C.load(i ,j) + xmm1 * factor );
7165 for( ; (j+2UL) <= N; j+=2UL )
7182 SIMDType a1( A.load(i ,k) );
7183 SIMDType a2( A.load(i+
SIMDSIZE ,k) );
7184 SIMDType a3( A.load(i+
SIMDSIZE*2UL,k) );
7185 SIMDType a4( A.load(i+
SIMDSIZE*3UL,k) );
7186 SIMDType a5( A.load(i+
SIMDSIZE*4UL,k) );
7187 SIMDType b1(
set( B(k,j ) ) );
7188 SIMDType b2(
set( B(k,j+1UL) ) );
7189 SIMDType xmm1 ( a1 * b1 );
7190 SIMDType xmm2 ( a2 * b1 );
7191 SIMDType xmm3 ( a3 * b1 );
7192 SIMDType xmm4 ( a4 * b1 );
7193 SIMDType xmm5 ( a5 * b1 );
7194 SIMDType xmm6 ( a1 * b2 );
7195 SIMDType xmm7 ( a2 * b2 );
7196 SIMDType xmm8 ( a3 * b2 );
7197 SIMDType xmm9 ( a4 * b2 );
7198 SIMDType xmm10( a5 * b2 );
7200 for( ++k; k<kend; ++k ) {
7206 b1 =
set( B(k,j ) );
7207 b2 =
set( B(k,j+1UL) );
7220 C.store( i , j , C.load(i ,j ) + xmm1 * factor );
7225 C.store( i , j+1UL, C.load(i ,j+1UL) + xmm6 * factor );
7227 C.store( i+
SIMDSIZE*2UL, j+1UL, C.load(i+
SIMDSIZE*2UL,j+1UL) + xmm8 * factor );
7228 C.store( i+
SIMDSIZE*3UL, j+1UL, C.load(i+
SIMDSIZE*3UL,j+1UL) + xmm9 * factor );
7229 C.store( i+
SIMDSIZE*4UL, j+1UL, C.load(i+
SIMDSIZE*4UL,j+1UL) + xmm10 * factor );
7246 SIMDType b1(
set( B(k,j) ) );
7247 SIMDType xmm1( A.load(i ,k) * b1 );
7248 SIMDType xmm2( A.load(i+
SIMDSIZE ,k) * b1 );
7249 SIMDType xmm3( A.load(i+
SIMDSIZE*2UL,k) * b1 );
7250 SIMDType xmm4( A.load(i+
SIMDSIZE*3UL,k) * b1 );
7251 SIMDType xmm5( A.load(i+
SIMDSIZE*4UL,k) * b1 );
7253 for( ++k; k<kend; ++k ) {
7255 xmm1 += A.load(i ,k) * b1;
7256 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
7257 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
7258 xmm4 += A.load(i+
SIMDSIZE*3UL,k) * b1;
7259 xmm5 += A.load(i+
SIMDSIZE*4UL,k) * b1;
7262 C.store( i , j, C.load(i ,j) + xmm1 * factor );
7275 for( ; (j+2UL) <= N; j+=2UL )
7292 SIMDType a1( A.load(i ,k) );
7293 SIMDType a2( A.load(i+
SIMDSIZE ,k) );
7294 SIMDType a3( A.load(i+
SIMDSIZE*2UL,k) );
7295 SIMDType a4( A.load(i+
SIMDSIZE*3UL,k) );
7296 SIMDType b1(
set( B(k,j ) ) );
7297 SIMDType b2(
set( B(k,j+1UL) ) );
7298 SIMDType xmm1( a1 * b1 );
7299 SIMDType xmm2( a2 * b1 );
7300 SIMDType xmm3( a3 * b1 );
7301 SIMDType xmm4( a4 * b1 );
7302 SIMDType xmm5( a1 * b2 );
7303 SIMDType xmm6( a2 * b2 );
7304 SIMDType xmm7( a3 * b2 );
7305 SIMDType xmm8( a4 * b2 );
7307 for( ++k; k<kend; ++k ) {
7312 b1 =
set( B(k,j ) );
7313 b2 =
set( B(k,j+1UL) );
7324 C.store( i , j , C.load(i ,j ) + xmm1 * factor );
7328 C.store( i , j+1UL, C.load(i ,j+1UL) + xmm5 * factor );
7330 C.store( i+
SIMDSIZE*2UL, j+1UL, C.load(i+
SIMDSIZE*2UL,j+1UL) + xmm7 * factor );
7331 C.store( i+
SIMDSIZE*3UL, j+1UL, C.load(i+
SIMDSIZE*3UL,j+1UL) + xmm8 * factor );
7348 SIMDType b1(
set( B(k,j) ) );
7349 SIMDType xmm1( A.load(i ,k) * b1 );
7350 SIMDType xmm2( A.load(i+
SIMDSIZE ,k) * b1 );
7351 SIMDType xmm3( A.load(i+
SIMDSIZE*2UL,k) * b1 );
7352 SIMDType xmm4( A.load(i+
SIMDSIZE*3UL,k) * b1 );
7354 for( ++k; k<kend; ++k ) {
7356 xmm1 += A.load(i ,k) * b1;
7357 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
7358 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
7359 xmm4 += A.load(i+
SIMDSIZE*3UL,k) * b1;
7362 C.store( i , j, C.load(i ,j) + xmm1 * factor );
7374 for( ; (j+2UL) <= N; j+=2UL )
7391 SIMDType a1( A.load(i ,k) );
7392 SIMDType a2( A.load(i+
SIMDSIZE ,k) );
7393 SIMDType a3( A.load(i+
SIMDSIZE*2UL,k) );
7394 SIMDType b1(
set( B(k,j ) ) );
7395 SIMDType b2(
set( B(k,j+1UL) ) );
7396 SIMDType xmm1( a1 * b1 );
7397 SIMDType xmm2( a2 * b1 );
7398 SIMDType xmm3( a3 * b1 );
7399 SIMDType xmm4( a1 * b2 );
7400 SIMDType xmm5( a2 * b2 );
7401 SIMDType xmm6( a3 * b2 );
7403 for( ++k; k<kend; ++k ) {
7407 b1 =
set( B(k,j ) );
7408 b2 =
set( B(k,j+1UL) );
7417 C.store( i , j , C.load(i ,j ) + xmm1 * factor );
7420 C.store( i , j+1UL, C.load(i ,j+1UL) + xmm4 * factor );
7422 C.store( i+
SIMDSIZE*2UL, j+1UL, C.load(i+
SIMDSIZE*2UL,j+1UL) + xmm6 * factor );
7439 SIMDType b1(
set( B(k,j) ) );
7440 SIMDType xmm1( A.load(i ,k) * b1 );
7441 SIMDType xmm2( A.load(i+
SIMDSIZE ,k) * b1 );
7442 SIMDType xmm3( A.load(i+
SIMDSIZE*2UL,k) * b1 );
7444 for( ++k; k<kend; ++k ) {
7446 xmm1 += A.load(i ,k) * b1;
7447 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
7448 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
7451 C.store( i , j, C.load(i ,j) + xmm1 * factor );
7460 const size_t jend( LOW ?
min(i+
SIMDSIZE*2UL,N) : N );
7461 size_t j( UPP ? i : 0UL );
7463 for( ; (j+4UL) <= jend; j+=4UL )
7480 SIMDType a1( A.load(i ,k) );
7481 SIMDType a2( A.load(i+
SIMDSIZE,k) );
7482 SIMDType b1(
set( B(k,j ) ) );
7483 SIMDType b2(
set( B(k,j+1UL) ) );
7484 SIMDType b3(
set( B(k,j+2UL) ) );
7485 SIMDType b4(
set( B(k,j+3UL) ) );
7486 SIMDType xmm1( a1 * b1 );
7487 SIMDType xmm2( a2 * b1 );
7488 SIMDType xmm3( a1 * b2 );
7489 SIMDType xmm4( a2 * b2 );
7490 SIMDType xmm5( a1 * b3 );
7491 SIMDType xmm6( a2 * b3 );
7492 SIMDType xmm7( a1 * b4 );
7493 SIMDType xmm8( a2 * b4 );
7495 for( ++k; k<kend; ++k ) {
7498 b1 =
set( B(k,j ) );
7499 b2 =
set( B(k,j+1UL) );
7500 b3 =
set( B(k,j+2UL) );
7501 b4 =
set( B(k,j+3UL) );
7512 C.store( i , j , C.load(i ,j ) + xmm1 * factor );
7514 C.store( i , j+1UL, C.load(i ,j+1UL) + xmm3 * factor );
7516 C.store( i , j+2UL, C.load(i ,j+2UL) + xmm5 * factor );
7518 C.store( i , j+3UL, C.load(i ,j+3UL) + xmm7 * factor );
7523 for( ; (j+3UL) <= jend; j+=3UL )
7540 SIMDType a1( A.load(i ,k) );
7541 SIMDType a2( A.load(i+
SIMDSIZE,k) );
7542 SIMDType b1(
set( B(k,j ) ) );
7543 SIMDType b2(
set( B(k,j+1UL) ) );
7544 SIMDType b3(
set( B(k,j+2UL) ) );
7545 SIMDType xmm1( a1 * b1 );
7546 SIMDType xmm2( a2 * b1 );
7547 SIMDType xmm3( a1 * b2 );
7548 SIMDType xmm4( a2 * b2 );
7549 SIMDType xmm5( a1 * b3 );
7550 SIMDType xmm6( a2 * b3 );
7552 for( ++k; k<kend; ++k ) {
7555 b1 =
set( B(k,j ) );
7556 b2 =
set( B(k,j+1UL) );
7557 b3 =
set( B(k,j+2UL) );
7566 C.store( i , j , C.load(i ,j ) + xmm1 * factor );
7568 C.store( i , j+1UL, C.load(i ,j+1UL) + xmm3 * factor );
7570 C.store( i , j+2UL, C.load(i ,j+2UL) + xmm5 * factor );
7575 for( ; (j+2UL) <= jend; j+=2UL )
7592 SIMDType a1( A.load(i ,k) );
7593 SIMDType a2( A.load(i+
SIMDSIZE,k) );
7594 SIMDType b1(
set( B(k,j ) ) );
7595 SIMDType b2(
set( B(k,j+1UL) ) );
7596 SIMDType xmm1( a1 * b1 );
7597 SIMDType xmm2( a2 * b1 );
7598 SIMDType xmm3( a1 * b2 );
7599 SIMDType xmm4( a2 * b2 );
7601 for( ++k; k<kend; ++k ) {
7604 b1 =
set( B(k,j ) );
7605 b2 =
set( B(k,j+1UL) );
7612 C.store( i , j , C.load(i ,j ) + xmm1 * factor );
7614 C.store( i , j+1UL, C.load(i ,j+1UL) + xmm3 * factor );
7632 SIMDType b1(
set( B(k,j) ) );
7633 SIMDType xmm1( A.load(i ,k) * b1 );
7634 SIMDType xmm2( A.load(i+
SIMDSIZE,k) * b1 );
7636 for( ++k; k<kend; ++k ) {
7638 xmm1 += A.load(i ,k) * b1;
7642 C.store( i , j, C.load(i ,j) + xmm1 * factor );
7650 const size_t jend( LOW && UPP ?
min(i+
SIMDSIZE,N) : N );
7651 size_t j( UPP ? i : 0UL );
7653 for( ; (j+4UL) <= jend; j+=4UL )
7668 SIMDType a1( A.load(i,k) );
7669 SIMDType xmm1( a1 *
set( B(k,j ) ) );
7670 SIMDType xmm2( a1 *
set( B(k,j+1UL) ) );
7671 SIMDType xmm3( a1 *
set( B(k,j+2UL) ) );
7672 SIMDType xmm4( a1 *
set( B(k,j+3UL) ) );
7674 for( ++k; k<kend; ++k ) {
7676 xmm1 += a1 *
set( B(k,j ) );
7677 xmm2 += a1 *
set( B(k,j+1UL) );
7678 xmm3 += a1 *
set( B(k,j+2UL) );
7679 xmm4 += a1 *
set( B(k,j+3UL) );
7682 C.store( i, j , C.load(i,j ) + xmm1 * factor );
7683 C.store( i, j+1UL, C.load(i,j+1UL) + xmm2 * factor );
7684 C.store( i, j+2UL, C.load(i,j+2UL) + xmm3 * factor );
7685 C.store( i, j+3UL, C.load(i,j+3UL) + xmm4 * factor );
7689 for( ; (j+3UL) <= jend; j+=3UL )
7704 SIMDType a1( A.load(i,k) );
7705 SIMDType xmm1( a1 *
set( B(k,j ) ) );
7706 SIMDType xmm2( a1 *
set( B(k,j+1UL) ) );
7707 SIMDType xmm3( a1 *
set( B(k,j+2UL) ) );
7709 for( ++k; k<kend; ++k ) {
7711 xmm1 += a1 *
set( B(k,j ) );
7712 xmm2 += a1 *
set( B(k,j+1UL) );
7713 xmm3 += a1 *
set( B(k,j+2UL) );
7716 C.store( i, j , C.load(i,j ) + xmm1 * factor );
7717 C.store( i, j+1UL, C.load(i,j+1UL) + xmm2 * factor );
7718 C.store( i, j+2UL, C.load(i,j+2UL) + xmm3 * factor );
7722 for( ; (j+2UL) <= jend; j+=2UL )
7737 SIMDType a1( A.load(i,k) );
7738 SIMDType xmm1( a1 *
set( B(k,j ) ) );
7739 SIMDType xmm2( a1 *
set( B(k,j+1UL) ) );
7741 for( ++k; k<kend; ++k ) {
7743 xmm1 += a1 *
set( B(k,j ) );
7744 xmm2 += a1 *
set( B(k,j+1UL) );
7747 C.store( i, j , C.load(i,j ) + xmm1 * factor );
7748 C.store( i, j+1UL, C.load(i,j+1UL) + xmm2 * factor );
7764 SIMDType xmm1( A.load(i,k) *
set( B(k,j) ) );
7766 for( ++k; k<K; ++k ) {
7767 xmm1 += A.load(i,k) *
set( B(k,j) );
7770 C.store( i, j, C.load(i,j) + xmm1 * factor );
7775 for( ; remainder && i<M; ++i )
7777 const size_t jend( LOW ? i+1UL : N );
7778 size_t j( UPP ? i : 0UL );
7780 for( ; (j+2UL) <= jend; j+=2UL )
7798 for( ++k; k<kend; ++k ) {
7799 value1 += A(i,k) * B(k,j );
7800 value2 += A(i,k) * B(k,j+1UL);
7803 C(i,j ) += value1 * scalar;
7804 C(i,j+1UL) += value2 * scalar;
7822 for( ++k; k<K; ++k ) {
7823 value += A(i,k) * B(k,j);
7826 C(i,j) += value * scalar;
7847 template<
typename MT3
7851 static inline auto selectLargeAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
7854 selectDefaultAddAssignKernel( C, A, B, scalar );
7873 template<
typename MT3
7877 static inline auto selectLargeAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
7881 lmmm( C, A, B, scalar, ST2(1) );
7883 ummm( C, A, B, scalar, ST2(1) );
7885 mmm( C, A, B, scalar, ST2(1) );
7904 template<
typename MT3
7908 static inline auto selectBlasAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
7911 selectLargeAddAssignKernel( C, A, B, scalar );
7916#if BLAZE_BLAS_MODE && BLAZE_USE_BLAS_MATRIX_MATRIX_MULTIPLICATION
7930 template<
typename MT3
7934 static inline auto selectBlasAddAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
7937 using ET = ElementType_t<MT3>;
7940 ResultType_t<MT3> tmp(
serial( B ) );
7941 trmm( tmp, A, CblasLeft, (
IsLower_v<MT4> )?( CblasLower ):( CblasUpper ),
ET(scalar) );
7942 addAssign( C, tmp );
7945 ResultType_t<MT3> tmp(
serial( A ) );
7946 trmm( tmp, B, CblasRight, (
IsLower_v<MT5> )?( CblasLower ):( CblasUpper ),
ET(scalar) );
7947 addAssign( C, tmp );
7950 gemm( C, A, B,
ET(scalar),
ET(1) );
7971 template<
typename MT >
7982 const ForwardFunctor fwd;
7987 addAssign( *lhs, fwd( A * B ) * rhs.scalar_ );
8007 template<
typename MT
8009 friend inline auto subAssign( DenseMatrix<MT,SO>& lhs,
const DMatScalarMultExpr& rhs )
8017 LeftOperand_t<MMM> left ( rhs.matrix_.leftOperand() );
8018 RightOperand_t<MMM> right( rhs.matrix_.rightOperand() );
8020 if( (*lhs).rows() == 0UL || (*lhs).columns() == 0UL || left.columns() == 0UL ) {
8034 DMatScalarMultExpr::selectSubAssignKernel( *lhs, A, B, rhs.scalar_ );
8049 template<
typename MT3
8053 static inline void selectSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
8056 ( !BLAZE_DEBUG_MODE && A.rows() <=
SIMDSIZE*10UL ) ||
8057 ( C.rows() * C.columns() < TDMATTDMATMULT_THRESHOLD ) )
8058 selectSmallSubAssignKernel( C, A, B, scalar );
8060 selectBlasSubAssignKernel( C, A, B, scalar );
8078 template<
typename MT3
8082 static inline auto selectDefaultSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
8086 subAssign( C, tmp );
8104 template<
typename MT3
8108 static inline auto selectDefaultSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
8113 const size_t M( A.rows() );
8114 const size_t N( B.columns() );
8116 for(
size_t j=0UL; j<N; ++j )
8126 const size_t inum( iend - ibegin );
8127 const size_t ipos( ibegin +
prevMultiple( inum, 2UL ) );
8130 for(
size_t i=ibegin; i<ipos; i+=2UL ) {
8131 C(i ,j) -= A(i ,j) * B(j,j) * scalar;
8132 C(i+1UL,j) -= A(i+1UL,j) * B(j,j) * scalar;
8135 C(ipos,j) -= A(ipos,j) * B(j,j) * scalar;
8155 template<
typename MT3
8159 static inline auto selectDefaultSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
8164 const size_t M( A.rows() );
8165 const size_t N( B.columns() );
8167 for(
size_t j=0UL; j<N; ++j )
8177 const size_t inum( iend - ibegin );
8178 const size_t ipos( ibegin +
prevMultiple( inum, 2UL ) );
8181 for(
size_t i=ibegin; i<ipos; i+=2UL ) {
8182 C(i ,j) -= A(i ,i ) * B(i ,j) * scalar;
8183 C(i+1UL,j) -= A(i+1UL,i+1UL) * B(i+1UL,j) * scalar;
8186 C(ipos,j) -= A(ipos,ipos) * B(ipos,j) * scalar;
8206 template<
typename MT3
8210 static inline auto selectDefaultSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
8215 for(
size_t i=0UL; i<A.rows(); ++i ) {
8216 C(i,i) -= A(i,i) * B(i,i) * scalar;
8235 template<
typename MT3
8239 static inline auto selectSmallSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
8242 selectDefaultSubAssignKernel( C, A, B, scalar );
8261 template<
typename MT3
8265 static inline auto selectSmallSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
8273 const ForwardFunctor fwd;
8276 const OppositeType_t<MT5> tmp(
serial( B ) );
8277 subAssign( C, fwd( A * tmp ) * scalar );
8280 const OppositeType_t<MT4> tmp(
serial( A ) );
8281 subAssign( C, fwd( tmp * B ) * scalar );
8283 else if( B.rows() * B.columns() <= A.rows() * A.columns() ) {
8284 const OppositeType_t<MT5> tmp(
serial( B ) );
8285 subAssign( C, fwd( A * tmp ) * scalar );
8288 const OppositeType_t<MT4> tmp(
serial( A ) );
8289 subAssign( C, fwd( tmp * B ) * scalar );
8309 template<
typename MT3
8313 static inline auto selectSmallSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
8318 const size_t M( A.rows() );
8319 const size_t N( B.columns() );
8320 const size_t K( A.columns() );
8327 const SIMDType factor(
set( scalar ) );
8334 for(
size_t j=0UL; j<N; ++j )
8351 SIMDType b1(
set( B(k,j) ) );
8352 SIMDType xmm1( A.load(i ,k) * b1 );
8353 SIMDType xmm2( A.load(i+
SIMDSIZE ,k) * b1 );
8354 SIMDType xmm3( A.load(i+
SIMDSIZE*2UL,k) * b1 );
8355 SIMDType xmm4( A.load(i+
SIMDSIZE*3UL,k) * b1 );
8356 SIMDType xmm5( A.load(i+
SIMDSIZE*4UL,k) * b1 );
8357 SIMDType xmm6( A.load(i+
SIMDSIZE*5UL,k) * b1 );
8358 SIMDType xmm7( A.load(i+
SIMDSIZE*6UL,k) * b1 );
8359 SIMDType xmm8( A.load(i+
SIMDSIZE*7UL,k) * b1 );
8361 for( ++k; k<kend; ++k ) {
8363 xmm1 += A.load(i ,k) * b1;
8364 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
8365 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
8366 xmm4 += A.load(i+
SIMDSIZE*3UL,k) * b1;
8367 xmm5 += A.load(i+
SIMDSIZE*4UL,k) * b1;
8368 xmm6 += A.load(i+
SIMDSIZE*5UL,k) * b1;
8369 xmm7 += A.load(i+
SIMDSIZE*6UL,k) * b1;
8370 xmm8 += A.load(i+
SIMDSIZE*7UL,k) * b1;
8373 C.store( i , j, C.load(i ,j) - xmm1 * factor );
8390 for( ; (j+2UL) <= N; j+=2UL )
8407 SIMDType a1( A.load(i ,k) );
8408 SIMDType a2( A.load(i+
SIMDSIZE ,k) );
8409 SIMDType a3( A.load(i+
SIMDSIZE*2UL,k) );
8410 SIMDType a4( A.load(i+
SIMDSIZE*3UL,k) );
8411 SIMDType a5( A.load(i+
SIMDSIZE*4UL,k) );
8412 SIMDType b1(
set( B(k,j ) ) );
8413 SIMDType b2(
set( B(k,j+1UL) ) );
8414 SIMDType xmm1 ( a1 * b1 );
8415 SIMDType xmm2 ( a2 * b1 );
8416 SIMDType xmm3 ( a3 * b1 );
8417 SIMDType xmm4 ( a4 * b1 );
8418 SIMDType xmm5 ( a5 * b1 );
8419 SIMDType xmm6 ( a1 * b2 );
8420 SIMDType xmm7 ( a2 * b2 );
8421 SIMDType xmm8 ( a3 * b2 );
8422 SIMDType xmm9 ( a4 * b2 );
8423 SIMDType xmm10( a5 * b2 );
8425 for( ++k; k<kend; ++k ) {
8431 b1 =
set( B(k,j ) );
8432 b2 =
set( B(k,j+1UL) );
8445 C.store( i , j , C.load(i ,j ) - xmm1 * factor );
8450 C.store( i , j+1UL, C.load(i ,j+1UL) - xmm6 * factor );
8452 C.store( i+
SIMDSIZE*2UL, j+1UL, C.load(i+
SIMDSIZE*2UL,j+1UL) - xmm8 * factor );
8453 C.store( i+
SIMDSIZE*3UL, j+1UL, C.load(i+
SIMDSIZE*3UL,j+1UL) - xmm9 * factor );
8454 C.store( i+
SIMDSIZE*4UL, j+1UL, C.load(i+
SIMDSIZE*4UL,j+1UL) - xmm10 * factor );
8471 SIMDType b1(
set( B(k,j) ) );
8472 SIMDType xmm1( A.load(i ,k) * b1 );
8473 SIMDType xmm2( A.load(i+
SIMDSIZE ,k) * b1 );
8474 SIMDType xmm3( A.load(i+
SIMDSIZE*2UL,k) * b1 );
8475 SIMDType xmm4( A.load(i+
SIMDSIZE*3UL,k) * b1 );
8476 SIMDType xmm5( A.load(i+
SIMDSIZE*4UL,k) * b1 );
8478 for( ++k; k<kend; ++k ) {
8480 xmm1 += A.load(i ,k) * b1;
8481 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
8482 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
8483 xmm4 += A.load(i+
SIMDSIZE*3UL,k) * b1;
8484 xmm5 += A.load(i+
SIMDSIZE*4UL,k) * b1;
8487 C.store( i , j, C.load(i ,j) - xmm1 * factor );
8500 for( ; (j+2UL) <= N; j+=2UL )
8517 SIMDType a1( A.load(i ,k) );
8518 SIMDType a2( A.load(i+
SIMDSIZE ,k) );
8519 SIMDType a3( A.load(i+
SIMDSIZE*2UL,k) );
8520 SIMDType a4( A.load(i+
SIMDSIZE*3UL,k) );
8521 SIMDType b1(
set( B(k,j ) ) );
8522 SIMDType b2(
set( B(k,j+1UL) ) );
8523 SIMDType xmm1( a1 * b1 );
8524 SIMDType xmm2( a2 * b1 );
8525 SIMDType xmm3( a3 * b1 );
8526 SIMDType xmm4( a4 * b1 );
8527 SIMDType xmm5( a1 * b2 );
8528 SIMDType xmm6( a2 * b2 );
8529 SIMDType xmm7( a3 * b2 );
8530 SIMDType xmm8( a4 * b2 );
8532 for( ++k; k<kend; ++k ) {
8537 b1 =
set( B(k,j ) );
8538 b2 =
set( B(k,j+1UL) );
8549 C.store( i , j , C.load(i ,j ) - xmm1 * factor );
8553 C.store( i , j+1UL, C.load(i ,j+1UL) - xmm5 * factor );
8555 C.store( i+
SIMDSIZE*2UL, j+1UL, C.load(i+
SIMDSIZE*2UL,j+1UL) - xmm7 * factor );
8556 C.store( i+
SIMDSIZE*3UL, j+1UL, C.load(i+
SIMDSIZE*3UL,j+1UL) - xmm8 * factor );
8573 SIMDType b1(
set( B(k,j) ) );
8574 SIMDType xmm1( A.load(i ,k) * b1 );
8575 SIMDType xmm2( A.load(i+
SIMDSIZE ,k) * b1 );
8576 SIMDType xmm3( A.load(i+
SIMDSIZE*2UL,k) * b1 );
8577 SIMDType xmm4( A.load(i+
SIMDSIZE*3UL,k) * b1 );
8579 for( ++k; k<kend; ++k ) {
8581 xmm1 += A.load(i ,k) * b1;
8582 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
8583 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
8584 xmm4 += A.load(i+
SIMDSIZE*3UL,k) * b1;
8587 C.store( i , j, C.load(i ,j) - xmm1 * factor );
8599 for( ; (j+2UL) <= N; j+=2UL )
8616 SIMDType a1( A.load(i ,k) );
8617 SIMDType a2( A.load(i+
SIMDSIZE ,k) );
8618 SIMDType a3( A.load(i+
SIMDSIZE*2UL,k) );
8619 SIMDType b1(
set( B(k,j ) ) );
8620 SIMDType b2(
set( B(k,j+1UL) ) );
8621 SIMDType xmm1( a1 * b1 );
8622 SIMDType xmm2( a2 * b1 );
8623 SIMDType xmm3( a3 * b1 );
8624 SIMDType xmm4( a1 * b2 );
8625 SIMDType xmm5( a2 * b2 );
8626 SIMDType xmm6( a3 * b2 );
8628 for( ++k; k<kend; ++k ) {
8632 b1 =
set( B(k,j ) );
8633 b2 =
set( B(k,j+1UL) );
8642 C.store( i , j , C.load(i ,j ) - xmm1 * factor );
8645 C.store( i , j+1UL, C.load(i ,j+1UL) - xmm4 * factor );
8647 C.store( i+
SIMDSIZE*2UL, j+1UL, C.load(i+
SIMDSIZE*2UL,j+1UL) - xmm6 * factor );
8664 SIMDType b1(
set( B(k,j) ) );
8665 SIMDType xmm1( A.load(i ,k) * b1 );
8666 SIMDType xmm2( A.load(i+
SIMDSIZE ,k) * b1 );
8667 SIMDType xmm3( A.load(i+
SIMDSIZE*2UL,k) * b1 );
8669 for( ++k; k<kend; ++k ) {
8671 xmm1 += A.load(i ,k) * b1;
8672 xmm2 += A.load(i+
SIMDSIZE ,k) * b1;
8673 xmm3 += A.load(i+
SIMDSIZE*2UL,k) * b1;
8676 C.store( i , j, C.load(i ,j) - xmm1 * factor );
8685 const size_t jend( LOW ?
min(i+
SIMDSIZE*2UL,N) : N );
8686 size_t j( UPP ? i : 0UL );
8688 for( ; (j+4UL) <= jend; j+=4UL )
8705 SIMDType a1( A.load(i ,k) );
8706 SIMDType a2( A.load(i+
SIMDSIZE,k) );
8707 SIMDType b1(
set( B(k,j ) ) );
8708 SIMDType b2(
set( B(k,j+1UL) ) );
8709 SIMDType b3(
set( B(k,j+2UL) ) );
8710 SIMDType b4(
set( B(k,j+3UL) ) );
8711 SIMDType xmm1( a1 * b1 );
8712 SIMDType xmm2( a2 * b1 );
8713 SIMDType xmm3( a1 * b2 );
8714 SIMDType xmm4( a2 * b2 );
8715 SIMDType xmm5( a1 * b3 );
8716 SIMDType xmm6( a2 * b3 );
8717 SIMDType xmm7( a1 * b4 );
8718 SIMDType xmm8( a2 * b4 );
8720 for( ++k; k<kend; ++k ) {
8723 b1 =
set( B(k,j ) );
8724 b2 =
set( B(k,j+1UL) );
8725 b3 =
set( B(k,j+2UL) );
8726 b4 =
set( B(k,j+3UL) );
8737 C.store( i , j , C.load(i ,j ) - xmm1 * factor );
8739 C.store( i , j+1UL, C.load(i ,j+1UL) - xmm3 * factor );
8741 C.store( i , j+2UL, C.load(i ,j+2UL) - xmm5 * factor );
8743 C.store( i , j+3UL, C.load(i ,j+3UL) - xmm7 * factor );
8748 for( ; (j+3UL) <= jend; j+=3UL )
8765 SIMDType a1( A.load(i ,k) );
8766 SIMDType a2( A.load(i+
SIMDSIZE,k) );
8767 SIMDType b1(
set( B(k,j ) ) );
8768 SIMDType b2(
set( B(k,j+1UL) ) );
8769 SIMDType b3(
set( B(k,j+2UL) ) );
8770 SIMDType xmm1( a1 * b1 );
8771 SIMDType xmm2( a2 * b1 );
8772 SIMDType xmm3( a1 * b2 );
8773 SIMDType xmm4( a2 * b2 );
8774 SIMDType xmm5( a1 * b3 );
8775 SIMDType xmm6( a2 * b3 );
8777 for( ++k; k<kend; ++k ) {
8780 b1 =
set( B(k,j ) );
8781 b2 =
set( B(k,j+1UL) );
8782 b3 =
set( B(k,j+2UL) );
8791 C.store( i , j , C.load(i ,j ) - xmm1 * factor );
8793 C.store( i , j+1UL, C.load(i ,j+1UL) - xmm3 * factor );
8795 C.store( i , j+2UL, C.load(i ,j+2UL) - xmm5 * factor );
8800 for( ; (j+2UL) <= jend; j+=2UL )
8817 SIMDType a1( A.load(i ,k) );
8818 SIMDType a2( A.load(i+
SIMDSIZE,k) );
8819 SIMDType b1(
set( B(k,j ) ) );
8820 SIMDType b2(
set( B(k,j+1UL) ) );
8821 SIMDType xmm1( a1 * b1 );
8822 SIMDType xmm2( a2 * b1 );
8823 SIMDType xmm3( a1 * b2 );
8824 SIMDType xmm4( a2 * b2 );
8826 for( ++k; k<kend; ++k ) {
8829 b1 =
set( B(k,j ) );
8830 b2 =
set( B(k,j+1UL) );
8837 C.store( i , j , C.load(i ,j ) - xmm1 * factor );
8839 C.store( i , j+1UL, C.load(i ,j+1UL) - xmm3 * factor );
8857 SIMDType b1(
set( B(k,j) ) );
8858 SIMDType xmm1( A.load(i ,k) * b1 );
8859 SIMDType xmm2( A.load(i+
SIMDSIZE,k) * b1 );
8861 for( ++k; k<kend; ++k ) {
8863 xmm1 += A.load(i ,k) * b1;
8867 C.store( i , j, C.load(i ,j) - xmm1 * factor );
8875 const size_t jend( LOW && UPP ?
min(i+
SIMDSIZE,N) : N );
8876 size_t j( UPP ? i : 0UL );
8878 for( ; (j+4UL) <= jend; j+=4UL )
8893 SIMDType a1( A.load(i,k) );
8894 SIMDType xmm1( a1 *
set( B(k,j ) ) );
8895 SIMDType xmm2( a1 *
set( B(k,j+1UL) ) );
8896 SIMDType xmm3( a1 *
set( B(k,j+2UL) ) );
8897 SIMDType xmm4( a1 *
set( B(k,j+3UL) ) );
8899 for( ++k; k<kend; ++k ) {
8901 xmm1 += a1 *
set( B(k,j ) );
8902 xmm2 += a1 *
set( B(k,j+1UL) );
8903 xmm3 += a1 *
set( B(k,j+2UL) );
8904 xmm4 += a1 *
set( B(k,j+3UL) );
8907 C.store( i, j , C.load(i,j ) - xmm1 * factor );
8908 C.store( i, j+1UL, C.load(i,j+1UL) - xmm2 * factor );
8909 C.store( i, j+2UL, C.load(i,j+2UL) - xmm3 * factor );
8910 C.store( i, j+3UL, C.load(i,j+3UL) - xmm4 * factor );
8914 for( ; (j+3UL) <= jend; j+=3UL )
8929 SIMDType a1( A.load(i,k) );
8930 SIMDType xmm1( a1 *
set( B(k,j ) ) );
8931 SIMDType xmm2( a1 *
set( B(k,j+1UL) ) );
8932 SIMDType xmm3( a1 *
set( B(k,j+2UL) ) );
8934 for( ++k; k<kend; ++k ) {
8936 xmm1 += a1 *
set( B(k,j ) );
8937 xmm2 += a1 *
set( B(k,j+1UL) );
8938 xmm3 += a1 *
set( B(k,j+2UL) );
8941 C.store( i, j , C.load(i,j ) - xmm1 * factor );
8942 C.store( i, j+1UL, C.load(i,j+1UL) - xmm2 * factor );
8943 C.store( i, j+2UL, C.load(i,j+2UL) - xmm3 * factor );
8947 for( ; (j+2UL) <= jend; j+=2UL )
8962 SIMDType a1( A.load(i,k) );
8963 SIMDType xmm1( a1 *
set( B(k,j ) ) );
8964 SIMDType xmm2( a1 *
set( B(k,j+1UL) ) );
8966 for( ++k; k<kend; ++k ) {
8968 xmm1 += a1 *
set( B(k,j ) );
8969 xmm2 += a1 *
set( B(k,j+1UL) );
8972 C.store( i, j , C.load(i,j ) - xmm1 * factor );
8973 C.store( i, j+1UL, C.load(i,j+1UL) - xmm2 * factor );
8989 SIMDType xmm1( A.load(i,k) *
set( B(k,j) ) );
8991 for( ++k; k<K; ++k ) {
8992 xmm1 += A.load(i,k) *
set( B(k,j) );
8995 C.store( i, j, C.load(i,j) - xmm1 * factor );
9000 for( ; remainder && i<M; ++i )
9002 const size_t jend( LOW ? i+1UL : N );
9003 size_t j( UPP ? i : 0UL );
9005 for( ; (j+2UL) <= jend; j+=2UL )
9023 for( ++k; k<kend; ++k ) {
9024 value1 += A(i,k) * B(k,j );
9025 value2 += A(i,k) * B(k,j+1UL);
9028 C(i,j ) -= value1 * scalar;
9029 C(i,j+1UL) -= value2 * scalar;
9047 for( ++k; k<K; ++k ) {
9048 value += A(i,k) * B(k,j);
9051 C(i,j) -= value * scalar;
9072 template<
typename MT3
9076 static inline auto selectLargeSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
9079 selectDefaultSubAssignKernel( C, A, B, scalar );
9098 template<
typename MT3
9102 static inline auto selectLargeSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
9106 lmmm( C, A, B, -scalar, ST2(1) );
9108 ummm( C, A, B, -scalar, ST2(1) );
9110 mmm( C, A, B, -scalar, ST2(1) );
9129 template<
typename MT3
9133 static inline auto selectBlasSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
9136 selectLargeSubAssignKernel( C, A, B, scalar );
9141#if BLAZE_BLAS_MODE && BLAZE_USE_BLAS_MATRIX_MATRIX_MULTIPLICATION
9155 template<
typename MT3
9159 static inline auto selectBlasSubAssignKernel( MT3& C,
const MT4& A,
const MT5& B, ST2 scalar )
9162 using ET = ElementType_t<MT3>;
9165 ResultType_t<MT3> tmp(
serial( B ) );
9166 trmm( tmp, A, CblasLeft, (
IsLower_v<MT4> )?( CblasLower ):( CblasUpper ),
ET(scalar) );
9167 subAssign( C, tmp );
9170 ResultType_t<MT3> tmp(
serial( A ) );
9171 trmm( tmp, B, CblasRight, (
IsLower_v<MT5> )?( CblasLower ):( CblasUpper ),
ET(scalar) );
9172 subAssign( C, tmp );
9175 gemm( C, A, B,
ET(-scalar),
ET(1) );
9195 template<
typename MT >
9206 const ForwardFunctor fwd;
9211 subAssign( *lhs, fwd( A * B ) * rhs.scalar_ );
9231 template<
typename MT
9233 friend inline void schurAssign( DenseMatrix<MT,SO>& lhs,
const DMatScalarMultExpr& rhs )
9245 schurAssign( *lhs, tmp );
9276 template<
typename MT
9286 LeftOperand_t<MMM> left ( rhs.matrix_.leftOperand() );
9287 RightOperand_t<MMM> right( rhs.matrix_.rightOperand() );
9289 if( (*lhs).rows() == 0UL || (*lhs).columns() == 0UL ) {
9292 else if( left.columns() == 0UL ) {
9326 template<
typename MT
9345 const ForwardFunctor fwd;
9347 const TmpType tmp( rhs );
9366 template<
typename MT >
9377 const ForwardFunctor fwd;
9382 smpAssign( *lhs, fwd( A * B ) * rhs.scalar_ );
9401 template<
typename MT
9411 LeftOperand_t<MMM> left ( rhs.matrix_.leftOperand() );
9412 RightOperand_t<MMM> right( rhs.matrix_.rightOperand() );
9414 if( (*lhs).rows() == 0UL || (*lhs).columns() == 0UL || left.columns() == 0UL ) {
9447 template<
typename MT >
9458 const ForwardFunctor fwd;
9486 template<
typename MT
9496 LeftOperand_t<MMM> left ( rhs.matrix_.leftOperand() );
9497 RightOperand_t<MMM> right( rhs.matrix_.rightOperand() );
9499 if( (*lhs).rows() == 0UL || (*lhs).columns() == 0UL || left.columns() == 0UL ) {
9532 template<
typename MT >
9543 const ForwardFunctor fwd;
9568 template<
typename MT
9648template<
typename MT1
9650inline decltype(
auto)
9655 if( (*lhs).columns() != (*rhs).rows() ) {
9660 return ReturnType( *lhs, *rhs );
9696template<
typename MT1
9702inline decltype(
auto)
declsym(
const TDMatTDMatMultExpr<MT1,MT2,SF,HF,LF,UF>& dm )
9711 return ReturnType( dm.leftOperand(), dm.rightOperand() );
9740template<
typename MT1
9755 return ReturnType( dm.leftOperand(), dm.rightOperand() );
9784template<
typename MT1
9799 return ReturnType( dm.leftOperand(), dm.rightOperand() );
9828template<
typename MT1
9870template<
typename MT1
9912template<
typename MT1
9927 return ReturnType( dm.leftOperand(), dm.rightOperand() );
9956template<
typename MT1
9998template<
typename MT1
10040template<
typename MT1
10055 return ReturnType( dm.leftOperand(), dm.rightOperand() );
10071template<
typename MT1,
typename MT2,
bool SF,
bool HF,
bool LF,
bool UF >
10073 :
public Size<MT1,0UL>
10076template<
typename MT1,
typename MT2,
bool SF,
bool HF,
bool LF,
bool UF >
10078 :
public Size<MT2,1UL>
10094template<
typename MT1,
typename MT2,
bool SF,
bool HF,
bool LF,
bool UF >
10096 :
public BoolConstant< IsAligned_v<MT1> && IsAligned_v<MT2> >
Header file for auxiliary alias declarations.
typename T::OppositeType OppositeType_t
Alias declaration for nested OppositeType type definitions.
Definition Aliases.h:310
typename T::CompositeType CompositeType_t
Alias declaration for nested CompositeType type definitions.
Definition Aliases.h:110
typename T::TransposeType TransposeType_t
Alias declaration for nested TransposeType type definitions.
Definition Aliases.h:550
typename T::ElementType ElementType_t
Alias declaration for nested ElementType type definitions.
Definition Aliases.h:190
typename T::ResultType ResultType_t
Alias declaration for nested ResultType type definitions.
Definition Aliases.h:450
Header file for run time assertion macros.
Header file for kernel specific block sizes.
Header file for the blaze::checked and blaze::unchecked instances.
Constraints on the storage order of matrix types.
Header file for the complex data type.
Header file for the conjugate shim.
Header file for the decldiag trait.
Header file for the DeclDiag functor.
Header file for the declherm trait.
Header file for the DeclHerm functor.
Header file for the decllow trait.
Header file for the DeclLow functor.
Header file for the declsym trait.
Header file for the DeclSym functor.
Header file for the declupp trait.
Header file for the DeclUpp functor.
Header file for the EnableIf class template.
Header file for the function trace functionality.
Header file for the HasConstDataAccess type trait.
Header file for the HasMutableDataAccess type trait.
Header file for the HasSIMDAdd type trait.
Header file for the HasSIMDMult type trait.
Header file for the If class template.
Header file for the IntegralConstant class template.
Header file for the IsAligned type trait.
Header file for the IsBLASCompatible type trait.
Header file for the IsBuiltin type trait.
Header file for the IsColumnMajorMatrix type trait.
Header file for the IsComplexDouble type trait.
Header file for the IsComplexFloat type trait.
Header file for the IsComplex type trait.
Header file for the IsComputation type trait class.
Header file for the IsContiguous type trait.
Header file for the IsDiagonal type trait.
Header file for the IsDouble type trait.
Header file for the IsExpression type trait class.
Header file for the IsFloat type trait.
Header file for the IsIntegral type trait.
Header file for the IsLower type trait.
Header file for the IsPadded type trait.
Header file for the IsResizable type trait.
Header file for the IsRowMajorMatrix type trait.
Header file for the IsSIMDCombinable type trait.
Header file for the IsSame and IsStrictlySame type traits.
Header file for the IsStrictlyLower type trait.
Header file for the IsStrictlyTriangular type trait.
Header file for the IsStrictlyUpper type trait.
Header file for the IsSymmetric type trait.
Header file for the IsTriangular type trait.
Header file for the IsUpper type trait.
Header file for the dense matrix multiplication kernels.
Header file for the multiplication trait.
Header file for the Noop functor.
Header file for the prevMultiple shim.
Constraints on the storage order of matrix types.
Header file for all SIMD functionality.
Constraint on the data type.
Constraint on the data type.
Expression object for dense matrix-scalar multiplications.
Definition DMatScalarMultExpr.h:103
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition DMatScalarMultExpr.h:592
LeftOperand leftOperand() const noexcept
Returns the left-hand side dense matrix operand.
Definition DMatScalarMultExpr.h:548
const If_t< returnExpr, ExprReturnType, ElementType > ReturnType
Return type for expression template evaluations.
Definition DMatScalarMultExpr.h:170
ElementType_t< MT > ET
Element type of the dense matrix expression.
Definition DMatScalarMultExpr.h:108
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition DMatScalarMultExpr.h:602
ElementType_t< ResultType > ElementType
Resulting element type.
Definition DMatScalarMultExpr.h:167
If_t< IsExpression_v< MT >, const MT, const MT & > LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition DMatScalarMultExpr.h:176
DMatScalarMultExpr< MT, ST, SO > This
Type of this DMatScalarMultExpr instance.
Definition DMatScalarMultExpr.h:159
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition DMatScalarMultExpr.h:474
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition DMatScalarMultExpr.h:570
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition DMatScalarMultExpr.h:538
ResultType_t< MT > RT
Result type of the dense matrix expression.
Definition DMatScalarMultExpr.h:106
MatScalarMultExpr< DenseMatrix< This, SO > > BaseType
Base type of this DMatScalarMultExpr instance.
Definition DMatScalarMultExpr.h:162
ST RightOperand
Composite type of the right-hand side scalar value.
Definition DMatScalarMultExpr.h:179
If_t< useAssign, const ResultType, const DMatScalarMultExpr & > CompositeType
Data type for composite expression templates.
Definition DMatScalarMultExpr.h:173
RightOperand scalar_
Right-hand side scalar of the multiplication expression.
Definition DMatScalarMultExpr.h:611
static constexpr bool simdEnabled
Compilation switch for the expression template evaluation strategy.
Definition DMatScalarMultExpr.h:427
RightOperand rightOperand() const noexcept
Returns the right-hand side scalar operand.
Definition DMatScalarMultExpr.h:558
static constexpr size_t SIMDSIZE
The number of elements packed within a single SIMD element.
Definition DMatScalarMultExpr.h:437
DMatScalarMultExpr(const MT &matrix, ST scalar) noexcept
Constructor for the DMatScalarMultExpr class.
Definition DMatScalarMultExpr.h:446
OppositeType_t< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition DMatScalarMultExpr.h:165
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition DMatScalarMultExpr.h:582
MultTrait_t< RT, ST > ResultType
Result type for expression template evaluations.
Definition DMatScalarMultExpr.h:164
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition DMatScalarMultExpr.h:528
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition DMatScalarMultExpr.h:459
TransposeType_t< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition DMatScalarMultExpr.h:166
LeftOperand matrix_
Left-hand side dense matrix of the multiplication expression.
Definition DMatScalarMultExpr.h:610
static constexpr bool smpAssignable
Compilation switch for the expression template assignment strategy.
Definition DMatScalarMultExpr.h:432
Base class for dense matrices.
Definition DenseMatrix.h:82
Expression object for transpose dense matrix-transpose dense matrix multiplications.
Definition TDMatTDMatMultExpr.h:152
static constexpr bool LOW
Flag for lower matrices.
Definition TDMatTDMatMultExpr.h:176
bool isAliased(const T *alias) const noexcept
Returns whether the expression is aliased with the given address alias.
Definition TDMatTDMatMultExpr.h:463
size_t rows() const noexcept
Returns the current number of rows of the matrix.
Definition TDMatTDMatMultExpr.h:409
MatMatMultExpr< DenseMatrix< This, true > > BaseType
Base type of this TDMatTDMatMultExpr instance.
Definition TDMatTDMatMultExpr.h:270
If_t< IsExpression_v< MT2 >, const MT2, const MT2 & > RightOperand
Composite type of the right-hand side dense matrix expression.
Definition TDMatTDMatMultExpr.h:296
size_t columns() const noexcept
Returns the current number of columns of the matrix.
Definition TDMatTDMatMultExpr.h:419
static constexpr size_t SIMDSIZE
The number of elements packed within a single SIMD element.
Definition TDMatTDMatMultExpr.h:320
TDMatTDMatMultExpr(const MT1 &lhs, const MT2 &rhs) noexcept
Constructor for the TDMatTDMatMultExpr class.
Definition TDMatTDMatMultExpr.h:329
CompositeType_t< MT2 > CT2
Composite type of the right-hand side dense matrix expression.
Definition TDMatTDMatMultExpr.h:160
ReturnType at(size_t i, size_t j) const
Checked access to the matrix elements.
Definition TDMatTDMatMultExpr.h:393
static constexpr bool UPP
Flag for upper matrices.
Definition TDMatTDMatMultExpr.h:177
ReturnType operator()(size_t i, size_t j) const
2D-access to the matrix elements.
Definition TDMatTDMatMultExpr.h:344
TDMatTDMatMultExpr< MT1, MT2, SF, HF, LF, UF > This
Type of this TDMatTDMatMultExpr instance.
Definition TDMatTDMatMultExpr.h:267
ResultType_t< MT1 > RT1
Result type of the left-hand side dense matrix expression.
Definition TDMatTDMatMultExpr.h:155
TransposeType_t< ResultType > TransposeType
Transpose type for expression template evaluations.
Definition TDMatTDMatMultExpr.h:286
const ElementType ReturnType
Return type for expression template evaluations.
Definition TDMatTDMatMultExpr.h:289
bool isAligned() const noexcept
Returns whether the operands of the expression are properly aligned in memory.
Definition TDMatTDMatMultExpr.h:473
static constexpr bool smpAssignable
Compilation switch for the expression template assignment strategy.
Definition TDMatTDMatMultExpr.h:314
LeftOperand lhs_
Left-hand side dense matrix of the multiplication expression.
Definition TDMatTDMatMultExpr.h:495
OppositeType_t< ResultType > OppositeType
Result type with opposite storage order for expression template evaluations.
Definition TDMatTDMatMultExpr.h:285
static constexpr bool evaluateLeft
Compilation switch for the composite type of the left-hand side dense matrix expression.
Definition TDMatTDMatMultExpr.h:165
static constexpr bool evaluateRight
Compilation switch for the composite type of the right-hand side dense matrix expression.
Definition TDMatTDMatMultExpr.h:170
bool canAlias(const T *alias) const noexcept
Returns whether the expression can alias with the given address alias.
Definition TDMatTDMatMultExpr.h:451
ResultType_t< MT2 > RT2
Result type of the right-hand side dense matrix expression.
Definition TDMatTDMatMultExpr.h:156
LeftOperand leftOperand() const noexcept
Returns the left-hand side transpose dense matrix operand.
Definition TDMatTDMatMultExpr.h:429
SIMDTrait_t< ElementType > SIMDType
Resulting SIMD element type.
Definition TDMatTDMatMultExpr.h:288
If_t< evaluateRight, const RT2, CT2 > RT
Type for the assignment of the right-hand side dense matrix operand.
Definition TDMatTDMatMultExpr.h:302
ElementType_t< ResultType > ElementType
Resulting element type.
Definition TDMatTDMatMultExpr.h:287
static constexpr bool SYM
Flag for symmetric matrices.
Definition TDMatTDMatMultExpr.h:174
ElementType_t< RT2 > ET2
Element type of the right-hand side dense matrix expression.
Definition TDMatTDMatMultExpr.h:158
If_t< evaluateLeft, const RT1, CT1 > LT
Type for the assignment of the left-hand side dense matrix operand.
Definition TDMatTDMatMultExpr.h:299
static constexpr bool HERM
Flag for Hermitian matrices.
Definition TDMatTDMatMultExpr.h:175
ElementType_t< RT1 > ET1
Element type of the left-hand side dense matrix expression.
Definition TDMatTDMatMultExpr.h:157
RightOperand rightOperand() const noexcept
Returns the right-hand side transpose dense matrix operand.
Definition TDMatTDMatMultExpr.h:439
static constexpr bool simdEnabled
Compilation switch for the expression template evaluation strategy.
Definition TDMatTDMatMultExpr.h:307
bool canSMPAssign() const noexcept
Returns whether the expression can be used in SMP assignments.
Definition TDMatTDMatMultExpr.h:483
RightOperand rhs_
Right-hand side dense matrix of the multiplication expression.
Definition TDMatTDMatMultExpr.h:496
const ResultType CompositeType
Data type for composite expression templates.
Definition TDMatTDMatMultExpr.h:290
If_t< IsExpression_v< MT1 >, const MT1, const MT1 & > LeftOperand
Composite type of the left-hand side dense matrix expression.
Definition TDMatTDMatMultExpr.h:293
typename If_t< HERM, DeclHermTrait< MultTrait_t< RT1, RT2 > >, If_t< SYM, DeclSymTrait< MultTrait_t< RT1, RT2 > >, If_t< LOW, If_t< UPP, DeclDiagTrait< MultTrait_t< RT1, RT2 > >, DeclLowTrait< MultTrait_t< RT1, RT2 > > >, If_t< UPP, DeclUppTrait< MultTrait_t< RT1, RT2 > >, MultTrait< RT1, RT2 > > > > >::Type ResultType
Result type for expression template evaluations.
Definition TDMatTDMatMultExpr.h:273
CompositeType_t< MT1 > CT1
Composite type of the left-hand side dense matrix expression.
Definition TDMatTDMatMultExpr.h:159
Constraint on the data type.
Constraint on the data type.
Constraint on the data type.
Header file for the Computation base class.
Header file for the DenseMatrix base class.
Header file for the MatMatMultExpr base class.
Header file for the MatScalarMultExpr base class.
Header file for BLAS general matrix/matrix multiplication functions (gemm).
decltype(auto) column(Matrix< MT, SO > &matrix, RCAs... args)
Creating a view on a specific column of the given matrix.
Definition Column.h:137
#define BLAZE_BLAS_MODE
Compilation switch for the BLAS mode.
Definition BLAS.h:68
#define BLAZE_BLAS_IS_PARALLEL
Compilation switch for the parallel BLAS mode.
Definition BLAS.h:136
#define BLAZE_CONSTRAINT_MUST_BE_SAME_TYPE(A, B)
Data type constraint.
Definition SameType.h:71
decltype(auto) transIf(const DenseMatrix< MT, SO > &dm)
Conditional calculation of the transpose of the given dense matrix.
Definition DMatTransExpr.h:832
decltype(auto) min(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise minimum of the dense matrices lhs and rhs.
Definition DMatDMatMapExpr.h:1339
decltype(auto) max(const DenseMatrix< MT1, SO1 > &lhs, const DenseMatrix< MT2, SO2 > &rhs)
Computes the componentwise maximum of the dense matrices lhs and rhs.
Definition DMatDMatMapExpr.h:1375
decltype(auto) declstrupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as strictly upper.
Definition DMatDeclStrUppExpr.h:1003
decltype(auto) conj(const DenseMatrix< MT, SO > &dm)
Returns a matrix containing the complex conjugate of each single element of dm.
Definition DMatMapExpr.h:1464
decltype(auto) decldiag(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as diagonal.
Definition DMatDeclDiagExpr.h:978
decltype(auto) declstrlow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as strictly lower.
Definition DMatDeclStrLowExpr.h:1003
decltype(auto) serial(const DenseMatrix< MT, SO > &dm)
Forces the serial evaluation of the given dense matrix expression dm.
Definition DMatSerialExpr.h:812
decltype(auto) declupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as upper.
Definition DMatDeclUppExpr.h:1004
decltype(auto) decllow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as lower.
Definition DMatDeclLowExpr.h:1004
decltype(auto) decluniupp(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as uniupper.
Definition DMatDeclUniUppExpr.h:1005
decltype(auto) declherm(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as Hermitian.
Definition DMatDeclHermExpr.h:1005
decltype(auto) declsym(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as symmetric.
Definition DMatDeclSymExpr.h:1005
decltype(auto) declunilow(const DenseMatrix< MT, SO > &dm)
Declares the given dense matrix expression dm as unilower.
Definition DMatDeclUniLowExpr.h:1004
#define BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE(T)
Constraint on the data type.
Definition Symmetric.h:79
#define BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.
Definition RowMajorMatrix.h:61
#define BLAZE_CONSTRAINT_MATRICES_MUST_HAVE_SAME_STORAGE_ORDER(T1, T2)
Constraint on the data type.
Definition StorageOrder.h:84
#define BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION(T)
Constraint on the data type.
Definition RequiresEvaluation.h:81
#define BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR(T1, T2)
Constraint on the data type.
Definition MatMatMultExpr.h:103
#define BLAZE_CONSTRAINT_MUST_BE_DENSE_MATRIX_TYPE(T)
Constraint on the data type.
Definition DenseMatrix.h:61
#define BLAZE_CONSTRAINT_MUST_BE_SCALAR_TYPE(T)
Constraint on the data type.
Definition Scalar.h:61
#define BLAZE_CONSTRAINT_MUST_BE_COLUMN_MAJOR_MATRIX_TYPE(T)
Constraint on the data type.
Definition ColumnMajorMatrix.h:61
constexpr bool RequiresEvaluation_v
Auxiliary variable template for the RequiresEvaluation type trait.
Definition RequiresEvaluation.h:91
constexpr bool IsStrictlyUpper_v
Auxiliary variable template for the IsStrictlyUpper type trait.
Definition IsStrictlyUpper.h:173
constexpr bool IsSymmetric_v
Auxiliary variable template for the IsSymmetric type trait.
Definition IsSymmetric.h:172
constexpr bool IsTriangular_v
Auxiliary variable template for the IsTriangular type trait.
Definition IsTriangular.h:106
constexpr bool HasMutableDataAccess_v
Auxiliary variable template for the HasMutableDataAccess type trait.
Definition HasMutableDataAccess.h:133
constexpr bool IsStrictlyLower_v
Auxiliary variable template for the IsStrictlyLower type trait.
Definition IsStrictlyLower.h:173
constexpr bool IsRowMajorMatrix_v
Auxiliary variable template for the IsRowMajorMatrix type trait.
Definition IsRowMajorMatrix.h:129
constexpr bool IsUpper_v
Auxiliary variable template for the IsUpper type trait.
Definition IsUpper.h:175
constexpr bool IsSIMDCombinable_v
Auxiliary variable template for the IsSIMDCombinable type trait.
Definition IsSIMDCombinable.h:137
constexpr bool IsDiagonal_v
Auxiliary variable template for the IsDiagonal type trait.
Definition IsDiagonal.h:148
constexpr bool IsComputation_v
Auxiliary variable template for the IsComputation type trait.
Definition IsComputation.h:90
constexpr bool HasConstDataAccess_v
Auxiliary variable template for the HasConstDataAccess type trait.
Definition HasConstDataAccess.h:133
constexpr bool HasSIMDAdd_v
Auxiliary variable template for the HasSIMDAdd type trait.
Definition HasSIMDAdd.h:187
constexpr bool HasSIMDMult_v
Auxiliary variable template for the HasSIMDMult type trait.
Definition HasSIMDMult.h:188
constexpr bool IsStrictlyTriangular_v
Auxiliary variable template for the IsStrictlyTriangular type trait.
Definition IsStrictlyTriangular.h:106
constexpr bool IsPadded_v
Auxiliary variable template for the IsPadded type trait.
Definition IsPadded.h:134
constexpr bool IsLower_v
Auxiliary variable template for the IsLower type trait.
Definition IsLower.h:175
constexpr bool IsBLASCompatible_v
Auxiliary variable template for the IsBLASCompatible type trait.
Definition IsBLASCompatible.h:99
constexpr bool IsContiguous_v
Auxiliary variable template for the IsContiguous type trait.
Definition IsContiguous.h:144
constexpr bool IsResizable_v
Auxiliary variable template for the IsResizable type trait.
Definition IsResizable.h:136
BLAZE_ALWAYS_INLINE constexpr auto prevMultiple(T1 value, T2 factor) noexcept
Rounds down an integral value to the previous multiple of a given factor.
Definition PrevMultiple.h:68
constexpr void reset(Matrix< MT, SO > &matrix)
Resetting the given matrix.
Definition Matrix.h:806
MT::Iterator end(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator just past the last element of row/column i.
Definition Matrix.h:584
MT::Iterator begin(Matrix< MT, SO > &matrix, size_t i)
Returns an iterator to the first element of row/column i.
Definition Matrix.h:518
bool isSquare(const Matrix< MT, SO > &matrix) noexcept
Checks if the given matrix is a square matrix.
Definition Matrix.h:1383
decltype(auto) row(Matrix< MT, SO > &, RRAs...)
Creating a view on a specific row of the given matrix.
Definition Row.h:137
#define BLAZE_INTERNAL_ASSERT(expr, msg)
Run time assertion macro for internal checks.
Definition Assert.h:101
BLAZE_ALWAYS_INLINE const EnableIf_t< IsIntegral_v< T > &&HasSize_v< T, 1UL >, If_t< IsSigned_v< T >, SIMDint8, SIMDuint8 > > set(T value) noexcept
Sets all values in the vector to the given 1-byte integral value.
Definition Set.h:75
typename SIMDTrait< T >::Type SIMDTrait_t
Auxiliary alias declaration for the SIMDTrait class template.
Definition SIMDTrait.h:315
auto smpSubAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP subtraction assignment of a matrix to dense matrix.
Definition DenseMatrix.h:162
auto smpAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP assignment of a matrix to a dense matrix.
Definition DenseMatrix.h:100
auto smpSchurAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP Schur product assignment of a matrix to dense matrix.
Definition DenseMatrix.h:194
auto smpAddAssign(Matrix< MT1, SO1 > &lhs, const Matrix< MT2, SO2 > &rhs) -> EnableIf_t< IsDenseMatrix_v< MT1 > >
Default implementation of the SMP addition assignment of a matrix to a dense matrix.
Definition DenseMatrix.h:131
decltype(auto) subvector(Vector< VT, TF > &, RSAs...)
Creating a view on a specific subvector of the given vector.
Definition Subvector.h:158
constexpr bool IsSame_v
Auxiliary variable template for the IsSame type trait.
Definition IsSame.h:159
constexpr bool IsBuiltin_v
Auxiliary variable template for the IsBuiltin type trait.
Definition IsBuiltin.h:95
constexpr bool IsIntegral_v
Auxiliary variable template for the IsIntegral type trait.
Definition IsIntegral.h:95
constexpr bool IsComplex_v
Auxiliary variable template for the IsComplex type trait.
Definition IsComplex.h:139
typename EnableIf< Condition, T >::Type EnableIf_t
Auxiliary type for the EnableIf class template.
Definition EnableIf.h:138
#define BLAZE_THROW_OUT_OF_RANGE(MESSAGE)
Macro for the emission of a std::out_of_range exception.
Definition Exception.h:331
#define BLAZE_THROW_INVALID_ARGUMENT(MESSAGE)
Macro for the emission of a std::invalid_argument exception.
Definition Exception.h:235
#define BLAZE_FUNCTION_TRACE
Function trace macro.
Definition FunctionTrace.h:94
typename If< Condition >::template Type< T1, T2 > If_t
Auxiliary alias template for the If class template.
Definition If.h:108
typename EnableIf<!Condition, T >::Type DisableIf_t
Auxiliary type for the EnableIf class template.
Definition EnableIf.h:175
IntegralConstant< bool, B > BoolConstant
Generic wrapper for a compile time constant boolean value.
Definition IntegralConstant.h:110
constexpr Unchecked unchecked
Global Unchecked instance.
Definition Check.h:146
constexpr decltype(auto) zero(size_t m, size_t n) noexcept
Creating a zero matrix.
Definition ZeroMatrix.h:1356
Header file for the exception macros of the math module.
Constraints on the storage order of matrix types.
Header file for all forward declarations for expression class templates.
Header file for the Size type trait.
Header file for the reset shim.
Header file for the serial shim.
Base class for all compute expression templates.
Definition Computation.h:68
Base template for the DeclDiagTrait class.
Definition DeclDiagTrait.h:127
Generic wrapper for the decldiag() function.
Definition DeclDiag.h:61
Base template for the DeclHermTrait class.
Definition DeclHermTrait.h:126
Generic wrapper for the declherm() function.
Definition DeclHerm.h:61
Base template for the DeclLowTrait class.
Definition DeclLowTrait.h:126
Generic wrapper for the decllow() function.
Definition DeclLow.h:61
Base template for the DeclSymTrait class.
Definition DeclSymTrait.h:126
Generic wrapper for the declsym() function.
Definition DeclSym.h:61
Base template for the DeclUppTrait class.
Definition DeclUppTrait.h:126
Generic wrapper for the declupp() function.
Definition DeclUpp.h:61
Compile time check for the alignment of data types.
Definition IsAligned.h:88
Base class for all matrix/matrix multiplication expression templates.
Definition MatMatMultExpr.h:71
Base class for all matrix/scalar multiplication expression templates.
Definition MatScalarMultExpr.h:77
Base template for the MultTrait class.
Definition MultTrait.h:130
Generic wrapper for the null function.
Definition Noop.h:62
Compile time evaluation of the size of vectors and matrices.
Definition Size.h:119
System settings for the BLAS mode.
System settings for the debugging policy of the Blaze library.
System settings for performance optimizations.
Header file for the thresholds for matrix/vector and matrix/matrix multiplications.
Header file for BLAS triangular matrix/matrix multiplication functions (trmm).
Header file for the RequiresEvaluation type trait.
Header file for basic type definitions.
Header file for the generic max algorithm.
Header file for the generic min algorithm.