Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
liyinrong
composable_kernel
Commits
f5de8b57
Unverified
Commit
f5de8b57
authored
3 years ago
by
Chao Liu
Committed by
GitHub
3 years ago
Browse files
Options
Download
Plain Diff
Merge branch 'develop' into modified_grouped_gemm_addressing_method
parents
e83c7061
fa9a0a5c
modified_grouped_gemm_addressing_method
No related merge requests found
Changes
76
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
20 changed files
CMakeLists.txt
+0
-7
CMakeLists.txt
client_example/03_gemm_layernorm/CMakeLists.txt
+2
-0
client_example/03_gemm_layernorm/CMakeLists.txt
client_example/03_gemm_layernorm/gemm_add_add_layernorm.cpp
+270
-0
client_example/03_gemm_layernorm/gemm_add_add_layernorm.cpp
client_example/CMakeLists.txt
+1
-0
client_example/CMakeLists.txt
example/16_gemm_reduce/gemm_reduce_xdl_max_fp16.cpp
+47
-42
example/16_gemm_reduce/gemm_reduce_xdl_max_fp16.cpp
example/16_gemm_reduce/gemm_reduce_xdl_mean_squaremean_fp16.cpp
+74
-67
...e/16_gemm_reduce/gemm_reduce_xdl_mean_squaremean_fp16.cpp
example/18_batched_gemm_reduce/batched_gemm_reduce_xdl_fp16.cpp
+62
-52
...e/18_batched_gemm_reduce/batched_gemm_reduce_xdl_fp16.cpp
example/19_binary_elementwise/broadcast_add_2d_amn_bn.cpp
+10
-8
example/19_binary_elementwise/broadcast_add_2d_amn_bn.cpp
example/19_binary_elementwise/broadcast_add_3d_am_bmnk.cpp
+17
-11
example/19_binary_elementwise/broadcast_add_3d_am_bmnk.cpp
example/19_binary_elementwise/elementwise_add_1d.cpp
+10
-8
example/19_binary_elementwise/elementwise_add_1d.cpp
example/19_binary_elementwise/elementwise_add_4d.cpp
+15
-9
example/19_binary_elementwise/elementwise_add_4d.cpp
example/21_gemm_layernorm/gemm_bias_relu_add_layernorm_xdl_fp16.cpp
+111
-108
..._gemm_layernorm/gemm_bias_relu_add_layernorm_xdl_fp16.cpp
example/21_gemm_layernorm/gemm_layernorm_xdl_fp16.cpp
+86
-79
example/21_gemm_layernorm/gemm_layernorm_xdl_fp16.cpp
example/23_softmax/softmax_blockwise.cpp
+6
-3
example/23_softmax/softmax_blockwise.cpp
example/25_gemm_bias_c_permute/CMakeLists.txt
+1
-0
example/25_gemm_bias_c_permute/CMakeLists.txt
example/25_gemm_bias_c_permute/gemm_bias_c_permute_xdl_fp16.cpp
+284
-0
...e/25_gemm_bias_c_permute/gemm_bias_c_permute_xdl_fp16.cpp
example/CMakeLists.txt
+1
-0
example/CMakeLists.txt
include/ck/tensor_operation/gpu/device/device_5ary_elementwise.hpp
+56
-42
...k/tensor_operation/gpu/device/device_5ary_elementwise.hpp
include/ck/tensor_operation/gpu/device/device_batched_gemm_reduce_xdl_cshuffle.hpp
+185
-115
...on/gpu/device/device_batched_gemm_reduce_xdl_cshuffle.hpp
include/ck/tensor_operation/gpu/device/device_binary_elementwise.hpp
+24
-16
...tensor_operation/gpu/device/device_binary_elementwise.hpp
with
1262 additions
and
567 deletions
+1262
-567
CMakeLists.txt
View file @
f5de8b57
...
...
@@ -71,13 +71,6 @@ if( DEFINED CK_OVERRIDE_HIP_VERSION_PATCH )
endif
()
message
(
STATUS
"Build with HIP
${
HIP_VERSION
}
"
)
rocm_create_package
(
NAME composablekernel
DESCRIPTION
"High Performance Composable Kernel for AMD GPUs"
MAINTAINER
"MIOpen Kernels Dev Team <dl.MIOpen@amd.com>"
LDCONFIG
)
## tidy
include
(
EnableCompilerWarnings
)
set
(
CK_TIDY_ERRORS ERRORS * -readability-inconsistent-declaration-parameter-name
)
...
...
This diff is collapsed.
Click to expand it.
client_example/03_gemm_layernorm/CMakeLists.txt
0 → 100644
View file @
f5de8b57
add_executable
(
gemm_add_add_reduce_normalize gemm_add_add_layernorm.cpp
)
target_link_libraries
(
gemm_add_add_reduce_normalize PRIVATE composable_kernel::device_operations
)
This diff is collapsed.
Click to expand it.
client_example/03_gemm_layernorm/gemm_add_add_layernorm.cpp
0 → 100644
View file @
f5de8b57
// SPDX-License-Identifier: MIT
// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
#include <iomanip>
#include <vector>
#include <iostream>
#include "ck/ck.hpp"
#include "ck/tensor_operation/gpu/device/tensor_layout.hpp"
#include "ck/tensor_operation/gpu/device/device_gemm_reduce.hpp"
#include "ck/tensor_operation/gpu/device/device_elementwise.hpp"
#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp"
#include "ck/library/tensor_operation_instance/gpu/device_elementwise_instance.hpp"
#include "ck/library/tensor_operation_instance/gpu/device_gemm_mean_squaremean_instance.hpp"
using
F16
=
ck
::
half_t
;
using
F32
=
float
;
using
ADataType
=
F16
;
using
BDataType
=
F16
;
using
BiasDataType
=
F32
;
using
CDataType
=
F16
;
using
D0DataType
=
F16
;
using
ReduceDataType
=
F32
;
using
GammaDataType
=
F16
;
using
BetaDataType
=
F16
;
using
LayerNormOutDataType
=
F16
;
using
ALayout
=
ck
::
tensor_layout
::
gemm
::
RowMajor
;
using
BLayout
=
ck
::
tensor_layout
::
gemm
::
ColumnMajor
;
using
CLayout
=
ck
::
tensor_layout
::
gemm
::
RowMajor
;
struct
SimpleDeviceMem
{
SimpleDeviceMem
()
=
delete
;
SimpleDeviceMem
(
std
::
size_t
mem_size
)
:
p_mem_
{}
{
(
void
)
hipMalloc
(
static_cast
<
void
**>
(
&
p_mem_
),
mem_size
);
}
void
*
GetDeviceBuffer
()
{
return
p_mem_
;
}
~
SimpleDeviceMem
()
{
(
void
)
hipFree
(
p_mem_
);
}
void
*
p_mem_
;
};
template
<
typename
gemm_reduce_op_ptr
>
bool
RunDeviceGemmMeanSquareMean
(
gemm_reduce_op_ptr
&
p_op
,
const
void
*
p_a
,
const
void
*
p_b
,
const
void
*
p_bias
,
const
void
*
p_d0
,
void
*
p_c
,
void
*
p_mean
,
void
*
p_square_mean
,
int
M
,
int
N
,
int
K
,
int
StrideA
,
int
StrideB
,
int
StrideC
,
int
StrideD0
,
bool
time_kernel
)
{
using
PassThrough
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
UnaryDivElementOp
=
ck
::
tensor_operation
::
element_wise
::
UnaryDivide
;
using
UnarySquareElementOp
=
ck
::
tensor_operation
::
element_wise
::
UnarySquare
;
auto
passOp
=
PassThrough
{};
auto
squareOp
=
UnarySquareElementOp
{};
auto
divOp
=
UnaryDivElementOp
{
N
};
auto
argument_ptr
=
p_op
->
MakeArgumentPointer
(
p_a
,
p_b
,
p_bias
,
{
p_d0
},
p_c
,
{
p_mean
,
p_square_mean
},
M
,
N
,
K
,
StrideA
,
StrideB
,
StrideC
,
{
StrideD0
},
{
&
passOp
,
&
passOp
,
&
passOp
},
// functor for a, b, c
{
&
passOp
},
// functor for d0
{
&
passOp
,
&
squareOp
},
// functor for inputs of reduction
{
&
divOp
,
&
divOp
});
// functor for outputs of reduction
if
(
p_op
->
IsSupportedArgument
(
argument_ptr
.
get
()))
{
auto
invoker_ptr
=
p_op
->
MakeInvokerPointer
();
// If we evaluate running time of gemm_reduce. The output may wrong.
// Because we need to initialize the reduction tensor before runing the kernel.
// However we run kernel many times for time_kernel = trie without reinitialize the out
// of reduction tensor.
float
ave_time
=
invoker_ptr
->
Run
(
argument_ptr
.
get
(),
StreamConfig
{
nullptr
,
time_kernel
});
if
(
time_kernel
)
std
::
cout
<<
"Gemm + reduce Perf: "
<<
std
::
setw
(
10
)
<<
ave_time
<<
" ms"
<<
std
::
endl
;
return
true
;
}
return
false
;
}
template
<
typename
normalize_op_ptr
>
bool
RunDeviceNormalize2D
(
normalize_op_ptr
&
p_op
,
const
void
*
p_x
,
const
void
*
p_mean
,
const
void
*
p_square_mean
,
const
void
*
p_gamma
,
const
void
*
p_beta
,
void
*
p_y
,
int
M
,
int
N
,
int
StrideX
,
bool
time_kernel
)
{
std
::
array
<
const
void
*
,
5
>
input
=
{
p_x
,
p_mean
,
p_square_mean
,
p_gamma
,
p_beta
};
std
::
array
<
void
*
,
1
>
output
=
{
p_y
};
auto
normalize_functor
=
ck
::
tensor_operation
::
element_wise
::
Normalize
{};
auto
argument_ptr
=
p_op
->
MakeArgumentPointer
(
input
,
output
,
{
M
,
N
},
{{
StrideX
,
1
},
{
1
,
0
},
{
1
,
0
},
{
0
,
1
},
{
0
,
1
}},
{{
StrideX
,
1
}},
ck
::
tensor_operation
::
element_wise
::
Normalize
{});
if
(
p_op
->
IsSupportedArgument
(
argument_ptr
.
get
()))
{
auto
invoker_ptr
=
p_op
->
MakeInvokerPointer
();
float
ave_time
=
invoker_ptr
->
Run
(
argument_ptr
.
get
(),
StreamConfig
{
nullptr
,
time_kernel
});
if
(
time_kernel
)
std
::
cout
<<
"Normalize Perf: "
<<
std
::
setw
(
10
)
<<
ave_time
<<
" ms"
<<
std
::
endl
;
return
true
;
}
return
false
;
}
int
main
()
{
ck
::
index_t
M
=
1024
;
ck
::
index_t
N
=
1024
;
ck
::
index_t
K
=
1024
;
ck
::
index_t
StrideA
=
1024
;
ck
::
index_t
StrideB
=
1024
;
ck
::
index_t
StrideC
=
1024
;
ck
::
index_t
StrideD0
=
1024
;
const
auto
gemm_reduce_ptrs
=
ck
::
tensor_operation
::
device
::
device_gemm_instance
::
get_device_gemm_add_add_mean_squaremean_instances
<
ADataType
,
BDataType
,
CDataType
,
ALayout
,
BLayout
,
CLayout
>
();
const
auto
normalize_ptrs
=
ck
::
tensor_operation
::
device
::
get_device_normalize_from_mean_meansquare_instances
<
CDataType
,
ReduceDataType
,
ReduceDataType
,
GammaDataType
,
BetaDataType
,
LayerNormOutDataType
>
();
std
::
cout
<<
"found "
<<
gemm_reduce_ptrs
.
size
()
<<
" gemm_reduceMean_reduceSquareMean instances"
<<
std
::
endl
;
std
::
cout
<<
"found "
<<
normalize_ptrs
.
size
()
<<
" normalize instances"
<<
std
::
endl
;
auto
f_matrix_space_size
=
[](
std
::
size_t
nRow
,
std
::
size_t
nCol
,
std
::
size_t
stride
,
auto
layout
)
{
using
Layout
=
decltype
(
layout
);
if
(
std
::
is_same
<
Layout
,
ck
::
tensor_layout
::
gemm
::
RowMajor
>::
value
)
{
return
(
nRow
-
1
)
*
stride
+
nCol
;
}
else
{
return
(
nCol
-
1
)
*
stride
+
nRow
;
}
};
SimpleDeviceMem
a_device_buf
(
sizeof
(
ADataType
)
*
f_matrix_space_size
(
M
,
K
,
StrideA
,
ALayout
{}));
SimpleDeviceMem
b_device_buf
(
sizeof
(
BDataType
)
*
f_matrix_space_size
(
K
,
N
,
StrideB
,
BLayout
{}));
SimpleDeviceMem
bias_device_buf
(
sizeof
(
BiasDataType
)
*
N
);
SimpleDeviceMem
c_device_buf
(
sizeof
(
CDataType
)
*
f_matrix_space_size
(
M
,
N
,
StrideC
,
CLayout
{}));
SimpleDeviceMem
d0_device_buf
(
sizeof
(
D0DataType
)
*
f_matrix_space_size
(
M
,
N
,
StrideD0
,
CLayout
{}));
SimpleDeviceMem
reduceMean_device_buf
(
sizeof
(
ReduceDataType
)
*
M
);
SimpleDeviceMem
reduceMeanSquare_device_buf
(
sizeof
(
ReduceDataType
)
*
M
);
SimpleDeviceMem
gamma_device_buf
(
sizeof
(
GammaDataType
)
*
N
);
SimpleDeviceMem
beta_device_buf
(
sizeof
(
BetaDataType
)
*
N
);
SimpleDeviceMem
layerNorm_device_buf
(
sizeof
(
LayerNormOutDataType
)
*
M
*
N
);
bool
b_time_kernel
=
true
;
bool
b_only_run_first_kernel
=
true
;
// layernorm => (1) + (2)
// (1). c = gemm(a, b), reduce_mean(c), reduce_square_mean(c)
// (2). normalize(c, mean, square_mean, gamma, beta)
for
(
auto
&
gemm_reduce_ptr
:
gemm_reduce_ptrs
)
{
// run first available kernel
if
(
RunDeviceGemmMeanSquareMean
(
gemm_reduce_ptr
,
a_device_buf
.
GetDeviceBuffer
(),
b_device_buf
.
GetDeviceBuffer
(),
bias_device_buf
.
GetDeviceBuffer
(),
d0_device_buf
.
GetDeviceBuffer
(),
c_device_buf
.
GetDeviceBuffer
(),
reduceMean_device_buf
.
GetDeviceBuffer
(),
reduceMeanSquare_device_buf
.
GetDeviceBuffer
(),
M
,
N
,
K
,
StrideA
,
StrideB
,
StrideC
,
StrideD0
,
b_time_kernel
))
{
if
(
b_only_run_first_kernel
)
break
;
}
else
{
std
::
cout
<<
gemm_reduce_ptr
->
GetTypeString
()
<<
" does not support this problem"
<<
std
::
endl
;
}
}
for
(
auto
&
normalize_ptr
:
normalize_ptrs
)
{
if
(
RunDeviceNormalize2D
(
normalize_ptr
,
c_device_buf
.
GetDeviceBuffer
(),
reduceMean_device_buf
.
GetDeviceBuffer
(),
reduceMeanSquare_device_buf
.
GetDeviceBuffer
(),
gamma_device_buf
.
GetDeviceBuffer
(),
beta_device_buf
.
GetDeviceBuffer
(),
layerNorm_device_buf
.
GetDeviceBuffer
(),
M
,
N
,
StrideC
,
b_time_kernel
))
{
if
(
b_only_run_first_kernel
)
break
;
}
else
{
std
::
cout
<<
normalize_ptr
->
GetTypeString
()
<<
" does not support this problem"
<<
std
::
endl
;
}
}
}
\ No newline at end of file
This diff is collapsed.
Click to expand it.
client_example/CMakeLists.txt
View file @
f5de8b57
...
...
@@ -7,3 +7,4 @@ find_package(hip REQUIRED PATHS /opt/rocm)
message
(
STATUS
"Build with HIP
${
hip_VERSION
}
"
)
add_subdirectory
(
02_gemm_add_add_fastgelu
)
add_subdirectory
(
03_gemm_layernorm
)
This diff is collapsed.
Click to expand it.
example/16_gemm_reduce/gemm_reduce_xdl_max_fp16.cpp
View file @
f5de8b57
...
...
@@ -33,19 +33,19 @@ using BDataType = F16;
using
CDataType
=
F16
;
using
GemmAccDataType
=
F32
;
using
ReduceAccDataType
=
F32
;
using
D
DataType
=
F64
;
using
D
PtrsGlobal
=
ck
::
Tuple
<
D
DataType
*>
;
using
Reduce
DataType
=
F64
;
using
Reduce
PtrsGlobal
=
ck
::
Tuple
<
Reduce
DataType
*>
;
using
ALayout
=
ck
::
tensor_layout
::
gemm
::
RowMajor
;
using
BLayout
=
ck
::
tensor_layout
::
gemm
::
ColumnMajor
;
using
CLayout
=
ck
::
tensor_layout
::
gemm
::
RowMajor
;
using
AElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
BElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
CElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
Ds
ReduceOp
=
ck
::
Tuple
<
ck
::
reduce
::
Max
>
;
using
Ds
ElementOp
=
ck
::
Tuple
<
ck
::
tensor_operation
::
element_wise
::
PassThrough
>
;
using
D
GlobalMemOp
=
using
AElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
BElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
CElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
ReduceOp
s
=
ck
::
Tuple
<
ck
::
reduce
::
Max
>
;
using
Reduce
ElementOp
s
=
ck
::
Tuple
<
ck
::
tensor_operation
::
element_wise
::
PassThrough
>
;
using
Reduce
GlobalMemOp
s
=
ck
::
InMemoryDataOperationEnumSequence
<
ck
::
InMemoryDataOperationEnum
::
AtomicMax
>
;
static
constexpr
auto
GemmSpecialization
=
...
...
@@ -53,11 +53,11 @@ static constexpr auto GemmSpecialization =
// clang-format off
using
DeviceGemmReduceInstance
=
ck
::
tensor_operation
::
device
::
DeviceGemmReduce_Xdl_CShuffle
//######| ALayout| BLayout| CLayout|AData| BData| CData| GemmAcc| CShuffle| ReduceAcc|
D
Data| A| B| C|
Dxs| Dxs
InEleOp|
Dxs
AccEleOp|
D
| GEMM| NumGemmK| Block| MPer| NPer| KPer| AK1| BK1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransferClusterLengths| CBlockTransfer| CReduce| CReduceThreadLds2VGprCopy| CReduceThreadVgpr2GlobalCopy|
//######| | | | Type| Type| Type| DataType| DataType| DataType| Type Tuple| Elementwise| Elementwise| Elementwise|
Reduce|
|
|
MemoryData| Spacialization| Prefetch| Size| Block| Block| Block| | | XDL| XDL| Per| Per| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraN| MXdlPerWave| NXdlPerWave| _MBlock_MPerBlock| ScalarPerVector| ThreadClusterLengths| SrcDstScalarPerVector| SrcDstScalarPerVector|
//######| | | | | | | | | |
|
Operation
| Operation| Operation| Operation|
| | Operation| | Stage| | | | | | | | | Wave| Wave| Lengths_K0_M_K1| ArrangeOrder| | | PerVector| PerVector_K1| | Lengths_K0_N_K1| ArrangeOrder| | | PerVector| PerVector_K1| | PerShuffle| PerShuffle| _NBlock_NPerBlock| _NPerBlock| _MPerBlock_NPerBlock| _NPerBlock| _MPerBlock|
//######| | | | | | | | | | | | | | |
|
| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
<
Row
,
Col
,
Row
,
F16
,
F16
,
F16
,
F32
,
F32
,
ReduceAccDataType
,
D
PtrsGlobal
,
AElementOp
,
BElementOp
,
CElementOp
,
Ds
ReduceOp
,
Ds
ElementOp
,
Ds
ElementOp
,
D
GlobalMemOp
,
GemmSpecialization
,
1
,
256
,
256
,
128
,
32
,
8
,
8
,
32
,
32
,
4
,
2
,
S
<
4
,
64
,
1
>
,
S
<
1
,
0
,
2
>
,
S
<
1
,
0
,
2
>
,
2
,
8
,
8
,
1
,
S
<
4
,
64
,
1
>
,
S
<
1
,
0
,
2
>
,
S
<
1
,
0
,
2
>
,
2
,
8
,
8
,
1
,
1
,
1
,
S
<
1
,
32
,
1
,
8
>
,
8
,
S
<
64
,
4
>
,
4
,
1
>
;
//######| ALayout| BLayout| CLayout|AData| BData| CData| GemmAcc| CShuffle| ReduceAcc|
Reduce
Data| A| B| C|
Reduce| Reduce
InEleOp|
Reduce
AccEleOp|
Reduce
| GEMM| NumGemmK| Block| MPer| NPer| KPer| AK1| BK1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransferClusterLengths| CBlockTransfer| CReduce| CReduceThreadLds2VGprCopy| CReduceThreadVgpr2GlobalCopy|
//######| | | | Type| Type| Type| DataType| DataType| DataType|
Type Tuple| Elementwise| Elementwise| Elementwise|
Operation|
|
|
MemoryData| Spacialization| Prefetch| Size| Block| Block| Block| | | XDL| XDL| Per| Per| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraN| MXdlPerWave| NXdlPerWave| _MBlock_MPerBlock| ScalarPerVector| ThreadClusterLengths| SrcDstScalarPerVector| SrcDstScalarPerVector|
//######| | | | | | | | | | | Operation| Operation| Operation| |
|
|
Operation| | Stage| | | | | | | | | Wave| Wave| Lengths_K0_M_K1| ArrangeOrder| | | PerVector| PerVector_K1| | Lengths_K0_N_K1| ArrangeOrder| | | PerVector| PerVector_K1| | PerShuffle| PerShuffle| _NBlock_NPerBlock| _NPerBlock| _MPerBlock_NPerBlock| _NPerBlock| _MPerBlock|
//######| | | | | | | | | |
| | | | |
|
|
| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
<
Row
,
Col
,
Row
,
F16
,
F16
,
F16
,
F32
,
F32
,
ReduceAccDataType
,
Reduce
PtrsGlobal
,
AElementOp
,
BElementOp
,
CElementOp
,
ReduceOp
s
,
Reduce
ElementOp
s
,
Reduce
ElementOp
s
,
Reduce
GlobalMemOp
s
,
GemmSpecialization
,
1
,
256
,
256
,
128
,
32
,
8
,
8
,
32
,
32
,
4
,
2
,
S
<
4
,
64
,
1
>
,
S
<
1
,
0
,
2
>
,
S
<
1
,
0
,
2
>
,
2
,
8
,
8
,
1
,
S
<
4
,
64
,
1
>
,
S
<
1
,
0
,
2
>
,
S
<
1
,
0
,
2
>
,
2
,
8
,
8
,
1
,
1
,
1
,
S
<
1
,
32
,
1
,
8
>
,
8
,
S
<
64
,
4
>
,
4
,
1
>
;
// clang-format on
using
ReferenceGemmInstance
=
ck
::
tensor_operation
::
host
::
ReferenceGemm
<
ADataType
,
...
...
@@ -68,12 +68,12 @@ using ReferenceGemmInstance = ck::tensor_operation::host::ReferenceGemm<ADataTyp
BElementOp
,
CElementOp
>
;
template
<
typename
ADataType
,
typename
BDataType
,
typename
CDataType
,
typename
D
DataType
>
template
<
typename
ADataType
,
typename
BDataType
,
typename
CDataType
,
typename
Reduce
DataType
>
void
DumpGemmLayerNormPerf
(
float
gemm_reduce_time
,
int
M
,
int
N
,
int
K
)
{
std
::
size_t
gemm_flop
=
std
::
size_t
(
2
)
*
M
*
N
*
K
;
std
::
size_t
gemm_num_byte
=
sizeof
(
ADataType
)
*
M
*
K
+
sizeof
(
BDataType
)
*
K
*
N
+
sizeof
(
CDataType
)
*
M
*
N
+
sizeof
(
D
DataType
)
*
M
;
sizeof
(
CDataType
)
*
M
*
N
+
sizeof
(
Reduce
DataType
)
*
M
;
float
tflops
=
static_cast
<
float
>
(
gemm_flop
)
/
1.E9
/
gemm_reduce_time
;
float
gemm_gb_per_sec
=
gemm_num_byte
/
1.E6
/
gemm_reduce_time
;
...
...
@@ -148,17 +148,17 @@ int main(int argc, char* argv[])
Tensor
<
BDataType
>
b_k_n
(
f_host_tensor_descriptor
(
K
,
N
,
StrideB
,
BLayout
{}));
Tensor
<
CDataType
>
c_m_n_host_result
(
f_host_tensor_descriptor
(
M
,
N
,
StrideC
,
CLayout
{}));
Tensor
<
D
DataType
>
d
_m_host_result
(
Tensor
<
Reduce
DataType
>
reduce
_m_host_result
(
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
({
static_cast
<
std
::
size_t
>
(
M
)})));
Tensor
<
CDataType
>
c_m_n_device_result
(
f_host_tensor_descriptor
(
M
,
N
,
StrideC
,
CLayout
{}));
Tensor
<
D
DataType
>
d
_m_device_result
(
Tensor
<
Reduce
DataType
>
reduce
_m_device_result
(
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
({
static_cast
<
std
::
size_t
>
(
M
)})));
std
::
cout
<<
"a_m_k: "
<<
a_m_k
.
mDesc
<<
std
::
endl
;
std
::
cout
<<
"b_k_n: "
<<
b_k_n
.
mDesc
<<
std
::
endl
;
std
::
cout
<<
"c_m_n: "
<<
c_m_n_host_result
.
mDesc
<<
std
::
endl
;
std
::
cout
<<
"
d
_m: "
<<
d
_m_host_result
.
mDesc
<<
std
::
endl
;
std
::
cout
<<
"
reduce
_m: "
<<
reduce
_m_host_result
.
mDesc
<<
std
::
endl
;
switch
(
init_method
)
{
...
...
@@ -176,35 +176,40 @@ int main(int argc, char* argv[])
DeviceMem
a_device_buf
(
sizeof
(
ADataType
)
*
a_m_k
.
mDesc
.
GetElementSpace
());
DeviceMem
b_device_buf
(
sizeof
(
BDataType
)
*
b_k_n
.
mDesc
.
GetElementSpace
());
DeviceMem
c_device_buf
(
sizeof
(
CDataType
)
*
c_m_n_device_result
.
mDesc
.
GetElementSpace
());
DeviceMem
d_device_buf
(
sizeof
(
DDataType
)
*
d_m_device_result
.
mDesc
.
GetElementSpace
());
DeviceMem
reduce_device_buf
(
sizeof
(
ReduceDataType
)
*
reduce_m_device_result
.
mDesc
.
GetElementSpace
());
a_device_buf
.
ToDevice
(
a_m_k
.
mData
.
data
());
b_device_buf
.
ToDevice
(
b_k_n
.
mData
.
data
());
auto
a_element_op
=
AElementOp
{};
auto
b_element_op
=
BElementOp
{};
auto
c_element_op
=
CElementOp
{};
auto
ds_element_op
=
DsElementOp
{};
auto
p_ds_global
=
ck
::
make_tuple
(
static_cast
<
DDataType
*>
(
d_device_buf
.
GetDeviceBuffer
()));
auto
a_element_op
=
AElementOp
{};
auto
b_element_op
=
BElementOp
{};
auto
c_element_op
=
CElementOp
{};
auto
reduce_element_op
=
ReduceElementOps
{}[
ck
::
Number
<
0
>
{}];
std
::
array
<
void
*
,
3
>
gemm_element_ops
=
{
&
a_element_op
,
&
b_element_op
,
&
c_element_op
};
std
::
array
<
void
*
,
1
>
reduce_element_ops
=
{
&
reduce_element_op
};
std
::
array
<
void
*
,
1
>
p_reduces
=
{
reduce_device_buf
.
GetDeviceBuffer
()};
// do GEMM
auto
gemm
=
DeviceGemmReduceInstance
{};
auto
invoker
=
gemm
.
MakeInvoker
();
auto
argument
=
gemm
.
MakeArgument
(
static_cast
<
ADataType
*>
(
a_device_buf
.
GetDeviceBuffer
()),
static_cast
<
BDataType
*>
(
b_device_buf
.
GetDeviceBuffer
()),
static_cast
<
CDataType
*>
(
c_device_buf
.
GetDeviceBuffer
()),
p_ds_global
,
auto
argument
=
gemm
.
MakeArgument
(
a_device_buf
.
GetDeviceBuffer
(),
b_device_buf
.
GetDeviceBuffer
(),
nullptr
,
{},
c_device_buf
.
GetDeviceBuffer
(),
p_reduces
,
M
,
N
,
K
,
StrideA
,
StrideB
,
StrideC
,
a_element_op
,
b
_element_op
,
c_element_op
,
ds
_element_op
,
ds
_element_op
);
{}
,
gemm
_element_op
s
,
{}
,
reduce
_element_op
s
,
reduce
_element_op
s
);
if
(
!
gemm
.
IsSupportedArgument
(
argument
))
{
...
...
@@ -215,7 +220,7 @@ int main(int argc, char* argv[])
// [CAUSION]: launch_and_time_kernel will not initialize D.
// If we evaluate kernel multiple time but without initialize D. Verification will fail
d
_device_buf
.
SetValue
(
ck
::
NumericLimits
<
D
DataType
>::
Lowest
());
reduce
_device_buf
.
SetValue
(
ck
::
NumericLimits
<
Reduce
DataType
>::
Lowest
());
invoker
.
Run
(
argument
,
StreamConfig
{
nullptr
,
false
});
bool
pass
=
true
;
...
...
@@ -223,7 +228,7 @@ int main(int argc, char* argv[])
if
(
do_verification
)
{
c_device_buf
.
FromDevice
(
c_m_n_device_result
.
mData
.
data
());
d
_device_buf
.
FromDevice
(
d
_m_device_result
.
mData
.
data
());
reduce
_device_buf
.
FromDevice
(
reduce
_m_device_result
.
mData
.
data
());
auto
ref_gemm
=
ReferenceGemmInstance
{};
auto
ref_invoker
=
ref_gemm
.
MakeInvoker
();
...
...
@@ -233,27 +238,27 @@ int main(int argc, char* argv[])
ref_invoker
.
Run
(
ref_argument
);
auto
d_
reduce_op
=
Ds
ReduceOp
{}[
ck
::
Number
<
0
>
{}];
auto
reduce_op
=
ReduceOp
s
{}[
ck
::
Number
<
0
>
{}];
for
(
int
m
=
0
;
m
<
M
;
++
m
)
{
ReduceAccDataType
d
_acc
=
d_
reduce_op
.
GetIdentityValue
<
ReduceAccDataType
>
();
ReduceAccDataType
reduce
_acc
=
reduce_op
.
GetIdentityValue
<
ReduceAccDataType
>
();
for
(
int
n
=
0
;
n
<
N
;
++
n
)
{
ReduceAccDataType
curr_val
=
ck
::
type_convert
<
ReduceAccDataType
>
(
c_m_n_host_result
(
m
,
n
));
d_
reduce_op
(
d
_acc
,
curr_val
);
reduce_op
(
reduce
_acc
,
curr_val
);
};
d
_m_host_result
(
m
)
=
d
_acc
;
reduce
_m_host_result
(
m
)
=
reduce
_acc
;
}
pass
=
ck
::
utils
::
check_err
(
c_m_n_device_result
.
mData
,
c_m_n_host_result
.
mData
,
"Error: Incorrect results c"
)
&&
ck
::
utils
::
check_err
(
d
_m_device_result
.
mData
,
d
_m_host_result
.
mData
,
ck
::
utils
::
check_err
(
reduce
_m_device_result
.
mData
,
reduce
_m_host_result
.
mData
,
"Error: Incorrect results d"
,
1e-3
,
1e-3
);
...
...
@@ -263,7 +268,7 @@ int main(int argc, char* argv[])
{
float
gemm_reduceMax_ave_time
=
invoker
.
Run
(
argument
,
StreamConfig
{
nullptr
,
true
});
DumpGemmLayerNormPerf
<
ADataType
,
BDataType
,
CDataType
,
D
DataType
>
(
DumpGemmLayerNormPerf
<
ADataType
,
BDataType
,
CDataType
,
Reduce
DataType
>
(
gemm_reduceMax_ave_time
,
M
,
N
,
K
);
}
...
...
This diff is collapsed.
Click to expand it.
example/16_gemm_reduce/gemm_reduce_xdl_mean_squaremean_fp16.cpp
View file @
f5de8b57
...
...
@@ -33,27 +33,27 @@ using BDataType = F16;
using
CDataType
=
F16
;
using
GemmAccDataType
=
F32
;
using
ReduceAccDataType
=
F32
;
using
D
DataType
=
F32
;
using
D
PtrsGlobal
=
ck
::
Tuple
<
D
DataType
*
,
D
DataType
*>
;
using
Reduce
DataType
=
F32
;
using
Reduce
PtrsGlobal
=
ck
::
Tuple
<
Reduce
DataType
*
,
Reduce
DataType
*>
;
using
ALayout
=
ck
::
tensor_layout
::
gemm
::
RowMajor
;
using
BLayout
=
ck
::
tensor_layout
::
gemm
::
ColumnMajor
;
using
CLayout
=
ck
::
tensor_layout
::
gemm
::
RowMajor
;
using
AElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
BElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
CElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
D0
ReduceOp
=
ck
::
reduce
::
Add
;
using
D1
ReduceOp
=
ck
::
reduce
::
Add
;
using
Dxs
ReduceOp
=
ck
::
Tuple
<
D0
ReduceOp
,
D1
ReduceOp
>
;
using
AElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
BElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
CElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
ReduceOp
0
=
ck
::
reduce
::
Add
;
using
ReduceOp
1
=
ck
::
reduce
::
Add
;
using
ReduceOp
s
=
ck
::
Tuple
<
ReduceOp
0
,
ReduceOp
1
>
;
using
UnaryIdenticElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
UnaryDivElementOp
=
ck
::
tensor_operation
::
element_wise
::
UnaryDivide
;
using
UnarySquareElementOp
=
ck
::
tensor_operation
::
element_wise
::
UnarySquare
;
using
Dxs
InElementOps
=
ck
::
Tuple
<
UnaryIdenticElementOp
,
UnarySquareElementOp
>
;
using
Dxs
OutElementOps
=
ck
::
Tuple
<
UnaryDivElementOp
,
UnaryDivElementOp
>
;
using
Reduce
InElementOps
=
ck
::
Tuple
<
UnaryIdenticElementOp
,
UnarySquareElementOp
>
;
using
Reduce
OutElementOps
=
ck
::
Tuple
<
UnaryDivElementOp
,
UnaryDivElementOp
>
;
using
D
GlobalMemOp
=
using
Reduce
GlobalMemOp
s
=
ck
::
InMemoryDataOperationEnumSequence
<
ck
::
InMemoryDataOperationEnum
::
AtomicAdd
,
ck
::
InMemoryDataOperationEnum
::
AtomicAdd
>
;
...
...
@@ -62,11 +62,11 @@ static constexpr auto GemmSpecialization =
// clang-format off
using
DeviceGemmReduceInstance
=
ck
::
tensor_operation
::
device
::
DeviceGemmReduce_Xdl_CShuffle
//######| ALayout| BLayout| CLayout|AData| BData| CData| GemmAcc| CShuffle| ReduceAcc|
DData| A| B| C|
Dxs| Dxs
InEleOp|
DxsAcc
EleOp|
D
| GEMM| NumGemmK| Block| MPer| NPer| KPer| AK1| BK1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransferClusterLengths| CBlockTransfer| CReduce| CReduceThreadLds2VGprCopy| CReduceThreadVgpr2GlobalCopy|
//######| | | | Type| Type| Type| DataType| DataType| DataType| Type Tuple| Elementwise| Elementwise| Elementwise|
Reduce|
|
|
MemoryData| Spacialization| Prefetch| Size| Block| Block| Block| | | XDL| XDL| Per| Per| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraN| MXdlPerWave| NXdlPerWave| _MBlock_MPerBlock| ScalarPerVector| ThreadClusterLengths| SrcDstScalarPerVector| SrcDstScalarPerVector|
//######| | | | | | | | | |
|
Operation
| Operation| Operation| Operation|
|
| Operation| | Stage| | | | | | | | | Wave| Wave| Lengths_K0_M_K1| ArrangeOrder| | | PerVector| PerVector_K1| | Lengths_K0_N_K1| ArrangeOrder| | | PerVector| PerVector_K1| | PerShuffle| PerShuffle| _NBlock_NPerBlock| _NPerBlock| _MPerBlock_NPerBlock| _NPerBlock| _MPerBlock|
//######| | | | | | | | | | | | | | |
|
| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
<
Row
,
Col
,
Row
,
F16
,
F16
,
F16
,
F32
,
F32
,
F32
,
D
PtrsGlobal
,
AElementOp
,
BElementOp
,
CElementOp
,
Dxs
ReduceOp
,
Dxs
InElementOps
,
Dxs
OutElementOps
,
D
GlobalMemOp
,
GemmSpecialization
,
1
,
256
,
256
,
128
,
32
,
8
,
8
,
32
,
32
,
4
,
2
,
S
<
4
,
64
,
1
>
,
S
<
1
,
0
,
2
>
,
S
<
1
,
0
,
2
>
,
2
,
8
,
8
,
1
,
S
<
4
,
64
,
1
>
,
S
<
1
,
0
,
2
>
,
S
<
1
,
0
,
2
>
,
2
,
8
,
8
,
1
,
1
,
1
,
S
<
1
,
32
,
1
,
8
>
,
8
,
S
<
64
,
4
>
,
4
,
1
>
;
//######| ALayout| BLayout| CLayout|AData| BData| CData| GemmAcc| CShuffle| ReduceAcc|
Reduce
DData| A| B| C|
Reduce| Reduce
InEleOp|
ReduceOut
EleOp|
Reduce
| GEMM| NumGemmK| Block| MPer| NPer| KPer| AK1| BK1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransferClusterLengths| CBlockTransfer| CReduce| CReduceThreadLds2VGprCopy| CReduceThreadVgpr2GlobalCopy|
//######| | | | Type| Type| Type| DataType| DataType| DataType|
Type Tuple| Elementwise| Elementwise| Elementwise|
Operation|
|
|
MemoryData| Spacialization| Prefetch| Size| Block| Block| Block| | | XDL| XDL| Per| Per| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraN| MXdlPerWave| NXdlPerWave| _MBlock_MPerBlock| ScalarPerVector| ThreadClusterLengths| SrcDstScalarPerVector| SrcDstScalarPerVector|
//######| | | | | | | | | | | Operation| Operation| Operation|
| |
|
Operation| | Stage| | | | | | | | | Wave| Wave| Lengths_K0_M_K1| ArrangeOrder| | | PerVector| PerVector_K1| | Lengths_K0_N_K1| ArrangeOrder| | | PerVector| PerVector_K1| | PerShuffle| PerShuffle| _NBlock_NPerBlock| _NPerBlock| _MPerBlock_NPerBlock| _NPerBlock| _MPerBlock|
//######| | | | | | | | | |
| | | | |
|
|
| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
<
Row
,
Col
,
Row
,
F16
,
F16
,
F16
,
F32
,
F32
,
F32
,
Reduce
PtrsGlobal
,
AElementOp
,
BElementOp
,
CElementOp
,
ReduceOp
s
,
Reduce
InElementOps
,
Reduce
OutElementOps
,
Reduce
GlobalMemOp
s
,
GemmSpecialization
,
1
,
256
,
256
,
128
,
32
,
8
,
8
,
32
,
32
,
4
,
2
,
S
<
4
,
64
,
1
>
,
S
<
1
,
0
,
2
>
,
S
<
1
,
0
,
2
>
,
2
,
8
,
8
,
1
,
S
<
4
,
64
,
1
>
,
S
<
1
,
0
,
2
>
,
S
<
1
,
0
,
2
>
,
2
,
8
,
8
,
1
,
1
,
1
,
S
<
1
,
32
,
1
,
8
>
,
8
,
S
<
64
,
4
>
,
4
,
1
>
;
// clang-format on
using
ReferenceGemmInstance
=
ck
::
tensor_operation
::
host
::
ReferenceGemm
<
ADataType
,
...
...
@@ -77,13 +77,13 @@ using ReferenceGemmInstance = ck::tensor_operation::host::ReferenceGemm<ADataTyp
BElementOp
,
CElementOp
>
;
template
<
typename
ADataType
,
typename
BDataType
,
typename
CDataType
,
typename
D
DataType
>
template
<
typename
ADataType
,
typename
BDataType
,
typename
CDataType
,
typename
Reduce
DataType
>
void
DumpGemmLayerNormPerf
(
float
gemm_reduce_time
,
int
M
,
int
N
,
int
K
)
{
std
::
size_t
gemm_flop
=
std
::
size_t
(
2
)
*
M
*
N
*
K
;
std
::
size_t
gemm_num_byte
=
sizeof
(
ADataType
)
*
M
*
K
+
sizeof
(
BDataType
)
*
K
*
N
+
sizeof
(
CDataType
)
*
M
*
N
+
sizeof
(
D
DataType
)
*
M
+
sizeof
(
D
DataType
)
*
M
;
sizeof
(
CDataType
)
*
M
*
N
+
sizeof
(
Reduce
DataType
)
*
M
+
sizeof
(
Reduce
DataType
)
*
M
;
float
tflops
=
static_cast
<
float
>
(
gemm_flop
)
/
1.E9
/
gemm_reduce_time
;
float
gemm_gb_per_sec
=
gemm_num_byte
/
1.E6
/
gemm_reduce_time
;
...
...
@@ -158,22 +158,22 @@ int main(int argc, char* argv[])
Tensor
<
BDataType
>
b_k_n
(
f_host_tensor_descriptor
(
K
,
N
,
StrideB
,
BLayout
{}));
Tensor
<
CDataType
>
c_m_n_host_result
(
f_host_tensor_descriptor
(
M
,
N
,
StrideC
,
CLayout
{}));
Tensor
<
D
DataType
>
d
0_m_host_result
(
Tensor
<
Reduce
DataType
>
reduce
0_m_host_result
(
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
({
static_cast
<
std
::
size_t
>
(
M
)})));
Tensor
<
D
DataType
>
d
1_m_host_result
(
Tensor
<
Reduce
DataType
>
reduce
1_m_host_result
(
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
({
static_cast
<
std
::
size_t
>
(
M
)})));
Tensor
<
CDataType
>
c_m_n_device_result
(
f_host_tensor_descriptor
(
M
,
N
,
StrideC
,
CLayout
{}));
Tensor
<
D
DataType
>
d
0_m_device_result
(
Tensor
<
Reduce
DataType
>
reduce
0_m_device_result
(
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
({
static_cast
<
std
::
size_t
>
(
M
)})));
Tensor
<
D
DataType
>
d
1_m_device_result
(
Tensor
<
Reduce
DataType
>
reduce
1_m_device_result
(
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
({
static_cast
<
std
::
size_t
>
(
M
)})));
std
::
cout
<<
"a_m_k: "
<<
a_m_k
.
mDesc
<<
std
::
endl
;
std
::
cout
<<
"b_k_n: "
<<
b_k_n
.
mDesc
<<
std
::
endl
;
std
::
cout
<<
"c_m_n: "
<<
c_m_n_host_result
.
mDesc
<<
std
::
endl
;
std
::
cout
<<
"
d
0_m: "
<<
d
0_m_host_result
.
mDesc
<<
std
::
endl
;
std
::
cout
<<
"
d
1_m: "
<<
d
1_m_host_result
.
mDesc
<<
std
::
endl
;
std
::
cout
<<
"
reduce
0_m: "
<<
reduce
0_m_host_result
.
mDesc
<<
std
::
endl
;
std
::
cout
<<
"
reduce
1_m: "
<<
reduce
1_m_host_result
.
mDesc
<<
std
::
endl
;
switch
(
init_method
)
{
...
...
@@ -191,39 +191,48 @@ int main(int argc, char* argv[])
DeviceMem
a_device_buf
(
sizeof
(
ADataType
)
*
a_m_k
.
mDesc
.
GetElementSpace
());
DeviceMem
b_device_buf
(
sizeof
(
BDataType
)
*
b_k_n
.
mDesc
.
GetElementSpace
());
DeviceMem
c_device_buf
(
sizeof
(
CDataType
)
*
c_m_n_device_result
.
mDesc
.
GetElementSpace
());
DeviceMem
d0_device_buf
(
sizeof
(
DDataType
)
*
d0_m_device_result
.
mDesc
.
GetElementSpace
());
DeviceMem
d1_device_buf
(
sizeof
(
DDataType
)
*
d1_m_device_result
.
mDesc
.
GetElementSpace
());
DeviceMem
reduce0_device_buf
(
sizeof
(
ReduceDataType
)
*
reduce0_m_device_result
.
mDesc
.
GetElementSpace
());
DeviceMem
reduce1_device_buf
(
sizeof
(
ReduceDataType
)
*
reduce1_m_device_result
.
mDesc
.
GetElementSpace
());
a_device_buf
.
ToDevice
(
a_m_k
.
mData
.
data
());
b_device_buf
.
ToDevice
(
b_k_n
.
mData
.
data
());
auto
a_element_op
=
AElementOp
{};
auto
b_element_op
=
BElementOp
{};
auto
c_element_op
=
CElementOp
{};
auto
dxs_global
=
ck
::
make_tuple
(
static_cast
<
DDataType
*>
(
d0_device_buf
.
GetDeviceBuffer
()),
static_cast
<
DDataType
*>
(
d1_device_buf
.
GetDeviceBuffer
()));
auto
a_element_op
=
AElementOp
{};
auto
b_element_op
=
BElementOp
{};
auto
c_element_op
=
CElementOp
{};
std
::
array
<
void
*
,
3
>
gemm_element_ops
=
{
&
a_element_op
,
&
b_element_op
,
&
c_element_op
};
auto
dxs_in_element_op
=
DxsInElementOps
{};
auto
dxs_out_element_op
=
DxsOutElementOps
{
N
,
N
};
auto
passthrough
=
UnaryIdenticElementOp
{};
auto
square
=
UnarySquareElementOp
{};
auto
div
=
UnaryDivElementOp
{
N
};
std
::
array
<
void
*
,
2
>
reduce_in_element_ops
=
{
&
passthrough
,
&
square
};
std
::
array
<
void
*
,
2
>
reduce_out_element_ops
=
{
&
div
,
&
div
};
std
::
array
<
void
*
,
2
>
p_reduces
=
{
reduce0_device_buf
.
GetDeviceBuffer
(),
reduce1_device_buf
.
GetDeviceBuffer
()};
// do GEMM
auto
gemm
=
DeviceGemmReduceInstance
{};
auto
invoker
=
gemm
.
MakeInvoker
();
auto
argument
=
gemm
.
MakeArgument
(
static_cast
<
ADataType
*>
(
a_device_buf
.
GetDeviceBuffer
()),
static_cast
<
BDataType
*>
(
b_device_buf
.
GetDeviceBuffer
()),
static_cast
<
CDataType
*>
(
c_device_buf
.
GetDeviceBuffer
()),
dxs_global
,
auto
argument
=
gemm
.
MakeArgument
(
a_device_buf
.
GetDeviceBuffer
(),
b_device_buf
.
GetDeviceBuffer
(),
nullptr
,
{},
c_device_buf
.
GetDeviceBuffer
(),
p_reduces
,
M
,
N
,
K
,
StrideA
,
StrideB
,
StrideC
,
a_element_op
,
b
_element_op
,
c_element_op
,
dxs
_in_element_op
,
dxs
_out_element_op
);
{}
,
gemm
_element_op
s
,
{}
,
reduce
_in_element_op
s
,
reduce
_out_element_op
s
);
if
(
!
gemm
.
IsSupportedArgument
(
argument
))
{
...
...
@@ -232,9 +241,9 @@ int main(int argc, char* argv[])
"not support this GEMM problem"
);
}
// init
DO, D1
to 0
d
0_device_buf
.
SetZero
();
d
1_device_buf
.
SetZero
();
// init
reducetion buffer
to 0
reduce
0_device_buf
.
SetZero
();
reduce
1_device_buf
.
SetZero
();
// if time_kernel == true, kernel will run multiple times. This kernel use atomic-add so result
// will not be correct. need to set time_kernel = false for correctness test
...
...
@@ -244,8 +253,8 @@ int main(int argc, char* argv[])
if
(
do_verification
)
{
c_device_buf
.
FromDevice
(
c_m_n_device_result
.
mData
.
data
());
d
0_device_buf
.
FromDevice
(
d
0_m_device_result
.
mData
.
data
());
d
1_device_buf
.
FromDevice
(
d
1_m_device_result
.
mData
.
data
());
reduce
0_device_buf
.
FromDevice
(
reduce
0_m_device_result
.
mData
.
data
());
reduce
1_device_buf
.
FromDevice
(
reduce
1_m_device_result
.
mData
.
data
());
auto
ref_gemm
=
ReferenceGemmInstance
{};
auto
ref_invoker
=
ref_gemm
.
MakeInvoker
();
...
...
@@ -255,42 +264,40 @@ int main(int argc, char* argv[])
ref_invoker
.
Run
(
ref_argument
);
auto
d0_
reduce_op
=
D0
ReduceOp
{};
auto
d1_
reduce_op
=
D1
ReduceOp
{};
auto
reduce
0
_op
=
ReduceOp
0
{};
auto
reduce
1
_op
=
ReduceOp
1
{};
for
(
int
m
=
0
;
m
<
M
;
++
m
)
{
auto
d
0_acc
=
d0_
reduce_op
.
GetIdentityValue
<
ReduceAccDataType
>
();
auto
d
1_acc
=
d1_
reduce_op
.
GetIdentityValue
<
ReduceAccDataType
>
();
auto
reduce
0_acc
=
reduce
0
_op
.
GetIdentityValue
<
ReduceAccDataType
>
();
auto
reduce
1_acc
=
reduce
1
_op
.
GetIdentityValue
<
ReduceAccDataType
>
();
for
(
int
n
=
0
;
n
<
N
;
++
n
)
{
auto
c_val
=
ck
::
type_convert
<
ReduceAccDataType
>
(
c_m_n_host_result
(
m
,
n
));
ReduceAccDataType
d0
_val
;
ReduceAccDataType
d1
_val
;
ReduceAccDataType
square_c
_val
;
square
(
square_c_val
,
c
_val
)
;
dxs_in_element_op
(
ck
::
Number
<
0
>
{})(
d0_val
,
c_val
);
dxs_in_element_op
(
ck
::
Number
<
1
>
{})(
d1_val
,
c_val
);
d0_reduce_op
(
d0_acc
,
d0_val
);
d1_reduce_op
(
d1_acc
,
d1_val
);
reduce0_op
(
reduce0_acc
,
c_val
);
reduce1_op
(
reduce1_acc
,
square_c_val
);
}
d
xs_out_element_op
(
ck
::
Number
<
0
>
{})(
d
0_acc
,
d
0_acc
);
d
xs_out_element_op
(
ck
::
Number
<
1
>
{})(
d
1_acc
,
d
1_acc
);
d
0_m_host_result
(
m
)
=
ck
::
type_convert
<
D
DataType
>
(
d
0_acc
);
d
1_m_host_result
(
m
)
=
ck
::
type_convert
<
D
DataType
>
(
d
1_acc
);
d
iv
(
reduce
0_acc
,
reduce
0_acc
);
d
iv
(
reduce
1_acc
,
reduce
1_acc
);
reduce
0_m_host_result
(
m
)
=
ck
::
type_convert
<
Reduce
DataType
>
(
reduce
0_acc
);
reduce
1_m_host_result
(
m
)
=
ck
::
type_convert
<
Reduce
DataType
>
(
reduce
1_acc
);
}
pass
=
ck
::
utils
::
check_err
(
c_m_n_device_result
.
mData
,
c_m_n_host_result
.
mData
,
"Error: Incorrect results c"
)
&&
ck
::
utils
::
check_err
(
d
0_m_device_result
.
mData
,
d
0_m_host_result
.
mData
,
ck
::
utils
::
check_err
(
reduce
0_m_device_result
.
mData
,
reduce
0_m_host_result
.
mData
,
"Error: Incorrect results d0"
,
1e-4
,
1e-5
)
&&
ck
::
utils
::
check_err
(
d
1_m_device_result
.
mData
,
d
1_m_host_result
.
mData
,
ck
::
utils
::
check_err
(
reduce
1_m_device_result
.
mData
,
reduce
1_m_host_result
.
mData
,
"Error: Incorrect results d1"
,
1e-3
,
1e-5
);
...
...
@@ -300,7 +307,7 @@ int main(int argc, char* argv[])
{
float
ave_time
=
invoker
.
Run
(
argument
,
StreamConfig
{
nullptr
,
true
});
DumpGemmLayerNormPerf
<
ADataType
,
BDataType
,
CDataType
,
D
DataType
>
(
ave_time
,
M
,
N
,
K
);
DumpGemmLayerNormPerf
<
ADataType
,
BDataType
,
CDataType
,
Reduce
DataType
>
(
ave_time
,
M
,
N
,
K
);
}
return
pass
?
0
:
1
;
...
...
This diff is collapsed.
Click to expand it.
example/18_batched_gemm_reduce/batched_gemm_reduce_xdl_fp16.cpp
View file @
f5de8b57
...
...
@@ -31,26 +31,26 @@ using ADataType = F16;
using
BDataType
=
F16
;
using
CDataType
=
F16
;
using
ReduceAccDataType
=
F32
;
using
D
DataType
=
F32
;
using
D
PtrsGlobal
=
ck
::
Tuple
<
D
DataType
*
,
D
DataType
*>
;
using
Reduce
DataType
=
F32
;
using
Reduce
PtrsGlobal
=
ck
::
Tuple
<
Reduce
DataType
*
,
Reduce
DataType
*>
;
using
ALayout
=
ck
::
tensor_layout
::
gemm
::
RowMajor
;
using
BLayout
=
ck
::
tensor_layout
::
gemm
::
ColumnMajor
;
using
CLayout
=
ck
::
tensor_layout
::
gemm
::
RowMajor
;
using
AElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
BElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
CElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
D0
ReduceOp
=
ck
::
reduce
::
Add
;
using
D1
ReduceOp
=
ck
::
reduce
::
Add
;
using
Dxs
ReduceOp
=
ck
::
Tuple
<
D0
ReduceOp
,
D1
ReduceOp
>
;
using
AElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
BElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
CElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
ReduceOp
0
=
ck
::
reduce
::
Add
;
using
ReduceOp
1
=
ck
::
reduce
::
Add
;
using
ReduceOp
s
=
ck
::
Tuple
<
ReduceOp
0
,
ReduceOp
1
>
;
using
UnaryIdenticElementOp
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
UnarySquareElementOp
=
ck
::
tensor_operation
::
element_wise
::
UnarySquare
;
using
Dxs
InElementOps
=
ck
::
Tuple
<
UnaryIdenticElementOp
,
UnarySquareElementOp
>
;
using
Dxs
OutElementOps
=
ck
::
Tuple
<
UnaryIdenticElementOp
,
UnaryIdenticElementOp
>
;
using
Reduce
InElementOps
=
ck
::
Tuple
<
UnaryIdenticElementOp
,
UnarySquareElementOp
>
;
using
Reduce
OutElementOps
=
ck
::
Tuple
<
UnaryIdenticElementOp
,
UnaryIdenticElementOp
>
;
using
D
GlobalMemOp
=
using
Reduce
GlobalMemOp
s
=
ck
::
InMemoryDataOperationEnumSequence
<
ck
::
InMemoryDataOperationEnum
::
AtomicAdd
,
ck
::
InMemoryDataOperationEnum
::
AtomicAdd
>
;
...
...
@@ -63,7 +63,7 @@ using DeviceBatchedGemmReduceInstance = ck::tensor_operation::device::DeviceBatc
//######| | | | Type| Type| Type| DataType| DataType| DataType| Type Tuple| Elementwise| Elementwise| Elementwise| Reduce| | | MemoryData| Spacialization| Prefetch| Size| Block| Block| Block| | | XDL| XDL| Per| Per| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| ExtraN| MXdlPerWave| NXdlPerWave| _MBlock_MPerBlock| ScalarPerVector| ThreadClusterLengths| SrcDstScalarPerVector| SrcDstScalarPerVector|
//######| | | | | | | | | | | Operation| Operation| Operation| Operation| | | Operation| | Stage| | | | | | | | | Wave| Wave| Lengths_K0_M_K1| ArrangeOrder| | | PerVector| PerVector_K1| | Lengths_K0_N_K1| ArrangeOrder| | | PerVector| PerVector_K1| | PerShuffle| PerShuffle| _NBlock_NPerBlock| _NPerBlock| _MPerBlock_NPerBlock| _NPerBlock| _MPerBlock|
//######| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
<
Row
,
Col
,
Row
,
F16
,
F16
,
F16
,
F32
,
F32
,
F32
,
D
PtrsGlobal
,
AElementOp
,
BElementOp
,
CElementOp
,
Dxs
ReduceOp
,
Dxs
InElementOps
,
Dxs
OutElementOps
,
D
GlobalMemOp
,
GemmSpecialization
,
1
,
256
,
256
,
128
,
32
,
8
,
8
,
32
,
32
,
4
,
2
,
S
<
4
,
64
,
1
>
,
S
<
1
,
0
,
2
>
,
S
<
1
,
0
,
2
>
,
2
,
8
,
8
,
1
,
S
<
4
,
64
,
1
>
,
S
<
1
,
0
,
2
>
,
S
<
1
,
0
,
2
>
,
2
,
8
,
8
,
1
,
1
,
1
,
S
<
1
,
32
,
1
,
8
>
,
8
,
S
<
64
,
4
>
,
4
,
1
>
;
<
Row
,
Col
,
Row
,
F16
,
F16
,
F16
,
F32
,
F32
,
F32
,
Reduce
PtrsGlobal
,
AElementOp
,
BElementOp
,
CElementOp
,
ReduceOp
s
,
Reduce
InElementOps
,
Reduce
OutElementOps
,
Reduce
GlobalMemOp
s
,
GemmSpecialization
,
1
,
256
,
256
,
128
,
32
,
8
,
8
,
32
,
32
,
4
,
2
,
S
<
4
,
64
,
1
>
,
S
<
1
,
0
,
2
>
,
S
<
1
,
0
,
2
>
,
2
,
8
,
8
,
1
,
S
<
4
,
64
,
1
>
,
S
<
1
,
0
,
2
>
,
S
<
1
,
0
,
2
>
,
2
,
8
,
8
,
1
,
1
,
1
,
S
<
1
,
32
,
1
,
8
>
,
8
,
S
<
64
,
4
>
,
4
,
1
>
;
// clang-format on
using
ReferenceBatchedGemmInstance
=
ck
::
tensor_operation
::
host
::
...
...
@@ -143,16 +143,16 @@ int main(int argc, char* argv[])
Tensor
<
CDataType
>
c_g_m_n_host_result
(
f_host_tensor_descriptor
(
BatchCount
,
M
,
N
,
StrideC
,
CLayout
{}));
Tensor
<
D
DataType
>
d0_g_m_host_result
(
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
(
Tensor
<
Reduce
DataType
>
d0_g_m_host_result
(
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
(
{
static_cast
<
std
::
size_t
>
(
BatchCount
),
static_cast
<
std
::
size_t
>
(
M
)})));
Tensor
<
D
DataType
>
d1_g_m_host_result
(
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
(
Tensor
<
Reduce
DataType
>
d1_g_m_host_result
(
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
(
{
static_cast
<
std
::
size_t
>
(
BatchCount
),
static_cast
<
std
::
size_t
>
(
M
)})));
Tensor
<
CDataType
>
c_g_m_n_device_result
(
f_host_tensor_descriptor
(
BatchCount
,
M
,
N
,
StrideC
,
CLayout
{}));
Tensor
<
D
DataType
>
d0_g_m_device_result
(
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
(
Tensor
<
Reduce
DataType
>
d0_g_m_device_result
(
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
(
{
static_cast
<
std
::
size_t
>
(
BatchCount
),
static_cast
<
std
::
size_t
>
(
M
)})));
Tensor
<
D
DataType
>
d1_g_m_device_result
(
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
(
Tensor
<
Reduce
DataType
>
d1_g_m_device_result
(
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
(
{
static_cast
<
std
::
size_t
>
(
BatchCount
),
static_cast
<
std
::
size_t
>
(
M
)})));
std
::
cout
<<
"a_g_m_k: "
<<
a_g_m_k
.
mDesc
<<
std
::
endl
;
...
...
@@ -177,38 +177,48 @@ int main(int argc, char* argv[])
DeviceMem
a_device_buf
(
sizeof
(
ADataType
)
*
a_g_m_k
.
mDesc
.
GetElementSpace
());
DeviceMem
b_device_buf
(
sizeof
(
BDataType
)
*
b_g_k_n
.
mDesc
.
GetElementSpace
());
DeviceMem
c_device_buf
(
sizeof
(
CDataType
)
*
c_g_m_n_device_result
.
mDesc
.
GetElementSpace
());
DeviceMem
d0_device_buf
(
sizeof
(
DDataType
)
*
d0_g_m_device_result
.
mDesc
.
GetElementSpace
());
DeviceMem
d1_device_buf
(
sizeof
(
DDataType
)
*
d1_g_m_device_result
.
mDesc
.
GetElementSpace
());
DeviceMem
reduce0_device_buf
(
sizeof
(
ReduceDataType
)
*
d0_g_m_device_result
.
mDesc
.
GetElementSpace
());
DeviceMem
reduce1_device_buf
(
sizeof
(
ReduceDataType
)
*
d1_g_m_device_result
.
mDesc
.
GetElementSpace
());
a_device_buf
.
ToDevice
(
a_g_m_k
.
mData
.
data
());
b_device_buf
.
ToDevice
(
b_g_k_n
.
mData
.
data
());
auto
a_element_op
=
AElementOp
{};
auto
b_element_op
=
BElementOp
{};
auto
c_element_op
=
CElementOp
{};
auto
dxs_global
=
ck
::
make_tuple
(
static_cast
<
DDataType
*>
(
d0_device_buf
.
GetDeviceBuffer
()),
static_cast
<
DDataType
*>
(
d1_device_buf
.
GetDeviceBuffer
()));
auto
a_element_op
=
AElementOp
{};
auto
b_element_op
=
BElementOp
{};
auto
c_element_op
=
CElementOp
{};
std
::
array
<
void
*
,
3
>
gemm_element_ops
=
{
&
a_element_op
,
&
b_element_op
,
&
c_element_op
};
auto
passthrough
=
UnaryIdenticElementOp
{};
auto
square
=
UnarySquareElementOp
{};
std
::
array
<
void
*
,
2
>
reduce_in_element_ops
=
{
&
passthrough
,
&
square
};
std
::
array
<
void
*
,
2
>
reduce_out_element_ops
=
{
&
passthrough
,
&
passthrough
};
std
::
array
<
void
*
,
2
>
p_reduces
=
{
reduce0_device_buf
.
GetDeviceBuffer
(),
reduce1_device_buf
.
GetDeviceBuffer
()};
// do GEMM
auto
batched_gemm
=
DeviceBatchedGemmReduceInstance
{};
auto
invoker
=
batched_gemm
.
MakeInvoker
();
auto
argument
=
batched_gemm
.
MakeArgument
(
static_cast
<
ADataType
*>
(
a_device_buf
.
GetDeviceBuffer
()),
static_cast
<
BDataType
*>
(
b_device_buf
.
GetDeviceBuffer
()),
static_cast
<
CDataType
*>
(
c_device_buf
.
GetDeviceBuffer
()),
dxs_global
,
M
,
N
,
K
,
StrideA
,
StrideB
,
StrideC
,
a_element_op
,
b_element_op
,
c_element_op
,
DxsInElementOps
{},
DxsOutElementOps
{},
BatchCount
);
auto
argument
=
batched_gemm
.
MakeArgument
(
a_device_buf
.
GetDeviceBuffer
(),
b_device_buf
.
GetDeviceBuffer
(),
nullptr
,
{},
c_device_buf
.
GetDeviceBuffer
(),
p_reduces
,
M
,
N
,
K
,
StrideA
,
StrideB
,
StrideC
,
{},
gemm_element_ops
,
{},
reduce_in_element_ops
,
reduce_out_element_ops
,
BatchCount
);
if
(
!
batched_gemm
.
IsSupportedArgument
(
argument
))
{
...
...
@@ -218,8 +228,8 @@ int main(int argc, char* argv[])
}
// init DO, D1 to 0
d
0_device_buf
.
SetZero
();
d
1_device_buf
.
SetZero
();
reduce
0_device_buf
.
SetZero
();
reduce
1_device_buf
.
SetZero
();
// if time_kernel == true, kernel will run multiple times. This kernel use atomic-add so result
// will not be correct. need to set time_kernel = false for correctness test
...
...
@@ -241,8 +251,8 @@ int main(int argc, char* argv[])
if
(
do_verification
)
{
c_device_buf
.
FromDevice
(
c_g_m_n_device_result
.
mData
.
data
());
d
0_device_buf
.
FromDevice
(
d0_g_m_device_result
.
mData
.
data
());
d
1_device_buf
.
FromDevice
(
d1_g_m_device_result
.
mData
.
data
());
reduce
0_device_buf
.
FromDevice
(
d0_g_m_device_result
.
mData
.
data
());
reduce
1_device_buf
.
FromDevice
(
d1_g_m_device_result
.
mData
.
data
());
auto
ref_batched_gemm
=
ReferenceBatchedGemmInstance
{};
auto
ref_invoker
=
ref_batched_gemm
.
MakeInvoker
();
...
...
@@ -252,15 +262,15 @@ int main(int argc, char* argv[])
ref_invoker
.
Run
(
ref_argument
);
auto
d0_
reduce_op
=
D0
ReduceOp
{};
auto
d1_
reduce_op
=
D1
ReduceOp
{};
auto
reduce
0
_op
=
ReduceOp
0
{};
auto
reduce
1
_op
=
ReduceOp
1
{};
for
(
int
batch
=
0
;
batch
<
BatchCount
;
++
batch
)
{
for
(
int
m
=
0
;
m
<
M
;
++
m
)
{
auto
d
0_acc
=
d0_
reduce_op
.
GetIdentityValue
<
ReduceAccDataType
>
();
auto
d
1_acc
=
d1_
reduce_op
.
GetIdentityValue
<
ReduceAccDataType
>
();
auto
reduce
0_acc
=
reduce
0
_op
.
GetIdentityValue
<
ReduceAccDataType
>
();
auto
reduce
1_acc
=
reduce
1
_op
.
GetIdentityValue
<
ReduceAccDataType
>
();
for
(
int
n
=
0
;
n
<
N
;
++
n
)
{
...
...
@@ -271,12 +281,12 @@ int main(int argc, char* argv[])
UnaryIdenticElementOp
{}(
d0_val
,
c_val
);
UnarySquareElementOp
{}(
d1_val
,
c_val
);
d0_
reduce_op
(
d
0_acc
,
d0_val
);
d1_
reduce_op
(
d
1_acc
,
d1_val
);
reduce
0
_op
(
reduce
0_acc
,
d0_val
);
reduce
1
_op
(
reduce
1_acc
,
d1_val
);
}
d0_g_m_host_result
(
batch
,
m
)
=
ck
::
type_convert
<
D
DataType
>
(
d
0_acc
);
d1_g_m_host_result
(
batch
,
m
)
=
ck
::
type_convert
<
D
DataType
>
(
d
1_acc
);
d0_g_m_host_result
(
batch
,
m
)
=
ck
::
type_convert
<
Reduce
DataType
>
(
reduce
0_acc
);
d1_g_m_host_result
(
batch
,
m
)
=
ck
::
type_convert
<
Reduce
DataType
>
(
reduce
1_acc
);
}
}
...
...
This diff is collapsed.
Click to expand it.
example/19_binary_elementwise/broadcast_add_2d_amn_bn.cpp
View file @
f5de8b57
...
...
@@ -99,15 +99,17 @@ int main()
a_m_n_device_buf
.
ToDevice
(
a_m_n
.
mData
.
data
());
b_n_device_buf
.
ToDevice
(
b_n
.
mData
.
data
());
std
::
array
<
const
void
*
,
2
>
input
=
{
a_m_n_device_buf
.
GetDeviceBuffer
(),
b_n_device_buf
.
GetDeviceBuffer
()};
std
::
array
<
void
*
,
1
>
output
=
{
c_m_n_device_buf
.
GetDeviceBuffer
()};
std
::
vector
<
ck
::
index_t
>
a_strides
=
{
Stride
,
1
};
std
::
vector
<
ck
::
index_t
>
b_strides
=
{
0
,
1
};
std
::
vector
<
ck
::
index_t
>
c_strides
=
{
Stride
,
1
};
auto
broadcastAdd
=
DeviceElementwiseAddInstance
{};
auto
argument
=
broadcastAdd
.
MakeArgumentPointer
(
a_m_n_device_buf
.
GetDeviceBuffer
(),
b_n_device_buf
.
GetDeviceBuffer
(),
c_m_n_device_buf
.
GetDeviceBuffer
(),
{
M
,
N
},
{
Stride
,
1
},
{
0
,
1
},
// broadcast in first dimension
{
Stride
,
1
},
Add
{});
auto
argument
=
broadcastAdd
.
MakeArgumentPointer
(
input
,
output
,
{
M
,
N
},
{
a_strides
,
b_strides
},
{
c_strides
},
Add
{});
if
(
!
broadcastAdd
.
IsSupportedArgument
(
argument
.
get
()))
{
...
...
This diff is collapsed.
Click to expand it.
example/19_binary_elementwise/broadcast_add_3d_am_bmnk.cpp
View file @
f5de8b57
...
...
@@ -81,18 +81,24 @@ int main()
a_m_device_buf
.
ToDevice
(
a_m
.
mData
.
data
());
b_m_n_k_device_buf
.
ToDevice
(
b_m_n_k
.
mData
.
data
());
std
::
array
<
const
void
*
,
2
>
input
=
{
a_m_device_buf
.
GetDeviceBuffer
(),
b_m_n_k_device_buf
.
GetDeviceBuffer
()};
std
::
array
<
void
*
,
1
>
output
=
{
c_m_n_k_device_buf
.
GetDeviceBuffer
()};
std
::
vector
<
ck
::
index_t
>
a_strides
=
{
1
,
0
,
0
};
std
::
vector
<
ck
::
index_t
>
b_strides
{
b_m_n_k
.
mDesc
.
GetStrides
().
begin
(),
b_m_n_k
.
mDesc
.
GetStrides
().
end
()};
std
::
vector
<
ck
::
index_t
>
c_strides
{
c_m_n_k
.
mDesc
.
GetStrides
().
begin
(),
c_m_n_k
.
mDesc
.
GetStrides
().
end
()};
auto
broadcastAdd
=
DeviceElementwiseAddInstance
{};
auto
argument
=
broadcastAdd
.
MakeArgumentPointer
(
a_m_device_buf
.
GetDeviceBuffer
(),
b_m_n_k_device_buf
.
GetDeviceBuffer
(),
c_m_n_k_device_buf
.
GetDeviceBuffer
(),
std
::
vector
<
ck
::
index_t
>
{
mnk
.
begin
(),
mnk
.
end
()},
{
1
,
0
,
0
},
// broadcast A on second and third dimension
std
::
vector
<
ck
::
index_t
>
{
b_m_n_k
.
mDesc
.
GetStrides
().
begin
(),
b_m_n_k
.
mDesc
.
GetStrides
().
end
()},
std
::
vector
<
ck
::
index_t
>
{
c_m_n_k
.
mDesc
.
GetStrides
().
begin
(),
c_m_n_k
.
mDesc
.
GetStrides
().
end
()},
Add
{});
auto
argument
=
broadcastAdd
.
MakeArgumentPointer
(
input
,
output
,
std
::
vector
<
ck
::
index_t
>
{
mnk
.
begin
(),
mnk
.
end
()},
{
a_strides
,
b_strides
},
{
c_strides
},
Add
{});
if
(
!
broadcastAdd
.
IsSupportedArgument
(
argument
.
get
()))
{
...
...
This diff is collapsed.
Click to expand it.
example/19_binary_elementwise/elementwise_add_1d.cpp
View file @
f5de8b57
...
...
@@ -79,15 +79,17 @@ int main()
a_m_device_buf
.
ToDevice
(
a_m
.
mData
.
data
());
b_m_device_buf
.
ToDevice
(
b_m
.
mData
.
data
());
std
::
array
<
const
void
*
,
2
>
input
=
{
a_m_device_buf
.
GetDeviceBuffer
(),
b_m_device_buf
.
GetDeviceBuffer
()};
std
::
array
<
void
*
,
1
>
output
=
{
c_m_device_buf
.
GetDeviceBuffer
()};
std
::
vector
<
ck
::
index_t
>
a_strides
=
{
1
};
std
::
vector
<
ck
::
index_t
>
b_strides
=
{
1
};
std
::
vector
<
ck
::
index_t
>
c_strides
=
{
1
};
auto
broadcastAdd
=
DeviceElementwiseAddInstance
{};
auto
argument
=
broadcastAdd
.
MakeArgumentPointer
(
a_m_device_buf
.
GetDeviceBuffer
(),
b_m_device_buf
.
GetDeviceBuffer
(),
c_m_device_buf
.
GetDeviceBuffer
(),
{
M
},
{
1
},
{
1
},
{
1
},
Add
{});
auto
argument
=
broadcastAdd
.
MakeArgumentPointer
(
input
,
output
,
{
M
},
{{
a_strides
},
b_strides
},
{
c_strides
},
Add
{});
if
(
!
broadcastAdd
.
IsSupportedArgument
(
argument
.
get
()))
{
...
...
This diff is collapsed.
Click to expand it.
example/19_binary_elementwise/elementwise_add_4d.cpp
View file @
f5de8b57
...
...
@@ -81,16 +81,22 @@ int main()
a_device_buf
.
ToDevice
(
a
.
mData
.
data
());
b_device_buf
.
ToDevice
(
b
.
mData
.
data
());
std
::
array
<
const
void
*
,
2
>
input
=
{
a_device_buf
.
GetDeviceBuffer
(),
b_device_buf
.
GetDeviceBuffer
()};
std
::
array
<
void
*
,
1
>
output
=
{
c_device_buf
.
GetDeviceBuffer
()};
std
::
vector
<
ck
::
index_t
>
a_strides
{
a
.
mDesc
.
GetStrides
().
begin
(),
a
.
mDesc
.
GetStrides
().
end
()};
std
::
vector
<
ck
::
index_t
>
b_strides
{
b
.
mDesc
.
GetStrides
().
begin
(),
b
.
mDesc
.
GetStrides
().
end
()};
std
::
vector
<
ck
::
index_t
>
c_strides
{
c
.
mDesc
.
GetStrides
().
begin
(),
c
.
mDesc
.
GetStrides
().
end
()};
auto
broadcastAdd
=
DeviceElementwiseAddInstance
{};
auto
argument
=
broadcastAdd
.
MakeArgumentPointer
(
a_device_buf
.
GetDeviceBuffer
(),
b_device_buf
.
GetDeviceBuffer
(),
c_device_buf
.
GetDeviceBuffer
(),
std
::
vector
<
ck
::
index_t
>
{
nchw
.
begin
(),
nchw
.
end
()},
std
::
vector
<
ck
::
index_t
>
{
a
.
mDesc
.
GetStrides
().
begin
(),
a
.
mDesc
.
GetStrides
().
end
()},
std
::
vector
<
ck
::
index_t
>
{
b
.
mDesc
.
GetStrides
().
begin
(),
b
.
mDesc
.
GetStrides
().
end
()},
std
::
vector
<
ck
::
index_t
>
{
c
.
mDesc
.
GetStrides
().
begin
(),
c
.
mDesc
.
GetStrides
().
end
()},
Add
{});
auto
argument
=
broadcastAdd
.
MakeArgumentPointer
(
input
,
output
,
std
::
vector
<
ck
::
index_t
>
{
nchw
.
begin
(),
nchw
.
end
()},
{{
a_strides
},
b_strides
},
{
c_strides
},
Add
{});
if
(
!
broadcastAdd
.
IsSupportedArgument
(
argument
.
get
()))
{
...
...
This diff is collapsed.
Click to expand it.
example/21_gemm_layernorm/gemm_bias_relu_add_layernorm_xdl_fp16.cpp
View file @
f5de8b57
This diff is collapsed.
Click to expand it.
example/21_gemm_layernorm/gemm_layernorm_xdl_fp16.cpp
View file @
f5de8b57
This diff is collapsed.
Click to expand it.
example/23_softmax/softmax_blockwise.cpp
View file @
f5de8b57
...
...
@@ -150,6 +150,9 @@ int main(int argc, char* argv[])
AccDataType
alpha
=
args
.
scales
[
0
];
AccDataType
beta
=
args
.
scales
[
1
];
std
::
cout
<<
"in: "
<<
in
.
mDesc
<<
std
::
endl
;
std
::
cout
<<
"out: "
<<
out
.
mDesc
<<
std
::
endl
;
std
::
size_t
num_thread
=
1
;
if
(
args
.
do_verification
)
...
...
@@ -195,7 +198,7 @@ int main(int argc, char* argv[])
using
ReferenceInstance
=
tensor_operation
::
host
::
ReferenceSoftmax
<
InDataType
,
OutDataType
,
AccDataType
>
;
ReferenceInstance
ref
;
auto
ref_arg
=
ref
.
MakeArgument
(
in
,
out_ref
,
alpha
,
beta
,
Rank
,
reduceDims
);
auto
ref_arg
=
ref
.
MakeArgument
(
in
,
out_ref
,
alpha
,
beta
,
reduceDims
);
auto
invoker
=
ref
.
MakeInvoker
();
invoker
.
Run
(
ref_arg
);
// LogRangeAsType<float>(std::cout << "tensor out_ref: ", out_ref.mData, ",") << std::endl;
...
...
@@ -212,8 +215,8 @@ int main(int argc, char* argv[])
auto
argument_ptr
=
device_instance
.
MakeArgumentPointer
(
i_inLengths
,
i_inStrides
,
reduceDims
,
alpha
,
beta
,
&
alpha
,
&
beta
,
in_dev
.
GetDeviceBuffer
(),
out_dev
.
GetDeviceBuffer
());
...
...
This diff is collapsed.
Click to expand it.
example/25_gemm_bias_c_permute/CMakeLists.txt
0 → 100644
View file @
f5de8b57
add_example_executable
(
example_gemm_bias_c_permute_xdl_fp16 gemm_bias_c_permute_xdl_fp16.cpp
)
This diff is collapsed.
Click to expand it.
example/25_gemm_bias_c_permute/gemm_bias_c_permute_xdl_fp16.cpp
0 → 100644
View file @
f5de8b57
// SPDX-License-Identifier: MIT
// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
#include <iostream>
#include <numeric>
#include <initializer_list>
#include <cstdlib>
#include "ck/ck.hpp"
#include "ck/tensor_operation/gpu/device/tensor_layout.hpp"
#include "ck/tensor_operation/gpu/device/gemm_specialization.hpp"
#include "ck/tensor_operation/gpu/device/device_gemm_bias_c_permute_xdl.hpp"
#include "ck/tensor_operation/gpu/element/binary_element_wise_operation.hpp"
#include "ck/library/host_tensor/device_memory.hpp"
#include "ck/library/host_tensor/host_tensor.hpp"
#include "ck/library/host_tensor/host_tensor_generator.hpp"
#include "ck/library/reference_tensor_operation/cpu/reference_gemm.hpp"
#include "ck/library/utility/check_err.hpp"
template
<
ck
::
index_t
...
Is
>
using
S
=
ck
::
Sequence
<
Is
...
>
;
using
F16
=
ck
::
half_t
;
using
F32
=
float
;
using
Row
=
ck
::
tensor_layout
::
gemm
::
RowMajor
;
using
Col
=
ck
::
tensor_layout
::
gemm
::
ColumnMajor
;
using
PassThrough
=
ck
::
tensor_operation
::
element_wise
::
PassThrough
;
using
Add
=
ck
::
tensor_operation
::
element_wise
::
Add
;
using
ADataType
=
F16
;
using
BDataType
=
F16
;
using
AccDataType
=
F32
;
using
CShuffleDataType
=
F32
;
using
DDataType
=
F16
;
using
EDataType
=
F16
;
using
ALayout
=
Row
;
using
BLayout
=
Col
;
using
DLayout
=
Row
;
using
ELayout
=
Row
;
using
AElementOp
=
PassThrough
;
using
BElementOp
=
PassThrough
;
using
CDEElementOp
=
Add
;
static
constexpr
auto
GemmDefault
=
ck
::
tensor_operation
::
device
::
GemmSpecialization
::
Default
;
// clang-format off
using
DeviceOpInstance
=
ck
::
tensor_operation
::
device
::
DeviceGemmBiasCPermute_Xdl
//######| ALayout| BLayout| ELayout| AData| BData| AccData| CShuffle| DsData| EData| A| B| CDE| GEMM| NumGemmK| Block| MPer| NPer| KPer| AK1| BK1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransferClusterLengths| CBlockTransfer|
//######| | | | Type| Type| Type| DataType| Type| Type| Elementwise| Elementwise| Elementwise| Spacialization| Prefetch| Size| Block| Block| Block| | | XDL| XDL| Per| Per| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraN| MXdlPerWave| NXdlPerWave| _MBlock_MWaveMPerXdl| ScalarPerVector|
//######| | | | | | | | | | Operation| Operation| Operation| | Stage| | | | | | | | | Wave| Wave| Lengths_K0_M_K1| ArrangeOrder| | | PerVector| PerVector_K1| | Lengths_K0_N_K1| ArrangeOrder| | | PerVector| PerVector_K1| | PerShuffle| PerShuffle| _NBlock_NWaveNPerXdl| _NWaveNPerXdl|
//######| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
<
ALayout
,
BLayout
,
ELayout
,
ADataType
,
BDataType
,
AccDataType
,
CShuffleDataType
,
DDataType
,
EDataType
,
AElementOp
,
BElementOp
,
CDEElementOp
,
GemmDefault
,
1
,
256
,
256
,
128
,
32
,
8
,
8
,
32
,
32
,
4
,
2
,
S
<
4
,
64
,
1
>
,
S
<
1
,
0
,
2
>
,
S
<
1
,
0
,
2
>
,
2
,
8
,
8
,
1
,
S
<
4
,
64
,
1
>
,
S
<
1
,
0
,
2
>
,
S
<
1
,
0
,
2
>
,
2
,
8
,
8
,
1
,
1
,
1
,
S
<
1
,
32
,
1
,
8
>
,
1
>
;
// clang-format on
int
main
(
int
argc
,
char
*
argv
[])
{
bool
do_verification
=
true
;
int
init_method
=
1
;
bool
time_kernel
=
false
;
ck
::
index_t
M0
=
4
;
ck
::
index_t
M1
=
32
;
ck
::
index_t
M2
=
128
;
ck
::
index_t
N0
=
16
;
ck
::
index_t
N1
=
256
;
// GEMM shape
ck
::
index_t
M
=
M0
*
M1
*
M2
;
ck
::
index_t
N
=
N0
*
N1
;
ck
::
index_t
K
=
128
;
ck
::
index_t
stride_A
=
K
;
ck
::
index_t
stride_B
=
K
;
#if 1
// E = [M0, N0, M1, N1, M2]
ck
::
index_t
stride_E_M0
=
N0
*
M1
*
N1
*
M2
;
ck
::
index_t
stride_E_M1
=
N1
*
M2
;
ck
::
index_t
stride_E_M2
=
1
;
ck
::
index_t
stride_E_N0
=
M1
*
N1
*
M2
;
ck
::
index_t
stride_E_N1
=
M2
;
// D = [0, N0, 0, N1, 0]
ck
::
index_t
stride_D_M0
=
0
;
ck
::
index_t
stride_D_M1
=
0
;
ck
::
index_t
stride_D_M2
=
0
;
ck
::
index_t
stride_D_N0
=
N1
;
ck
::
index_t
stride_D_N1
=
1
;
#else
// D = [0, 0, 0, N0, N1]
ck
::
index_t
stride_D_M0
=
0
;
ck
::
index_t
stride_D_M1
=
0
;
ck
::
index_t
stride_D_M2
=
0
;
ck
::
index_t
stride_D_N0
=
N1
;
ck
::
index_t
stride_D_N1
=
1
;
// E = [M0, M1, M2, N0, N1]
ck
::
index_t
stride_E_M0
=
M1
*
M2
*
N0
*
N1
;
ck
::
index_t
stride_E_M1
=
M2
*
N0
*
N1
;
ck
::
index_t
stride_E_M2
=
N0
*
N1
;
ck
::
index_t
stride_E_N0
=
N1
;
ck
::
index_t
stride_E_N1
=
1
;
#endif
const
ck
::
tensor_operation
::
device
::
DEGridDesc_M0_M1_M2_N0_N1
d_grid_desc
{
M0
,
M1
,
M2
,
N0
,
N1
,
stride_D_M0
,
stride_D_M1
,
stride_D_M2
,
stride_D_N0
,
stride_D_N1
};
const
ck
::
tensor_operation
::
device
::
DEGridDesc_M0_M1_M2_N0_N1
e_grid_desc
{
M0
,
M1
,
M2
,
N0
,
N1
,
stride_E_M0
,
stride_E_M1
,
stride_E_M2
,
stride_E_N0
,
stride_E_N1
};
if
(
argc
==
1
)
{
// use default case
}
else
if
(
argc
==
4
)
{
do_verification
=
std
::
stoi
(
argv
[
1
]);
init_method
=
std
::
stoi
(
argv
[
2
]);
time_kernel
=
std
::
stoi
(
argv
[
3
]);
}
else
{
printf
(
"arg1: verification (0=no, 1=yes)
\n
"
);
printf
(
"arg2: initialization (0=no init, 1=integer value, 2=decimal value)
\n
"
);
printf
(
"arg3: time kernel (0=no, 1=yes)
\n
"
);
exit
(
0
);
}
auto
f_host_tensor_descriptor
=
[](
std
::
size_t
row
,
std
::
size_t
col
,
std
::
size_t
stride
,
auto
layout
)
{
if
(
std
::
is_same
<
decltype
(
layout
),
ck
::
tensor_layout
::
gemm
::
RowMajor
>::
value
)
{
return
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
({
row
,
col
}),
std
::
vector
<
std
::
size_t
>
({
stride
,
1
}));
}
else
{
return
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
({
row
,
col
}),
std
::
vector
<
std
::
size_t
>
({
1
,
stride
}));
}
};
auto
f_host_de_tensor_descriptor
=
[](
ck
::
tensor_operation
::
device
::
DEGridDesc_M0_M1_M2_N0_N1
de_grid_desc
)
{
std
::
size_t
m0
=
de_grid_desc
.
M0_
;
std
::
size_t
m1
=
de_grid_desc
.
M1_
;
std
::
size_t
m2
=
de_grid_desc
.
M2_
;
std
::
size_t
n0
=
de_grid_desc
.
N0_
;
std
::
size_t
n1
=
de_grid_desc
.
N1_
;
std
::
size_t
stride_m0
=
de_grid_desc
.
stride_M0_
;
std
::
size_t
stride_m1
=
de_grid_desc
.
stride_M1_
;
std
::
size_t
stride_m2
=
de_grid_desc
.
stride_M2_
;
std
::
size_t
stride_n0
=
de_grid_desc
.
stride_N0_
;
std
::
size_t
stride_n1
=
de_grid_desc
.
stride_N1_
;
return
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
({
m0
,
m1
,
m2
,
n0
,
n1
}),
std
::
vector
<
std
::
size_t
>
({
stride_m0
,
stride_m1
,
stride_m2
,
stride_n0
,
stride_n1
}));
};
Tensor
<
ADataType
>
a_m_k
(
f_host_tensor_descriptor
(
M
,
K
,
stride_A
,
ALayout
{}));
Tensor
<
BDataType
>
b_k_n
(
f_host_tensor_descriptor
(
K
,
N
,
stride_B
,
BLayout
{}));
Tensor
<
DDataType
>
d_m0_m1_m2_n0_n1
(
f_host_de_tensor_descriptor
(
d_grid_desc
));
Tensor
<
EDataType
>
e_m0_m1_m2_n0_n1_host_result
(
f_host_de_tensor_descriptor
(
e_grid_desc
));
Tensor
<
EDataType
>
e_m0_m1_m2_n0_n1_device_result
(
f_host_de_tensor_descriptor
(
e_grid_desc
));
std
::
cout
<<
"a_m_k: "
<<
a_m_k
.
mDesc
<<
std
::
endl
;
std
::
cout
<<
"b_k_n: "
<<
b_k_n
.
mDesc
<<
std
::
endl
;
std
::
cout
<<
"d_m0_m1_m2_n0_n1: "
<<
d_m0_m1_m2_n0_n1
.
mDesc
<<
std
::
endl
;
std
::
cout
<<
"e_m0_m1_m2_n0_n1: "
<<
e_m0_m1_m2_n0_n1_host_result
.
mDesc
<<
std
::
endl
;
switch
(
init_method
)
{
case
0
:
break
;
case
1
:
a_m_k
.
GenerateTensorValue
(
GeneratorTensor_2
<
ADataType
>
{
-
5
,
5
});
b_k_n
.
GenerateTensorValue
(
GeneratorTensor_2
<
BDataType
>
{
-
5
,
5
});
d_m0_m1_m2_n0_n1
.
GenerateTensorValue
(
GeneratorTensor_2
<
DDataType
>
{
-
5
,
5
});
break
;
default:
a_m_k
.
GenerateTensorValue
(
GeneratorTensor_3
<
ADataType
>
{
0.0
,
1.0
});
b_k_n
.
GenerateTensorValue
(
GeneratorTensor_3
<
BDataType
>
{
-
0.5
,
0.5
});
d_m0_m1_m2_n0_n1
.
GenerateTensorValue
(
GeneratorTensor_3
<
DDataType
>
{
0.0
,
1.0
});
}
DeviceMem
a_m_k_device_buf
(
sizeof
(
ADataType
)
*
a_m_k
.
mDesc
.
GetElementSpace
());
DeviceMem
b_k_n_device_buf
(
sizeof
(
BDataType
)
*
b_k_n
.
mDesc
.
GetElementSpace
());
DeviceMem
d_m0_m1_m2_n0_n1_device_buf
(
sizeof
(
DDataType
)
*
d_m0_m1_m2_n0_n1
.
mDesc
.
GetElementSpace
());
DeviceMem
e_m0_m1_m2_n0_n1_device_buf
(
sizeof
(
EDataType
)
*
e_m0_m1_m2_n0_n1_device_result
.
mDesc
.
GetElementSpace
());
a_m_k_device_buf
.
ToDevice
(
a_m_k
.
mData
.
data
());
b_k_n_device_buf
.
ToDevice
(
b_k_n
.
mData
.
data
());
d_m0_m1_m2_n0_n1_device_buf
.
ToDevice
(
d_m0_m1_m2_n0_n1
.
mData
.
data
());
auto
a_element_op
=
AElementOp
{};
auto
b_element_op
=
BElementOp
{};
auto
cde_element_op
=
CDEElementOp
{};
// do GEMM
auto
device_op
=
DeviceOpInstance
{};
auto
invoker
=
device_op
.
MakeInvoker
();
auto
argument
=
device_op
.
MakeArgument
(
a_m_k_device_buf
.
GetDeviceBuffer
(),
b_k_n_device_buf
.
GetDeviceBuffer
(),
d_m0_m1_m2_n0_n1_device_buf
.
GetDeviceBuffer
(),
e_m0_m1_m2_n0_n1_device_buf
.
GetDeviceBuffer
(),
M
,
N
,
K
,
stride_A
,
stride_B
,
d_grid_desc
,
e_grid_desc
,
a_element_op
,
b_element_op
,
cde_element_op
);
if
(
!
device_op
.
IsSupportedArgument
(
argument
))
{
throw
std
::
runtime_error
(
"wrong! this device_op instance does not support this problem"
);
}
float
ave_time
=
invoker
.
Run
(
argument
,
StreamConfig
{
nullptr
,
time_kernel
});
std
::
size_t
flop
=
std
::
size_t
(
2
)
*
M
*
N
*
K
;
std
::
size_t
num_btype
=
sizeof
(
ADataType
)
*
M
*
K
+
sizeof
(
BDataType
)
*
K
*
N
+
sizeof
(
DDataType
)
*
N
+
sizeof
(
EDataType
)
*
M
*
N
;
float
tflops
=
static_cast
<
float
>
(
flop
)
/
1.E9
/
ave_time
;
float
gb_per_sec
=
num_btype
/
1.E6
/
ave_time
;
std
::
cout
<<
"Perf: "
<<
ave_time
<<
" ms, "
<<
tflops
<<
" TFlops, "
<<
gb_per_sec
<<
" GB/s, "
<<
device_op
.
GetTypeString
()
<<
std
::
endl
;
if
(
do_verification
)
{
Tensor
<
AccDataType
>
c_m_n
(
HostTensorDescriptor
(
std
::
vector
<
std
::
size_t
>
{
static_cast
<
std
::
size_t
>
(
M
),
static_cast
<
std
::
size_t
>
(
N
)}));
using
ReferenceGemmInstance
=
ck
::
tensor_operation
::
host
::
ReferenceGemm
<
ADataType
,
BDataType
,
AccDataType
,
AccDataType
,
AElementOp
,
BElementOp
,
PassThrough
>
;
auto
ref_gemm
=
ReferenceGemmInstance
{};
auto
ref_invoker
=
ref_gemm
.
MakeInvoker
();
auto
ref_argument
=
ref_gemm
.
MakeArgument
(
a_m_k
,
b_k_n
,
c_m_n
,
a_element_op
,
b_element_op
,
PassThrough
{});
ref_invoker
.
Run
(
ref_argument
);
for
(
int
m0
=
0
;
m0
<
M0
;
++
m0
)
for
(
int
m1
=
0
;
m1
<
M1
;
++
m1
)
for
(
int
m2
=
0
;
m2
<
M2
;
++
m2
)
for
(
int
n0
=
0
;
n0
<
N0
;
++
n0
)
for
(
int
n1
=
0
;
n1
<
N1
;
++
n1
)
{
int
m
=
m0
*
M1
*
M2
+
m1
*
M2
+
m2
;
int
n
=
n0
*
N1
+
n1
;
cde_element_op
(
e_m0_m1_m2_n0_n1_host_result
(
m0
,
m1
,
m2
,
n0
,
n1
),
ck
::
type_convert
<
EDataType
>
(
c_m_n
(
m
,
n
)),
d_m0_m1_m2_n0_n1
(
m0
,
m1
,
m2
,
n0
,
n1
));
}
e_m0_m1_m2_n0_n1_device_buf
.
FromDevice
(
e_m0_m1_m2_n0_n1_device_result
.
mData
.
data
());
return
ck
::
utils
::
check_err
(
e_m0_m1_m2_n0_n1_device_result
.
mData
,
e_m0_m1_m2_n0_n1_host_result
.
mData
)
?
0
:
1
;
}
return
0
;
}
This diff is collapsed.
Click to expand it.
example/CMakeLists.txt
View file @
f5de8b57
...
...
@@ -42,3 +42,4 @@ add_subdirectory(20_convnd_bwd_weight_xdl)
add_subdirectory
(
21_gemm_layernorm
)
add_subdirectory
(
22_cgemm
)
add_subdirectory
(
23_softmax
)
add_subdirectory
(
25_gemm_bias_c_permute
)
This diff is collapsed.
Click to expand it.
include/ck/tensor_operation/gpu/device/device_5ary_elementwise.hpp
View file @
f5de8b57
...
...
@@ -10,7 +10,7 @@
#include "ck/utility/common_header.hpp"
#include "ck/tensor_description/tensor_descriptor.hpp"
#include "ck/tensor_description/tensor_descriptor_helper.hpp"
#include "ck/tensor_operation/gpu/device/device_
ba
se.hpp"
#include "ck/tensor_operation/gpu/device/device_
elementwi
se.hpp"
#include "ck/tensor_operation/gpu/grid/gridwise_5ary_Elementwise_1d.hpp"
#include "ck/device_utility/device_prop.hpp"
#include "ck/device_utility/kernel_launch.hpp"
...
...
@@ -35,7 +35,7 @@ template <typename ADataType,
index_t
DScalarPerVector
,
index_t
EScalarPerVector
,
index_t
FScalarPerVector
>
struct
Device5AryElementwise
:
public
BaseOpera
tor
struct
Device5AryElementwise
:
public
DeviceElementwise
<
5
,
1
,
NDim
,
ElementwiseFunc
tor
>
{
static
constexpr
auto
I0
=
Number
<
0
>
{};
...
...
@@ -268,12 +268,8 @@ struct Device5AryElementwise : public BaseOperator
return
true
;
};
static
auto
MakeArgument
(
const
ADataType
*
p_a
,
const
BDataType
*
p_b
,
const
CDataType
*
p_c
,
const
DDataType
*
p_d
,
const
EDataType
*
p_e
,
FDataType
*
p_f
,
static
auto
MakeArgument
(
std
::
array
<
const
void
*
,
5
>
p_inputs
,
std
::
array
<
void
*
,
1
>
p_outputs
,
std
::
vector
<
index_t
>
lengths
,
std
::
vector
<
index_t
>
a_strides
,
std
::
vector
<
index_t
>
b_strides
,
...
...
@@ -283,12 +279,12 @@ struct Device5AryElementwise : public BaseOperator
std
::
vector
<
index_t
>
f_strides
,
ElementwiseFunctor
functor
)
{
return
Argument
{
p_a
,
p_b
,
p_c
,
p_d
,
p_e
,
p_f
,
return
Argument
{
static_cast
<
const
ADataType
*>
(
p_inputs
[
0
])
,
static_cast
<
const
BDataType
*>
(
p_inputs
[
1
])
,
static_cast
<
const
CDataType
*>
(
p_inputs
[
2
])
,
static_cast
<
const
DDataType
*>
(
p_inputs
[
3
])
,
static_cast
<
const
EDataType
*>
(
p_inputs
[
4
])
,
static_cast
<
FDataType
*>
(
p_outputs
[
0
])
,
lengths
,
a_strides
,
b_strides
,
...
...
@@ -299,40 +295,58 @@ struct Device5AryElementwise : public BaseOperator
functor
};
}
std
::
unique_ptr
<
BaseArgument
>
MakeArgumentPointer
(
const
void
*
p_a
,
const
void
*
p_b
,
const
void
*
p_c
,
const
void
*
p_d
,
const
void
*
p_e
,
void
*
p_f
,
std
::
vector
<
index_t
>
lengths
,
std
::
vector
<
index_t
>
a_strides
,
std
::
vector
<
index_t
>
b_strides
,
std
::
vector
<
index_t
>
c_strides
,
std
::
vector
<
index_t
>
d_strides
,
std
::
vector
<
index_t
>
e_strides
,
std
::
vector
<
index_t
>
f_strides
,
ElementwiseFunctor
functor
)
std
::
unique_ptr
<
BaseArgument
>
MakeArgumentPointer
(
std
::
array
<
const
void
*
,
5
>
p_inputs
,
std
::
array
<
void
*
,
1
>
p_outputs
,
std
::
vector
<
index_t
>
lengths
,
std
::
vector
<
std
::
vector
<
index_t
>>
input_strides
,
std
::
vector
<
std
::
vector
<
index_t
>>
output_strides
,
ElementwiseFunctor
functor
)
override
{
return
std
::
make_unique
<
Argument
>
(
static_cast
<
const
ADataType
*>
(
p_
a
),
static_cast
<
const
BDataType
*>
(
p_
b
),
static_cast
<
const
CDataType
*>
(
p_
c
),
static_cast
<
const
DDataType
*>
(
p_
d
),
static_cast
<
const
EDataType
*>
(
p_
e
),
static_cast
<
FDataType
*>
(
p_
f
),
return
std
::
make_unique
<
Argument
>
(
static_cast
<
const
ADataType
*>
(
p_
inputs
[
0
]
),
static_cast
<
const
BDataType
*>
(
p_
inputs
[
1
]
),
static_cast
<
const
CDataType
*>
(
p_
inputs
[
2
]
),
static_cast
<
const
DDataType
*>
(
p_
inputs
[
3
]
),
static_cast
<
const
EDataType
*>
(
p_
inputs
[
4
]
),
static_cast
<
FDataType
*>
(
p_
outputs
[
0
]
),
lengths
,
a
_strides
,
b
_strides
,
c
_strides
,
d
_strides
,
e
_strides
,
f
_strides
,
input
_strides
[
0
]
,
input
_strides
[
1
]
,
input
_strides
[
2
]
,
input
_strides
[
3
]
,
input
_strides
[
4
]
,
output
_strides
[
0
]
,
functor
);
}
static
auto
MakeInvoker
()
{
return
Invoker
{};
}
std
::
unique_ptr
<
BaseInvoker
>
MakeInvokerPointer
()
{
return
std
::
make_unique
<
Invoker
>
();
}
};
std
::
unique_ptr
<
BaseInvoker
>
MakeInvokerPointer
()
override
{
return
std
::
make_unique
<
Invoker
>
();
}
// polymorphic
std
::
string
GetTypeString
()
const
override
{
auto
str
=
std
::
stringstream
();
// clang-format off
str
<<
"Device5aryElementwise"
<<
"<"
<<
"NDim = "
<<
NDim
<<
"MPerThread = "
<<
MPerThread
<<
"AScalarPerVector = "
<<
AScalarPerVector
<<
"BScalarPerVector = "
<<
BScalarPerVector
<<
"CScalarPerVector = "
<<
CScalarPerVector
<<
"DScalarPerVector = "
<<
DScalarPerVector
<<
"EScalarPerVector = "
<<
EScalarPerVector
<<
"FScalarPerVector = "
<<
FScalarPerVector
<<
">"
;
// clang-format on
return
str
.
str
();
}
};
// namespace device
}
// namespace device
}
// namespace tensor_operation
...
...
This diff is collapsed.
Click to expand it.
include/ck/tensor_operation/gpu/device/device_batched_gemm_reduce_xdl_cshuffle.hpp
View file @
f5de8b57
This diff is collapsed.
Click to expand it.
include/ck/tensor_operation/gpu/device/device_binary_elementwise.hpp
View file @
f5de8b57
...
...
@@ -9,6 +9,7 @@
#include "ck/device_utility/device_prop.hpp"
#include "ck/device_utility/kernel_launch.hpp"
#include "ck/tensor_operation/gpu/device/device_base.hpp"
#include "ck/tensor_operation/gpu/device/device_elementwise.hpp"
#include "ck/tensor_operation/gpu/grid/gridwise_binary_elementwise_1d.hpp"
namespace
ck
{
...
...
@@ -25,7 +26,7 @@ template <typename ADataType,
index_t
AScalarPerVector
,
index_t
BScalarPerVector
,
index_t
CScalarPerVector
>
struct
DeviceBinaryElementwise
:
public
BaseOpera
tor
struct
DeviceBinaryElementwise
:
public
DeviceElementwise
<
2
,
1
,
NDim
,
ElementwiseFunc
tor
>
{
static
constexpr
auto
I0
=
Number
<
0
>
{};
...
...
@@ -198,27 +199,30 @@ struct DeviceBinaryElementwise : public BaseOperator
return
true
;
};
std
::
unique_ptr
<
BaseArgument
>
MakeArgumentPointer
(
const
void
*
p_a
,
const
void
*
p_b
,
void
*
p_c
,
std
::
vector
<
index_t
>
lengths
,
std
::
vector
<
index_t
>
a_strides
,
std
::
vector
<
index_t
>
b_strides
,
std
::
vector
<
index_t
>
c_strides
,
ElementwiseFunctor
functor
)
virtual
std
::
unique_ptr
<
BaseArgument
>
MakeArgumentPointer
(
std
::
array
<
const
void
*
,
2
>
p_inputs
,
std
::
array
<
void
*
,
1
>
p_outputs
,
std
::
vector
<
index_t
>
lengths
,
std
::
vector
<
std
::
vector
<
index_t
>>
input_strides
,
std
::
vector
<
std
::
vector
<
index_t
>>
output_strides
,
ElementwiseFunctor
functor
)
override
{
return
std
::
make_unique
<
Argument
>
(
static_cast
<
const
ADataType
*>
(
p_
a
),
static_cast
<
const
BDataType
*>
(
p_
b
),
static_cast
<
CDataType
*>
(
p_
c
),
return
std
::
make_unique
<
Argument
>
(
static_cast
<
const
ADataType
*>
(
p_
inputs
[
0
]
),
static_cast
<
const
BDataType
*>
(
p_
inputs
[
1
]
),
static_cast
<
CDataType
*>
(
p_
outputs
[
0
]
),
lengths
,
a
_strides
,
b
_strides
,
c
_strides
,
input
_strides
[
0
]
,
input
_strides
[
1
]
,
output
_strides
[
0
]
,
functor
);
}
std
::
unique_ptr
<
BaseInvoker
>
MakeInvokerPointer
()
{
return
std
::
make_unique
<
Invoker
>
();
}
std
::
unique_ptr
<
BaseInvoker
>
MakeInvokerPointer
()
override
{
return
std
::
make_unique
<
Invoker
>
();
}
// polymorphic
std
::
string
GetTypeString
()
const
override
{
auto
str
=
std
::
stringstream
();
...
...
@@ -226,7 +230,11 @@ struct DeviceBinaryElementwise : public BaseOperator
// clang-format off
str
<<
"DeviceBinaryElementwise"
<<
"<"
<<
"NDim = "
<<
NDim
<<
"MPerThread = "
<<
MPerThread
<<
"AScalarPerVector = "
<<
AScalarPerVector
<<
"BScalarPerVector = "
<<
BScalarPerVector
<<
"CScalarPerVector = "
<<
CScalarPerVector
<<
">"
;
// clang-format on
...
...
This diff is collapsed.
Click to expand it.
Prev
1
2
3
4
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment
Menu
Projects
Groups
Snippets
Help