Skip to content

[CPU] Add test case of setting model_distribution_policy to TENSOR_PARALLEL #29322

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 25 commits into from
Jul 8, 2025
Merged

[CPU] Add test case of setting model_distribution_policy to TENSOR_PARALLEL #29322

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
f976de7
add test case of Tensor Parallel
sunxiaoxia2022 Mar 7, 2025
6526aee
add accuracy test case
sunxiaoxia2022 Mar 11, 2025
285d66e
enable TP on single socket
wangleis Mar 11, 2025
8f4f399
fix accuracy issue
sunxiaoxia2022 Mar 13, 2025
f3619cc
Merge branch 'master' into xiaoxia/tensor_parallel_test_case
sunxiaoxia2022 Mar 13, 2025
1266258
change test case
sunxiaoxia2022 Mar 13, 2025
6d469f4
Merge branch 'master' into xiaoxia/tensor_parallel_test_case
wangleis Mar 19, 2025
f1df5f7
Merge branch 'master' into xiaoxia/tensor_parallel_test_case
sunxiaoxia2022 Mar 24, 2025
3073dc1
fix ci test
sunxiaoxia2022 Mar 24, 2025
96ac699
add test cases to MatMul and MatMulSharedCompressedWeights
sunxiaoxia2022 Mar 26, 2025
808a4d1
fix test case issue
sunxiaoxia2022 Mar 27, 2025
15ded70
fix conflict
sunxiaoxia2022 Apr 18, 2025
f30463f
add property is_test for testing PIPELINE_PARALLEL
sunxiaoxia2022 Apr 21, 2025
af92bd0
code style
sunxiaoxia2022 Apr 21, 2025
db136d2
change ov::hint::is_test to ov::intel_cpu::is_test
sunxiaoxia2022 Apr 24, 2025
639ebbd
change is_test to enable_tensor_parallel
sunxiaoxia2022 May 15, 2025
8b6b2d9
move enable_tensor_parallel to openvino/src/plugins/intel_cpu/src/int…
sunxiaoxia2022 May 26, 2025
08dda20
fix confilct
sunxiaoxia2022 May 26, 2025
137029f
Merge branch 'master' into xiaoxia/tensor_parallel_test_case
wangleis May 27, 2025
4480b46
fix conflict
sunxiaoxia2022 May 28, 2025
bd080a5
Merge branch 'xiaoxia/tensor_parallel_test_case' of https://github.co…
sunxiaoxia2022 May 28, 2025
e4abb5b
code style
sunxiaoxia2022 May 28, 2025
af900d2
remove smoke_Model_Distribution_MatMulSharedCompressedWeights on non …
sunxiaoxia2022 May 28, 2025
66cd885
fix conflict
sunxiaoxia2022 Jul 8, 2025
6200ef8
Merge branch 'master' into xiaoxia/tensor_parallel_test_case
sunxiaoxia2022 Jul 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/inference/dev_api/openvino/runtime/threading/istreams_executor.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,8 @@ class OPENVINO_RUNTIME_API IStreamsExecutor : virtual public ITaskExecutor {
bool operator==(const Config& config) {
if (_name == config._name && _streams == config._streams &&
_threads_per_stream == config._threads_per_stream &&
_thread_preferred_core_type == config._thread_preferred_core_type) {
_thread_preferred_core_type == config._thread_preferred_core_type &&
_rank == config._rank) {
return true;
} else {
return false;
Expand Down
10 changes: 10 additions & 0 deletions src/plugins/intel_cpu/src/cpu_streams_calculation.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -500,6 +500,16 @@ std::vector<std::vector<int>> get_streams_info_table(
stream_table_size = streams_info_table.size();
}
}

if ((total_streams == 1) && (proc_type_table.size() == 1) &&
(hint_model_distribution_policy.find(ov::hint::ModelDistributionPolicy::TENSOR_PARALLEL) !=
hint_model_distribution_policy.end())) {
streams_info_table.push_back(streams_info_table[0]);
streams_info_table.push_back(streams_info_table[0]);
streams_info_table[0][THREADS_PER_STREAM] = streams_info_table[0][THREADS_PER_STREAM] * 2;
streams_info_table[1][NUMBER_OF_STREAMS] = -1;
streams_info_table[2][NUMBER_OF_STREAMS] = -1;
}
}
} else if (proc_type_table.size() == 1) {
if (stream_info[PROC_TYPE] == ALL_PROC) {
Expand Down
4 changes: 2 additions & 2 deletions src/plugins/intel_cpu/src/infer_request.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -621,8 +621,8 @@ void SyncInferRequest::sub_streams_infer() {

if (requests.size() > 0) {
for (const auto& output : outputs) {
auto tensor = requests[0]->get_tensor(output);
set_tensor(output, tensor);
auto tensor = get_tensor(output);
requests[0]->set_tensor(output, tensor);
}
for (size_t i = 0; i < requests_num; i++) {
for (auto& input : inputs) {
Expand Down
61 changes: 61 additions & 0 deletions src/plugins/intel_cpu/tests/functional/custom/behavior/ov_executable_network/properties.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@

#include <gtest/gtest.h>

#include "common_test_utils/ov_tensor_utils.hpp"
#include "common_test_utils/subgraph_builders/matmul_bias.hpp"
#include "openvino/runtime/compiled_model.hpp"
#include "openvino/runtime/core.hpp"
#include "openvino/runtime/intel_cpu/properties.hpp"
Expand Down Expand Up @@ -518,4 +520,63 @@ TEST_F(OVClassConfigTestCPU, smoke_CpuExecNetworkCheckCPURuntimOptionsWithCorePr
ASSERT_EQ(valueCacheType.as<ov::element::Type>(), ov::element::bf16);
}

TEST_F(OVClassConfigTestCPU, smoke_CpuModelDistributionPolicyTensorParallel) {
ov::Core core;
std::shared_ptr<ov::Model> model = ov::test::utils::make_matmul_bias();
std::set<ov::hint::ModelDistributionPolicy> setModels = {ov::hint::ModelDistributionPolicy::TENSOR_PARALLEL};
ov::AnyMap config = {{ov::hint::model_distribution_policy.name(), setModels},
{ov::num_streams.name(), 1},
{ov::inference_num_threads.name(), 1}};

core.set_property(deviceName, config);
ov::CompiledModel compiledModel = core.compile_model(model, deviceName);

std::set<ov::hint::ModelDistributionPolicy> model_distribution_policy_value = {};
OV_ASSERT_NO_THROW(model_distribution_policy_value = compiledModel.get_property(ov::hint::model_distribution_policy));
ASSERT_EQ(model_distribution_policy_value, setModels);
}

TEST_F(OVClassConfigTestCPU, smoke_CpuModelDistributionPolicyTensorParallelAccurcay) {
Copy link

@dmitry-gorokhov dmitry-gorokhov Mar 24, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It has to be usual functional subgraph test, which already have all the infra for output comparision and inputs filling.
Besides that we have to cover cases like matmul with compressed weights.
It also worth to verify execution graph to make sure TP was actually applied

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, I added test case about TP to mat_mul.cpp and shared_matmul_weights_decompression.cpp. Please have a look.

ov::Core core;
std::shared_ptr<ov::Model> model = ov::test::utils::make_matmul_bias();
std::set<ov::hint::ModelDistributionPolicy> setModels = {ov::hint::ModelDistributionPolicy::TENSOR_PARALLEL};
ov::AnyMap config_model = {{ov::hint::model_distribution_policy.name(), setModels},
{ov::num_streams.name(), 1},
{ov::inference_num_threads.name(), 1}};

core.set_property(deviceName, config_model);

std::map<ov::Output<ov::Node>, ov::Tensor> inputs;
for (const auto& input : model->inputs()) {
auto tensor = ov::test::utils::create_and_fill_tensor_normal_distribution(input.get_element_type(),
input.get_shape(),
0.0f,
0.2f,
7235346);
inputs.insert({input, tensor});
}

auto getOutputBlob = [&](ov::Core& core) {
auto compiled_model = core.compile_model(model, deviceName);
auto req = compiled_model.create_infer_request();
for (const auto& input : inputs) {
req.set_tensor(input.first, input.second);
}
auto output_tensor = ov::Tensor(model->output().get_element_type(), model->output().get_shape());
req.set_output_tensor(output_tensor);
req.infer();
return output_tensor;
};

auto outputActual = getOutputBlob(core);

{
ov::Core coreRef;
ov::AnyMap config = {{ov::num_streams.name(), 1}, {ov::inference_num_threads.name(), 1}};
coreRef.set_property(deviceName, config);
auto outputRef = getOutputBlob(coreRef);
ov::test::utils::compare(outputActual, outputRef);
}
}

} // namespace
30 changes: 29 additions & 1 deletion src/plugins/intel_cpu/tests/unit/streams_info/streams_info_table_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2643,6 +2643,32 @@ StreamsCalculationTestCase _2sockets_mock_latency_51 = {
{{16, 16, 0, 0, -1, -1}, {8, 8, 0, 0, 0, 0}, {8, 8, 0, 0, 1, 1}},
{{1, ALL_PROC, 16, -1, -1}, {0, MAIN_CORE_PROC, 8, 0, 0}, {0, MAIN_CORE_PROC, 8, 1, 1}},
};
StreamsCalculationTestCase _1sockets_mock_TP_1 = {
1,
false,
0,
0,
0,
"LATENCY",
{ov::hint::ModelDistributionPolicy::TENSOR_PARALLEL},
{{8, 8, 0, 0, 0, 0}},
{{1, MAIN_CORE_PROC, 16, 0, 0},
{-1, MAIN_CORE_PROC, 8, 0, 0},
{-1, MAIN_CORE_PROC, 8, 0, 0}},
};
StreamsCalculationTestCase _1sockets_mock_TP_2 = {
1,
false,
1,
0,
0,
"LATENCY",
{ov::hint::ModelDistributionPolicy::TENSOR_PARALLEL},
{{8, 8, 0, 0, 0, 0}},
{{1, MAIN_CORE_PROC, 2, 0, 0},
{-1, MAIN_CORE_PROC, 1, 0, 0},
{-1, MAIN_CORE_PROC, 1, 0, 0}},
};

TEST_P(StreamsCalculationTests, StreamsCalculation) {}

Expand Down Expand Up @@ -2843,6 +2869,8 @@ INSTANTIATE_TEST_SUITE_P(StreamsInfoTable,
_1sockets_mock_latency_3,
_1sockets_mock_latency_4,
_1sockets_mock_latency_5,
_1sockets_mock_latency_6));
_1sockets_mock_latency_6,
_1sockets_mock_TP_1,
_1sockets_mock_TP_2));

} // namespace
Loading