Skip to content

Commit 6a8af51

Browse files
oeqqwqoeqqwq
authored
repo-sync-2024-10-22T19:34:07+0800 (#121)
* repo-sync-2024-10-22T19:34:07+0800 * fix build * limit bazel jobs * fix build * fix build
1 parent f2a6b02 commit 6a8af51

36 files changed

+648
-253
lines changed

‎.ci/inferencer_test.py

+37-23
Original file line numberDiff line numberDiff line change
@@ -16,27 +16,12 @@
1616
from importlib import resources
1717
import asyncio
1818
import os
19+
import pandas as pd
20+
import numpy as np
1921

2022
current_file_path = os.path.abspath(__file__)
2123
code_dir = os.path.dirname(os.path.dirname(current_file_path))
2224

23-
alice_serving_config_file_path = os.path.join(
24-
code_dir,
25-
"secretflow_serving/tools/inferencer/example/alice/serving.config",
26-
)
27-
alice_inference_config_file_path = os.path.join(
28-
code_dir,
29-
"secretflow_serving/tools/inferencer/example/alice/inference.config",
30-
)
31-
bob_serving_config_file_path = os.path.join(
32-
code_dir,
33-
"secretflow_serving/tools/inferencer/example/bob/serving.config",
34-
)
35-
bob_inference_config_file_path = os.path.join(
36-
code_dir,
37-
"secretflow_serving/tools/inferencer/example/bob/inference.config",
38-
)
39-
4025

4126
async def run_process(command):
4227
process = await asyncio.create_subprocess_exec(
@@ -53,26 +38,55 @@ async def run_process(command):
5338
)
5439

5540

56-
async def main():
41+
async def run_inferencer_example(exmaple_dir: str, result_path: str, target_path: str):
42+
print(f"====begin example: {exmaple_dir}=====")
43+
5744
with resources.path(
5845
'secretflow_serving.tools.inferencer', 'inferencer'
5946
) as tool_path:
6047
alice_command = [
6148
str(tool_path),
62-
f'--serving_config_file={alice_serving_config_file_path}',
63-
f'--inference_config_file={alice_inference_config_file_path}',
49+
f'--serving_config_file={exmaple_dir}/alice/serving.config',
50+
f'--inference_config_file={exmaple_dir}/alice/inference.config',
6451
]
6552
bob_command = [
6653
str(tool_path),
67-
f'--serving_config_file={bob_serving_config_file_path}',
68-
f'--inference_config_file={bob_inference_config_file_path}',
54+
f'--serving_config_file={exmaple_dir}/bob/serving.config',
55+
f'--inference_config_file={exmaple_dir}/bob/inference.config',
6956
]
7057
commands = [alice_command, bob_command]
7158

7259
tasks = [run_process(command) for command in commands]
7360

7461
await asyncio.gather(*tasks)
7562

63+
result_df = pd.read_csv(result_path)
64+
target_df = pd.read_csv(target_path)
65+
66+
score_col = result_df['score']
67+
pred_col = target_df['pred']
68+
69+
assert len(score_col) == len(pred_col)
70+
71+
are_close = np.isclose(score_col, pred_col, atol=0.0001)
72+
73+
for i, match in enumerate(are_close):
74+
assert match, f"row {i} mismatch: {score_col[i]} != {pred_col[i]}"
75+
7676

7777
if __name__ == '__main__':
78-
asyncio.run(main())
78+
asyncio.run(
79+
run_inferencer_example(
80+
"secretflow_serving/tools/inferencer/example/normal",
81+
"tmp/alice/score.csv",
82+
".ci/test_data/bin_onehot_glm/predict.csv",
83+
)
84+
)
85+
86+
asyncio.run(
87+
run_inferencer_example(
88+
"secretflow_serving/tools/inferencer/example/one_party_no_feature",
89+
"tmp/bob/score.csv",
90+
".ci/test_data/bin_onehot_glm_alice_no_feature/predict.csv",
91+
)
92+
)

‎.circleci/continue-config.yml

+2
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ commands:
5050
- run:
5151
name: "build"
5252
command: |
53+
export BAZEL_MAX_JOBS=16
54+
5355
sh ./build_wheel_entrypoint.sh
5456
bazel build <<parameters.bazel_targets>> -c opt --jobs 16
5557
- run:

‎.circleci/release-config.yml

+1
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ commands:
5555
conda create -n build python=<< parameters.python_ver >> -y
5656
conda activate build
5757
58+
export BAZEL_MAX_JOBS=16
5859
sh ./build_wheel_entrypoint.sh
5960
python3 -m pip install twine
6061
ls python_lib/dist/*.whl

‎bazel/ipp.BUILD

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Copyright 2024 Ant Group Co., Ltd.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
load("@yacl//bazel:yacl.bzl", "yacl_cmake_external")
16+
17+
package(default_visibility = ["//visibility:public"])
18+
19+
filegroup(
20+
name = "all_srcs",
21+
srcs = glob(["**"]),
22+
)
23+
24+
yacl_cmake_external(
25+
name = "ipp",
26+
cache_entries = {
27+
"ARCH": "intel64",
28+
"OPENSSL_INCLUDE_DIR": "$EXT_BUILD_DEPS/openssl/include",
29+
"OPENSSL_LIBRARIES": "$EXT_BUILD_DEPS/openssl/lib",
30+
"OPENSSL_ROOT_DIR": "$EXT_BUILD_DEPS/openssl",
31+
"CMAKE_BUILD_TYPE": "Release",
32+
},
33+
lib_source = ":all_srcs",
34+
out_static_libs = [
35+
"intel64/libippcp.a",
36+
"intel64/libcrypto_mb.a",
37+
],
38+
deps = [
39+
"@com_github_openssl_openssl//:openssl",
40+
],
41+
)

‎bazel/patches/ippcp.patch

+172
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
diff --git a/sources/cmake/linux/GNU8.2.0.cmake b/sources/cmake/linux/GNU8.2.0.cmake
2+
index 24d7e0f..15dd433 100644
3+
--- a/sources/cmake/linux/GNU8.2.0.cmake
4+
+++ b/sources/cmake/linux/GNU8.2.0.cmake
5+
@@ -32,7 +32,7 @@ set(LINK_FLAG_DYNAMIC_LINUX "${LINK_FLAG_SECURITY} -nostdlib")
6+
# Dynamically link lib c (libdl is for old apps)
7+
set(LINK_FLAG_DYNAMIC_LINUX "${LINK_FLAG_DYNAMIC_LINUX} -Wl,-call_shared,-lc")
8+
# Create a shared library
9+
-set(LINK_FLAG_DYNAMIC_LINUX "-Wl,-shared")
10+
+set(LINK_FLAG_DYNAMIC_LINUX "-Wl,-shared,-fuse-ld=bfd")
11+
if(${ARCH} MATCHES "ia32")
12+
# Tells the compiler to generate code for a specific architecture (32)
13+
set(LINK_FLAG_DYNAMIC_LINUX "${LINK_FLAG_DYNAMIC_LINUX} -m32")
14+
@@ -74,7 +74,7 @@ if ((${ARCH} MATCHES "ia32") OR (NOT NONPIC_LIB))
15+
endif()
16+
17+
# Security flag that adds compile-time and run-time checks
18+
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_FORTIFY_SOURCE=2")
19+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2")
20+
21+
if(NOT NONPIC_LIB)
22+
# Position Independent Execution (PIE)
23+
@@ -95,6 +95,8 @@ if(${ARCH} MATCHES "ia32")
24+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")
25+
endif(${ARCH} MATCHES "ia32")
26+
27+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unknown-pragmas -Wno-missing-braces -Wno-comment -Wno-strict-aliasing -Wno-parentheses -Wno-array-parameter")
28+
+
29+
# Optimization level = 3, no-debug definition (turns off asserts), warnings=errors
30+
set (CMAKE_C_FLAGS_RELEASE " -O3 -DNDEBUG -Werror")
31+
32+
diff --git a/sources/cmake/macosx/AppleClang11.0.0.cmake b/sources/cmake/macosx/AppleClang11.0.0.cmake
33+
index 5b92877..ccb963e 100644
34+
--- a/sources/cmake/macosx/AppleClang11.0.0.cmake
35+
+++ b/sources/cmake/macosx/AppleClang11.0.0.cmake
36+
@@ -20,12 +20,6 @@
37+
38+
# Security Linker flags
39+
set(LINK_FLAG_SECURITY "")
40+
-# Disallows undefined symbols in object files. Undefined symbols in shared libraries are still allowed
41+
-set(LINK_FLAG_SECURITY "${LINK_FLAG_SECURITY} -Wl,-z,defs")
42+
-# Stack execution protection
43+
-set(LINK_FLAG_SECURITY "${LINK_FLAG_SECURITY} -Wl,-z,noexecstack")
44+
-# Data relocation and protection (RELRO)
45+
-set(LINK_FLAG_SECURITY "${LINK_FLAG_SECURITY} -Wl,-z,relro -Wl,-z,now")
46+
# Prevents the compiler from using standard libraries and startup files when linking.
47+
set(LINK_FLAG_DYNAMIC_MACOSX "${LINK_FLAG_SECURITY} -nostdlib")
48+
# Dynamically link lib c (libdl is for old apps)
49+
@@ -79,7 +73,7 @@ if ((${ARCH} MATCHES "ia32") OR (NOT NONPIC_LIB))
50+
endif()
51+
52+
# Security flag that adds compile-time and run-time checks
53+
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_FORTIFY_SOURCE=2")
54+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2")
55+
56+
if(NOT NONPIC_LIB)
57+
# Position Independent Execution (PIE)
58+
@@ -98,6 +92,8 @@ if(${ARCH} MATCHES "ia32")
59+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")
60+
endif(${ARCH} MATCHES "ia32")
61+
62+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unknown-pragmas -Wno-missing-braces -Wno-unused-command-line-argument -Wno-unused-but-set-variable -Wno-unknown-warning-option")
63+
+
64+
# Optimization level = 3, no-debug definition (turns off asserts), warnings=errors
65+
set (CMAKE_C_FLAGS_RELEASE " -O3 -DNDEBUG -Werror")
66+
67+
@@ -115,3 +111,5 @@ set(l9_opt "${l9_opt} -march=haswell -mavx2 -maes -mpclmul -msha -mrdrnd -mrdsee
68+
set(n0_opt "${n0_opt} -march=knl -mavx2 -maes -mavx512f -mavx512cd -mavx512pf -mavx512er -mpclmul -msha -mrdrnd -mrdseed")
69+
set(k0_opt "${k0_opt} -march=skylake-avx512")
70+
set(k0_opt "${k0_opt} -maes -mavx512f -mavx512cd -mavx512vl -mavx512bw -mavx512dq -mavx512ifma -mpclmul -msha -mrdrnd -mrdseed -madx -mgfni -mvaes -mvpclmulqdq -mavx512vbmi -mavx512vbmi2")
71+
+set(k1_opt "${k1_opt} -march=skylake-avx512")
72+
+set(k1_opt "${k1_opt} -maes -mavx512f -mavx512cd -mavx512vl -mavx512bw -mavx512dq -mavx512ifma -mpclmul -msha -mrdrnd -mrdseed -madx -mgfni -mvaes -mvpclmulqdq -mavx512vbmi -mavx512vbmi2")
73+
diff --git a/sources/cmake/macosx/common.cmake b/sources/cmake/macosx/common.cmake
74+
index 85ec3ad..67bb9f9 100644
75+
--- a/sources/cmake/macosx/common.cmake
76+
+++ b/sources/cmake/macosx/common.cmake
77+
@@ -18,7 +18,7 @@
78+
# Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography)
79+
#
80+
81+
-set(OS_DEFAULT_COMPILER Intel19.0.0)
82+
+set(OS_DEFAULT_COMPILER AppleClang11.0.0)
83+
84+
set(LIBRARY_DEFINES "${LIBRARY_DEFINES} -DIPP_PIC -DOSXEM64T -DLINUX32E -D_ARCH_EM64T")
85+
#set(LIBRARY_DEFINES "${LIBRARY_DEFINES} -DBN_OPENSSL_DISABLE")
86+
\ No newline at end of file
87+
diff --git a/sources/ippcp/crypto_mb/src/cmake/linux/GNU.cmake b/sources/ippcp/crypto_mb/src/cmake/linux/GNU.cmake
88+
index a2abeeb..67aca8b 100644
89+
--- a/sources/ippcp/crypto_mb/src/cmake/linux/GNU.cmake
90+
+++ b/sources/ippcp/crypto_mb/src/cmake/linux/GNU.cmake
91+
@@ -31,7 +31,7 @@ set(CMAKE_C_FLAGS_SECURITY "${CMAKE_C_FLAGS_SECURITY} -Wformat -Wformat-security
92+
if(${CMAKE_BUILD_TYPE} STREQUAL "Release")
93+
if(NOT DEFINED NO_FORTIFY_SOURCE)
94+
# Security flag that adds compile-time and run-time checks.
95+
- set(CMAKE_C_FLAGS_SECURITY "${CMAKE_C_FLAGS_SECURITY} -D_FORTIFY_SOURCE=2")
96+
+ set(CMAKE_C_FLAGS_SECURITY "${CMAKE_C_FLAGS_SECURITY} -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2")
97+
endif()
98+
endif()
99+
100+
@@ -51,7 +51,7 @@ set(CMAKE_C_FLAGS_SECURITY "${CMAKE_C_FLAGS_SECURITY} -Werror")
101+
# Linker flags
102+
103+
# Create shared library
104+
-set(LINK_FLAGS_DYNAMIC " -Wl,-shared")
105+
+set(LINK_FLAGS_DYNAMIC " -Wl,-shared,-fuse-ld=bfd")
106+
# Add export files
107+
set(LINK_FLAGS_DYNAMIC "${LINK_FLAGS_DYNAMIC} ${CRYPTO_MB_SOURCES_DIR}/cmake/dll_export/crypto_mb.linux.lib-export")
108+
109+
@@ -69,6 +69,7 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
110+
111+
# Suppress warnings from casts from a pointer to an integer type of a different size
112+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-pointer-to-int-cast")
113+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unknown-pragmas -Wno-missing-braces -Wno-comment -Wno-strict-aliasing -Wno-parentheses")
114+
115+
# Optimization level = 3, no-debug definition (turns off asserts)
116+
set(CMAKE_C_FLAGS_RELEASE " -O3 -DNDEBUG")
117+
diff --git a/sources/ippcp/crypto_mb/src/cmake/macosx/AppleClang.cmake b/sources/ippcp/crypto_mb/src/cmake/macosx/AppleClang.cmake
118+
index ea1641d..f98fc2d 100644
119+
--- a/sources/ippcp/crypto_mb/src/cmake/macosx/AppleClang.cmake
120+
+++ b/sources/ippcp/crypto_mb/src/cmake/macosx/AppleClang.cmake
121+
@@ -17,10 +17,6 @@
122+
# Security Linker flags
123+
124+
set(LINK_FLAG_SECURITY "")
125+
-# Data relocation and protection (RELRO)
126+
-set(LINK_FLAG_SECURITY "${LINK_FLAG_SECURITY} -Wl,-z,relro -Wl,-z,now")
127+
-# Stack execution protection
128+
-set(LINK_FLAG_SECURITY "${LINK_FLAG_SECURITY} -Wl,-z,noexecstack")
129+
130+
# Security Compiler flags
131+
132+
@@ -30,7 +26,7 @@ set(CMAKE_C_FLAGS_SECURITY "${CMAKE_C_FLAGS_SECURITY} -Wformat -Wformat-security
133+
134+
if(${CMAKE_BUILD_TYPE} STREQUAL "Release")
135+
# Security flag that adds compile-time and run-time checks.
136+
- set(CMAKE_C_FLAGS_SECURITY "${CMAKE_C_FLAGS_SECURITY} -D_FORTIFY_SOURCE=2")
137+
+ set(CMAKE_C_FLAGS_SECURITY "${CMAKE_C_FLAGS_SECURITY} -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2")
138+
endif()
139+
140+
# Stack-based Buffer Overrun Detection
141+
@@ -65,6 +61,8 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
142+
# Suppress warnings from casts from a pointer to an integer type of a different size
143+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-pointer-to-int-cast")
144+
145+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unknown-pragmas -Wno-missing-braces -Wno-unknown-warning-option")
146+
+
147+
# Optimization level = 3, no-debug definition (turns off asserts)
148+
set(CMAKE_C_FLAGS_RELEASE " -O3 -DNDEBUG")
149+
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
150+
151+
diff --git a/sources/cmake/linux/Clang9.0.0.cmake b/sources/cmake/linux/Clang9.0.0.cmake
152+
index 0015431..f93411c 100644
153+
--- a/sources/cmake/linux/Clang9.0.0.cmake
154+
+++ b/sources/cmake/linux/Clang9.0.0.cmake
155+
@@ -79,7 +79,7 @@ endif()
156+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fcf-protection=full")
157+
158+
# Security flag that adds compile-time and run-time checks
159+
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_FORTIFY_SOURCE=2")
160+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
161+
162+
if(NOT NONPIC_LIB)
163+
# Position Independent Execution (PIE)
164+
@@ -107,7 +107,7 @@ if(SANITIZERS)
165+
endif(SANITIZERS)
166+
167+
# Optimization level = 3, no-debug definition (turns off asserts), warnings=errors
168+
-set (CMAKE_C_FLAGS_RELEASE " -O3 -DNDEBUG -Werror")
169+
+set (CMAKE_C_FLAGS_RELEASE " -O3 -DNDEBUG -Werror -Wno-unused-function -Wno-missing-braces -Wno-unused-but-set-variable -Wno-unknown-pragmas")
170+
171+
# DEBUG flags - optimization level = 0, generation GDB information (-g)
172+
set (CMAKE_C_FLAGS_DEBUG " -O0 -g")

‎bazel/repositories.bzl

+18
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ def sf_serving_deps():
3333
_com_aws_checksums()
3434
_com_aws_sdk()
3535

36+
_com_github_intel_ipp()
37+
3638
_heu()
3739
_dataproxy()
3840

@@ -209,3 +211,19 @@ def _com_github_brpc_brpc():
209211
"https://github.com/apache/brpc/archive/refs/tags/1.9.0.tar.gz",
210212
],
211213
)
214+
215+
def _com_github_intel_ipp():
216+
maybe(
217+
http_archive,
218+
name = "com_github_intel_ipp",
219+
sha256 = "d70f42832337775edb022ca8ac1ac418f272e791ec147778ef7942aede414cdc",
220+
strip_prefix = "cryptography-primitives-ippcp_2021.8",
221+
build_file = "@sf_serving//bazel:ipp.BUILD",
222+
patch_args = ["-p1"],
223+
patches = [
224+
"@sf_serving//bazel:patches/ippcp.patch",
225+
],
226+
urls = [
227+
"https://github.com/intel/ipp-crypto/archive/refs/tags/ippcp_2021.8.tar.gz",
228+
],
229+
)

0 commit comments

Comments
 (0)