repo-sync-2025-01-02T18:56:23+0800 (#954)

- [SPU] Migrating to Bazel Modules and update minimum Python version to
3.10.
- [Feature] Add soPRF (LowMC cipher) for SEMI2K
- [API] Add Permute/InvPermute support in HLO
- [Feature] Add SSL configuration to the TTP server
- [Feature] Support quick sort for semi2k and aby3

---------

Co-authored-by: tongke6 <[email protected]>

Author: w-gc

.bazeliskrc

@@ -0,0 +1 @@
+USE_BAZEL_VERSION=7.4.1

.bazelrc

@@ -15,6 +15,10 @@
 common --experimental_repo_remote_exec
 common --experimental_cc_shared_library
 
+common --registry=https://raw.githubusercontent.com/secretflow/bazel-registry/main
+common --registry=https://bcr.bazel.build
+common --registry=https://baidu.github.io/babylon/registry
+
 # Required by OpenXLA
 build --nocheck_visibility
 
@@ -64,3 +68,6 @@ build:macos --action_env MACOSX_DEPLOYMENT_TARGET=13.0
 
 build:linux --copt=-fopenmp
 build:linux --linkopt=-fopenmp
+
+# NOTE: fix build on linux_aarch64, since `build:linux_aarch64` works only if `--config linux_aarch64` option included.
+build:linux --cxxopt=-Wno-mismatched-new-delete

.bazelversion

@@ -46,10 +46,6 @@ jobs:
       - run:
           name: Checkout devtools
           command: git clone https://github.com/secretflow/devtools.git ../devtools
-      - run:
-          name: "Install dependencies"
-          command: |
-            python3 -m pip install numpy
       - run:
           name: "test"
           command: |

.circleci/asan-config.yml

@@ -47,8 +47,6 @@ jobs:
                 "python3 -m pip install -U pip && \
                    python3 -m pip install spu && \
                    cd /home/admin/dev/ && \
-                   python3 -m pip install -r requirements-dev.txt && \
-                   python3 -m pip install -r examples/python/ml/requirements.txt && \
                    export PYTHONPATH="${PWD}:$PYTHONPATH" && \
                    bash .circleci/run-nn.sh" | tee benchmark_results.log
       - run:

.circleci/benchmark-config.yml

@@ -52,19 +52,17 @@ commands:
       - run:
           name: Checkout devtools
           command: git clone https://github.com/secretflow/devtools.git ../devtools
-      - run:
-          name: "Install dependencies"
-          command: |
-            python3 -m pip install -r requirements.txt
-            python3 -m pip install -r requirements-dev.txt
       - run:
           name: Setup GCS
           command: |
             echo ${gcs_content} > ../gcs.data
             ../devtools/bazel_cache_setup.py --in_file=../gcs.data --out_file=../gcs.json --min_download
       - run:
           name: "build"
-          command: bazel build <<parameters.targets>> -c opt --ui_event_filters=-info,-debug,-warning
+          command: |
+            bazel --version && python3 --version
+            if [ ! -e "/usr/bin/python3" ]; then ln -s `which python3` /usr/bin/python3; fi
+            bazel run //:requirements-dev.update && bazel build <<parameters.targets>> -c opt --ui_event_filters=-info,-debug,-warning
       - run:
           name: "test"
           command: |
@@ -114,7 +112,6 @@ jobs:
           name: Install extra deps
           command: |
             python3 -m pip install tzdata
-            python3 -m pip install -r examples/python/ml/requirements.txt
       - build_and_test:
           targets: //examples/python/ml:ml_test
           extra_bazel_test_args: --test_env LD_LIBRARY_PATH=/root/miniconda3/lib/
@@ -134,8 +131,6 @@ jobs:
             bash ~/miniconda.sh -b -p $HOME/miniconda
             source $HOME/miniconda/bin/activate
             conda init bash zsh
-            pip install -r requirements.txt
-            pip install -r requirements-dev.txt
       - build_and_test
 # Invoke jobs via workflows
 # See: https://circleci.com/docs/2.0/configuration-reference/#workflows

.circleci/continue-config.yml

@@ -52,11 +52,11 @@ commands:
             conda create -n build python=<< parameters.python_ver >> -y
             conda activate build
 
-            sh ./build_wheel_entrypoint.sh
+            bazel build //:spu_wheel -c opt --@rules_python//python/config_settings:python_version=<< parameters.python_ver >>
             python3 -m pip install twine
-            ls dist/*.whl
+            ls bazel-bin/spu*.whl
 
-            python3 -m twine upload -r pypi -u __token__ -p ${PYPI_TWINE_TOKEN} dist/*.whl
+            python3 -m twine upload -r pypi -u __token__ -p ${PYPI_TWINE_TOKEN} bazel-bin/spu*.whl
 
 # Define a job to be invoked later in a workflow.
 # See: https://circleci.com/docs/2.0/configuration-reference/#jobs
@@ -103,15 +103,15 @@ workflows:
       - linux_publish:
           matrix:
             parameters:
-              python_ver: ["3.9", "3.10", "3.11"]
+              python_ver: ["3.10", "3.11"]
               executor: ["linux_x64_executor", "linux_aarch64_executor"]
           filters:
             tags:
               only: /.*(?<!dev\d{8})$/
       - macOS_publish:
           matrix:
             parameters:
-              python_ver: ["3.9", "3.10", "3.11"]
+              python_ver: ["3.10", "3.11"]
           filters:
             tags:
               only: /.*(?<!dev\d{8})$/

.circleci/release-config.yml

@@ -25,6 +25,7 @@ _build
 
 # bazel
 /bazel-*
+MODULE.bazel.lock
 
 # cmake related
 abseil-cpp

.gitignore

@@ -43,14 +43,14 @@ header: # <1>
     - 'libspu/compiler/tests/interpret/template/**.template'
     - 'LICENSE'
     - 'NOTICE'
-    - '.bazelversion'
+    - 'MODULE.bazel.lock'
+    - '.bazeliskrc'
     - '.clang-format'
     - '.clang-tidy'
     - '.gitattributes'
     - '.gitignore'
     - '.gitmodules'
     - 'pyproject.toml'
-    - 'setup.cfg'
     - 'libspu/core/half.h' # MIT
 
   comment: never # <9>

.licenserc.yaml

@@ -11,3 +11,114 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+load("@python_versions//3.10:defs.bzl", compile_pip_requirements_3_10 = "compile_pip_requirements")
+load("@python_versions//3.11:defs.bzl", compile_pip_requirements_3_11 = "compile_pip_requirements")
+
+# load("@python_versions//3.9:defs.bzl", compile_pip_requirements_3_9 = "compile_pip_requirements")
+load("@rules_python//python:packaging.bzl", "py_package", "py_wheel")
+load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+load("//:version.bzl", "SPU_VERSION")
+
+# compile_pip_requirements_3_9(
+#     name = "requirements_3_9",
+#     src = "requirements_3_9.txt",
+#     requirements_txt = "requirements_lock_3_9.txt",
+#     tags = ["manual"],
+# )
+
+compile_pip_requirements_3_10(
+    name = "requirements_3_10",
+    src = "requirements.txt",
+    requirements_txt = "requirements_lock_3_10.txt",
+    tags = ["manual"],
+)
+
+compile_pip_requirements_3_11(
+    name = "requirements_3_11",
+    src = "requirements.txt",
+    requirements_txt = "requirements_lock_3_11.txt",
+    tags = ["manual"],
+)
+
+compile_pip_requirements(
+    name = "requirements-dev",
+    src = "requirements-dev.txt",
+    requirements_txt = "requirements_dev_lock.txt",
+    tags = ["manual"],
+)
+
+exports_files([
+    "README.md",
+])
+
+# https://rules-python.readthedocs.io/en/latest/api/rules_python/python/packaging.html#py_wheel_rule
+py_wheel(
+    name = "spu_wheel",
+    abi = select(
+        {
+            "@rules_python//python/config_settings:is_python_3.10": "cp310",
+            "@rules_python//python/config_settings:is_python_3.11": "cp311",
+            # "@rules_python//python/config_settings:is_python_3.9": "cp39",
+            "//conditions:default": "none",
+        },
+    ),
+    author = "SecretFlow Team",
+    author_email = "[email protected]",
+    classifiers = [
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+    ],
+    description_content_type = "text/markdown",
+    description_file = "README.md",
+    distribution = "spu",
+    extra_distinfo_files = {
+        "//:LICENSE": "LICENSE",
+    },
+    homepage = "https://github.com/secretflow/spu",
+    license = "Apache License 2.0",
+    # TODO: add other fields.
+    platform = select(
+        {
+            "@bazel_tools//src/conditions:linux_x86_64": "manylinux2014_x86_64",
+            "@bazel_tools//src/conditions:darwin_arm64": "macosx_13_0_arm64",
+            "@bazel_tools//src/conditions:linux_aarch64": "manylinux_2_28_aarch64",
+            "//conditions:default": "any",
+        },
+    ),
+    python_requires = ">=3.9",
+    python_tag = select(
+        {
+            "@rules_python//python/config_settings:is_python_3.10": "cp310",
+            "@rules_python//python/config_settings:is_python_3.11": "cp311",
+            # "@rules_python//python/config_settings:is_python_3.9": "cp39",
+            "//conditions:default": "py3",
+        },
+    ),
+    requires_file = "requirements.txt",
+    summary = "SPU aims to be a 'provable', 'measurable' secure computation device.",
+    tags = ["manual"],
+    twine = None,
+    version = SPU_VERSION,
+    deps = [
+        ":spu_pkg",
+    ],
+)
+
+py_package(
+    name = "spu_pkg",
+    packages = [
+        "libspu",
+        "spu",
+    ],
+    visibility = ["//visibility:private"],
+    deps = [
+        "//spu:api",
+        "//spu:init",
+        "//spu:libpsi",
+        "//spu:libspu",
+        "//spu/ops/groupby",
+        "//spu/utils:distributed",
+    ],
+)

BUILD.bazel

@@ -10,6 +10,12 @@
 >
 > please add your unreleased change here.
 
+- [SPU] Migrating to Bazel Modules and update minimum Python version to 3.10.
+- [Feature] Add soPRF (LowMC cipher) for SEMI2K
+- [API] Add Permute/InvPermute support in HLO
+- [Feature] Add SSL configuration to the TTP server
+- [Feature] Support quick sort for semi2k and aby3
+
 ## 20241219
 
 - [SPU] 0.9.3b0 release

CHANGELOG.md

@@ -62,13 +62,6 @@ docker exec -it spu-dev-$(whoami) bash
 Install gcc>=11.2, cmake>=3.26, ninja, nasm>=2.15, python>=3.9, bazelisk, xxd, lld
 ```
 
-About the commands used to install the above dependencies, you can follow [Ubuntu docker file](https://github.com/secretflow/devtools/blob/main/dockerfiles/ubuntu-base-ci.DockerFile).
-
-```sh
-python3 -m pip install -r requirements.txt
-python3 -m pip install -r requirements-dev.txt
-```
-
 #### macOS
 
 ```sh
@@ -90,10 +83,6 @@ brew install bazelisk cmake ninja libomp wget
 
 # For Intel mac only
 brew install nasm
-
-# Install python dependencies
-pip install -r requirements.txt
-pip install -r requirements-dev.txt
 ```
 
 ### Build & UnitTest
@@ -117,6 +106,7 @@ bazel test //... --features=ubsan
 
 - `--define gperf=on` enable gperf
 - `--define tracelog=on` enable link trace log.
+- `--@rules_python//python/config_settings:python_version=3.10` set the Python version as 3.10, the default version is 3.11
 
 ### Build docs
 

CONTRIBUTING.md

@@ -38,8 +38,8 @@ pip install spu
 - At the root of repo, run
 
 ```bash
-python setup.py bdist_wheel
-pip install dist/*.whl --force-reinstall
+bazel build //:spu_wheel -c opt
+pip install bazel-bin/spu-*.whl --force-reinstall
 ```
 
 - Once GCC/bazel/python/Xcode version or other environment settings have changed, please run the following command to ensure a clean build
@@ -51,5 +51,15 @@ bazel clean --expunge
 #### Build with GPU support
 
 ```bash
-export ENABLE_GPU_BUILD=1 && python setup.py bdist_wheel
+bazel build //:spu_wheel -c opt --config=gpu
+```
+
+#### Build with specified python version
+
+```bash
+# build with python 3.10
+bazel build //:spu_wheel -c opt --@rules_python//python/config_settings:python_version=3.10
+
+# build with python 3.11
+bazel build //:spu_wheel -c opt --@rules_python//python/config_settings:python_version=3.11
 ```