diff options
author | Matthew Sotoudeh <masotoudeh@ucdavis.edu> | 2020-08-24 09:01:16 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-08-24 09:01:16 -0700 |
commit | 06fa0707035bd646d26f9477b93be83a54dae425 (patch) | |
tree | fa2c3db2ebfb5c63f2b8d0c16ada06ad0e00bbc1 | |
parent | 38c22236f1c460f16c27443e7d37c8d33a81a5f8 (diff) |
Switch to External Bazel-Python (#5)
Refactors the Bazel Python code into a separate repository. Also updates the CI to use the new Python approach.
-rw-r--r-- | ._dummy_.py | 2 | ||||
-rw-r--r-- | .github/workflows/test_syrenn.yml | 4 | ||||
-rw-r--r-- | BUILD | 153 | ||||
-rw-r--r-- | Dockerfile | 12 | ||||
-rw-r--r-- | README.md | 44 | ||||
-rw-r--r-- | WORKSPACE | 25 | ||||
-rwxr-xr-x | coverage_report.sh | 10 | ||||
-rw-r--r-- | experiments/BUILD | 11 | ||||
-rw-r--r-- | external/openssl.BUILD | 31 | ||||
-rw-r--r-- | external/python.BUILD | 30 | ||||
-rw-r--r-- | models/BUILD | 12 | ||||
-rw-r--r-- | pysyrenn/frontend/BUILD | 12 | ||||
-rw-r--r-- | pysyrenn/frontend/network.py | 5 | ||||
-rw-r--r-- | pysyrenn/frontend/tests/BUILD | 14 | ||||
-rw-r--r-- | pysyrenn/frontend/transformer_client.py | 2 | ||||
-rw-r--r-- | pysyrenn/helpers/BUILD | 7 | ||||
-rw-r--r-- | pysyrenn/helpers/tests/BUILD | 5 | ||||
-rwxr-xr-x | pywrapper.sh | 7 | ||||
-rw-r--r-- | requirements.txt | 3 | ||||
-rw-r--r-- | third_party/eran_preconditions/BUILD | 16 | ||||
-rw-r--r-- | third_party/eran_preconditions/Dockerfile | 3 | ||||
-rwxr-xr-x | third_party/eran_preconditions/experiment.sh | 2 | ||||
-rw-r--r-- | third_party/reluplex_model_checking/BUILD | 13 |
23 files changed, 116 insertions, 307 deletions
diff --git a/._dummy_.py b/._dummy_.py deleted file mode 100644 index 252ad38..0000000 --- a/._dummy_.py +++ /dev/null @@ -1,2 +0,0 @@ -# This file exists because the py_library rule in BUILD (that acts as a dummy -# target for the pip imports) needs a src. diff --git a/.github/workflows/test_syrenn.yml b/.github/workflows/test_syrenn.yml index 3ea52b5..ed0d668 100644 --- a/.github/workflows/test_syrenn.yml +++ b/.github/workflows/test_syrenn.yml @@ -43,4 +43,8 @@ jobs: chmod +x install.sh ./install.sh --user rm -f install.sh + git clone https://github.com/95616ARG/bazel_python.git + cd bazel_python + echo y | ./setup_python.sh 3.7.4 $HOME/.bazel_python + cd .. && rm -rf bazel_python /home/runner/bin/bazel test pysyrenn/... @@ -1,130 +1,37 @@ -sh_library( - name = "pywrapper", - srcs = ["pywrapper.sh"], - visibility = ["//:__subpackages__"], -) - -# https://stackoverflow.com/questions/47036855 -py_runtime( - name = "python-3", - files = ["@python_3//:installdir"], - interpreter = "pywrapper.sh", - python_version = "PY3", -) - -# https://github.com/bazelbuild/rules_python/blob/master/proposals/2019-02-12-design-for-a-python-toolchain.md -load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") - -constraint_value( - name = "python3_constraint", - constraint_setting = "@bazel_tools//tools/python:py3_interpreter_path", -) - -platform( - name = "python3_platform", - constraint_values = [ - ":python3_constraint", - ], -) - -py_runtime_pair( - name = "python3_runtime_pair", - py3_runtime = ":python-3", -) - -toolchain( - name = "python3-toolchain", - # Since the Python interpreter is invoked at runtime on the target - # platform, there's no need to specify execution platform constraints here. - target_compatible_with = [ - # Make sure this toolchain is only selected for a target platform that - # advertises that it has interpreters available under /usr/weirdpath. - # ":python3_constraint", - ], - toolchain = "//:python3_runtime_pair", - toolchain_type = "@bazel_tools//tools/python:toolchain_type", -) - -# This seems to be the way Google is doing it now: -# https://github.com/bazelbuild/rules_python/issues/119 -# See https://github.com/pypa/pip/issues/3826 for why we need the --system flag -# on the pip call. For newer versions of pip, you may have to add -# --no-build-isolation. -genrule( - name = "install-pip-packages", - srcs = ["requirements.txt"], - outs = ["pip_packages"], - cmd = """ - PYTHON=$(location pywrapper.sh) - PIP="$$PYTHON -m pip" - - DUMMY_HOME=/tmp/$$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8) - rm -rf $$DUMMY_HOME - - export HOME=$$DUMMY_HOME - PIP_INSTALL="$$PIP \ - install --no-cache-dir --disable-pip-version-check \ - --target=$@" - - # Setup the environment to point to the right Python installation. - mkdir -p $$DUMMY_HOME - INSTALLDIR=$$PWD/$$(find **/** -name "installdir" | head -n 1) - export LD_LIBRARY_PATH=$$INSTALLDIR/lib/:$$INSTALLDIR/lib64/ - ln -s $$INSTALLDIR/python3.7 $$DUMMY_HOME/python - export PYTHONPATH=$$PWD/$@ - export PATH=$$DUMMY_HOME:$$PATH - - # Install the correct version of Torch - $$PIP_INSTALL torch==1.2.0+cpu -f https://download.pytorch.org/whl/torch_stable.html - - # Install the other requirements. - $$PIP_INSTALL --upgrade setuptools - $$PIP_INSTALL -r requirements.txt - - # The custom typing package installed as a dependency doesn't seem to work - # well. - rm -rf $@/typing-* - rm -rf $@/typing.py - rm -rf $$DUMMY_HOME +load("@bazel_python//:bazel_python.bzl", "bazel_python_interpreter") + +bazel_python_interpreter( + name = "bazel_python_venv", + python_version = "3.7.4", + requirements_file = "requirements.txt", + run_after_pip = """ + pip3 install torch==1.2.0+cpu -f https://download.pytorch.org/whl/torch_stable.html """, - tools = [ - ":pywrapper.sh", - "@python_3//:installdir", - ], - visibility = ["//:__subpackages__"], -) - -py_library( - name = "pip-packages", - srcs = ["._dummy_.py"], - data = [":install-pip-packages"], - imports = ["pip_packages"], visibility = ["//:__subpackages__"], ) # Make the thicker-bordered plane SVG. genrule( name = "thicker-plane", - srcs = [ - "@plane_svg//file", - "pywrapper.sh", - ], + srcs = ["@plane_svg//file"], outs = ["plane.png"], cmd = """ PLANESVG=$(location @plane_svg//file) - PYTHON=$(location pywrapper.sh) - export PYTHONPATH=$(location //:pip_packages) + + PYTHON_VENV=$(location //:bazel_python_venv) + pushd $$PYTHON_VENV/.. + source bazel_python_venv_installed/bin/activate + popd sed -i -e \ 's/id="path5724" /id="path5724" stroke="white" fill="black" stroke-width="10" /' \ $$PLANESVG - $$PYTHON -m cairosvg $$PLANESVG -o plane.png --output-width 4965 + python3 -m cairosvg $$PLANESVG -o plane.png --output-width 4965 cp plane.png $@ """, tools = [ - "//:pip_packages", - "@python_3//:installdir", + "//:bazel_python_venv", ], visibility = ["//:__subpackages__"], ) @@ -133,11 +40,17 @@ genrule( sh_binary( name = "coverage_report", srcs = ["coverage_report.sh"], - data = [ - "//:pip_packages", - "//:pywrapper", - "@python_3//:installdir", - ], + deps = [":_dummy_coverage_report"], +) + +# This is only to get bazel_python_venv as a data dependency for +# coverage_report above. For some reason, this doesn't work if we directly put +# it on the sh_binary. This is a known issue: +# https://github.com/bazelbuild/bazel/issues/1147#issuecomment-428698802 +sh_library( + name = "_dummy_coverage_report", + srcs = ["coverage_report.sh"], + data = ["//:bazel_python_venv"], ) # For wheel-ifying the Python code. @@ -156,23 +69,23 @@ genrule( ], outs = ["pysyrenn.dist"], cmd = """ - PYTHON=$(location pywrapper) - export PYTHONPATH=$$PWD:$(location //:pip_packages) + PYTHON_VENV=$(location //:bazel_python_venv) + pushd $$PYTHON_VENV/.. + source bazel_python_venv_installed/bin/activate + popd mkdir -p syrenn_proto cp -Lr $(locations //syrenn_proto:syrenn_py_grpc) syrenn_proto cp -Lr $(locations //syrenn_proto:syrenn_py_proto) syrenn_proto touch syrenn_proto/__init__.py cp pip_info/* . - $$PYTHON setup.py sdist bdist_wheel + python3 setup.py sdist bdist_wheel cp -r dist $@ """, tools = [ - "pywrapper", - "//:pip_packages", + "//:bazel_python_venv", "//syrenn_proto:syrenn_py_grpc", "//syrenn_proto:syrenn_py_proto", - "@python_3//:installdir", ], ) @@ -12,7 +12,7 @@ RUN apt-get update && apt-get install -y \ pkg-config zip g++ zlib1g-dev unzip \ python2.7 python-pip python3 python3-pip \ build-essential curl git cmake libcairo2 libffi-dev libgmp3-dev \ - zlib1g-dev zip \ + zlib1g-dev libssl-dev libbz2-dev zip \ apt-transport-https ca-certificates gnupg-agent software-properties-common # Install Docker. @@ -30,6 +30,14 @@ RUN curl -OL https://github.com/bazelbuild/bazel/releases/download/1.1.0/bazel-1 ./bazel-1.1.0-installer-linux-x86_64.sh && \ rm bazel-1.1.0-installer-linux-x86_64.sh +# Set up bazel_python +RUN git clone https://github.com/95616ARG/bazel_python.git && \ + cd bazel_python && \ + echo y | ./setup_python.sh 3.7.4 /.bazel_python && \ + cp $HOME/.bazelrc /etc/bazel.bazelrc && \ + cd .. && \ + rm -rf bazel_python + # Install benchexec. RUN curl -OL https://github.com/sosy-lab/benchexec/releases/download/2.2/benchexec_2.2-1_all.deb && \ apt install -y --install-recommends ./benchexec_2.2-1_all.deb && \ @@ -39,7 +47,7 @@ RUN rm -rf /var/lib/apt/lists/* # Store Bazel outputs on /iovol/.docker_bazel, which will be loaded at runtime. # This allows us to cache Bazel build artifacts across Docker invocations. -RUN echo "startup --output_user_root=/iovol/.docker_bazel" > /etc/bazel.bazelrc +RUN echo "startup --output_user_root=/iovol/.docker_bazel" >> /etc/bazel.bazelrc RUN useradd -u $UID -ms /bin/bash docker_user RUN adduser docker_user benchexec @@ -103,26 +103,39 @@ is already freezing/running out of memory, consider removing the existing ``-j#`` flags from ``external/*.BUILD``. #### Local Builds -You must install [Bazel](https://bazel.build/) 1.1.0 and have binaries for -building arbitrary C++ packages (eg. ``build-essential`` for Ubuntu). -Furthermore, the ``libcairo2``, ``libffi-dev``, ``zlib1g-dev``, ``zip``, and -``libgmp3-dev`` packages are required for the Python code (but usually come -pre-installed with most desktop Linux distributions). A -[Dockerfile](Dockerfile) is provided with a compatible setup for reference. -Note that Bazel will _automatically_ download and build copies of OpenSSL, -Python 3.7.4, Intel TBB, Intel MKLDNN, Eigen, OpenBLAS, PyTorch, and all PIP -dependencies to the Bazel environment when applicable --- they do not need to -be manually installed. +1. You must install [Bazel](https://bazel.build/) 1.1.0 and have binaries for + building arbitrary C++ packages (eg. ``build-essential`` for Ubuntu). +2. Furthermore, the ``libcairo2``, ``libffi-dev``, ``zlib1g-dev``, ``zip``, and + ``libgmp3-dev`` packages are required for the Python code (but usually come + pre-installed with most desktop Linux distributions). +3. You must follow the "One-Time Setup" instructions in + [bazel_python](https://github.com/95616ARG/bazel_python) to install a fresh + copy of Python 3.7.4. This can be done per-user without root permissions. + This is **not** necessary if you only want to run the server, e.g., if you + just want to use the `pysyrenn` package from PyPI. It **is**, however, + required to run any of the Python code in this repository with Bazel, + including the Python tests. +4. A [Dockerfile](Dockerfile) is provided with a compatible setup for + reference. Note that Bazel will _automatically_ download and build copies + of Intel TBB, Intel MKLDNN, Eigen, OpenBLAS, PyTorch, and all PIP + dependencies to the Bazel environment when applicable --- they do not need + to be manually installed. #### Docker Builds Alternatively, a Docker container is provided to simplify the build and running process. To use it, first build the image with ``./docker_build.sh`` then prepend ``./docker_run.sh`` to all of the commands below. For example, instead of ``make start_server``, use ``./docker_run.sh make start_server``. Everything -should be handled transparently. **NOTE:** Benchexec is currently not supported -under the Docker container due to permission errors. It is possible to resolve -this by removing all of the user-related lines from [Dockerfile](Dockerfile), -but that will likely cause issues with Bazel and generated file permissions. +should be handled transparently. + +**NOTE:** Benchexec is currently not supported under the Docker container due +to permission errors. It is possible to resolve this by removing all of the +user-related lines from [Dockerfile](Dockerfile), but that will likely cause +issues with Bazel and generated file permissions. + +**NOTE:** Docker-in-docker does not seem to currently be working. This means +some builds will fail, although you should be able to run `make start_server`, +`bazel test //pysyrenn/...`, and `bazel test //syrenn_server/...`. ### Configuration There are two major things to configure before using the code: @@ -217,8 +230,7 @@ There are three main pieces to the library: A few other directories also exist: -- [external/](external) contains Bazel BUILD rules for GTEST, Python 3.7.4 (including - OpenSSL), and TBB. +- [external/](external) contains Bazel BUILD rules for GTEST, Eigen, and TBB. - [models/](models) contains Bazel rules either generating, downloading, or referencing trained models from the ERAN, ACAS Xu, and VRL projects (prior work). ERAN models are downloaded in WORKSPACE and referenced in @@ -1,28 +1,17 @@ workspace(name = "SyReNN") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") -# Get the Python interpreter & build dependency (OpenSSL). -# https://stackoverflow.com/questions/47036855 -http_archive( - name = "openssl", - build_file = "openssl.BUILD", - sha256 = "cabd5c9492825ce5bd23f3c3aeed6a97f8142f606d893df216411f07d1abab96", - strip_prefix = "openssl-1.0.2s", - urls = ["https://www.openssl.org/source/old/1.0.2/openssl-1.0.2s.tar.gz"], -) - -http_archive( - name = "python_3", - build_file = "python.BUILD", - sha256 = "d63e63e14e6d29e17490abbe6f7d17afb3db182dbd801229f14e55f4157c4ba3", - strip_prefix = "Python-3.7.4", - urls = ["https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz"], +git_repository( + name = "bazel_python", + commit = "5edcf3b97c4c77b654af177bfa27558d9b88b52f", + remote = "https://github.com/95616ARG/bazel_python.git", ) -register_toolchains("//:python3-toolchain") +load("@bazel_python//:bazel_python.bzl", "bazel_python") -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +bazel_python() # EIGEN SUPPORT # See the README in: https://github.com/bazelbuild/rules_foreign_cc diff --git a/coverage_report.sh b/coverage_report.sh index 5edbf50..87bdf6c 100755 --- a/coverage_report.sh +++ b/coverage_report.sh @@ -2,11 +2,7 @@ COVTEMP=$PWD/coverage_tmp rm -rf $COVTEMP mkdir $COVTEMP -# Make sure we know where to find the Python script, and that it knows where to -# find its pip packages. -export INSTALLDIR=$PWD/external/python_3/installdir -PYTHON=$PWD/pywrapper.sh -export PYTHONPATH=$PYTHONPATH:$PWD/pip_packages +source bazel_python_venv_installed/bin/activate # Go to the main workspace directory and run the coverage-report. pushd $BUILD_WORKSPACE_DIRECTORY @@ -25,8 +21,8 @@ done rm -rf .coverage htmlcov # Then we build a new .coverage as well as export to HTML -$PYTHON -m coverage combine $COVTEMP/*.cov -$PYTHON -m coverage html pysyrenn/*/*.py +python3 -m coverage combine $COVTEMP/*.cov +python3 -m coverage html pysyrenn/*/*.py # Remove temporaries and go back to where Bazel started us. rm -r $COVTEMP diff --git a/experiments/BUILD b/experiments/BUILD index 203583d..beef753 100644 --- a/experiments/BUILD +++ b/experiments/BUILD @@ -6,7 +6,6 @@ py_library( "//models:datasets", ], deps = [ - "//:pip-packages", "//pysyrenn", "//syrenn_proto:syrenn_py_grpc", ], @@ -15,7 +14,6 @@ py_library( py_library( name = "polar_image", srcs = ["polar_image.py"], - deps = ["//:pip-packages"], ) filegroup( @@ -29,7 +27,6 @@ filegroup( py_library( name = "integral_approximations", srcs = ["integral_approximations.py"], - deps = ["//:pip-packages"], ) py_binary( @@ -38,7 +35,6 @@ py_binary( deps = [ ":experiment", ":integral_approximations", - "//:pip-packages", "//pysyrenn", ], ) @@ -51,7 +47,6 @@ py_binary( ], deps = [ ":experiment", - "//:pip-packages", "//pysyrenn", ], ) @@ -61,7 +56,6 @@ py_binary( srcs = ["linearity_hypothesis.py"], deps = [ ":experiment", - "//:pip-packages", "//pysyrenn", ], ) @@ -75,7 +69,6 @@ py_binary( deps = [ ":acas_lines", ":polar_image", - "//:pip-packages", "//pysyrenn", ], ) @@ -84,7 +77,6 @@ py_library( name = "vrl_model", srcs = ["vrl_model.py"], visibility = ["//:__subpackages__"], - deps = ["//:pip-packages"], ) py_binary( @@ -94,7 +86,6 @@ py_binary( deps = [ ":experiment", ":vrl_model", - "//:pip-packages", "//pysyrenn", ], ) @@ -105,7 +96,6 @@ py_binary( deps = [ ":acas_planes", ":polar_image", - "//:pip-packages", "//pysyrenn", ], ) @@ -115,7 +105,6 @@ py_binary( srcs = ["toy_examples.py"], deps = [ ":acas_planes", - "//:pip-packages", "//pysyrenn", ], ) diff --git a/external/openssl.BUILD b/external/openssl.BUILD deleted file mode 100644 index b1c0856..0000000 --- a/external/openssl.BUILD +++ /dev/null @@ -1,31 +0,0 @@ -# https://stackoverflow.com/questions/2537271 -genrule( - name = "build_openssl", - srcs = glob(["**"]), - outs = ["openssl-installdir"], - cmd = """ - # First, we need to copy the OpenSSL source to a writeable location - # (following links). If we try to do it directly in the original - # OPENSSLDIR, it consistently throws errors about file system permissions - # (I'm not entirely sure why, but this fixes it). - HOMEDIR=$$PWD - SOURCEDIR=$$PWD/$$(dirname $(location config)) - cp -Lr $$SOURCEDIR openssl - SOURCEDIR=$$PWD/openssl - - # Then create the install dir. - INSTALLDIR=$$PWD/openssl-installdir - mkdir -p $$INSTALLDIR - - # Install OpenSSL to installdir. - cd $$SOURCEDIR - ./config shared -fPIC --prefix=$$INSTALLDIR --openssldir=$$INSTALLDIR/openssl > /dev/null - make -j2 > /dev/null - make install > /dev/null - cd $$HOMEDIR - - # And copy to the output directory. - cp -r $$INSTALLDIR $@ - """, - visibility = ["//visibility:public"], -) diff --git a/external/python.BUILD b/external/python.BUILD deleted file mode 100644 index 0741461..0000000 --- a/external/python.BUILD +++ /dev/null @@ -1,30 +0,0 @@ -# https://superuser.com/questions/1346141 -# There's another one I lost about finding the current script's directory. -genrule( - name = "build_python", - srcs = glob(["**"]) + ["@openssl//:openssl-installdir"], - outs = ["installdir"], - cmd = """ - HOMEDIR=$$PWD - INSTALLDIR=$$PWD/installdir - cp -Lr $(location @openssl//:openssl-installdir) $$INSTALLDIR - - # See external/openssl.BUILD for notes on why we do this weird copy. - SOURCEDIR=$$PWD/$$(dirname $(location configure)) - cp -Lr $$SOURCEDIR $$PWD/python - SOURCEDIR=$$PWD/python - - export LDFLAGS="-L$$INSTALLDIR/lib/ -L$$INSTALLDIR/lib64/" - export LD_LIBRARY_PATH="$$INSTALLDIR/lib/:$$INSTALLDIR/lib64/" - export CPPFLAGS="-I$$INSTALLDIR/include -I$$INSTALLDIR/include/openssl" - - # Install Python to installdir. - cd $$SOURCEDIR - ./configure --prefix=$$INSTALLDIR --with-openssl=$$INSTALLDIR > /dev/null - make -j2 > /dev/null - make install > /dev/null - cd $$HOMEDIR - cp -r installdir $(@D) - """, - visibility = ["//visibility:public"], -) diff --git a/models/BUILD b/models/BUILD index fc829bf..1ad15e0 100644 --- a/models/BUILD +++ b/models/BUILD @@ -46,15 +46,17 @@ genrule( ], outs = ["acas_models"], cmd = """ - PYTHON=$(location //:pywrapper) - export PYTHONPATH=$(location //:pip_packages) + PYTHON_VENV=$(location //:bazel_python_venv) + pushd $$PYTHON_VENV/.. + source bazel_python_venv_installed/bin/activate + popd cp -L external/reluplex/nnet/*.nnet . all_files=$$(ls | grep .nnet) for file in $$all_files do - $$PYTHON $(location translate_acas_model.py) $$file + python3 $(location translate_acas_model.py) $$file done mkdir acas_models @@ -62,9 +64,7 @@ genrule( mv acas_models $@ """, tools = [ - "//:pip_packages", - "//:pywrapper", - "@python_3//:installdir", + "//:bazel_python_venv", ], visibility = ["//:__subpackages__"], ) diff --git a/pysyrenn/frontend/BUILD b/pysyrenn/frontend/BUILD index 8ddf4ee..c52a68a 100644 --- a/pysyrenn/frontend/BUILD +++ b/pysyrenn/frontend/BUILD @@ -22,7 +22,6 @@ py_library( srcs = ["transformer_client.py"], visibility = ["//pysyrenn/frontend:__subpackages__"], deps = [ - "//:pip-packages", "//syrenn_proto:syrenn_py_grpc", ], ) @@ -42,7 +41,6 @@ py_library( ":relu_layer", ":strided_window_data", ":transformer_client", - "//:pip-packages", "//syrenn_proto:syrenn_py_grpc", ], ) @@ -60,7 +58,6 @@ py_library( visibility = ["//:__subpackages__"], deps = [ ":layer", - "//:pip-packages", "//syrenn_proto:syrenn_py_grpc", ], ) @@ -71,7 +68,6 @@ py_library( visibility = ["//:__subpackages__"], deps = [ ":layer", - "//:pip-packages", "//syrenn_proto:syrenn_py_grpc", ], ) @@ -82,7 +78,6 @@ py_library( visibility = ["//:__subpackages__"], deps = [ ":layer", - "//:pip-packages", "//syrenn_proto:syrenn_py_grpc", ], ) @@ -93,7 +88,6 @@ py_library( visibility = ["//:__subpackages__"], deps = [ ":layer", - "//:pip-packages", "//syrenn_proto:syrenn_py_grpc", ], ) @@ -104,7 +98,6 @@ py_library( visibility = ["//:__subpackages__"], deps = [ ":layer", - "//:pip-packages", "//syrenn_proto:syrenn_py_grpc", ], ) @@ -114,7 +107,6 @@ py_library( srcs = ["strided_window_data.py"], visibility = ["//:__subpackages__"], deps = [ - "//:pip-packages", "//syrenn_proto:syrenn_py_grpc", ], ) @@ -126,7 +118,6 @@ py_library( deps = [ ":layer", ":strided_window_data", - "//:pip-packages", "//syrenn_proto:syrenn_py_grpc", ], ) @@ -138,7 +129,6 @@ py_library( deps = [ ":layer", ":strided_window_data", - "//:pip-packages", "//syrenn_proto:syrenn_py_grpc", ], ) @@ -150,7 +140,6 @@ py_library( deps = [ ":layer", ":strided_window_data", - "//:pip-packages", "//syrenn_proto:syrenn_py_grpc", ], ) @@ -161,7 +150,6 @@ py_library( visibility = ["//:__subpackages__"], deps = [ ":layer", - "//:pip-packages", "//syrenn_proto:syrenn_py_grpc", ], ) diff --git a/pysyrenn/frontend/network.py b/pysyrenn/frontend/network.py index c3c8247..fb9623e 100644 --- a/pysyrenn/frontend/network.py +++ b/pysyrenn/frontend/network.py @@ -151,7 +151,10 @@ class Network: Returns a list of UPolytopes (one per plane in @planes), which are themselves lists of either tuples of Numpy arrays (pre, post) or just - Numpy arrays pre depending on @include_post. + Numpy arrays pre depending on @include_post. Note that + @compute_preimages=False still returns (pre, post), except the @pre + matrices are *combinations* of the original vertices, not points + themselves. """ planes = np.asarray(planes) transformed = transformer_client.transform_planes(self, planes) diff --git a/pysyrenn/frontend/tests/BUILD b/pysyrenn/frontend/tests/BUILD index ac45fee..2f89338 100644 --- a/pysyrenn/frontend/tests/BUILD +++ b/pysyrenn/frontend/tests/BUILD @@ -2,7 +2,6 @@ py_library( name = "helpers", srcs = ["helpers.py"], visibility = ["//:__subpackages__"], - deps = ["//:pip-packages"], ) py_test( @@ -11,7 +10,6 @@ py_test( srcs = ["relu_layer.py"], deps = [ ":helpers", - "//:pip-packages", "//pysyrenn/frontend:relu_layer", ], ) @@ -22,7 +20,6 @@ py_test( srcs = ["hard_tanh_layer.py"], deps = [ ":helpers", - "//:pip-packages", "//pysyrenn/frontend:hard_tanh_layer", ], ) @@ -33,7 +30,6 @@ py_test( srcs = ["argmax_layer.py"], deps = [ ":helpers", - "//:pip-packages", "//pysyrenn/frontend:argmax_layer", ], ) @@ -44,7 +40,6 @@ py_test( srcs = ["fullyconnected_layer.py"], deps = [ ":helpers", - "//:pip-packages", "//pysyrenn/frontend:fullyconnected_layer", ], ) @@ -55,7 +50,6 @@ py_test( srcs = ["normalize_layer.py"], deps = [ ":helpers", - "//:pip-packages", "//pysyrenn/frontend:normalize_layer", ], ) @@ -66,7 +60,6 @@ py_test( srcs = ["averagepool_layer.py"], deps = [ ":helpers", - "//:pip-packages", "//pysyrenn/frontend:averagepool_layer", "//pysyrenn/frontend:strided_window_data", ], @@ -78,7 +71,6 @@ py_test( srcs = ["maxpool_layer.py"], deps = [ ":helpers", - "//:pip-packages", "//pysyrenn/frontend:maxpool_layer", "//pysyrenn/frontend:strided_window_data", ], @@ -90,7 +82,6 @@ py_test( srcs = ["conv2d_layer.py"], deps = [ ":helpers", - "//:pip-packages", "//pysyrenn/frontend:conv2d_layer", "//pysyrenn/frontend:strided_window_data", ], @@ -102,7 +93,6 @@ py_test( srcs = ["concat_layer.py"], deps = [ ":helpers", - "//:pip-packages", "//pysyrenn/frontend:averagepool_layer", "//pysyrenn/frontend:concat_layer", "//pysyrenn/frontend:conv2d_layer", @@ -118,7 +108,6 @@ py_test( srcs = ["strided_window_data.py"], deps = [ ":helpers", - "//:pip-packages", "//pysyrenn/frontend:strided_window_data", ], ) @@ -130,7 +119,6 @@ py_test( data = ["@onnx_squeezenet//:all"], deps = [ ":helpers", - "//:pip-packages", "//pysyrenn/frontend:argmax_layer", "//pysyrenn/frontend:fullyconnected_layer", "//pysyrenn/frontend:network", @@ -145,7 +133,6 @@ py_test( srcs = ["layer.py"], deps = [ ":helpers", - "//:pip-packages", "//pysyrenn/frontend:layer", ], ) @@ -156,7 +143,6 @@ py_test( srcs = ["transformer_client.py"], deps = [ ":helpers", - "//:pip-packages", "//pysyrenn/frontend:fullyconnected_layer", "//pysyrenn/frontend:network", "//pysyrenn/frontend:relu_layer", diff --git a/pysyrenn/frontend/transformer_client.py b/pysyrenn/frontend/transformer_client.py index bdf748a..adfe9c2 100644 --- a/pysyrenn/frontend/transformer_client.py +++ b/pysyrenn/frontend/transformer_client.py @@ -5,6 +5,8 @@ these methods. """ import os import numpy as np +import sys +sys.path = [path for path in sys.path if "/com_github_grpc_grpc/" not in path] import grpc import syrenn_proto.syrenn_pb2_grpc as grpc_pb import syrenn_proto.syrenn_pb2 as transformer_pb diff --git a/pysyrenn/helpers/BUILD b/pysyrenn/helpers/BUILD index d071306..5e97de8 100644 --- a/pysyrenn/helpers/BUILD +++ b/pysyrenn/helpers/BUILD @@ -15,9 +15,6 @@ py_library( name = "integrated_gradients", srcs = ["integrated_gradients.py"], visibility = [":__subpackages__"], - deps = [ - "//:pip-packages", - ], ) py_library( @@ -25,7 +22,6 @@ py_library( srcs = ["classify_lines.py"], visibility = [":__subpackages__"], deps = [ - "//:pip-packages", "//pysyrenn/frontend:argmax_layer", "//pysyrenn/frontend:network", ], @@ -36,7 +32,6 @@ py_library( srcs = ["classify_planes.py"], visibility = [":__subpackages__"], deps = [ - "//:pip-packages", "//pysyrenn/frontend:argmax_layer", "//pysyrenn/frontend:network", ], @@ -48,7 +43,6 @@ py_library( visibility = [":__subpackages__"], deps = [ ":masking_network", - "//:pip-packages", "//pysyrenn/frontend:fullyconnected_layer", "//pysyrenn/frontend:network", "//pysyrenn/frontend:relu_layer", @@ -60,7 +54,6 @@ py_library( srcs = ["masking_network.py"], visibility = [":__subpackages__"], deps = [ - "//:pip-packages", "//pysyrenn/frontend:fullyconnected_layer", "//pysyrenn/frontend:network", "//pysyrenn/frontend:relu_layer", diff --git a/pysyrenn/helpers/tests/BUILD b/pysyrenn/helpers/tests/BUILD index dd425e1..2800717 100644 --- a/pysyrenn/helpers/tests/BUILD +++ b/pysyrenn/helpers/tests/BUILD @@ -3,7 +3,6 @@ py_test( size = "small", srcs = ["classify_lines.py"], deps = [ - "//:pip-packages", "//pysyrenn/frontend", "//pysyrenn/frontend/tests:helpers", "//pysyrenn/helpers:classify_lines", @@ -15,7 +14,6 @@ py_test( size = "small", srcs = ["classify_planes.py"], deps = [ - "//:pip-packages", "//pysyrenn/frontend", "//pysyrenn/frontend/tests:helpers", "//pysyrenn/helpers:classify_planes", @@ -27,7 +25,6 @@ py_test( size = "small", srcs = ["integrated_gradients.py"], deps = [ - "//:pip-packages", "//pysyrenn/frontend", "//pysyrenn/frontend/tests:helpers", "//pysyrenn/helpers:integrated_gradients", @@ -39,7 +36,6 @@ py_test( size = "small", srcs = ["masking_network.py"], deps = [ - "//:pip-packages", "//pysyrenn/frontend", "//pysyrenn/frontend/tests:helpers", "//pysyrenn/helpers:masking_network", @@ -51,7 +47,6 @@ py_test( size = "small", srcs = ["netpatch.py"], deps = [ - "//:pip-packages", "//pysyrenn/frontend", "//pysyrenn/frontend/tests:helpers", "//pysyrenn/helpers:netpatch", diff --git a/pywrapper.sh b/pywrapper.sh deleted file mode 100755 index 379b17a..0000000 --- a/pywrapper.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -if [ -z $INSTALLDIR ]; then - INSTALLDIR=$PWD/$(find **/** -name "installdir" | head -n 1) -fi -export LD_LIBRARY_PATH=$INSTALLDIR/lib/:$INSTALLDIR/lib64/ -$INSTALLDIR/bin/python3.7 $@ diff --git a/requirements.txt b/requirements.txt index 5a89f8c..23be3dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,8 @@ protobuf==3.9.2 grpcio==1.24.0 onnx==1.5.0 tqdm==4.36.1 -# Torch is installed in the BUILD script and specified in pip_info/setup.py. +# TORCH-CPU IS INSTALLED SEPARATELY. For some reason it doesn't work if we +# install it here. It is also manually included in pip-info. aenum==2.2.1 # END coverage==4.5.4 diff --git a/third_party/eran_preconditions/BUILD b/third_party/eran_preconditions/BUILD index fb3e96b..6204d7e 100644 --- a/third_party/eran_preconditions/BUILD +++ b/third_party/eran_preconditions/BUILD @@ -3,17 +3,18 @@ genrule( srcs = [ "Dockerfile", "install.sh", - "//:plane.png", ], - outs = ["eran_preconditions.tgz"], + outs = ["eran_image.tgz"], cmd = """ - cp $(location //:plane.png) plane.png cp $(location Dockerfile) Dockerfile cp $(location install.sh) install.sh - docker build --force-rm -t eran_preconditions . - docker save -o eran_preconditions.tgz eran_preconditions - cp eran_preconditions.tgz $@ + docker build --force-rm -t eran_image . + docker save -o eran_image.tgz eran_image + cp eran_image.tgz $@ """, + visibility = [ + "//third_party/eran_bmc:__pkg__", + ], ) sh_binary( @@ -21,7 +22,8 @@ sh_binary( srcs = ["experiment.sh"], data = [ "experiment.py", - ":eran_preconditions.tgz", + ":eran_image.tgz", + "//:plane.png", "//experiments:polar_image_only", "//models:acas_models", ], diff --git a/third_party/eran_preconditions/Dockerfile b/third_party/eran_preconditions/Dockerfile index 9691177..45d1a57 100644 --- a/third_party/eran_preconditions/Dockerfile +++ b/third_party/eran_preconditions/Dockerfile @@ -7,14 +7,13 @@ RUN apt-get update && apt-get install -y \ wget unzip build-essential python3 \ python3-distutils python3-pip git gdb -# Get the Reluplex source +# Get the ERAN source WORKDIR / RUN wget https://github.com/eth-sri/eran/archive/31f7784bea1cf00ab2e358b12e68414893122034.zip -O eran.zip RUN unzip eran.zip -d eran && rm eran.zip RUN mv eran/**/* eran && rm -r eran/eran-* WORKDIR /eran -COPY plane.png plane.png # We use our own install.sh to specify the ELINA version (for reproducibility) # and use multi-threaded Make. COPY install.sh install.sh diff --git a/third_party/eran_preconditions/experiment.sh b/third_party/eran_preconditions/experiment.sh index 5a623a0..8b060e0 100755 --- a/third_party/eran_preconditions/experiment.sh +++ b/third_party/eran_preconditions/experiment.sh @@ -5,7 +5,7 @@ outdir=$builddir/third_party/eran_preconditions model="$PWD/models/acas_models/1_1.eran" polar_image_py="$PWD/experiments/polar_image.py" experiment_py="$PWD/third_party/eran_preconditions/experiment.py" -image_file="$PWD/third_party/eran_preconditions/eran_preconditions.tgz" +image_file="$PWD/third_party/eran_preconditions/eran_image.tgz" # Copy everything to the local directory and load the Docker image. cp $model model.eran diff --git a/third_party/reluplex_model_checking/BUILD b/third_party/reluplex_model_checking/BUILD index 8f4307b..a432dfa 100644 --- a/third_party/reluplex_model_checking/BUILD +++ b/third_party/reluplex_model_checking/BUILD @@ -24,19 +24,18 @@ genrule( ], outs = ["specs.tgz"], cmd = """ - SCRIPT=$(location export_specs.py) - PYTHON=$(location //:pywrapper) - export PYTHONPATH=$(location //:pip_packages) + PYTHON_VENV=$(location //:bazel_python_venv) + pushd $$PYTHON_VENV/.. + source bazel_python_venv_installed/bin/activate + popd - $$PYTHON $$SCRIPT + python3 $(location export_specs.py) tar -czvf specs.tgz pendulum_continuous satelite quadcopter cp specs.tgz $@ """, tools = [ - "//:pip_packages", - "//:pywrapper", + "//:bazel_python_venv", "//experiments:vrl_model", - "@python_3//:installdir", ], visibility = ["//:__subpackages__"], ) |