summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMatthew Sotoudeh <masotoudeh@ucdavis.edu>2020-08-24 09:01:16 -0700
committerGitHub <noreply@github.com>2020-08-24 09:01:16 -0700
commit06fa0707035bd646d26f9477b93be83a54dae425 (patch)
treefa2c3db2ebfb5c63f2b8d0c16ada06ad0e00bbc1
parent38c22236f1c460f16c27443e7d37c8d33a81a5f8 (diff)
Switch to External Bazel-Python (#5)
Refactors the Bazel Python code into a separate repository. Also updates the CI to use the new Python approach.
-rw-r--r--._dummy_.py2
-rw-r--r--.github/workflows/test_syrenn.yml4
-rw-r--r--BUILD153
-rw-r--r--Dockerfile12
-rw-r--r--README.md44
-rw-r--r--WORKSPACE25
-rwxr-xr-xcoverage_report.sh10
-rw-r--r--experiments/BUILD11
-rw-r--r--external/openssl.BUILD31
-rw-r--r--external/python.BUILD30
-rw-r--r--models/BUILD12
-rw-r--r--pysyrenn/frontend/BUILD12
-rw-r--r--pysyrenn/frontend/network.py5
-rw-r--r--pysyrenn/frontend/tests/BUILD14
-rw-r--r--pysyrenn/frontend/transformer_client.py2
-rw-r--r--pysyrenn/helpers/BUILD7
-rw-r--r--pysyrenn/helpers/tests/BUILD5
-rwxr-xr-xpywrapper.sh7
-rw-r--r--requirements.txt3
-rw-r--r--third_party/eran_preconditions/BUILD16
-rw-r--r--third_party/eran_preconditions/Dockerfile3
-rwxr-xr-xthird_party/eran_preconditions/experiment.sh2
-rw-r--r--third_party/reluplex_model_checking/BUILD13
23 files changed, 116 insertions, 307 deletions
diff --git a/._dummy_.py b/._dummy_.py
deleted file mode 100644
index 252ad38..0000000
--- a/._dummy_.py
+++ /dev/null
@@ -1,2 +0,0 @@
-# This file exists because the py_library rule in BUILD (that acts as a dummy
-# target for the pip imports) needs a src.
diff --git a/.github/workflows/test_syrenn.yml b/.github/workflows/test_syrenn.yml
index 3ea52b5..ed0d668 100644
--- a/.github/workflows/test_syrenn.yml
+++ b/.github/workflows/test_syrenn.yml
@@ -43,4 +43,8 @@ jobs:
chmod +x install.sh
./install.sh --user
rm -f install.sh
+ git clone https://github.com/95616ARG/bazel_python.git
+ cd bazel_python
+ echo y | ./setup_python.sh 3.7.4 $HOME/.bazel_python
+ cd .. && rm -rf bazel_python
/home/runner/bin/bazel test pysyrenn/...
diff --git a/BUILD b/BUILD
index 8a75ead..4300013 100644
--- a/BUILD
+++ b/BUILD
@@ -1,130 +1,37 @@
-sh_library(
- name = "pywrapper",
- srcs = ["pywrapper.sh"],
- visibility = ["//:__subpackages__"],
-)
-
-# https://stackoverflow.com/questions/47036855
-py_runtime(
- name = "python-3",
- files = ["@python_3//:installdir"],
- interpreter = "pywrapper.sh",
- python_version = "PY3",
-)
-
-# https://github.com/bazelbuild/rules_python/blob/master/proposals/2019-02-12-design-for-a-python-toolchain.md
-load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")
-
-constraint_value(
- name = "python3_constraint",
- constraint_setting = "@bazel_tools//tools/python:py3_interpreter_path",
-)
-
-platform(
- name = "python3_platform",
- constraint_values = [
- ":python3_constraint",
- ],
-)
-
-py_runtime_pair(
- name = "python3_runtime_pair",
- py3_runtime = ":python-3",
-)
-
-toolchain(
- name = "python3-toolchain",
- # Since the Python interpreter is invoked at runtime on the target
- # platform, there's no need to specify execution platform constraints here.
- target_compatible_with = [
- # Make sure this toolchain is only selected for a target platform that
- # advertises that it has interpreters available under /usr/weirdpath.
- # ":python3_constraint",
- ],
- toolchain = "//:python3_runtime_pair",
- toolchain_type = "@bazel_tools//tools/python:toolchain_type",
-)
-
-# This seems to be the way Google is doing it now:
-# https://github.com/bazelbuild/rules_python/issues/119
-# See https://github.com/pypa/pip/issues/3826 for why we need the --system flag
-# on the pip call. For newer versions of pip, you may have to add
-# --no-build-isolation.
-genrule(
- name = "install-pip-packages",
- srcs = ["requirements.txt"],
- outs = ["pip_packages"],
- cmd = """
- PYTHON=$(location pywrapper.sh)
- PIP="$$PYTHON -m pip"
-
- DUMMY_HOME=/tmp/$$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8)
- rm -rf $$DUMMY_HOME
-
- export HOME=$$DUMMY_HOME
- PIP_INSTALL="$$PIP \
- install --no-cache-dir --disable-pip-version-check \
- --target=$@"
-
- # Setup the environment to point to the right Python installation.
- mkdir -p $$DUMMY_HOME
- INSTALLDIR=$$PWD/$$(find **/** -name "installdir" | head -n 1)
- export LD_LIBRARY_PATH=$$INSTALLDIR/lib/:$$INSTALLDIR/lib64/
- ln -s $$INSTALLDIR/python3.7 $$DUMMY_HOME/python
- export PYTHONPATH=$$PWD/$@
- export PATH=$$DUMMY_HOME:$$PATH
-
- # Install the correct version of Torch
- $$PIP_INSTALL torch==1.2.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
-
- # Install the other requirements.
- $$PIP_INSTALL --upgrade setuptools
- $$PIP_INSTALL -r requirements.txt
-
- # The custom typing package installed as a dependency doesn't seem to work
- # well.
- rm -rf $@/typing-*
- rm -rf $@/typing.py
- rm -rf $$DUMMY_HOME
+load("@bazel_python//:bazel_python.bzl", "bazel_python_interpreter")
+
+bazel_python_interpreter(
+ name = "bazel_python_venv",
+ python_version = "3.7.4",
+ requirements_file = "requirements.txt",
+ run_after_pip = """
+ pip3 install torch==1.2.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
""",
- tools = [
- ":pywrapper.sh",
- "@python_3//:installdir",
- ],
- visibility = ["//:__subpackages__"],
-)
-
-py_library(
- name = "pip-packages",
- srcs = ["._dummy_.py"],
- data = [":install-pip-packages"],
- imports = ["pip_packages"],
visibility = ["//:__subpackages__"],
)
# Make the thicker-bordered plane SVG.
genrule(
name = "thicker-plane",
- srcs = [
- "@plane_svg//file",
- "pywrapper.sh",
- ],
+ srcs = ["@plane_svg//file"],
outs = ["plane.png"],
cmd = """
PLANESVG=$(location @plane_svg//file)
- PYTHON=$(location pywrapper.sh)
- export PYTHONPATH=$(location //:pip_packages)
+
+ PYTHON_VENV=$(location //:bazel_python_venv)
+ pushd $$PYTHON_VENV/..
+ source bazel_python_venv_installed/bin/activate
+ popd
sed -i -e \
's/id="path5724" /id="path5724" stroke="white" fill="black" stroke-width="10" /' \
$$PLANESVG
- $$PYTHON -m cairosvg $$PLANESVG -o plane.png --output-width 4965
+ python3 -m cairosvg $$PLANESVG -o plane.png --output-width 4965
cp plane.png $@
""",
tools = [
- "//:pip_packages",
- "@python_3//:installdir",
+ "//:bazel_python_venv",
],
visibility = ["//:__subpackages__"],
)
@@ -133,11 +40,17 @@ genrule(
sh_binary(
name = "coverage_report",
srcs = ["coverage_report.sh"],
- data = [
- "//:pip_packages",
- "//:pywrapper",
- "@python_3//:installdir",
- ],
+ deps = [":_dummy_coverage_report"],
+)
+
+# This is only to get bazel_python_venv as a data dependency for
+# coverage_report above. For some reason, this doesn't work if we directly put
+# it on the sh_binary. This is a known issue:
+# https://github.com/bazelbuild/bazel/issues/1147#issuecomment-428698802
+sh_library(
+ name = "_dummy_coverage_report",
+ srcs = ["coverage_report.sh"],
+ data = ["//:bazel_python_venv"],
)
# For wheel-ifying the Python code.
@@ -156,23 +69,23 @@ genrule(
],
outs = ["pysyrenn.dist"],
cmd = """
- PYTHON=$(location pywrapper)
- export PYTHONPATH=$$PWD:$(location //:pip_packages)
+ PYTHON_VENV=$(location //:bazel_python_venv)
+ pushd $$PYTHON_VENV/..
+ source bazel_python_venv_installed/bin/activate
+ popd
mkdir -p syrenn_proto
cp -Lr $(locations //syrenn_proto:syrenn_py_grpc) syrenn_proto
cp -Lr $(locations //syrenn_proto:syrenn_py_proto) syrenn_proto
touch syrenn_proto/__init__.py
cp pip_info/* .
- $$PYTHON setup.py sdist bdist_wheel
+ python3 setup.py sdist bdist_wheel
cp -r dist $@
""",
tools = [
- "pywrapper",
- "//:pip_packages",
+ "//:bazel_python_venv",
"//syrenn_proto:syrenn_py_grpc",
"//syrenn_proto:syrenn_py_proto",
- "@python_3//:installdir",
],
)
diff --git a/Dockerfile b/Dockerfile
index 7005ce6..5d565b1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -12,7 +12,7 @@ RUN apt-get update && apt-get install -y \
pkg-config zip g++ zlib1g-dev unzip \
python2.7 python-pip python3 python3-pip \
build-essential curl git cmake libcairo2 libffi-dev libgmp3-dev \
- zlib1g-dev zip \
+ zlib1g-dev libssl-dev libbz2-dev zip \
apt-transport-https ca-certificates gnupg-agent software-properties-common
# Install Docker.
@@ -30,6 +30,14 @@ RUN curl -OL https://github.com/bazelbuild/bazel/releases/download/1.1.0/bazel-1
./bazel-1.1.0-installer-linux-x86_64.sh && \
rm bazel-1.1.0-installer-linux-x86_64.sh
+# Set up bazel_python
+RUN git clone https://github.com/95616ARG/bazel_python.git && \
+ cd bazel_python && \
+ echo y | ./setup_python.sh 3.7.4 /.bazel_python && \
+ cp $HOME/.bazelrc /etc/bazel.bazelrc && \
+ cd .. && \
+ rm -rf bazel_python
+
# Install benchexec.
RUN curl -OL https://github.com/sosy-lab/benchexec/releases/download/2.2/benchexec_2.2-1_all.deb && \
apt install -y --install-recommends ./benchexec_2.2-1_all.deb && \
@@ -39,7 +47,7 @@ RUN rm -rf /var/lib/apt/lists/*
# Store Bazel outputs on /iovol/.docker_bazel, which will be loaded at runtime.
# This allows us to cache Bazel build artifacts across Docker invocations.
-RUN echo "startup --output_user_root=/iovol/.docker_bazel" > /etc/bazel.bazelrc
+RUN echo "startup --output_user_root=/iovol/.docker_bazel" >> /etc/bazel.bazelrc
RUN useradd -u $UID -ms /bin/bash docker_user
RUN adduser docker_user benchexec
diff --git a/README.md b/README.md
index 3cefdf5..e073a12 100644
--- a/README.md
+++ b/README.md
@@ -103,26 +103,39 @@ is already freezing/running out of memory, consider removing the existing
``-j#`` flags from ``external/*.BUILD``.
#### Local Builds
-You must install [Bazel](https://bazel.build/) 1.1.0 and have binaries for
-building arbitrary C++ packages (eg. ``build-essential`` for Ubuntu).
-Furthermore, the ``libcairo2``, ``libffi-dev``, ``zlib1g-dev``, ``zip``, and
-``libgmp3-dev`` packages are required for the Python code (but usually come
-pre-installed with most desktop Linux distributions). A
-[Dockerfile](Dockerfile) is provided with a compatible setup for reference.
-Note that Bazel will _automatically_ download and build copies of OpenSSL,
-Python 3.7.4, Intel TBB, Intel MKLDNN, Eigen, OpenBLAS, PyTorch, and all PIP
-dependencies to the Bazel environment when applicable --- they do not need to
-be manually installed.
+1. You must install [Bazel](https://bazel.build/) 1.1.0 and have binaries for
+ building arbitrary C++ packages (eg. ``build-essential`` for Ubuntu).
+2. Furthermore, the ``libcairo2``, ``libffi-dev``, ``zlib1g-dev``, ``zip``, and
+ ``libgmp3-dev`` packages are required for the Python code (but usually come
+ pre-installed with most desktop Linux distributions).
+3. You must follow the "One-Time Setup" instructions in
+ [bazel_python](https://github.com/95616ARG/bazel_python) to install a fresh
+ copy of Python 3.7.4. This can be done per-user without root permissions.
+ This is **not** necessary if you only want to run the server, e.g., if you
+ just want to use the `pysyrenn` package from PyPI. It **is**, however,
+ required to run any of the Python code in this repository with Bazel,
+ including the Python tests.
+4. A [Dockerfile](Dockerfile) is provided with a compatible setup for
+ reference. Note that Bazel will _automatically_ download and build copies
+ of Intel TBB, Intel MKLDNN, Eigen, OpenBLAS, PyTorch, and all PIP
+ dependencies to the Bazel environment when applicable --- they do not need
+ to be manually installed.
#### Docker Builds
Alternatively, a Docker container is provided to simplify the build and running
process. To use it, first build the image with ``./docker_build.sh`` then
prepend ``./docker_run.sh`` to all of the commands below. For example, instead
of ``make start_server``, use ``./docker_run.sh make start_server``. Everything
-should be handled transparently. **NOTE:** Benchexec is currently not supported
-under the Docker container due to permission errors. It is possible to resolve
-this by removing all of the user-related lines from [Dockerfile](Dockerfile),
-but that will likely cause issues with Bazel and generated file permissions.
+should be handled transparently.
+
+**NOTE:** Benchexec is currently not supported under the Docker container due
+to permission errors. It is possible to resolve this by removing all of the
+user-related lines from [Dockerfile](Dockerfile), but that will likely cause
+issues with Bazel and generated file permissions.
+
+**NOTE:** Docker-in-docker does not seem to currently be working. This means
+some builds will fail, although you should be able to run `make start_server`,
+`bazel test //pysyrenn/...`, and `bazel test //syrenn_server/...`.
### Configuration
There are two major things to configure before using the code:
@@ -217,8 +230,7 @@ There are three main pieces to the library:
A few other directories also exist:
-- [external/](external) contains Bazel BUILD rules for GTEST, Python 3.7.4 (including
- OpenSSL), and TBB.
+- [external/](external) contains Bazel BUILD rules for GTEST, Eigen, and TBB.
- [models/](models) contains Bazel rules either generating, downloading, or
referencing trained models from the ERAN, ACAS Xu, and VRL projects (prior
work). ERAN models are downloaded in WORKSPACE and referenced in
diff --git a/WORKSPACE b/WORKSPACE
index 357d01a..09c71ae 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -1,28 +1,17 @@
workspace(name = "SyReNN")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-# Get the Python interpreter & build dependency (OpenSSL).
-# https://stackoverflow.com/questions/47036855
-http_archive(
- name = "openssl",
- build_file = "openssl.BUILD",
- sha256 = "cabd5c9492825ce5bd23f3c3aeed6a97f8142f606d893df216411f07d1abab96",
- strip_prefix = "openssl-1.0.2s",
- urls = ["https://www.openssl.org/source/old/1.0.2/openssl-1.0.2s.tar.gz"],
-)
-
-http_archive(
- name = "python_3",
- build_file = "python.BUILD",
- sha256 = "d63e63e14e6d29e17490abbe6f7d17afb3db182dbd801229f14e55f4157c4ba3",
- strip_prefix = "Python-3.7.4",
- urls = ["https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz"],
+git_repository(
+ name = "bazel_python",
+ commit = "5edcf3b97c4c77b654af177bfa27558d9b88b52f",
+ remote = "https://github.com/95616ARG/bazel_python.git",
)
-register_toolchains("//:python3-toolchain")
+load("@bazel_python//:bazel_python.bzl", "bazel_python")
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+bazel_python()
# EIGEN SUPPORT
# See the README in: https://github.com/bazelbuild/rules_foreign_cc
diff --git a/coverage_report.sh b/coverage_report.sh
index 5edbf50..87bdf6c 100755
--- a/coverage_report.sh
+++ b/coverage_report.sh
@@ -2,11 +2,7 @@ COVTEMP=$PWD/coverage_tmp
rm -rf $COVTEMP
mkdir $COVTEMP
-# Make sure we know where to find the Python script, and that it knows where to
-# find its pip packages.
-export INSTALLDIR=$PWD/external/python_3/installdir
-PYTHON=$PWD/pywrapper.sh
-export PYTHONPATH=$PYTHONPATH:$PWD/pip_packages
+source bazel_python_venv_installed/bin/activate
# Go to the main workspace directory and run the coverage-report.
pushd $BUILD_WORKSPACE_DIRECTORY
@@ -25,8 +21,8 @@ done
rm -rf .coverage htmlcov
# Then we build a new .coverage as well as export to HTML
-$PYTHON -m coverage combine $COVTEMP/*.cov
-$PYTHON -m coverage html pysyrenn/*/*.py
+python3 -m coverage combine $COVTEMP/*.cov
+python3 -m coverage html pysyrenn/*/*.py
# Remove temporaries and go back to where Bazel started us.
rm -r $COVTEMP
diff --git a/experiments/BUILD b/experiments/BUILD
index 203583d..beef753 100644
--- a/experiments/BUILD
+++ b/experiments/BUILD
@@ -6,7 +6,6 @@ py_library(
"//models:datasets",
],
deps = [
- "//:pip-packages",
"//pysyrenn",
"//syrenn_proto:syrenn_py_grpc",
],
@@ -15,7 +14,6 @@ py_library(
py_library(
name = "polar_image",
srcs = ["polar_image.py"],
- deps = ["//:pip-packages"],
)
filegroup(
@@ -29,7 +27,6 @@ filegroup(
py_library(
name = "integral_approximations",
srcs = ["integral_approximations.py"],
- deps = ["//:pip-packages"],
)
py_binary(
@@ -38,7 +35,6 @@ py_binary(
deps = [
":experiment",
":integral_approximations",
- "//:pip-packages",
"//pysyrenn",
],
)
@@ -51,7 +47,6 @@ py_binary(
],
deps = [
":experiment",
- "//:pip-packages",
"//pysyrenn",
],
)
@@ -61,7 +56,6 @@ py_binary(
srcs = ["linearity_hypothesis.py"],
deps = [
":experiment",
- "//:pip-packages",
"//pysyrenn",
],
)
@@ -75,7 +69,6 @@ py_binary(
deps = [
":acas_lines",
":polar_image",
- "//:pip-packages",
"//pysyrenn",
],
)
@@ -84,7 +77,6 @@ py_library(
name = "vrl_model",
srcs = ["vrl_model.py"],
visibility = ["//:__subpackages__"],
- deps = ["//:pip-packages"],
)
py_binary(
@@ -94,7 +86,6 @@ py_binary(
deps = [
":experiment",
":vrl_model",
- "//:pip-packages",
"//pysyrenn",
],
)
@@ -105,7 +96,6 @@ py_binary(
deps = [
":acas_planes",
":polar_image",
- "//:pip-packages",
"//pysyrenn",
],
)
@@ -115,7 +105,6 @@ py_binary(
srcs = ["toy_examples.py"],
deps = [
":acas_planes",
- "//:pip-packages",
"//pysyrenn",
],
)
diff --git a/external/openssl.BUILD b/external/openssl.BUILD
deleted file mode 100644
index b1c0856..0000000
--- a/external/openssl.BUILD
+++ /dev/null
@@ -1,31 +0,0 @@
-# https://stackoverflow.com/questions/2537271
-genrule(
- name = "build_openssl",
- srcs = glob(["**"]),
- outs = ["openssl-installdir"],
- cmd = """
- # First, we need to copy the OpenSSL source to a writeable location
- # (following links). If we try to do it directly in the original
- # OPENSSLDIR, it consistently throws errors about file system permissions
- # (I'm not entirely sure why, but this fixes it).
- HOMEDIR=$$PWD
- SOURCEDIR=$$PWD/$$(dirname $(location config))
- cp -Lr $$SOURCEDIR openssl
- SOURCEDIR=$$PWD/openssl
-
- # Then create the install dir.
- INSTALLDIR=$$PWD/openssl-installdir
- mkdir -p $$INSTALLDIR
-
- # Install OpenSSL to installdir.
- cd $$SOURCEDIR
- ./config shared -fPIC --prefix=$$INSTALLDIR --openssldir=$$INSTALLDIR/openssl > /dev/null
- make -j2 > /dev/null
- make install > /dev/null
- cd $$HOMEDIR
-
- # And copy to the output directory.
- cp -r $$INSTALLDIR $@
- """,
- visibility = ["//visibility:public"],
-)
diff --git a/external/python.BUILD b/external/python.BUILD
deleted file mode 100644
index 0741461..0000000
--- a/external/python.BUILD
+++ /dev/null
@@ -1,30 +0,0 @@
-# https://superuser.com/questions/1346141
-# There's another one I lost about finding the current script's directory.
-genrule(
- name = "build_python",
- srcs = glob(["**"]) + ["@openssl//:openssl-installdir"],
- outs = ["installdir"],
- cmd = """
- HOMEDIR=$$PWD
- INSTALLDIR=$$PWD/installdir
- cp -Lr $(location @openssl//:openssl-installdir) $$INSTALLDIR
-
- # See external/openssl.BUILD for notes on why we do this weird copy.
- SOURCEDIR=$$PWD/$$(dirname $(location configure))
- cp -Lr $$SOURCEDIR $$PWD/python
- SOURCEDIR=$$PWD/python
-
- export LDFLAGS="-L$$INSTALLDIR/lib/ -L$$INSTALLDIR/lib64/"
- export LD_LIBRARY_PATH="$$INSTALLDIR/lib/:$$INSTALLDIR/lib64/"
- export CPPFLAGS="-I$$INSTALLDIR/include -I$$INSTALLDIR/include/openssl"
-
- # Install Python to installdir.
- cd $$SOURCEDIR
- ./configure --prefix=$$INSTALLDIR --with-openssl=$$INSTALLDIR > /dev/null
- make -j2 > /dev/null
- make install > /dev/null
- cd $$HOMEDIR
- cp -r installdir $(@D)
- """,
- visibility = ["//visibility:public"],
-)
diff --git a/models/BUILD b/models/BUILD
index fc829bf..1ad15e0 100644
--- a/models/BUILD
+++ b/models/BUILD
@@ -46,15 +46,17 @@ genrule(
],
outs = ["acas_models"],
cmd = """
- PYTHON=$(location //:pywrapper)
- export PYTHONPATH=$(location //:pip_packages)
+ PYTHON_VENV=$(location //:bazel_python_venv)
+ pushd $$PYTHON_VENV/..
+ source bazel_python_venv_installed/bin/activate
+ popd
cp -L external/reluplex/nnet/*.nnet .
all_files=$$(ls | grep .nnet)
for file in $$all_files
do
- $$PYTHON $(location translate_acas_model.py) $$file
+ python3 $(location translate_acas_model.py) $$file
done
mkdir acas_models
@@ -62,9 +64,7 @@ genrule(
mv acas_models $@
""",
tools = [
- "//:pip_packages",
- "//:pywrapper",
- "@python_3//:installdir",
+ "//:bazel_python_venv",
],
visibility = ["//:__subpackages__"],
)
diff --git a/pysyrenn/frontend/BUILD b/pysyrenn/frontend/BUILD
index 8ddf4ee..c52a68a 100644
--- a/pysyrenn/frontend/BUILD
+++ b/pysyrenn/frontend/BUILD
@@ -22,7 +22,6 @@ py_library(
srcs = ["transformer_client.py"],
visibility = ["//pysyrenn/frontend:__subpackages__"],
deps = [
- "//:pip-packages",
"//syrenn_proto:syrenn_py_grpc",
],
)
@@ -42,7 +41,6 @@ py_library(
":relu_layer",
":strided_window_data",
":transformer_client",
- "//:pip-packages",
"//syrenn_proto:syrenn_py_grpc",
],
)
@@ -60,7 +58,6 @@ py_library(
visibility = ["//:__subpackages__"],
deps = [
":layer",
- "//:pip-packages",
"//syrenn_proto:syrenn_py_grpc",
],
)
@@ -71,7 +68,6 @@ py_library(
visibility = ["//:__subpackages__"],
deps = [
":layer",
- "//:pip-packages",
"//syrenn_proto:syrenn_py_grpc",
],
)
@@ -82,7 +78,6 @@ py_library(
visibility = ["//:__subpackages__"],
deps = [
":layer",
- "//:pip-packages",
"//syrenn_proto:syrenn_py_grpc",
],
)
@@ -93,7 +88,6 @@ py_library(
visibility = ["//:__subpackages__"],
deps = [
":layer",
- "//:pip-packages",
"//syrenn_proto:syrenn_py_grpc",
],
)
@@ -104,7 +98,6 @@ py_library(
visibility = ["//:__subpackages__"],
deps = [
":layer",
- "//:pip-packages",
"//syrenn_proto:syrenn_py_grpc",
],
)
@@ -114,7 +107,6 @@ py_library(
srcs = ["strided_window_data.py"],
visibility = ["//:__subpackages__"],
deps = [
- "//:pip-packages",
"//syrenn_proto:syrenn_py_grpc",
],
)
@@ -126,7 +118,6 @@ py_library(
deps = [
":layer",
":strided_window_data",
- "//:pip-packages",
"//syrenn_proto:syrenn_py_grpc",
],
)
@@ -138,7 +129,6 @@ py_library(
deps = [
":layer",
":strided_window_data",
- "//:pip-packages",
"//syrenn_proto:syrenn_py_grpc",
],
)
@@ -150,7 +140,6 @@ py_library(
deps = [
":layer",
":strided_window_data",
- "//:pip-packages",
"//syrenn_proto:syrenn_py_grpc",
],
)
@@ -161,7 +150,6 @@ py_library(
visibility = ["//:__subpackages__"],
deps = [
":layer",
- "//:pip-packages",
"//syrenn_proto:syrenn_py_grpc",
],
)
diff --git a/pysyrenn/frontend/network.py b/pysyrenn/frontend/network.py
index c3c8247..fb9623e 100644
--- a/pysyrenn/frontend/network.py
+++ b/pysyrenn/frontend/network.py
@@ -151,7 +151,10 @@ class Network:
Returns a list of UPolytopes (one per plane in @planes), which are
themselves lists of either tuples of Numpy arrays (pre, post) or just
- Numpy arrays pre depending on @include_post.
+ Numpy arrays pre depending on @include_post. Note that
+ @compute_preimages=False still returns (pre, post), except the @pre
+ matrices are *combinations* of the original vertices, not points
+ themselves.
"""
planes = np.asarray(planes)
transformed = transformer_client.transform_planes(self, planes)
diff --git a/pysyrenn/frontend/tests/BUILD b/pysyrenn/frontend/tests/BUILD
index ac45fee..2f89338 100644
--- a/pysyrenn/frontend/tests/BUILD
+++ b/pysyrenn/frontend/tests/BUILD
@@ -2,7 +2,6 @@ py_library(
name = "helpers",
srcs = ["helpers.py"],
visibility = ["//:__subpackages__"],
- deps = ["//:pip-packages"],
)
py_test(
@@ -11,7 +10,6 @@ py_test(
srcs = ["relu_layer.py"],
deps = [
":helpers",
- "//:pip-packages",
"//pysyrenn/frontend:relu_layer",
],
)
@@ -22,7 +20,6 @@ py_test(
srcs = ["hard_tanh_layer.py"],
deps = [
":helpers",
- "//:pip-packages",
"//pysyrenn/frontend:hard_tanh_layer",
],
)
@@ -33,7 +30,6 @@ py_test(
srcs = ["argmax_layer.py"],
deps = [
":helpers",
- "//:pip-packages",
"//pysyrenn/frontend:argmax_layer",
],
)
@@ -44,7 +40,6 @@ py_test(
srcs = ["fullyconnected_layer.py"],
deps = [
":helpers",
- "//:pip-packages",
"//pysyrenn/frontend:fullyconnected_layer",
],
)
@@ -55,7 +50,6 @@ py_test(
srcs = ["normalize_layer.py"],
deps = [
":helpers",
- "//:pip-packages",
"//pysyrenn/frontend:normalize_layer",
],
)
@@ -66,7 +60,6 @@ py_test(
srcs = ["averagepool_layer.py"],
deps = [
":helpers",
- "//:pip-packages",
"//pysyrenn/frontend:averagepool_layer",
"//pysyrenn/frontend:strided_window_data",
],
@@ -78,7 +71,6 @@ py_test(
srcs = ["maxpool_layer.py"],
deps = [
":helpers",
- "//:pip-packages",
"//pysyrenn/frontend:maxpool_layer",
"//pysyrenn/frontend:strided_window_data",
],
@@ -90,7 +82,6 @@ py_test(
srcs = ["conv2d_layer.py"],
deps = [
":helpers",
- "//:pip-packages",
"//pysyrenn/frontend:conv2d_layer",
"//pysyrenn/frontend:strided_window_data",
],
@@ -102,7 +93,6 @@ py_test(
srcs = ["concat_layer.py"],
deps = [
":helpers",
- "//:pip-packages",
"//pysyrenn/frontend:averagepool_layer",
"//pysyrenn/frontend:concat_layer",
"//pysyrenn/frontend:conv2d_layer",
@@ -118,7 +108,6 @@ py_test(
srcs = ["strided_window_data.py"],
deps = [
":helpers",
- "//:pip-packages",
"//pysyrenn/frontend:strided_window_data",
],
)
@@ -130,7 +119,6 @@ py_test(
data = ["@onnx_squeezenet//:all"],
deps = [
":helpers",
- "//:pip-packages",
"//pysyrenn/frontend:argmax_layer",
"//pysyrenn/frontend:fullyconnected_layer",
"//pysyrenn/frontend:network",
@@ -145,7 +133,6 @@ py_test(
srcs = ["layer.py"],
deps = [
":helpers",
- "//:pip-packages",
"//pysyrenn/frontend:layer",
],
)
@@ -156,7 +143,6 @@ py_test(
srcs = ["transformer_client.py"],
deps = [
":helpers",
- "//:pip-packages",
"//pysyrenn/frontend:fullyconnected_layer",
"//pysyrenn/frontend:network",
"//pysyrenn/frontend:relu_layer",
diff --git a/pysyrenn/frontend/transformer_client.py b/pysyrenn/frontend/transformer_client.py
index bdf748a..adfe9c2 100644
--- a/pysyrenn/frontend/transformer_client.py
+++ b/pysyrenn/frontend/transformer_client.py
@@ -5,6 +5,8 @@ these methods.
"""
import os
import numpy as np
+import sys
+sys.path = [path for path in sys.path if "/com_github_grpc_grpc/" not in path]
import grpc
import syrenn_proto.syrenn_pb2_grpc as grpc_pb
import syrenn_proto.syrenn_pb2 as transformer_pb
diff --git a/pysyrenn/helpers/BUILD b/pysyrenn/helpers/BUILD
index d071306..5e97de8 100644
--- a/pysyrenn/helpers/BUILD
+++ b/pysyrenn/helpers/BUILD
@@ -15,9 +15,6 @@ py_library(
name = "integrated_gradients",
srcs = ["integrated_gradients.py"],
visibility = [":__subpackages__"],
- deps = [
- "//:pip-packages",
- ],
)
py_library(
@@ -25,7 +22,6 @@ py_library(
srcs = ["classify_lines.py"],
visibility = [":__subpackages__"],
deps = [
- "//:pip-packages",
"//pysyrenn/frontend:argmax_layer",
"//pysyrenn/frontend:network",
],
@@ -36,7 +32,6 @@ py_library(
srcs = ["classify_planes.py"],
visibility = [":__subpackages__"],
deps = [
- "//:pip-packages",
"//pysyrenn/frontend:argmax_layer",
"//pysyrenn/frontend:network",
],
@@ -48,7 +43,6 @@ py_library(
visibility = [":__subpackages__"],
deps = [
":masking_network",
- "//:pip-packages",
"//pysyrenn/frontend:fullyconnected_layer",
"//pysyrenn/frontend:network",
"//pysyrenn/frontend:relu_layer",
@@ -60,7 +54,6 @@ py_library(
srcs = ["masking_network.py"],
visibility = [":__subpackages__"],
deps = [
- "//:pip-packages",
"//pysyrenn/frontend:fullyconnected_layer",
"//pysyrenn/frontend:network",
"//pysyrenn/frontend:relu_layer",
diff --git a/pysyrenn/helpers/tests/BUILD b/pysyrenn/helpers/tests/BUILD
index dd425e1..2800717 100644
--- a/pysyrenn/helpers/tests/BUILD
+++ b/pysyrenn/helpers/tests/BUILD
@@ -3,7 +3,6 @@ py_test(
size = "small",
srcs = ["classify_lines.py"],
deps = [
- "//:pip-packages",
"//pysyrenn/frontend",
"//pysyrenn/frontend/tests:helpers",
"//pysyrenn/helpers:classify_lines",
@@ -15,7 +14,6 @@ py_test(
size = "small",
srcs = ["classify_planes.py"],
deps = [
- "//:pip-packages",
"//pysyrenn/frontend",
"//pysyrenn/frontend/tests:helpers",
"//pysyrenn/helpers:classify_planes",
@@ -27,7 +25,6 @@ py_test(
size = "small",
srcs = ["integrated_gradients.py"],
deps = [
- "//:pip-packages",
"//pysyrenn/frontend",
"//pysyrenn/frontend/tests:helpers",
"//pysyrenn/helpers:integrated_gradients",
@@ -39,7 +36,6 @@ py_test(
size = "small",
srcs = ["masking_network.py"],
deps = [
- "//:pip-packages",
"//pysyrenn/frontend",
"//pysyrenn/frontend/tests:helpers",
"//pysyrenn/helpers:masking_network",
@@ -51,7 +47,6 @@ py_test(
size = "small",
srcs = ["netpatch.py"],
deps = [
- "//:pip-packages",
"//pysyrenn/frontend",
"//pysyrenn/frontend/tests:helpers",
"//pysyrenn/helpers:netpatch",
diff --git a/pywrapper.sh b/pywrapper.sh
deleted file mode 100755
index 379b17a..0000000
--- a/pywrapper.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-if [ -z $INSTALLDIR ]; then
- INSTALLDIR=$PWD/$(find **/** -name "installdir" | head -n 1)
-fi
-export LD_LIBRARY_PATH=$INSTALLDIR/lib/:$INSTALLDIR/lib64/
-$INSTALLDIR/bin/python3.7 $@
diff --git a/requirements.txt b/requirements.txt
index 5a89f8c..23be3dd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,8 @@ protobuf==3.9.2
grpcio==1.24.0
onnx==1.5.0
tqdm==4.36.1
-# Torch is installed in the BUILD script and specified in pip_info/setup.py.
+# TORCH-CPU IS INSTALLED SEPARATELY. For some reason it doesn't work if we
+# install it here. It is also manually included in pip-info.
aenum==2.2.1
# END
coverage==4.5.4
diff --git a/third_party/eran_preconditions/BUILD b/third_party/eran_preconditions/BUILD
index fb3e96b..6204d7e 100644
--- a/third_party/eran_preconditions/BUILD
+++ b/third_party/eran_preconditions/BUILD
@@ -3,17 +3,18 @@ genrule(
srcs = [
"Dockerfile",
"install.sh",
- "//:plane.png",
],
- outs = ["eran_preconditions.tgz"],
+ outs = ["eran_image.tgz"],
cmd = """
- cp $(location //:plane.png) plane.png
cp $(location Dockerfile) Dockerfile
cp $(location install.sh) install.sh
- docker build --force-rm -t eran_preconditions .
- docker save -o eran_preconditions.tgz eran_preconditions
- cp eran_preconditions.tgz $@
+ docker build --force-rm -t eran_image .
+ docker save -o eran_image.tgz eran_image
+ cp eran_image.tgz $@
""",
+ visibility = [
+ "//third_party/eran_bmc:__pkg__",
+ ],
)
sh_binary(
@@ -21,7 +22,8 @@ sh_binary(
srcs = ["experiment.sh"],
data = [
"experiment.py",
- ":eran_preconditions.tgz",
+ ":eran_image.tgz",
+ "//:plane.png",
"//experiments:polar_image_only",
"//models:acas_models",
],
diff --git a/third_party/eran_preconditions/Dockerfile b/third_party/eran_preconditions/Dockerfile
index 9691177..45d1a57 100644
--- a/third_party/eran_preconditions/Dockerfile
+++ b/third_party/eran_preconditions/Dockerfile
@@ -7,14 +7,13 @@ RUN apt-get update && apt-get install -y \
wget unzip build-essential python3 \
python3-distutils python3-pip git gdb
-# Get the Reluplex source
+# Get the ERAN source
WORKDIR /
RUN wget https://github.com/eth-sri/eran/archive/31f7784bea1cf00ab2e358b12e68414893122034.zip -O eran.zip
RUN unzip eran.zip -d eran && rm eran.zip
RUN mv eran/**/* eran && rm -r eran/eran-*
WORKDIR /eran
-COPY plane.png plane.png
# We use our own install.sh to specify the ELINA version (for reproducibility)
# and use multi-threaded Make.
COPY install.sh install.sh
diff --git a/third_party/eran_preconditions/experiment.sh b/third_party/eran_preconditions/experiment.sh
index 5a623a0..8b060e0 100755
--- a/third_party/eran_preconditions/experiment.sh
+++ b/third_party/eran_preconditions/experiment.sh
@@ -5,7 +5,7 @@ outdir=$builddir/third_party/eran_preconditions
model="$PWD/models/acas_models/1_1.eran"
polar_image_py="$PWD/experiments/polar_image.py"
experiment_py="$PWD/third_party/eran_preconditions/experiment.py"
-image_file="$PWD/third_party/eran_preconditions/eran_preconditions.tgz"
+image_file="$PWD/third_party/eran_preconditions/eran_image.tgz"
# Copy everything to the local directory and load the Docker image.
cp $model model.eran
diff --git a/third_party/reluplex_model_checking/BUILD b/third_party/reluplex_model_checking/BUILD
index 8f4307b..a432dfa 100644
--- a/third_party/reluplex_model_checking/BUILD
+++ b/third_party/reluplex_model_checking/BUILD
@@ -24,19 +24,18 @@ genrule(
],
outs = ["specs.tgz"],
cmd = """
- SCRIPT=$(location export_specs.py)
- PYTHON=$(location //:pywrapper)
- export PYTHONPATH=$(location //:pip_packages)
+ PYTHON_VENV=$(location //:bazel_python_venv)
+ pushd $$PYTHON_VENV/..
+ source bazel_python_venv_installed/bin/activate
+ popd
- $$PYTHON $$SCRIPT
+ python3 $(location export_specs.py)
tar -czvf specs.tgz pendulum_continuous satelite quadcopter
cp specs.tgz $@
""",
tools = [
- "//:pip_packages",
- "//:pywrapper",
+ "//:bazel_python_venv",
"//experiments:vrl_model",
- "@python_3//:installdir",
],
visibility = ["//:__subpackages__"],
)
generated by cgit on debian on lair
contact matthew@masot.net with questions or feedback