Commit 6b434820 authored by Stanley Clark's avatar Stanley Clark
Browse files

Moved around many files

parent 19ca64e8
SAQP_SCALE_FACTOR=1
SAQP_QUERIES=3,7,11,19,25,26,46,90,96
SAQP_USER_IDS=1,2,3,4,5,6,7,8,9,10
SAQP_NUM_TRIALS=25
\ No newline at end of file
......@@ -5,21 +5,33 @@ This is the project containing the prototype for a security-aware query processo
## Setup & Execution
We use [Docker](https://www.docker.com/) as a wrapper to uniformly handle the whole experiment pipeline for better reproducibility.
Other commands related to Docker are based on shell scripts and should be run on a LINUX environment e.g. WSL on Windows.
To start, install [Git](https://git-scm.com/), and Docker.
Run the following:
```bash
```shell
git clone git@gitlab.tue.nl:stanrogo/saqp/saqp-optimiser.git
cd saqp-optimiser
git submodule update --init
```
Then copy the `.env.example` file to a file named `.env`.
Change the defaults if you are not happy with them (for testing you might want to run one trial with one query and one user id only).
Finally, run:
# Build and launch services
docker-compose up -d
# Next, let's set up the database for a scale factor 1 TPC-DS dataset:
docker-compose exec postgres ./gen_tpcds.sh 1
docker-compose exec postgres ./populate_db.sh tpcds1
# Now, let's build the Java project:
docker-compose run builder mvn initialize
docker-compose run builder mvn package
# Copy the environment files (and change defaults if needed):
cp experiments/.env.example experiments/.env
cp pg-cuckoo/PgCuckoo/config-blank.ini pg-cuckoo/PgCuckoo/config.ini
```bash
docker-compose up
# We are now able to run the compiled files using a dedicated java container:
docker-compose run optimiser
```
This will first build two images and start two respective containers: one containing a PostrgeSQL 10 database instance, and the other containing haskell, java and maven build dependencies.
......
#!/usr/bin/env sh
for query in ../../queries-spjr/*.sql
for query in ../../queries/*.sql
do
for i in $(seq 1 10);
do
......
version: '3'
services:
db:
postgres:
image: stanrogo/postgres-tpc:latest
container_name: db
container_name: postgres
build:
context: ./postgres
context: .
dockerfile: postgres/Dockerfile
ports:
- "5444:5432"
- "5432:5432"
volumes:
- ./postgres/sql:/home/app/sql/
- ./postgres/populate-db.sh:/docker-entrypoint-initdb.d/populate-db.sh
- data-store:/var/lib/postgresql/data
- PostgreSqlTpcDB:/var/lib/postgresql/data
- ./tpc-ds:/home/app/tpc-ds
environment:
POSTGRES_USER: ubuntu
POSTGRES_PASSWORD: ubuntu
POSTGRES_DB: ubuntu
experiments:
image: stanrogo/saqp-spjr:latest
container_name: experiments
env_file:
- .env
builder:
image: stanrogo/saqp-builder:latest
container_name: builder
build:
context: ./experiments
args:
SAQP_SCALE_FACTOR: ${SAQP_SCALE_FACTOR}
SAQP_QUERIES: ${SAQP_QUERIES}
SAQP_USER_IDS: ${SAQP_USER_IDS}
SAQP_NUM_TRIALS: ${SAQP_NUM_TRIALS}
command: java -Duser.dir="/home/app" -jar /home/app/target/SAQP-1.0.0.jar
context: .
dockerfile: maven/Dockerfile
volumes:
- ./experiments/app/results:/home/app/results
depends_on:
- "db"
- MavenM2:/root/.m2
- ./experiments:/usr/src/app
optimiser:
image: stanrogo/saqp-optimiser:latest
container_name: optimiser
build:
context: .
dockerfile: experiments/Dockerfile
volumes:
- ./experiments:/home/app
- ./queries:/home/queries
volumes:
data-store:
PostgreSqlTpcDB:
external: false
MavenM2:
external: false
pg-cuckoo/PgCuckoo/.stack-work
pg-cuckoo/PgCuckoo/PgCuckoo.cabal
pg-cuckoo/PgCuckoo/stack.yaml.lock
pg-cuckoo/PgCuckoo/app/Test.hs
app/target
app/.env
app/.idea
\ No newline at end of file
......@@ -14,55 +14,12 @@ RUN curl -O https://download.java.net/java/GA/jdk14.0.1/664493ef4a6946b186ff29eb
&& rm openjdk-14.0.1_linux-x64_bin.tar.gz \
&& ln -s /opt/jdk-14.0.1/bin/java /usr/bin/java
# Install Maven
RUN mkdir -p /usr/share/maven /usr/share/maven/ref \
&& curl -fsSL -o /tmp/apache-maven.tar.gz https://apache.osuosl.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
&& echo "c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0 /tmp/apache-maven.tar.gz" | sha512sum -c - \
&& tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \
&& rm -f /tmp/apache-maven.tar.gz \
&& ln -s /usr/share/maven/bin/mvn /usr/bin/mvn
# Set environment variables
ENV MAVEN_HOME=/usr/share/maven MAVEN_CONFIG=/root/.m2 JAVA_HOME=/opt/jdk-14.0.1
# First time setup of stack
COPY pg-cuckoo/PgCuckoo /home/pg-cuckoo/PgCuckoo
RUN cd /home/pg-cuckoo/PgCuckoo && stack install --allow-different-user || true
# Link Z3 dependencies for SMT XACML policy translator
COPY app/libz3.so app/libz3java.so /home/app/
RUN ln -s /home/app/libz3.so /usr/lib/libz3.so && ln -s /home/app/libz3java.so /usr/lib/libz3java.so
# Initialise maven dependencies once
COPY app/.env.example app/pom.xml /home/app/
COPY app/src/main/resources /home/app/src/main/resources
COPY app/src/main/xsd /home/app/src/main/xsd
RUN cd /home/app && mvn initialize && mvn generate-sources && mvn process-sources && mvn generate-resources && mvn process-resources && mvn dependency:resolve-plugins
# Copy application code -- repeated everytime a code change happens
COPY app/src/main/java /home/app/src/main/java
# Get arguments to populate configuration files
ARG SAQP_SCALE_FACTOR
ARG SAQP_QUERIES
ARG SAQP_USER_IDS
ARG SAQP_NUM_TRIALS
# Edit the environment
RUN cd /home/app \
&& cp .env.example .env \
&& sed -i "s|^saqp_db_conn=.*$|saqp_db_conn=jdbc:postgresql://db:5432/tpcds$SAQP_SCALE_FACTOR|" .env \
&& sed -i "s|^saqp_db_user=.*$|saqp_db_user=ubuntu|" .env \
&& sed -i "s|^saqp_db_pass=.*$|saqp_db_pass=ubuntu|" .env \
&& sed -i "s|^saqp_results_file=.*$|saqp_results_file=results$SAQP_SCALE_FACTOR.csv|" .env \
&& sed -i "s|^saqp_enable_logging=.*$|saqp_enable_logging=file|" .env \
&& sed -i "s|^saqp_queries=.*$|saqp_queries=$SAQP_QUERIES|" .env \
&& sed -i "s|^saqp_user_ids=.*$|saqp_user_ids=$SAQP_USER_IDS|" .env \
&& sed -i "s|^saqp_num_trials=.*$|saqp_num_trials=$SAQP_NUM_TRIALS|" .env \
&& cd /home/pg-cuckoo/PgCuckoo \
&& cp config-blank.ini config.ini \
&& sed -i "s|^dbauth :.*$|dbauth: host=db dbname=tpcds$SAQP_SCALE_FACTOR port=5432 password=ubuntu user=ubuntu|" config.ini
# Package the jar
RUN cd /home/app && mvn package
WORKDIR /home/app/
CMD java -Duser.dir="/home/app" -jar /home/app/target/SAQP-1.0.0.jar
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
......@@ -98,53 +98,6 @@
</execution>
</executions>
</plugin>
<!-- For running tests -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.2</version>
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
<artifactId>surefire-junit47</artifactId>
<version>2.22.2</version>
</dependency>
</dependencies>
<configuration>
<includes>
<include>**/*.class</include>
</includes>
<excludedGroups>saqp.IntegrationTest</excludedGroups>
</configuration>
</plugin>
<!-- Run integration tests -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.22.2</version>
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
<artifactId>surefire-junit47</artifactId>
<version>2.22.2</version>
</dependency>
</dependencies>
<configuration>
<groups>saqp.IntegrationTest</groups>
</configuration>
<executions>
<execution>
<goals>
<goal>integration-test</goal>
</goals>
<configuration>
<includes>
<include>**/*.class</include>
</includes>
</configuration>
</execution>
</executions>
</plugin>
<!-- Execute java classes -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
......@@ -233,12 +186,6 @@
</build>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.github.cdimascio</groupId>
<artifactId>java-dotenv</artifactId>
......
......@@ -112,7 +112,7 @@ public class TestSuite {
InfoLogger.info("Query " + query);
InfoLogger.addLevel();
String wd = System.getProperty("user.dir");
Path path = Paths.get(wd, "src", "main", "resources", "queries-spjr", "query" + query + ".sql");
Path path = Paths.get(wd, "..", "queries", "query" + query + ".sql");
String queryPath = path.toString();
String sqlQuery = FileReader.readSql(queryPath);
currentQuery = query;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment