Commit fb3eb952 authored by Stanley Clark's avatar Stanley Clark
Browse files

Made SAQP Optimiser into a server

parent 7836010f
......@@ -1433,7 +1433,7 @@ CREATE OR REPLACE FUNCTION web_site_ROW_ACCESS(
AS
return_val VARCHAR2 (400);
BEGIN
return_val := '1 IN (SELECT 1 FROM (1,2))';
return_val := '((SYS_CONTEXT(''USERENV'', ''SESSION_USER'') = ''ADMIN1'') OR (EXISTS (SELECT 1 FROM employee WHERE (web_country = ''United States'') AND (employee.e_name = SYS_CONTEXT(''USERENV'', ''SESSION_USER'')))))';
RETURN return_val;
END;
/
......
......@@ -16,26 +16,18 @@ services:
POSTGRES_USER: ubuntu
POSTGRES_PASSWORD: ubuntu
POSTGRES_DB: ubuntu
builder:
image: maven:latest
container_name: builder
volumes:
- MavenM2:/root/.m2
- ./experiments:/usr/src/app
working_dir: /usr/src/app
command: mvn package
optimiser:
image: stanrogo/saqp-optimiser:latest
container_name: optimiser
build:
context: .
dockerfile: experiments/Dockerfile
ports:
- "4567:4567"
volumes:
- ./pg-cuckoo/PgCuckoo/config.ini:/home/app/pg-cuckoo/PgCuckoo/config.ini
- ./experiments:/home/app
- ./experiments/results:/home/app/results
- ./queries:/home/queries
volumes:
PostgreSqlTpcDB:
external: false
MavenM2:
external: false
......@@ -14,6 +14,17 @@ RUN curl -O https://download.java.net/java/GA/jdk14.0.1/664493ef4a6946b186ff29eb
&& rm openjdk-14.0.1_linux-x64_bin.tar.gz \
&& ln -s /opt/jdk-14.0.1/bin/java /usr/bin/java
# Install Maven
RUN mkdir -p /usr/share/maven /usr/share/maven/ref \
&& curl -fsSL -o /tmp/apache-maven.tar.gz https://apache.osuosl.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
&& echo "c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0 /tmp/apache-maven.tar.gz" | sha512sum -c - \
&& tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \
&& rm -f /tmp/apache-maven.tar.gz \
&& ln -s /usr/share/maven/bin/mvn /usr/bin/mvn
# Set environment variables
ENV MAVEN_HOME=/usr/share/maven MAVEN_CONFIG=/root/.m2 JAVA_HOME=/opt/jdk-14.0.1
# First time setup of stack
COPY pg-cuckoo/PgCuckoo/app /home/pg-cuckoo/PgCuckoo/app/
COPY pg-cuckoo/PgCuckoo/src /home/pg-cuckoo/PgCuckoo/src/
......@@ -22,8 +33,18 @@ COPY pg-cuckoo/PgCuckoo/Makefile pg-cuckoo/PgCuckoo/package.yaml pg-cuckoo/PgCuc
RUN cd /home/pg-cuckoo/PgCuckoo && stack install --allow-different-user || true
# Link Z3 dependencies for SMT XACML policy translator
COPY experiments/libz3.so /home/app/libz3.so
COPY experiments/libz3java.so /home/app/libz3java.so
RUN ln -s /home/app/libz3.so /usr/lib/libz3.so && ln -s /home/app/libz3java.so /usr/lib/libz3java.so
# Install maven dependencies
COPY experiments/pom.xml /home/app/pom.xml
RUN mvn -f /home/app/pom.xml clean package -Dmaven.main.skip -Dmaven.test.skip && rm -r /home/app/target
# Build maven project
COPY experiments/src /home/app/src
RUN mvn -f /home/app/pom.xml clean package
WORKDIR /home/app/
COPY pg-cuckoo/PgCuckoo/config.ini /home/pg-cuckoo/PgCuckoo
CMD java -Duser.dir="/home/app" -jar /home/app/target/SAQP-1.0.0.jar
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
......@@ -25,43 +25,6 @@
<target>14</target>
</configuration>
</plugin>
<!-- Enforce code style -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<version>3.1.1</version>
<configuration>
<configLocation>google_checks.xml</configLocation>
<encoding>UTF-8</encoding>
<consoleOutput>true</consoleOutput>
<failsOnError>true</failsOnError>
<linkXRef>false</linkXRef>
<excludes>saqp/jdbc/*</excludes>
</configuration>
<executions>
<execution>
<id>validate</id>
<phase>validate</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Generate XACML schema classes -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>jaxb2-maven-plugin</artifactId>
<version>2.5.0</version>
<executions>
<execution>
<id>xjc</id>
<goals>
<goal>xjc</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Install local jars to the global maven repo -->
<plugin>
<groupId>com.googlecode.addjars-maven-plugin</groupId>
......@@ -75,10 +38,10 @@
<configuration>
<resources>
<resource>
<directory>src/main/resources/xacml-smt</directory>
<directory>src/main/resources</directory>
<includes>
<include>**/z3-4.4.1.jar</include>
<include>**/XACMLSMT.jar</include>
<include>z3-4.4.1.jar</include>
<include>XACMLSMT.jar</include>
</includes>
</resource>
</resources>
......@@ -155,7 +118,6 @@
</execution>
</executions>
</plugin>
<!-- Make a main manifest upon packaging -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
......@@ -165,7 +127,7 @@
<manifest>
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
<mainClass>app.pipeline.Pipeline</mainClass>
<mainClass>Main</mainClass>
</manifest>
</archive>
</configuration>
......@@ -174,16 +136,6 @@
</build>
<dependencies>
<dependency>
<groupId>io.github.cdimascio</groupId>
<artifactId>java-dotenv</artifactId>
<version>5.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<version>3.1.1</version>
</dependency>
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
......@@ -209,17 +161,6 @@
<artifactId>guava</artifactId>
<version>29.0-jre</version>
</dependency>
<!-- XACML to SMT translation dependencies -->
<dependency>
<groupId>com.stanrogo</groupId>
<artifactId>xacmlsmt</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>com.stanrogo</groupId>
<artifactId>z3</artifactId>
<version>4.4.1</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
......@@ -235,34 +176,43 @@
<artifactId>commons-math3</artifactId>
<version>3.6.1</version>
</dependency>
<!-- XACML policy generation -->
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.1</version>
</dependency>
<dependency>
<groupId>org.glassfish.jaxb</groupId>
<artifactId>jaxb-runtime</artifactId>
<version>2.3.3</version>
</dependency>
<!-- XACML schema to Java Class generation -->
<dependency>
<groupId>org.glassfish.jaxb</groupId>
<artifactId>jaxb-xjc</artifactId>
<version>2.3.3</version>
</dependency>
<!-- Remove errors from the console for sql queries -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>1.7.30</version>
</dependency>
<!-- CVS file helper -->
<!-- CSV file helper -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>1.8</version>
</dependency>
<!-- All Spark dependencies -->
<dependency>
<groupId>com.sparkjava</groupId>
<artifactId>spark-core</artifactId>
<version>2.9.3</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>9.4.33.v20201020</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-webapp</artifactId>
<version>9.4.33.v20201020</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.websocket</groupId>
<artifactId>websocket-server</artifactId>
<version>9.4.33.v20201020</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.websocket</groupId>
<artifactId>websocket-servlet</artifactId>
<version>9.4.33.v20201020</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
import static spark.Spark.get;
import app.TestSuite;
import app.pipeline.Pipeline;
import saqp.Middleware;
import spark.Request;
import spark.Response;
public class Main {
public static void main(String[] args) {
get("/getQueryPlan", (Request request, Response response) -> {
RequestProps rp = new RequestProps(request);
Middleware mw = rp.getMiddleware();
mw.buildRaTree().makeApriori().plan().printRaTree();
return mw.getRAJson();
}
);
get("/time", (Request request, Response response) -> {
RequestProps rp = new RequestProps(request);
String[] headers = {"case", "num_results", "user_id", "ra", "apriori", "plan", "expand", "string", "db"};
TestSuite testSuite = new TestSuite(new Pipeline(rp.getUserIds()), headers, rp.getQueries(), rp.getNumTrials());
testSuite.run();
return "";
});
}
}
import app.FileReader;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.calcite.plan.RelOptRule;
import saqp.Environment;
import saqp.Middleware;
import saqp.Rules;
import spark.Request;
public class RequestProps {
private int userId = 0;
private List<Integer> userIds;
private boolean secRARules = false;
private int queryNum = 0;
private List<Integer> queries;
private int numTrials = 0;
RequestProps(Request request) {
if (request.queryParams("userId") != null) {
userId = Integer.parseInt(request.queryParams("userId"));
}
if (request.queryParams("userIds") != null) {
userIds = Arrays.stream(request.queryParams("userIds").split(",")).map(Integer::parseInt)
.collect(Collectors.toList());
}
if (request.queryParams("secRA") != null) {
secRARules = request.queryParams("secRA").equals("true");
}
if (request.queryParams("queryNum") != null) {
queryNum = Integer.parseInt(request.queryParams("queryNum"));
}
if (request.queryParams("queries") != null) {
queries = Arrays.stream(request.queryParams("queries").split(",")).map(Integer::parseInt)
.collect(Collectors.toList());
}
if (request.queryParams("numTrials") != null) {
numTrials = Integer.parseInt(request.queryParams("numTrials"));
}
}
private String getQuery() {
String wd = System.getProperty("user.dir");
Path path = Paths.get(wd, "..", "queries", "query" + queryNum + ".sql");
String queryPath = path.toString();
return FileReader.readSql(queryPath);
}
public Middleware getMiddleware() {
List<List<RelOptRule>> relOptRules = secRARules ? List.of(Rules.secRaPass1, Rules.secRaPass2) :
List.of(Rules.defaultRewriteRules);
return app.TestSuite.getMiddleware(new Environment(userId), relOptRules).reset(getQuery());
}
public List<Integer> getUserIds() {
return userIds;
}
public List<Integer> getQueries() {
return queries;
}
public int getNumTrials() {
return numTrials;
}
}
......@@ -15,14 +15,14 @@ import saqp.Middleware;
import saqp.db.DB;
import saqp.db.ExplainStats;
import saqp.policy.Policy;
import utils.EnvironmentVariables;
import utils.InfoLogger;
public class TestSuite {
private static final String[] DEFAULT_HEADERS = {"query", "trial"};
private final DB db;
private final int[] queryFiles;
private final List<Integer> queryFiles;
private final int numTrials;
private final IQueryRun queryRunner;
private final FileReader fileReader;
private final String[] headers;
......@@ -38,13 +38,14 @@ public class TestSuite {
* @param queryRun The function to run against each query
* @param appendHeaders The headers to append to the results file
*/
public TestSuite(IQueryRun queryRun, String[] appendHeaders) {
queryFiles = EnvironmentVariables.getInstance().saqpQueries;
public TestSuite(IQueryRun queryRun, String[] appendHeaders, List<Integer> queryFiles, int numTrials) {
this.queryFiles = queryFiles;
this.numTrials = numTrials;
db = new DB();
queryRunner = queryRun;
headers = Stream.of(DEFAULT_HEADERS, appendHeaders).flatMap(Stream::of).toArray(String[]::new);
String wd = System.getProperty("user.dir");
Path path = Paths.get(wd, "results", EnvironmentVariables.getInstance().saqpResultsFile);
Path path = Paths.get(wd, "results", "results.csv");
fileReader = new FileReader(path.toString(), headers);
}
......@@ -128,7 +129,7 @@ public class TestSuite {
* Execute N runs of the given function.
*/
public void runN() {
for (int i = 0; i < EnvironmentVariables.getInstance().saqpNumTrials; i++) {
for (int i = 0; i < numTrials; i++) {
InfoLogger.info("Trial " + i);
InfoLogger.addLevel();
newRun(i);
......
......@@ -21,17 +21,6 @@ public class ExhaustiveEnumeration implements IQueryRun {
mw = TestSuite.getMiddleware(new Environment(1), List.of(Rules.secRaPass1, Rules.secRaPass2));
}
/**
* Run exhaustive enumeration.
*
* @param args Arguments to pass in (none)
*/
public static void main(String[] args) {
String[] headers = {"permutation", "num_results", "db"};
TestSuite test = new TestSuite(new ExhaustiveEnumeration(), headers);
test.run();
}
private void buildPermutations(String sql) {
RelSubset root = mw.reset(sql).buildRaTree().makeApriori().plan().getRelRoot();
PlanConstructor planConstructor = new PlanConstructor();
......
......@@ -7,7 +7,6 @@ import org.apache.calcite.plan.RelOptRule;
import saqp.Environment;
import saqp.Middleware;
import saqp.Rules;
import utils.EnvironmentVariables;
import utils.InfoLogger;
public class Pipeline implements IQueryRun {
......@@ -16,15 +15,10 @@ public class Pipeline implements IQueryRun {
private int caseNum;
private int userId;
/**
* Run the testing pipeline.
*
* @param args Arguments to pass in (None)
*/
public static void main(String[] args) {
String[] headers = {"case", "num_results", "user_id", "ra", "apriori", "plan", "expand", "string", "db"};
TestSuite testSuite = new TestSuite(new Pipeline(), headers);
testSuite.run();
private final List<Integer> userIds;
public Pipeline(List<Integer> userIds) {
this.userIds = userIds;
}
/**
......@@ -40,7 +34,7 @@ public class Pipeline implements IQueryRun {
public void testQuery(String sql, TestSuite testSuite) {
this.sql = sql;
for (int userId : EnvironmentVariables.getInstance().saqpUserIds) {
for (int userId : userIds) {
InfoLogger.info("User ID " + userId);
InfoLogger.addLevel();
this.userId = userId;
......
......@@ -6,24 +6,19 @@ import java.util.List;
import saqp.Environment;
import saqp.Middleware;
import saqp.Rules;
import utils.EnvironmentVariables;
public class Planning implements IQueryRun {
private String sql ="";
private String sql = "";
private int userId;
/**
* Run planning tests.
* @param args Arguments to pass in (None)
*/
public static void main(String[] args) {
String[] headers = {"ra", "apriori", "plan"};
TestSuite testSuite = new TestSuite(new Planning(), headers);
testSuite.run();
Planning(int userId) {
this.userId = userId;
}
/**
* Test a single query.
* @param sql The query content
*
* @param sql The query content
* @param testSuite TestSuite class instance
*/
public void testQuery(String sql, TestSuite testSuite) {
......@@ -34,9 +29,8 @@ public class Planning implements IQueryRun {
public void runN(TestSuite testSuite) {
Middleware mw;
int userId = EnvironmentVariables.getInstance().saqpUserIds[0];
mw = TestSuite.getMiddleware(new Environment(userId), List.of(Rules.defaultRewriteRules)).reset(sql);
mw = TestSuite.getMiddleware(new Environment(userId), List.of(Rules.defaultRewriteRules))
.reset(sql);
testSuite.newRun(0);
testSuite.time("ra", mw::buildRaTree);
testSuite.time("apriori", mw::makeApriori);
......@@ -44,7 +38,9 @@ public class Planning implements IQueryRun {
testSuite.recordResultsForRun();
mw.printRaTree();
mw = TestSuite.getMiddleware(new Environment(userId), List.of(Rules.secRaPass1, Rules.secRaPass2)).reset(sql);
mw = TestSuite
.getMiddleware(new Environment(userId), List.of(Rules.secRaPass1, Rules.secRaPass2))
.reset(sql);
testSuite.newRun(1);
testSuite.time("ra", mw::buildRaTree);
testSuite.time("apriori", mw::makeApriori);
......
package saqp;
import app.FileReader;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.apache.calcite.adapter.enumerable.EnumerableConvention;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.plan.volcano.RelSubset;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelRoot;
import org.apache.calcite.rel.externalize.RelJsonReader;
import org.apache.calcite.rel.externalize.RelJsonWriter;
import org.apache.calcite.rel.externalize.RelWriterImpl;
import org.apache.calcite.sql.SqlExplainFormat;
import org.apache.calcite.sql.SqlExplainLevel;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.dialect.PostgresqlSqlDialect;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.tools.Frameworks;
import org.apache.calcite.tools.Planner;
import org.apache.calcite.tools.RelConversionException;
import org.apache.calcite.tools.ValidationException;
......@@ -122,6 +130,11 @@ public class Middleware {
StandardCharsets.UTF_8))));
}
public String getRAJson() {
return RelOptUtil.dumpPlan("", raParseTree, SqlExplainFormat.TEXT,
SqlExplainLevel.EXPPLAN_ATTRIBUTES);
}
/**
* Generate an SQL query from the store ra tree.
*
......
package saqp.db;
import app.FileReader;
import io.github.cdimascio.dotenv.Dotenv;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Connection;
......@@ -11,7 +10,6 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import utils.EnvironmentVariables;
public class DB {
private final String dbConn;
......@@ -25,10 +23,9 @@ public class DB {
* Create a new database instance.
*/
public DB() {
Dotenv dotenv = Dotenv.load();
dbConn = dotenv.get("saqp_db_conn");
dbUser = dotenv.get("saqp_db_user");
dbPass = dotenv.get("saqp_db_pass");
dbConn = "jdbc:postgresql://postgres:5432/tpcds1";
dbUser = "ubuntu";
dbPass = "ubuntu";
}
private Connection getConnection() throws SQLException {
......@@ -37,6 +34,7 @@ public class DB {
/**
* Create a new connection and blank statement.
*
* @throws SQLException If either step fails
*/
public void open() throws SQLException {
......@@ -46,6 +44,7 @@ public class DB {
/**
* Close the current connection.
*
* @throws SQLException If closing the connection fails
*/
public void close() throws SQLException {
......
package saqp.schema;
import io.github.cdimascio.dotenv.Dotenv;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
......@@ -21,10 +20,9 @@ public class Schema {
*/
public static SchemaPlus getSchema() {
if (tpcdsSchema == null) {
Dotenv dotenv = Dotenv.load();
String dbConn = dotenv.get("saqp_db_conn");
String dbUser = dotenv.get("saqp_db_user");
String dbPass = dotenv.get("saqp_db_pass");
String dbConn = "jdbc:postgresql://postgres:5432/tpcds1";
String dbUser = <