Commit cf8ea036 authored by Stanley Clark's avatar Stanley Clark
Browse files

WIP

parent da17d44a
......@@ -28,9 +28,6 @@ docker-compose exec postgres /home/app/postgres/populate_db.sh tpcds1
docker-compose run builder mvn initialize
docker-compose run builder mvn package
# Setup Haskell Stack dependencies for the first time
docker-compose run optimiser /bin/sh -c "cd /home/pg-cuckoo/PgCuckoo && stack install --allow-different-user"
# We are now able to run the compiled files using a dedicated java container:
docker-compose run optimiser
```
......
......@@ -14,9 +14,16 @@ This will start both Oracle DB and IBM DB2.
The next thing to do is to populate both databases with TPC-DS data:
```shell
docker-compose exec DB2 /home/app/ibm/populate.sh
docker-compose exec Oracle /home/app/oracle/populate.sh
docker-compose exec ibm ./users.sh
docker-compose exec ibm su -c "/home/app/comparison/ibm/populate.sh" - db2inst1
docker-compose exec oracle ./populate.sh
```
Now we can run experiments by querying the two databases as normal and logging in
Notice that for IBM DB2 we first create user accounts before populating the database with data.
We can now run experiments by querying the two databases as normal and logging in
as different users.
```shell
docker-compose exec oracle ./run.sh
docker-compose exec ibm su -c "/home/app/comparison/ibm/run.sh" - db2inst1
```
......@@ -7,12 +7,15 @@ services:
- "50000:50000"
volumes:
- DB2Data:/database
- ./ibm:/home/app/ibm
- ./ibm:/home/app/comparison/ibm
- ./../tpc-ds:/home/app/tpc-ds
- ./../queries:/home/app/queries
environment:
- LICENSE=accept
- DB2INST1_PASSWORD=password
- DBNAME=testdb
privileged: true
working_dir: /home/app/comparison/ibm
oracle:
image: store/oracle/database-enterprise:12.2.0.1
container_name: Oracle
......@@ -21,11 +24,14 @@ services:
- "5500:5500"
volumes:
- OracleDBData:/ORCL
- ./oracle:/home/app/oracle
- ./oracle:/home/app/comparison/oracle
- ./../tpc-ds:/home/app/tpc-ds
- ./../queries:/home/app/queries
environment:
- ORACLE_SID=ORCLCDB
- ORACLE_PDB=ORCLPDB1
- ORACLE_PWD=password
working_dir: /home/app/comparison/oracle
volumes:
DB2Data:
external: false
......
#!/usr/bin/env sh
# Connect to DB2
db2 catalog tcpip node docker remote 0.0.0.0 server 50000
db2 catalog database testdb as testdb at node docker
docker exec -it DB2 useradd -m db2sec -g db2iadm1 -p "$(openssl passwd -crypt passwd1)"
db2 CONNECT TO testdb user db2inst1 using password
db2 grant secadm on database to user db2sec
......@@ -14,23 +9,16 @@ db2 grant secadm on database to user db2inst1
db2 CONNECT TO testdb user db2inst1 using password
# DB2 users are OS users, so create them here
for i in $(seq 1 10);
do
docker exec -it DB2 useradd -m "user$i" -g db2iadm1 -p "$(openssl passwd -crypt passwd1)"
db2 grant dataaccess on database to user "user$i"
done
# Create TPC-DS schema and users
db2 -t -f sql/employee.sql
db2 -t -f ../../tpc-ds/tools/tpcds.sql
db2 -t -f /home/app/comparison/ibm/sql/employee.sql
db2 -t -f /home/app/tpc-ds/tools/tpcds.sql
# Import generated data to postgres instance
for tableScript in ../../tpc-ds/tools/*.dat
for tableScript in /home/app/tpc-ds/tools/*.dat
do
db2 TRUNCATE "$(basename "$tableScript" .dat)" IMMEDIATE;
db2 LOAD CLIENT FROM "$(realpath "$tableScript")" OF DEL MODIFIED BY COLDEL\| INSERT INTO "$(basename "$tableScript" .dat)" NONRECOVERABLE;
done
# Import and activate policies
db2 -t -f sql/policies.sql
db2 -t -f /home/app/comparison/ibm/sql/policies.sql
#!/usr/bin/env sh
for query in ../../queries/*.sql
for query in /home/app/queries/*.sql
do
for i in $(seq 1 10);
do
......
#!/usr/bin/env sh
# DB2 users are OS users, so create them here
useradd -m db2sec -g db2iadm1 -p "$(openssl passwd -crypt passwd1)"
for i in $(seq 1 10);
do
useradd -m "user$i" -g db2iadm1 -p "$(openssl passwd -crypt passwd1)"
done
for i in $(seq 1 10);
do
su -c "db2 CONNECT TO testdb user db2inst1 using password && db2 grant dataaccess on database to user user$i" - db2inst1
done
#!/usr/bin/env sh
# Oracle doesn't support the 'time' format
cp ../../tpc-ds/tools/tpcds.sql sql/tpcds.sql
sed -i 's/dv_create_time time/dv_create_time date/g' sql/tpcds.sql
. /home/oracle/.bashrc;
# Create TPC-DS schema and users
echo @sql/users.sql | sqlplus sys/Oradoc_db1@ORCLCDB as sysdba
echo @sql/clean.sql | sqlplus admin1/admin1@ORCLCDB
echo @sql/employee.sql | sqlplus admin1/admin1@ORCLCDB
echo @sql/tpcds.sql | sqlplus admin1/admin1@ORCLCDB
# Import generated data to instance
for tableScript in ctl/*.ctl
do
sqlldr admin1/admin1@ORCLCDB control="$tableScript" log="log/$(basename "$tableScript" .ctl).log" direct=true
done
## Create TPC-DS schema and users
#echo @sql/users.sql | sqlplus sys/Oradoc_db1@ORCLCDB as sysdba
#echo @sql/clean.sql | sqlplus admin1/admin1@ORCLCDB
#echo @sql/employee.sql | sqlplus admin1/admin1@ORCLCDB
#echo @sql/tpcds.sql | sqlplus admin1/admin1@ORCLCDB
#
## Import generated data to instance
#for tableScript in ctl/*.ctl
#do
# sqlldr admin1/admin1@ORCLCDB control="$tableScript" log="log/$(basename "$tableScript" .ctl).log" direct=true
#done
# Attach policies
echo @sql/policies.sql | sqlplus admin1/admin1@ORCLCDB
#!/usr/bin/env sh
for query in ../../queries/*.sql
do
for i in $(seq 1 10);
do
alt_ses="ALTER SESSION SET CURRENT_SCHEMA = ADMIN1;"
query_text="$(cat "$query")"
printf "%s \n %s" "$alt_ses" "$query_text" | sqlplus "user$i"/password@ORCLCDB
done
done
. /home/oracle/.bashrc;
#for query in ../../queries/*.sql
#do
# for i in $(seq 1 10);
# do
# alt_ses="ALTER SESSION SET CURRENT_SCHEMA = ADMIN1;"
# query_text="$(cat "$query")"
# printf "%s \n %s" "$alt_ses" "$query_text" | sqlplus "user$i"/password@ORCLCDB
# done
#done
for i in $(seq 1 10);
do
alt_ses="ALTER SESSION SET CURRENT_SCHEMA = ADMIN1;"
query_text="$(cat ../../queries/query3.sql)"
query_text="SELECT * FROM item OFFSET 20 ROWS FETCH NEXT 10 ROWS ONLY;"
printf "%s \n %s" "$alt_ses" "$query_text" | sqlplus "user$i"/password@ORCLCDB
done
This diff is collapsed.
......@@ -26,6 +26,7 @@ create user admin1 identified by admin1;
GRANT CONNECT, RESOURCE, DBA TO admin1;
GRANT execute on DBMS_RLS TO admin1;
GRANT CREATE PROCEDURE TO admin1;
GRANT CREATE PACKAGE TO admin1;
GRANT CREATE ROLE TO admin1;
GRANT CREATE SESSION TO admin1;
\ No newline at end of file
GRANT CREATE SESSION TO admin1;
GRANT execute on sys.dbms_session TO admin1, user1, user2, user3, user4, user5, user6, user7, user8, user9, user10;
\ No newline at end of file
saqp_db_conn=jdbc:postgresql://hostname/database
saqp_db_user=db_user
saqp_db_pass=db_password
saqp_db_conn=jdbc:postgresql://postgres:5432/tpcds1
saqp_db_user=ubuntu
saqp_db_pass=ubuntu
saqp_results_file=results.csv
saqp_enable_logging=file|console|none
saqp_enable_logging=file
saqp_queries=3,7,11,19,25,26,46,90,96
saqp_user_ids=1,2,3,4,5,6,7,8,9,10
saqp_num_trials=25
\ No newline at end of file
table|attribute|policy|permission|note
call_center|cc_closed_date_sk||deny|Mask the close date to pretend no call centres were ever closed
catalog_page|cp_catalog_page_number|13 < catalog_page.cp_catalog_page_number|deny|Hide unlucky page number 13
catalog_returns|cr_fee|0 <= catalog_returns.cr_reason_sk|deny|Deny access to the fee when there was a specific reason for returning
catalog_sales|cs_net_profit|0 > catalog_sales.cs_net_profit AND 'sales' = employee.e_role|deny|The company would like to keep their books looking clean and therefore deny access to those cells where there is a negative profit and the person works in sales
customer|c_first_name|customer.c_birth_year = employee.e_year|deny|Deny access if birth year is same as date employee joined the company
catalog_page|cp_catalog_page_number|13 < cp_catalog_page_number|deny|Hide unlucky page number 13
catalog_returns|cr_fee|0 <= cr_reason_sk|deny|Deny access to the fee when there was a specific reason for returning
catalog_sales|cs_net_profit|0 > cs_net_profit AND 'sales' = employee.e_role|deny|The company would like to keep their books looking clean and therefore deny access to those cells where there is a negative profit and the person works in sales
customer|c_first_name|c_birth_year = employee.e_year|deny|Deny access if birth year is same as date employee joined the company
customer_demographics|cd_credit_rating|customer.c_birth_country = 'UNITED KINGDOM'|deny|Deny access to credit rating of customers from 'UNITED KINGDOM'
household_demographics|hd_vehicle_count|110000 < income_band.ib_upper_bound AND household_demographics.hd_income_band_sk = income_band.ib_income_band_sk|deny|Mask the vehicle count of customers in an income band with an 'ib_upper_bound > 110000'
household_demographics|hd_vehicle_count|110000 < income_band.ib_upper_bound AND hd_income_band_sk = income_band.ib_income_band_sk|deny|Mask the vehicle count of customers in an income band with an 'ib_upper_bound > 110000'
income_band|ib_lower_bound|employee.e_role = 'manager'|permit|Deny access to lower income band if not a manager
inventory|inv_quantity_on_hand|warehouse.w_warehouse_sk = inventory.inv_warehouse_sk AND 'United States' = warehouse.w_country|deny|Mask quantities from warehouses in a specific country
item|i_brand_id|1000000 = item.i_brand_id|deny|
inventory|inv_quantity_on_hand|warehouse.w_warehouse_sk = inv_warehouse_sk AND 'United States' = warehouse.w_country|deny|Mask quantities from warehouses in a specific country
item|i_brand_id|1000000 = i_brand_id|deny|
reason|r_reason_desc|employee.e_role = 'support'|deny|Deny access to support staff
store_returns|sr_fee|0 <= store_returns.sr_reason_sk|deny|Deny access to the fee when there was a specific reason for returning
store_sales|ss_quantity|13 < store_sales.ss_quantity|deny|store_sales.ss_quantity > 13
store_returns|sr_fee|0 <= sr_reason_sk|deny|Deny access to the fee when there was a specific reason for returning
store_sales|ss_quantity|13 < ss_quantity|deny|store_sales.ss_quantity > 13
web_page|wp_customer_sk||deny|Cells in this column are not allowed to be seen
web_returns|wr_fee|0 <= web_returns.wr_reason_sk|deny|Deny access to the fee when there was a specific reason for returning
web_sales|ws_coupon_amt|1000 < web_sales.ws_net_paid AND 80001 <= income_band.ib_lower_bound AND web_sales.ws_ship_customer_sk = customer.c_customer_sk AND customer.c_current_hdemo_sk = household_demographics.hd_demo_sk AND household_demographics.hd_income_band_sk = income_band.ib_income_band_sk|permit|To protect the privacy of customers who are in a higher income bracket,the value of the coupon used for purchases is only allowed when 'ws_net_paid > 1000' and the customer income band is high ('ib_lower_bound >= 80001')
call_center||employee.e_role = 'sales' OR (date_dim.d_date_sk = call_center.cc_open_date_sk AND 1997 > call_center.cc_open_date_sk)|permit|
web_returns|wr_fee|0 <= wr_reason_sk|deny|Deny access to the fee when there was a specific reason for returning
web_sales|ws_coupon_amt|1000 < ws_net_paid AND 80001 <= income_band.ib_lower_bound AND ws_ship_customer_sk = customer.c_customer_sk AND customer.c_current_hdemo_sk = household_demographics.hd_demo_sk AND household_demographics.hd_income_band_sk = income_band.ib_income_band_sk|permit|To protect the privacy of customers who are in a higher income bracket,the value of the coupon used for purchases is only allowed when 'ws_net_paid > 1000' and the customer income band is high ('ib_lower_bound >= 80001')
call_center||employee.e_role = 'sales' OR (date_dim.d_date_sk = cc_open_date_sk AND 1997 > cc_open_date_sk)|permit|
catalog_page|||permit|
catalog_returns||(catalog_returns.cr_refunded_customer_sk = customer.c_customer_sk AND customer.c_current_hdemo_sk = household_demographics.hd_demo_sk) AND ((household_demographics.hd_income_band_sk = income_band.ib_income_band_sk AND income_band.ib_lower_bound = 0) OR (household_demographics.hd_buy_potential = '0-500'))|permit|
catalog_sales||(catalog_sales.cs_ship_customer_sk = customer.c_customer_sk AND customer.c_current_hdemo_sk = household_demographics.hd_demo_sk) AND ((household_demographics.hd_income_band_sk = income_band.ib_income_band_sk AND income_band.ib_lower_bound = 0) OR (household_demographics.hd_buy_potential = '0-500'))|permit|Only sales of customers in low income bands or with low buying potential
customer||customer.c_current_addr_sk = customer_address.ca_address_sk AND customer.c_birth_country = customer_address.ca_country|permit|Only customers with an address country the same as their birth country
catalog_returns||(cr_refunded_customer_sk = customer.c_customer_sk AND customer.c_current_hdemo_sk = household_demographics.hd_demo_sk) AND ((household_demographics.hd_income_band_sk = income_band.ib_income_band_sk AND income_band.ib_lower_bound = 0) OR (household_demographics.hd_buy_potential = '0-500'))|permit|
catalog_sales||(cs_ship_customer_sk = customer.c_customer_sk AND customer.c_current_hdemo_sk = household_demographics.hd_demo_sk) AND ((household_demographics.hd_income_band_sk = income_band.ib_income_band_sk AND income_band.ib_lower_bound = 0) OR (household_demographics.hd_buy_potential = '0-500'))|permit|Only sales of customers in low income bands or with low buying potential
customer||c_current_addr_sk = customer_address.ca_address_sk AND c_birth_country = customer_address.ca_country|permit|Only customers with an address country the same as their birth country
customer_address||employee.e_role = 'sales' AND 2010 >= employee.e_year|permit|Only employees with role sales who have worked at the company since 2010
customer_demographics|||permit|
date_dim|||permit|
......@@ -27,17 +27,17 @@ dbgen_version|||deny|
employee|||deny|
household_demographics|||permit|
income_band|||permit|
inventory||inventory.inv_warehouse_sk = warehouse.w_warehouse_sk AND 300000 >= warehouse.w_warehouse_sq_ft|permit|Only inventory for warehouses with small storage space (<= 300000)
item||store_returns.sr_item_sk = item.i_item_sk AND employee.e_store_sk = store_returns.sr_store_sk|permit|Only items which have been returned to the employees own store
promotion||store_sales.ss_promo_sk = promotion.p_promo_sk|permit|Only promotions which are involved in at least one store sale
inventory||inv_warehouse_sk = warehouse.w_warehouse_sk AND 300000 >= warehouse.w_warehouse_sq_ft|permit|Only inventory for warehouses with small storage space (<= 300000)
item||store_returns.sr_item_sk = i_item_sk AND employee.e_store_sk = store_returns.sr_store_sk|permit|Only items which have been returned to the employees own store
promotion||store_sales.ss_promo_sk = p_promo_sk|permit|Only promotions which are involved in at least one store sale
reason|||permit|
ship_mode|||permit|
store||0 > store.s_closed_date_sk OR 'manager' = employee.e_role|permit|Only stores which are closed or if the employee has role manager
store_returns||store_returns.sr_store_sk = employee.e_store_sk|permit|Only records from employees own store
store_sales||store_sales.ss_store_sk = employee.e_store_sk|permit|Only records from employees own store
store||0 > s_closed_date_sk OR 'manager' = employee.e_role|permit|Only stores which are closed or if the employee has role manager
store_returns||sr_store_sk = employee.e_store_sk|permit|Only records from employees own store
store_sales||ss_store_sk = employee.e_store_sk|permit|Only records from employees own store
time_dim|||permit|
warehouse|||permit|
web_page||0 < web_page.wp_customer_sk|permit|Only web pages with a customer
web_returns||date_dim.d_year > employee.e_year AND web_returns.wr_returned_date_sk = date_dim.d_date_sk|permit|Only returns made after the date the employee joined the company
web_sales||date_dim.d_year > employee.e_year AND web_sales.ws_sold_date_sk = date_dim.d_date_sk|permit|Only sales made after the date the employee joined the company
web_site||web_site.web_country = 'United States'|permit|Only sites in "United States"
web_page||0 < wp_customer_sk|permit|Only web pages with a customer
web_returns||date_dim.d_year > employee.e_year AND wr_returned_date_sk = date_dim.d_date_sk|permit|Only returns made after the date the employee joined the company
web_sales||date_dim.d_year > employee.e_year AND ws_sold_date_sk = date_dim.d_date_sk|permit|Only sales made after the date the employee joined the company
web_site||web_country = 'United States'|permit|Only sites in "United States"
......@@ -16,7 +16,7 @@ public class OraclePolicies extends PolicyParser {
" AS\n" +
" return_val VARCHAR2 (400);\n" +
" BEGIN\n" +
" return_val := '" + existsClause + "';\n" +
" return_val := '(" + existsClause + ")';\n" +
" RETURN return_val;\n" +
"END;\n" +
"/\n" +
......@@ -28,6 +28,9 @@ public class OraclePolicies extends PolicyParser {
" IF SQLCODE != -28102 THEN\n" +
" RAISE;\n" +
" END IF;\n" +
"END;\n" +
"/\n" +
"BEGIN\n" +
" DBMS_RLS.ADD_POLICY(\n" +
props + "\n" +
" );\n" +
......@@ -81,15 +84,15 @@ public class OraclePolicies extends PolicyParser {
throws JSQLParserException {
if (policy.isEmpty()) {
String boolPermission = permission.equals("deny") ? "1 = 0" : "1 = 1";
return "SYS_CONTEXT(''sec_app_context'', ''e_name'') = ''ADMIN1'' OR " + boolPermission;
return "(SYS_CONTEXT(''USERENV'', ''SESSION_USER'') = ''ADMIN1'') OR " + boolPermission;
}
List<String> tableList = getTablesUsed(baseTable, policy);
String negative = permission.equals("deny") ? "NOT " : "";
return "SYS_CONTEXT(''USERENV'', ''SESSION_USER'') = ''ADMIN1'' OR " +
negative + "EXISTS (SELECT 1 FROM " + String.join(", ", tableList) +
return "(SYS_CONTEXT(''USERENV'', ''SESSION_USER'') = ''ADMIN1'') OR " +
negative + "(EXISTS (SELECT 1 FROM " + String.join(", ", tableList) +
" WHERE (" + policy.replaceAll("'", "''") +
") AND employee.e_name = SYS_CONTEXT(''sec_app_context'', ''e_name''))";
") AND (employee.e_name = SYS_CONTEXT(''USERENV'', ''SESSION_USER''))))";
}
public static void main(String[] args) throws IOException, JSQLParserException {
......
DROP TABLE IF EXISTS employee;
CREATE TABLE employee
(
e_employee_id integer, -- employee id
......
......@@ -11,5 +11,5 @@ for tableScript in /home/app/tpc-ds/tools/*.dat
do
tbl=$(basename "$tableScript" ".dat")
echo "Copying data for table $tbl"
psql -U ubuntu -d "$1" -c "\copy $tbl FROM './$tableScript' (DELIMITER '|', ENCODING LATIN1, FORMAT csv);"
psql -U ubuntu -d "$1" -c "\copy $tbl FROM '$tableScript' (DELIMITER '|', ENCODING LATIN1, FORMAT csv);"
done
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment