Commit 0fe4cb06 authored by Carlos Giraldo's avatar Carlos Giraldo
Browse files

updated from github

parent a64389a0
File mode changed from 100644 to 100755
# It will wait until 3 am, just to be sure
# issue the requests,
# success: memorize the id of the 2 requests
# Failure: log the wrong behaviour
# a missing request - try again every 10 minutes maximum until 10 times, then stop
# Check periodically untill those requests are all successful
# download the files,
# success: memorize the names of the files and update the table with the latest interval
# Failure: log the wrong behaviour and stop and delete all the downloaded documents
# Update database
from datetime import date, datetime, timedelta
import time
import sys
import os
import tools
#import logging
......@@ -51,3 +29,5 @@ tools.update_jsonfile(RANGE_LAST,sys.argv[1],CONFIG_JSON)
# to run the script that refreshes the materialized views
# sudo -u www-data /srv/apps/coursera/venv/bin/python /srv/apps/coursera/code/manage.py refreshviews
# Only works in the server with the analytics API installed
# os.system('sudo -u www-data /srv/apps/coursera/venv/bin/python /srv/apps/coursera/code/manage.py refreshviews')
\ No newline at end of file
echo 'Hola'
unzip $1 -d .
psql -d $3 -1 -f setup_tables_unique.sql
psql -d $3 -1 -f load.sql
......
{"first_time": "no", "initial_download_location": ".", "database_name": "coursera", "table_export_location": "tables/", "last_clickstream_date": "2018-10-24", "fist_clickstream_date": "2018-07-01", "eit_digital_id": "253", "clickstream_purpose": "of the clickstream data.", "clickstream_export_location": "events/", "purpose_for_requests": "For self study and analysis ", "tables_purpose": "of the tables data.", "database_user": "www-data"}
\ No newline at end of file
{"first_time": "no", "initial_download_location": ".", "database_name": "coursera", "table_export_location": "tables/", "last_clickstream_date": "2018-11-15", "fist_clickstream_date": "2018-10-15", "eit_digital_id": "253", "clickstream_purpose": "of the clickstream data.", "clickstream_export_location": "events/", "purpose_for_requests": "For self study and analysis ", "tables_purpose": "of the tables data.", "database_user": "www-data"}
\ No newline at end of file
File mode changed from 100644 to 100755
ALTER TABLE IF EXISTS users RENAME COLUMN eitdigital_user_id TO organization_user_id;
ALTER TABLE IF EXISTS peer_reviews RENAME COLUMN eitdigital_peer_assignments_user_id TO organization_peer_assignments_user_id;
ALTER TABLE IF EXISTS programming_submissions RENAME COLUMN eitdigital_programming_assignments_user_id TO organization_programming_assignments_user_id;
ALTER TABLE IF EXISTS discussion_answers RENAME COLUMN eitdigital_discussions_user_id TO organization_discussions_user_id;
ALTER TABLE IF EXISTS peer_submissions RENAME COLUMN eitdigital_peer_assignments_user_id TO organization_peer_assignments_user_id;
ALTER TABLE IF EXISTS users_courses__certificate_payments RENAME COLUMN eitdigital_user_id TO organization_user_id;
ALTER TABLE IF EXISTS notebook_workspaces RENAME COLUMN eitdigital_user_id TO organization_user_id;
ALTER TABLE IF EXISTS course_grades RENAME COLUMN eitdigital_user_id TO organization_user_id;
ALTER TABLE IF EXISTS course_memberships RENAME COLUMN eitdigital_user_id TO organization_user_id;
ALTER TABLE IF EXISTS feedback_item_comments RENAME COLUMN eitdigital_feedback_user_id TO organization_feedback_user_id;
ALTER TABLE IF EXISTS discussion_question_votes RENAME COLUMN eitdigital_discussions_user_id TO organization_discussions_user_id;
ALTER TABLE IF EXISTS peer_skips RENAME COLUMN eitdigital_peer_assignments_user_id TO organization_peer_assignments_user_id;
ALTER TABLE IF EXISTS feedback_course_comments RENAME COLUMN eitdigital_feedback_user_id TO organization_feedback_user_id;
ALTER TABLE IF EXISTS course_progress RENAME COLUMN eitdigital_user_id TO organization_user_id;
ALTER TABLE IF EXISTS feedback_course_ratings RENAME COLUMN eitdigital_feedback_user_id TO organization_feedback_user_id;
ALTER TABLE IF EXISTS discussion_answer_flags RENAME COLUMN eitdigital_discussions_user_id TO organization_discussions_user_id;
ALTER TABLE IF EXISTS discussion_answer_votes RENAME COLUMN eitdigital_discussions_user_id TO organization_discussions_user_id;
ALTER TABLE IF EXISTS feedback_item_ratings RENAME COLUMN eitdigital_feedback_user_id TO organization_feedback_user_id;
ALTER TABLE IF EXISTS course_branch_grades RENAME COLUMN eitdigital_user_id TO organization_user_id;
ALTER TABLE IF EXISTS discussion_questions RENAME COLUMN eitdigital_discussions_user_id TO organization_discussions_user_id;
ALTER TABLE IF EXISTS ecb_evaluation_requests RENAME COLUMN eitdigital_ecb_user_id TO organization_ecb_user_id;
ALTER TABLE IF EXISTS peer_comments RENAME COLUMN eitdigital_peer_assignments_user_id TO organization_peer_assignments_user_id;
ALTER TABLE IF EXISTS organization_enrollments RENAME COLUMN eitdigital_user_id TO organization_user_id;
ALTER TABLE IF EXISTS course_item_grades RENAME COLUMN eitdigital_user_id TO organization_user_id;
ALTER TABLE IF EXISTS on_demand_session_memberships RENAME COLUMN eitdigital_user_id TO organization_user_id;
ALTER TABLE IF EXISTS assessment_actions RENAME COLUMN eitdigital_assessments_user_id TO organization_assessments_user_id;
ALTER TABLE IF EXISTS course_formative_quiz_grades RENAME COLUMN eitdigital_user_id TO organization_user_id;
ALTER TABLE IF EXISTS discussion_question_followings RENAME COLUMN eitdigital_discussions_user_id TO organization_discussions_user_id;
ALTER TABLE IF EXISTS discussion_question_flags RENAME COLUMN eitdigital_discussions_user_id TO organization_discussions_user_id;
ALTER TABLE IF EXISTS demographics_answers RENAME COLUMN eitdigital_demographics_user_id TO organization_demographics_user_id;
ALTER TABLE IF EXISTS eitdigital_course_user_ids DROP CONSTRAINT IF EXISTS eitdigital_course_user_ids_pkey;
ALTER TABLE IF EXISTS eitdigital_course_user_ids DROP CONSTRAINT IF EXISTS eitdigital_course_user_ids_fkey;
ALTER TABLE IF EXISTS eitdigital_course_user_ids RENAME COLUMN eitdigital_user_id TO organization_user_id;
ALTER TABLE IF EXISTS eitdigital_course_user_ids ADD PRIMARY KEY (organization_user_id);
ALTER TABLE IF EXISTS eitdigital_course_user_ids ADD FOREIGN KEY (organization_user_id) REFERENCES users (organization_user_id);
ALTER TABLE IF EXISTS eitdigital_course_user_ids RENAME TO organization_course_user_ids;
\ No newline at end of file
File mode changed from 100644 to 100755
This diff is collapsed.
......@@ -75,17 +75,7 @@ def get_from_todays(tab_req, click_req):
return_clickstream_req = get_clickstream_request(today_requests, click_req)
return return_tab_req, return_clickstream_req
'''
def get_latest_table_request():
print "ENTERING METHOD: get_latest_table_request"
today_requests = get_today_requests()
latest = None
for r in today_requests:
if r.to_json()['exportType'] == 'RESEARCH_WITH_SCHEMAS':
latest = r
break
return latest
'''
def issue_requests(config_json):
......@@ -124,10 +114,7 @@ def issue_requests(config_json):
print 'THAT WAS THE MESSAGE'
if int(str(e.message.split(' ')[0])) != 429 :
#time to wait
#print e.message
#print 'TIME TO WAIT for the latest request! ' + str(datetime.fromtimestamp(get_latest_table_request().to_json()['metadata']['startedAt']/ 1e3))
#print 'MIGHT NEED TO WAIT UNTIL: ' + str(datetime.fromtimestamp(get_latest_table_request().to_json()['metadata']['startedAt']/ 1e3) + timedelta(hours=1))
#print 'ONLY ' + str((datetime.fromtimestamp(get_latest_table_request().to_json()['metadata']['startedAt']/ 1e3) + timedelta(hours=1) - datetime.today()).seconds)
exit_with_error = True
error_message += 'tables: ' + e.message
break
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment