import threading import secrets import os import tempfile import pandas as pd from flask import Flask, render_template, url_for, request, Blueprint, flash, redirect, abort, current_app, send_file from flask_login import login_required, current_user from surveyapp.surveys.forms import UploadForm, EditForm from surveyapp import mongo from bson.objectid import ObjectId from surveyapp.graphs.utils import delete_image, graphs_to_excel from surveyapp.analysis.utils import run_all_tests, tests_to_excel from surveyapp.surveys.utils import save_file, read_file, delete_file, generate_filepath from xlsxwriter import Workbook surveys = Blueprint("surveys", __name__) # Home page, displaying all the user's surveys as well as notifications @surveys.route('/home', methods=['GET']) @login_required def home(): # Get all surveys related to the current_user surveys=mongo.db.surveys.find({"user":current_user._id}) # Initialise an empty list that will contain the number of graphs and tests for each survey survey_list = [] # Loop through each survey, counting the number of graphs and tests for survey in surveys: graphs = mongo.db.graphs.count_documents({"surveyId":str(survey["_id"])}) tests = mongo.db.tests.count_documents({"surveyId":str(survey["_id"])}) survey_list.append({"title": survey["title"],\ "_id": survey["_id"],\ "numGraphs": graphs,\ "numTests": tests}) # Get the number of notifications (if any) notifications = mongo.db.temp_results.count_documents({"user": current_user._id}) return render_template("surveys/home.html", title="Home", surveys=survey_list, notifications=notifications) # Dasboard page for each survey # Renders a page with all graphs and surveys relating to the chosen survey @surveys.route('/home/', methods=['GET']) @login_required def dashboard(survey_id): # Get the current survey survey = mongo.db.surveys.find_one_or_404({"_id": ObjectId(survey_id)}) # Get the graphs and tests associated with that survey graphs = mongo.db.graphs.find({"surveyId":survey_id}) tests = mongo.db.tests.find({"surveyId":survey_id}) return render_template("surveys/dashboard.html", title="Dashboard", graphs=list(graphs), tests=list(tests), survey=survey) # Importing a file (CSV or Excel spreadsheet) @surveys.route('/import', methods=['GET', 'POST']) @login_required def import_file(): form = UploadForm() # Checks validation when form is submitted with submit button if form.validate_on_submit(): # Save the file file_name = save_file(form.file.data) # And add to the database survey_id = mongo.db.surveys.insert_one({ "fileName" : file_name, "user" : current_user._id, "title" : form.title.data}).inserted_id # Get the id of the survey just inserted flash("File uploaded successfully!", "success") # Running all the statistical tests on the data can take a lot of time. Therefore I # carry it out using a python thread. It is important to pass the current application, # so that the threaded function can be carried out from the current application context thread = threading.Thread(target=run_all_tests, args=(str(survey_id), current_user._id, current_app._get_current_object()), daemon=True) thread.start() # Redirect user to the input/table page so they can view their uploaded data in tabular form return redirect(url_for("surveys.input", survey_id=survey_id)) return render_template("surveys/import.html", title = "Import", form=form) # # This page displays data in tabular form. It can be used for entering data from # # scratch or for editing a survey that is already uploaded @surveys.route('/input', methods=['GET', 'POST']) @login_required def input(): # Initialise variables for handsontable (2d array for values, 1d array for column headers) value_list = [[]] header_list = [] survey_id = request.args.get("survey_id") form = EditForm() # Handsontable data cannot be posted using WTForms POST methods - Post needs to be from combined WTForm and javascript AJAX if form.validate_on_submit(): # get the file_obj (if one exists yet) file_obj = mongo.db.surveys.find_one({"_id":ObjectId(survey_id)}) # if file already exists we can simply get the name of the file if file_obj: file_name = file_obj["fileName"] file = os.path.join(current_app.root_path, "uploads", file_name) # Else we need to generate a new filename with a new random hex. else: # Generate a random hex to be the new filename file_name = generate_filepath() file = os.path.join(current_app.root_path, "uploads", file_name) # write/overwrite the table values to the file with open(file, "w") as file_to_write: file_to_write.write(request.form["table"]) # Update/insert into the database survey = mongo.db.surveys.update_one({"_id": ObjectId(survey_id)},\ {"$set": {"fileName" : file_name,\ "user" : current_user._id,\ "title" : form.title.data}}, upsert=True) if not survey_id: survey_id = survey.upserted_id # Respond to the jquery POST with the survey_id. This is so that if the survey was new, it # can now be incorporated into subsequent POST requests to avoid multiple surveys being saved return str(survey_id) # If GET request and the survey already exists (i.e. editing an existing survey) elif request.method == "GET" and survey_id: file_obj = mongo.db.surveys.find_one_or_404({"_id":ObjectId(survey_id)}) if file_obj["user"] != current_user._id: flash("You do not have access to that page", "danger") return redirect(url_for("main.index")) # Read the file and extract the cell values and column headers df = read_file(file_obj["fileName"]) value_list = df.values.tolist() header_list = df.columns.values.tolist() form.title.data = file_obj["title"] data = {"values": value_list, "headers": header_list} return render_template("surveys/input.html", title="Input", data=data, survey_id=survey_id, form=form) # Helper function called by client javascript. Will run all statistical tests on their # chosen survey and save significant findings in their findings/notification page @surveys.route('/home/run_tests/', methods=['GET', 'POST']) @login_required def run_tests(survey_id): file_obj = mongo.db.surveys.find_one_or_404({"_id":ObjectId(survey_id)}) if file_obj["user"] != current_user._id: flash("You do not have access to that survey", "danger") abort(403) # Running all the statistical tests on the data can take a lot of time. Therefore I # carry it out using a python thread. It is important to pass the current application, # so that the threaded function can be carried out from the current application context thread = threading.Thread(target=run_all_tests, args=(str(survey_id), current_user._id, current_app._get_current_object()), daemon=True) thread.start() # Redirect user to the input/table page so they can view their uploaded data in tabular form flash("Depending on the size of your survey, running all statistical tests may take some time. \ Try refreshing the page in a bit or come back again later.", "success") return redirect(url_for("surveys.home")) @surveys.route('/findings', methods=['GET', 'POST']) @login_required def findings(): form = EditForm() notifications = mongo.db.temp_results.find({"user": current_user._id}) # If user chooses to save a test if form.validate_on_submit(): result_id = request.args.get("result_id") mongo.db.tests.insert_one({ "surveyId" : request.args.get("survey_id"), "user" : current_user._id, "title" : form.title.data, "test" : request.args.get("test"), "independentVariable" : request.args.get("independent_variable"), "dependentVariable" : request.args.get("dependent_variable"), "p" : request.args.get("p")}) mongo.db.temp_results.delete_one({'_id': ObjectId(result_id)}) notifications = mongo.db.temp_results.find({"user": current_user._id}) flash("Statistical test saved to your survey dashboard", "success") return redirect(url_for("surveys.findings")) return render_template("surveys/findings.html", title="Findings", form=form, notifications=notifications, count=notifications.count()) # Delete a temporary result @surveys.route('/findings//delete', methods=['POST']) @login_required def delete_temp_result(result_id): file_obj = mongo.db.temp_results.find_one_or_404({"_id":ObjectId(result_id)}) if file_obj["user"] != current_user._id: flash("You do not have access to that page", "danger") abort(403) mongo.db.temp_results.delete_one(file_obj) return redirect(url_for('surveys.findings')) # Delete all temporary results @surveys.route('/findings/delete', methods=['POST']) @login_required def delete_findings(): mongo.db.temp_results.delete_many({"user":current_user._id}) return redirect(url_for('surveys.findings')) # DELETE A SURVEY @surveys.route("/survey//delete", methods=['POST']) @login_required def delete_survey(survey_id): file_obj = mongo.db.surveys.find_one_or_404({"_id":ObjectId(survey_id)}) if file_obj["user"] != current_user._id: flash("You do not have access to that page", "danger") abort(403) # First loop through all graphs, tests and remp results associated with that survey and delete them graphs = mongo.db.graphs.find({"surveyId":survey_id}) # Loop through and delete all associated graphs and tests for graph in graphs: delete_image(graph["image"]) mongo.db.graphs.delete_one(graph) mongo.db.tests.delete_many({"surveyId":survey_id}) mongo.db.temp_results.delete_many({"survey_id":survey_id}) # Delete the file delete_file(file_obj["fileName"]) # finally delete from the surveys database mongo.db.surveys.delete_one(file_obj) return redirect(url_for('surveys.home')) # EXPORT A SURVEY @surveys.route("/survey//export", methods=['GET']) @login_required def export_survey(survey_id): file_obj = mongo.db.surveys.find_one_or_404({"_id":ObjectId(survey_id)}) if file_obj["user"] != current_user._id: flash("You do not have access to that survey", "danger") abort(403) # Get all graphs and tests relating to this file tests = mongo.db.tests.find({"surveyId":survey_id}) graphs = mongo.db.graphs.find({"surveyId":survey_id}) # Use a temp file so that it can be deleted after with tempfile.NamedTemporaryFile() as f: # Create a Pandas Excel writer using XlsxWriter as the engine. writer = pd.ExcelWriter(f.name, engine='xlsxwriter') # Convert the dataframe to an XlsxWriter Excel object. df = read_file(file_obj["fileName"]) df.to_excel(writer, sheet_name='Survey data') wb = writer.book # Get a worksheet for the statistical tests ws = wb.add_worksheet("Statistical Tests") tests_to_excel(ws, tests) # Get a worksheet for the graphs ws2 = wb.add_worksheet("Graphs") graphs_to_excel(ws2, graphs) writer.close() wb.close() return send_file(f.name, attachment_filename=file_obj["title"] + ".xlsx", as_attachment=True)