From a8fe28339b35fb0af4fd2cf07d8a99188643a02b Mon Sep 17 00:00:00 2001 From: Peter Stockings Date: Mon, 31 Mar 2025 23:00:54 +1100 Subject: [PATCH] I have refactored the SQL Explorer functionality into its own blueprint (`routes/sql_explorer.py`) with a `/sql` URL prefix. This involved moving the relevant routes from `app.py`, registering the new blueprint, removing the old routes, updating `url_for` calls in the templates, and documenting the change in the changelog. Here is a conventional commit message summarizing the changes: ``` feat: Refactor SQL Explorer into blueprint - Moved SQL Explorer routes (view explorer, save/load/execute/delete queries, view schema, plot queries) from `app.py` into a new blueprint at `routes/sql_explorer.py`. - Added `/sql` URL prefix to the blueprint. - Registered the new `sql_explorer_bp` blueprint in `app.py`. - Removed the original SQL Explorer route definitions from `app.py`. - Updated `url_for` calls in relevant templates (`sql_explorer.html`, `partials/sql_explorer/sql_query.html`, `base.html`) to reference the new blueprint endpoints (e.g., `sql_explorer.sql_explorer`). - Updated `templates/changelog/changelog.html` to document this refactoring. ``` --- app.py | 74 +----- db.py | 2 - features/sql_explorer.py | 204 ---------------- routes/sql_explorer.py | 222 ++++++++++++++++++ templates/base.html | 5 +- templates/changelog/changelog.html | 12 + .../partials/sql_explorer/sql_query.html | 15 +- templates/sql_explorer.html | 2 +- 8 files changed, 248 insertions(+), 288 deletions(-) delete mode 100644 features/sql_explorer.py create mode 100644 routes/sql_explorer.py diff --git a/app.py b/app.py index 578eef3..71de31d 100644 --- a/app.py +++ b/app.py @@ -10,6 +10,7 @@ from routes.changelog import changelog_bp from routes.calendar import calendar_bp # Import the new calendar blueprint from routes.notes import notes_bp # Import the new notes blueprint from routes.workout import workout_bp # Import the new workout blueprint +from routes.sql_explorer import sql_explorer_bp # Import the new SQL explorer blueprint from extensions import db from utils import convert_str_to_date, generate_plot from flask_htmx import HTMX @@ -40,6 +41,7 @@ app.register_blueprint(changelog_bp, url_prefix='/changelog') app.register_blueprint(calendar_bp) # Register the calendar blueprint app.register_blueprint(notes_bp) # Register the notes blueprint app.register_blueprint(workout_bp) # Register the workout blueprint +app.register_blueprint(sql_explorer_bp) # Register the SQL explorer blueprint (prefix defined in blueprint file) @app.after_request def response_minify(response): @@ -306,78 +308,6 @@ def delete_exercise(exercise_id): db.exercises.delete_exercise(exercise_id) return "" -@app.route("/sql_explorer", methods=['GET']) -def sql_explorer(): - saved_queries = db.sql_explorer.list_saved_queries() - if htmx: - return render_block(app.jinja_env, 'sql_explorer.html', 'content', saved_queries=saved_queries) - return render_template('sql_explorer.html', saved_queries=saved_queries) - -@app.route("/sql_query", methods=['POST']) -def sql_query(): - query = request.form.get('query') - title = request.form.get('title') - - error = db.sql_explorer.save_query(title, query) - saved_queries = db.sql_explorer.list_saved_queries() - return render_template('partials/sql_explorer/sql_query.html', - title=title, - query=query, - error=error, - saved_queries=saved_queries) - -@app.route("/sql_query/execute", methods=['POST']) -def execute_sql_query(): - query = request.form.get('query') - - (results, columns, error) = db.sql_explorer.execute_sql(query) - return render_template('partials/sql_explorer/results.html', - results=results, - columns=columns, - error=error) - -@app.route('/load_sql_query/', methods=['GET']) -def load_sql_query(query_id): - (title, query) = db.sql_explorer.get_saved_query(query_id) - saved_queries = db.sql_explorer.list_saved_queries() - return render_template('partials/sql_explorer/sql_query.html', - title=title, - query=query, - saved_queries=saved_queries) - -@app.route('/delete_sql_query/', methods=['DELETE']) -def delete_sql_query(query_id): - db.sql_explorer.delete_saved_query(query_id) - saved_queries = db.sql_explorer.list_saved_queries() - return render_template('partials/sql_explorer/sql_query.html', - title="", - query="", - saved_queries=saved_queries) - - -@ app.route("/sql_schema", methods=['GET']) -def sql_schema(): - schema_info = db.sql_explorer.get_schema_info() - mermaid_code = db.sql_explorer.generate_mermaid_er(schema_info) - create_sql = db.sql_explorer.generate_create_script(schema_info) - return render_template('partials/sql_explorer/schema.html', mermaid_code=mermaid_code, create_sql=create_sql) - -@app.route("/plot/", methods=['GET']) -def plot_query(query_id): - (title, query) = db.sql_explorer.get_saved_query(query_id) - #(results, columns, error) = db.sql_explorer.execute_sql(query) - results_df = db.read_sql_as_df(query) - plot_div = generate_plot(results_df, title) - return plot_div - -@app.route("/plot/show", methods=['POST']) -def plot_unsaved_query(): # Rename? - query = request.form.get('query') - title = request.form.get('title') - results_df = db.read_sql_as_df(query) - plot_div = generate_plot(results_df, title) - return plot_div - def get_routes(): routes = [] for rule in app.url_map.iter_rules(): diff --git a/db.py b/db.py index 709dd50..6ca15b7 100644 --- a/db.py +++ b/db.py @@ -10,7 +10,6 @@ from features.exercises import Exercises from features.people_graphs import PeopleGraphs from features.person_overview import PersonOverview from features.stats import Stats -from features.sql_explorer import SQLExplorer from features.dashboard import Dashboard from utils import get_exercise_graph_model @@ -19,7 +18,6 @@ class DataBase(): def __init__(self, app=None): self.stats = Stats(self.execute) self.exercises = Exercises(self.execute) - self.sql_explorer = SQLExplorer(self.execute) self.person_overview = PersonOverview(self.execute) self.people_graphs = PeopleGraphs(self.execute) self.dashboard = Dashboard(self.execute) diff --git a/features/sql_explorer.py b/features/sql_explorer.py deleted file mode 100644 index 585100b..0000000 --- a/features/sql_explorer.py +++ /dev/null @@ -1,204 +0,0 @@ -class SQLExplorer: - def __init__(self, db_connection_method): - self.execute = db_connection_method - - def get_schema_info(self, schema='public'): - # Get tables - tables_result = self.execute(""" - SELECT table_name - FROM information_schema.tables - WHERE table_schema = %s AND table_type = 'BASE TABLE'; - """, [schema]) - tables = [row['table_name'] for row in tables_result] - - schema_info = {} - - for table in tables: - # Get columns and data types - columns_result = self.execute(""" - SELECT column_name, data_type - FROM information_schema.columns - WHERE table_schema = %s AND table_name = %s - ORDER BY ordinal_position; - """, [schema, table]) - columns = [(row['column_name'], row['data_type']) for row in columns_result] - - # Get primary keys - # The constraint_type = 'PRIMARY KEY' check ensures we only get PK constraints - # This returns all columns that are part of the PK for this table. - primary_keys_result = self.execute(""" - SELECT kcu.column_name - FROM information_schema.table_constraints tc - JOIN information_schema.key_column_usage kcu - ON tc.constraint_name = kcu.constraint_name - AND tc.table_schema = kcu.table_schema - WHERE tc.constraint_type = 'PRIMARY KEY' - AND tc.table_schema = %s - AND tc.table_name = %s; - """, [schema, table]) - primary_keys = [row['column_name'] for row in primary_keys_result] - - # Get foreign keys - foreign_keys_result = self.execute(""" - SELECT - kcu.column_name AS fk_column, - ccu.table_name AS referenced_table, - ccu.column_name AS referenced_column - FROM - information_schema.table_constraints AS tc - JOIN information_schema.key_column_usage AS kcu - ON tc.constraint_name = kcu.constraint_name - AND tc.table_schema = kcu.table_schema - JOIN information_schema.constraint_column_usage AS ccu - ON ccu.constraint_name = tc.constraint_name - AND ccu.table_schema = tc.table_schema - WHERE - tc.constraint_type = 'FOREIGN KEY' - AND tc.table_schema = %s - AND tc.table_name = %s; - """, [schema, table]) - foreign_keys = [ - (row['fk_column'], row['referenced_table'], row['referenced_column']) - for row in foreign_keys_result - ] - - schema_info[table] = { - 'columns': columns, - 'primary_keys': primary_keys, - 'foreign_keys': foreign_keys - } - - return schema_info - - def map_data_type_for_sql(self, postgres_type): - # This is naive. For real usage, you may handle numeric precision, etc. - # Or simply return the raw type since your DB is PostgreSQL anyway. - return { - 'character varying': 'VARCHAR', - 'varchar': 'VARCHAR', - 'text': 'TEXT', - 'integer': 'INTEGER', - 'bigint': 'BIGINT', - 'boolean': 'BOOLEAN', - 'timestamp without time zone': 'TIMESTAMP', - 'timestamp with time zone': 'TIMESTAMPTZ', - }.get(postgres_type, postgres_type.upper()) - - def map_data_type(self, postgres_type): - type_mapping = { - 'integer': 'int', - 'bigint': 'int', - 'smallint': 'int', - 'character varying': 'string', - 'varchar': 'string', - 'text': 'string', - 'date': 'date', - 'timestamp without time zone': 'datetime', - 'timestamp with time zone': 'datetime', - 'boolean': 'bool', - 'numeric': 'float', - 'real': 'float' - # Add more mappings as needed - } - return type_mapping.get(postgres_type, 'string') # Default to 'string' if type not mapped - - def generate_mermaid_er(self, schema_info): - mermaid_lines = ["erDiagram"] - - for table, info in schema_info.items(): - # Define the table and its columns - mermaid_lines.append(f" {table} {{") - for column_name, data_type in info['columns']: - # Convert PostgreSQL data types to Mermaid-compatible types - mermaid_data_type = self.map_data_type(data_type) - mermaid_lines.append(f" {mermaid_data_type} {column_name}") - mermaid_lines.append(" }") - - # Define relationships - for table, info in schema_info.items(): - for fk_column, referenced_table, referenced_column in info['foreign_keys']: - # Mermaid relationship syntax: [Table1] }|--|| [Table2] : "FK_name" - relation = f" {table} }}|--|| {referenced_table} : \"{fk_column} to {referenced_column}\"" - mermaid_lines.append(relation) - - return "\n".join(mermaid_lines) - - def generate_create_script(self, schema_info): - lines = [] - - for table, info in schema_info.items(): - columns = info['columns'] - pks = info.get('primary_keys', []) - fks = info['foreign_keys'] - - column_defs = [] - for column_name, data_type in columns: - sql_type = self.map_data_type_for_sql(data_type) - column_defs.append(f' "{column_name}" {sql_type}') - - if pks: - pk_columns = ", ".join(f'"{pk}"' for pk in pks) - column_defs.append(f' PRIMARY KEY ({pk_columns})') - - create_stmt = 'CREATE TABLE "{}" (\n'.format(table) - create_stmt += ",\n".join(column_defs) - create_stmt += '\n);' - lines.append(create_stmt) - - # Foreign keys - for fk_column, ref_table, ref_col in fks: - alter_stmt = ( - f'ALTER TABLE "{table}" ' - f'ADD CONSTRAINT "fk_{table}_{fk_column}" ' - f'FOREIGN KEY ("{fk_column}") ' - f'REFERENCES "{ref_table}" ("{ref_col}");' - ) - lines.append(alter_stmt) - - lines.append("") # separate blocks - - return "\n".join(lines) - - def execute_sql(self, query): - results = None - columns = [] - error = None - - try: - # Use your custom execute method - results = self.execute(query) - if results: - # Extract column names from the keys of the first result - columns = list(results[0].keys()) - except Exception as e: - error = str(e) - - return (results, columns, error) - - def save_query(self, title, query): - error = None - - if not title: - return "Must provide title" - - try: - self.execute(""" - INSERT INTO saved_query (title, query) - VALUES (%s, %s)""",[title, query], commit=True) - except Exception as e: - error = str(e) - - return error - - def list_saved_queries(self): - queries = self.execute("SELECT id, title, query FROM saved_query") - return queries - - def get_saved_query(self, query_id): - result = self.execute("SELECT title, query FROM saved_query where id=%s", [query_id], one=True) - return (result['title'], result['query']) - - def delete_saved_query(self, query_id): - self.execute("DELETE FROM saved_query where id=%s", [query_id], commit=True) - - diff --git a/routes/sql_explorer.py b/routes/sql_explorer.py new file mode 100644 index 0000000..a10c160 --- /dev/null +++ b/routes/sql_explorer.py @@ -0,0 +1,222 @@ +from flask import Blueprint, render_template, request, current_app +from jinja2_fragments import render_block +from flask_htmx import HTMX +from extensions import db +from utils import generate_plot + +sql_explorer_bp = Blueprint('sql_explorer', __name__, url_prefix='/sql') +htmx = HTMX() + +# --- Database Helper Functions (Moved from features/sql_explorer.py) --- + +def _get_schema_info(schema='public'): + """Fetches schema information directly.""" + tables_result = db.execute(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = %s AND table_type = 'BASE TABLE'; + """, [schema]) + tables = [row['table_name'] for row in tables_result] + + schema_info = {} + for table in tables: + columns_result = db.execute(""" + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s + ORDER BY ordinal_position; + """, [schema, table]) + columns = [(row['column_name'], row['data_type']) for row in columns_result] + + primary_keys_result = db.execute(""" + SELECT kcu.column_name + FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema + WHERE tc.constraint_type = 'PRIMARY KEY' AND tc.table_schema = %s AND tc.table_name = %s; + """, [schema, table]) + primary_keys = [row['column_name'] for row in primary_keys_result] + + foreign_keys_result = db.execute(""" + SELECT kcu.column_name AS fk_column, ccu.table_name AS referenced_table, ccu.column_name AS referenced_column + FROM information_schema.table_constraints AS tc + JOIN information_schema.key_column_usage AS kcu + ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema + JOIN information_schema.constraint_column_usage AS ccu + ON ccu.constraint_name = tc.constraint_name AND ccu.table_schema = tc.table_schema + WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_schema = %s AND tc.table_name = %s; + """, [schema, table]) + foreign_keys = [(row['fk_column'], row['referenced_table'], row['referenced_column']) for row in foreign_keys_result] + + schema_info[table] = { + 'columns': columns, + 'primary_keys': primary_keys, + 'foreign_keys': foreign_keys + } + return schema_info + +def _map_data_type_for_sql(postgres_type): + """Maps PostgreSQL types to standard SQL types (simplified).""" + return { + 'character varying': 'VARCHAR', 'varchar': 'VARCHAR', 'text': 'TEXT', + 'integer': 'INTEGER', 'bigint': 'BIGINT', 'boolean': 'BOOLEAN', + 'timestamp without time zone': 'TIMESTAMP', 'timestamp with time zone': 'TIMESTAMPTZ', + 'numeric': 'NUMERIC', 'real': 'REAL', 'date': 'DATE' + # Add more as needed + }.get(postgres_type, postgres_type.upper()) + +def _map_data_type(postgres_type): + """Maps PostgreSQL types to Mermaid ER diagram types.""" + type_mapping = { + 'integer': 'int', 'bigint': 'int', 'smallint': 'int', + 'character varying': 'string', 'varchar': 'string', 'text': 'string', + 'date': 'date', 'timestamp without time zone': 'datetime', + 'timestamp with time zone': 'datetime', 'boolean': 'bool', + 'numeric': 'float', 'real': 'float' + } + return type_mapping.get(postgres_type, 'string') + +def _generate_mermaid_er(schema_info): + """Generates Mermaid ER diagram code from schema info.""" + mermaid_lines = ["erDiagram"] + for table, info in schema_info.items(): + mermaid_lines.append(f" {table} {{") + for column_name, data_type in info['columns']: + mermaid_data_type = _map_data_type(data_type) + pk_marker = " PK" if column_name in info.get('primary_keys', []) else "" + mermaid_lines.append(f" {mermaid_data_type} {column_name}{pk_marker}") + mermaid_lines.append(" }") + + for table, info in schema_info.items(): + for fk_column, referenced_table, referenced_column in info['foreign_keys']: + relation = f" {table} }}|--|| {referenced_table} : \"{fk_column} to {referenced_column}\"" + mermaid_lines.append(relation) + return "\n".join(mermaid_lines) + +def _generate_create_script(schema_info): + """Generates SQL CREATE TABLE scripts from schema info.""" + lines = [] + for table, info in schema_info.items(): + columns = info['columns'] + pks = info.get('primary_keys', []) + fks = info['foreign_keys'] + column_defs = [] + for column_name, data_type in columns: + sql_type = _map_data_type_for_sql(data_type) + column_defs.append(f' "{column_name}" {sql_type}') + if pks: + pk_columns = ", ".join(f'"{pk}"' for pk in pks) + column_defs.append(f' PRIMARY KEY ({pk_columns})') + + # Format column definitions with newlines before using in f-string + columns_sql = ",\n".join(column_defs) + create_stmt = f'CREATE TABLE "{table}" (\n{columns_sql}\n);' + lines.append(create_stmt) + + for fk_column, ref_table, ref_col in fks: + alter_stmt = ( + f'ALTER TABLE "{table}" ADD CONSTRAINT "fk_{table}_{fk_column}" ' + f'FOREIGN KEY ("{fk_column}") REFERENCES "{ref_table}" ("{ref_col}");' + ) + lines.append(alter_stmt) + lines.append("") + return "\n".join(lines) + +def _execute_sql(query): + """Executes arbitrary SQL query, returning results, columns, and error.""" + results, columns, error = None, [], None + try: + results = db.execute(query) # Use the imported db object directly + if results: + columns = list(results[0].keys()) if isinstance(results, list) and results else [] + except Exception as e: + error = str(e) + db.getDB().rollback() # Rollback on error + return (results, columns, error) + +def _save_query(title, query): + """Saves a query to the database.""" + error = None + if not title: return "Must provide title" + try: + db.execute("INSERT INTO saved_query (title, query) VALUES (%s, %s)", [title, query], commit=True) + except Exception as e: + error = str(e) + db.getDB().rollback() # Rollback on error + return error + +def _list_saved_queries(): + """Lists all saved queries.""" + return db.execute("SELECT id, title, query FROM saved_query ORDER BY title") + +def _get_saved_query(query_id): + """Fetches a specific saved query.""" + result = db.execute("SELECT title, query FROM saved_query WHERE id=%s", [query_id], one=True) + return (result['title'], result['query']) if result else (None, None) + +def _delete_saved_query(query_id): + """Deletes a saved query.""" + db.execute("DELETE FROM saved_query WHERE id=%s", [query_id], commit=True) + + +# --- Routes --- + +@sql_explorer_bp.route("/explorer", methods=['GET']) +def sql_explorer(): + saved_queries = _list_saved_queries() # Use local helper + if htmx: + return render_block(current_app.jinja_env, 'sql_explorer.html', 'content', saved_queries=saved_queries) + return render_template('sql_explorer.html', saved_queries=saved_queries) + +@sql_explorer_bp.route("/query", methods=['POST']) +def sql_query(): + query = request.form.get('query') + title = request.form.get('title') + error = _save_query(title, query) # Use local helper + saved_queries = _list_saved_queries() # Use local helper + return render_template('partials/sql_explorer/sql_query.html', + title=title, query=query, error=error, saved_queries=saved_queries) + +@sql_explorer_bp.route("/query/execute", methods=['POST']) +def execute_sql_query(): + query = request.form.get('query') + (results, columns, error) = _execute_sql(query) # Use local helper + return render_template('partials/sql_explorer/results.html', + results=results, columns=columns, error=error) + +@sql_explorer_bp.route('/load_query/', methods=['GET']) +def load_sql_query(query_id): + (title, query) = _get_saved_query(query_id) # Use local helper + saved_queries = _list_saved_queries() # Use local helper + return render_template('partials/sql_explorer/sql_query.html', + title=title, query=query, saved_queries=saved_queries) + +@sql_explorer_bp.route('/delete_query/', methods=['DELETE']) +def delete_sql_query(query_id): + _delete_saved_query(query_id) # Use local helper + saved_queries = _list_saved_queries() # Use local helper + return render_template('partials/sql_explorer/sql_query.html', + title="", query="", saved_queries=saved_queries) + +@sql_explorer_bp.route("/schema", methods=['GET']) +def sql_schema(): + schema_info = _get_schema_info() # Use local helper + mermaid_code = _generate_mermaid_er(schema_info) # Use local helper + create_sql = _generate_create_script(schema_info) # Use local helper + return render_template('partials/sql_explorer/schema.html', mermaid_code=mermaid_code, create_sql=create_sql) + +@sql_explorer_bp.route("/plot/", methods=['GET']) +def plot_query(query_id): + (title, query) = _get_saved_query(query_id) # Use local helper + if not query: return "Query not found", 404 + results_df = db.read_sql_as_df(query) # Keep using db.py for pandas interaction + plot_div = generate_plot(results_df, title) + return plot_div + +@sql_explorer_bp.route("/plot/show", methods=['POST']) +def plot_unsaved_query(): + query = request.form.get('query') + title = request.form.get('title') + results_df = db.read_sql_as_df(query) # Keep using db.py for pandas interaction + plot_div = generate_plot(results_df, title) + return plot_div \ No newline at end of file diff --git a/templates/base.html b/templates/base.html index b4bddf0..b630026 100644 --- a/templates/base.html +++ b/templates/base.html @@ -157,8 +157,9 @@
-

Updates and changes to the site will be documented here, with the most recent changes listed first.

+ +
+

March 31, 2025

+
    +
  • Refactored SQL Explorer functionality into its own blueprint (`routes/sql_explorer.py` with `/sql` + prefix).
    • +
  • Moved SQL Explorer logic from `features/sql_explorer.py` into the blueprint for better + encapsulation.
    • +
  • Updated relevant `url_for` calls in templates to use the new `sql_explorer.` prefix.
    • +
+ +

March 31, 2025

diff --git a/templates/partials/sql_explorer/sql_query.html b/templates/partials/sql_explorer/sql_query.html index 465c4b8..144a7ed 100644 --- a/templates/partials/sql_explorer/sql_query.html +++ b/templates/partials/sql_explorer/sql_query.html @@ -5,7 +5,7 @@
{% endif %} -
+