diff --git a/.gitignore b/.gitignore index c01352b..604daa4 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,8 @@ backup/ history.json # Temporary files +.pytest_cache/ +.pytest-tmp/ *.tmp *.bak *_backup.* diff --git a/README.md b/README.md index a33e246..c31ce36 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,16 @@ flask --app app create-admin python app.py # Visit: http://localhost:8084 ``` + +For a fresh or unmanaged installation, `init-db` is safe to run again: it +creates missing tables without deleting existing data. + +For an installation already managed by Alembic, apply schema updates before +restarting: + +```bash +flask --app app db upgrade +``` ## Dependencies diff --git a/app.py b/app.py index 25b8aa8..1220c5d 100644 --- a/app.py +++ b/app.py @@ -1,6 +1,5 @@ import argparse import datetime -import json import logging import os from pathlib import Path @@ -10,13 +9,16 @@ from flask import Flask, render_template from flask_login import LoginManager from flask_migrate import Migrate +from flask_wtf.csrf import CSRFError, CSRFProtect from models import User, db, initialize_postgres_extensions from utils.email_service import init_mail +from utils.model_catalog import load_model_catalog BASE_DIR = Path(__file__).resolve().parent load_dotenv(BASE_DIR / ".env") +csrf = CSRFProtect() def _is_production() -> bool: @@ -47,16 +49,7 @@ def _secret_key() -> str: def _load_model_database() -> dict: - model_db_path = BASE_DIR / "data" / "model_database.json" - if not model_db_path.is_file(): - raise RuntimeError(f"Required model database is missing: {model_db_path}") - - with model_db_path.open(encoding="utf-8") as model_db_file: - models_data = json.load(model_db_file) - - if not isinstance(models_data, dict) or not models_data: - raise RuntimeError("The model database must be a non-empty JSON object.") - return models_data + return load_model_catalog(BASE_DIR) def _register_cli_commands(app: Flask) -> None: @@ -108,6 +101,16 @@ def create_app() -> Flask: SQLALCHEMY_TRACK_MODIFICATIONS=False, SQLALCHEMY_ENGINE_OPTIONS={"pool_pre_ping": True}, MAX_CONTENT_LENGTH=4 * 1024 * 1024, + SESSION_COOKIE_HTTPONLY=True, + SESSION_COOKIE_SAMESITE="Lax", + SESSION_COOKIE_SECURE=_is_production(), + REMEMBER_COOKIE_HTTPONLY=True, + REMEMBER_COOKIE_SAMESITE="Lax", + REMEMBER_COOKIE_SECURE=_is_production(), + PERMANENT_SESSION_LIFETIME=datetime.timedelta(days=7), + WTF_CSRF_ENABLED=os.environ.get( + "WTF_CSRF_ENABLED", "true" + ).lower() == "true", MAIL_SERVER=os.environ.get("MAIL_SERVER", "smtp.gmail.com"), MAIL_PORT=int(os.environ.get("MAIL_PORT", 587)), MAIL_USE_TLS=os.environ.get("MAIL_USE_TLS", "true").lower() == "true", @@ -131,6 +134,7 @@ def create_app() -> Flask: db.init_app(app) Migrate(app, db) + csrf.init_app(app) init_mail(app) login_manager = LoginManager() @@ -182,6 +186,14 @@ def upload_too_large(_error): "error.html", error="Uploads must be smaller than 4 MB." ), 413 + @app.errorhandler(CSRFError) + def handle_csrf_error(error): + logger.warning("CSRF validation failed: %s", error.description) + return render_template( + "error.html", + error="This form expired or could not be verified. Please try again.", + ), 400 + @app.errorhandler(500) def internal_server_error(error): logger.exception("Unhandled application error: %s", error) diff --git a/data/model_expansions.json b/data/model_expansions.json new file mode 100644 index 0000000..0899e77 --- /dev/null +++ b/data/model_expansions.json @@ -0,0 +1,370 @@ +{ + "Negative Binomial Regression": { + "description": "A generalized linear model for overdispersed count outcomes whose variance is larger than its mean.", + "when_to_use": "Use for event or incident counts when a Poisson model shows substantial overdispersion.", + "assumptions": ["Independent observations", "Count-valued outcome", "Correct mean structure", "Negative-binomial variance is appropriate"], + "use_cases": ["overdispersed counts", "event rates"], + "analysis_goals": ["predict", "explain"], + "dependent_variable": ["count"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "random"], + "data_distribution": ["non_normal"], + "relationship_type": ["linear"], + "implementation": { + "python": { + "code": "import statsmodels.api as sm\nmodel = sm.GLM(y, sm.add_constant(X), family=sm.families.NegativeBinomial()).fit()", + "documentation": "https://www.statsmodels.org/stable/generated/statsmodels.genmod.families.family.NegativeBinomial.html" + }, + "r": { + "code": "library(MASS)\nmodel <- glm.nb(y ~ x1 + x2, data=df)", + "documentation": "https://stat.ethz.ch/R-manual/R-devel/library/MASS/html/glm.nb.html" + } + } + }, + "Zero-Inflated Poisson": { + "description": "A two-part count model that separates structural zeros from counts generated by a Poisson process.", + "when_to_use": "Use when count data contain more zeros than an ordinary Poisson or negative-binomial model can explain.", + "assumptions": ["Independent observations", "Two distinct zero-generating processes", "Poisson count component is appropriate", "Correct predictors for both components"], + "use_cases": ["excess-zero counts", "event occurrence and frequency"], + "analysis_goals": ["predict", "explain"], + "dependent_variable": ["count"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "random"], + "data_distribution": ["non_normal"], + "relationship_type": ["linear"], + "implementation": { + "python": { + "code": "from statsmodels.discrete.count_model import ZeroInflatedPoisson\nmodel = ZeroInflatedPoisson(y, X, exog_infl=X).fit()", + "documentation": "https://www.statsmodels.org/stable/generated/statsmodels.discrete.count_model.ZeroInflatedPoisson.html" + }, + "r": { + "code": "library(pscl)\nmodel <- zeroinfl(y ~ x1 + x2 | z1, data=df, dist='poisson')", + "documentation": "https://cran.r-project.org/package=pscl" + } + } + }, + "Quantile Regression": { + "description": "Models a selected conditional quantile of a continuous outcome instead of only its conditional mean.", + "when_to_use": "Use to study effects in the median or tails, especially with heteroscedastic or skewed outcomes.", + "assumptions": ["Independent observations", "Linear conditional quantile unless nonlinear terms are added", "Adequate observations near the selected quantile"], + "use_cases": ["median regression", "tail effects", "heteroscedastic outcomes"], + "analysis_goals": ["predict", "explore", "explain"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "random"], + "data_distribution": ["non_normal", "asymmetric_laplace"], + "relationship_type": ["linear"], + "implementation": { + "python": { + "code": "import statsmodels.formula.api as smf\nmodel = smf.quantreg('y ~ x1 + x2', df).fit(q=0.5)", + "documentation": "https://www.statsmodels.org/stable/generated/statsmodels.regression.quantile_regression.QuantReg.html" + }, + "r": { + "code": "library(quantreg)\nmodel <- rq(y ~ x1 + x2, tau=0.5, data=df)", + "documentation": "https://cran.r-project.org/package=quantreg" + } + } + }, + "Partial Least Squares": { + "description": "Builds latent components that maximize covariance between predictors and outcomes, supporting prediction with many correlated predictors.", + "when_to_use": "Use for continuous outcomes with numerous, highly collinear predictors and limited observations.", + "assumptions": ["Observations are independent", "Components capture relevant covariance", "Predictors are scaled when units differ", "Component count is validated"], + "use_cases": ["multicollinearity", "high-dimensional prediction", "latent components"], + "analysis_goals": ["predict", "reduce dimensions", "explore relationships"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["small", "medium", "large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["linear"], + "implementation": { + "python": { + "code": "from sklearn.cross_decomposition import PLSRegression\nmodel = PLSRegression(n_components=2).fit(X, y)", + "documentation": "https://scikit-learn.org/stable/modules/generated/sklearn.cross_decomposition.PLSRegression.html" + }, + "r": { + "code": "library(pls)\nmodel <- plsr(y ~ ., data=df, validation='CV')", + "documentation": "https://cran.r-project.org/package=pls" + } + } + }, + "Generalized Additive Model": { + "description": "A regression model that learns smooth nonlinear effects while retaining an interpretable additive structure.", + "when_to_use": "Use when relationships are nonlinear but the contribution of each predictor should remain inspectable.", + "assumptions": ["Independent observations unless correlation is modeled", "Appropriate response family and link", "Smoothness is selected or validated", "Additive structure is adequate"], + "use_cases": ["smooth nonlinear effects", "interpretable prediction"], + "analysis_goals": ["predict", "explore", "explain"], + "dependent_variable": ["continuous", "binary", "count"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "random"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "from pygam import LinearGAM, s\nmodel = LinearGAM(s(0) + s(1)).fit(X, y)", + "documentation": "https://pygam.readthedocs.io/" + }, + "r": { + "code": "library(mgcv)\nmodel <- gam(y ~ s(x1) + s(x2), data=df, method='REML')", + "documentation": "https://cran.r-project.org/package=mgcv" + } + } + }, + "Bayesian Linear Regression": { + "description": "Linear regression with prior distributions on coefficients and a posterior distribution for parameters and predictions.", + "when_to_use": "Use when prior knowledge, direct uncertainty statements, or partial pooling of weak information is valuable.", + "assumptions": ["Linear mean relationship", "Appropriate residual likelihood", "Priors are defensible", "Posterior sampling converges"], + "use_cases": ["probabilistic prediction", "prior information", "uncertainty quantification"], + "analysis_goals": ["predict", "explain", "robust_prediction"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["small", "medium", "large"], + "missing_data": ["none", "random"], + "data_distribution": ["normal", "non_normal (with alternative likelihoods)"], + "relationship_type": ["linear"], + "implementation": { + "python": { + "code": "import bambi as bmb\nmodel = bmb.Model('y ~ x1 + x2', df)\ntrace = model.fit()", + "documentation": "https://bambinos.github.io/bambi/" + }, + "r": { + "code": "library(brms)\nmodel <- brm(y ~ x1 + x2, data=df, family=gaussian())", + "documentation": "https://cran.r-project.org/package=brms" + } + } + }, + "ARIMAX": { + "description": "An ARIMA model augmented with external predictors so temporal dependence and explanatory variables are modeled together.", + "when_to_use": "Use for a single time series affected by known external regressors, interventions, holidays, or policy variables.", + "assumptions": ["Series is stationary after differencing", "Residuals are approximately white noise", "External regressors are known for the forecast horizon", "Parameters are stable"], + "use_cases": ["dynamic regression", "forecasting with external drivers"], + "analysis_goals": ["time_series", "predict"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["linear"], + "implementation": { + "python": { + "code": "from statsmodels.tsa.statespace.sarimax import SARIMAX\nmodel = SARIMAX(y, exog=X, order=(1,1,1)).fit()", + "documentation": "https://www.statsmodels.org/stable/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html" + }, + "r": { + "code": "library(forecast)\nmodel <- auto.arima(y, xreg=X)", + "documentation": "https://cran.r-project.org/package=forecast" + } + } + }, + "Exponential Smoothing": { + "description": "Forecasts a series using recursively weighted level, trend, and seasonal components, with recent observations receiving more weight.", + "when_to_use": "Use for univariate forecasting with evolving level, trend, or seasonality and no required external predictors.", + "assumptions": ["Regularly spaced observations", "Level, trend, and seasonality are reasonably stable", "Residual forecast errors are uncorrelated"], + "use_cases": ["short-term forecasting", "trend and seasonal series"], + "analysis_goals": ["time_series", "predict"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["small", "medium", "large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "from statsmodels.tsa.holtwinters import ExponentialSmoothing\nmodel = ExponentialSmoothing(y, trend='add', seasonal='add', seasonal_periods=12).fit()", + "documentation": "https://www.statsmodels.org/stable/generated/statsmodels.tsa.holtwinters.ExponentialSmoothing.html" + }, + "r": { + "code": "library(forecast)\nmodel <- ets(y)", + "documentation": "https://cran.r-project.org/package=forecast" + } + } + }, + "Prophet": { + "description": "An additive forecasting model with piecewise trend, multiple seasonalities, holidays, and robust handling of missing observations.", + "when_to_use": "Use for business time series with strong calendar effects, trend changes, holidays, and a reasonably long history.", + "assumptions": ["Future seasonality resembles historical seasonality", "Trend changes can be represented by changepoints", "Calendar effects are specified appropriately"], + "use_cases": ["business forecasting", "holiday effects", "multiple seasonality"], + "analysis_goals": ["time_series", "predict"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous", "binary"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "random"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "from prophet import Prophet\nmodel = Prophet().fit(df[['ds', 'y']])", + "documentation": "https://facebook.github.io/prophet/" + }, + "r": { + "code": "library(prophet)\nmodel <- prophet(df)", + "documentation": "https://facebook.github.io/prophet/" + } + } + }, + "GARCH": { + "description": "Models time-varying conditional variance and volatility clustering in a return or residual series.", + "when_to_use": "Use when volatility changes over time and large movements tend to be followed by other large movements.", + "assumptions": ["Mean dynamics are adequately modeled", "Conditional variance parameters imply a valid process", "Standardized residuals have little remaining dependence"], + "use_cases": ["volatility forecasting", "financial returns", "risk estimation"], + "analysis_goals": ["time_series", "estimate financial risk"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["non_normal", "heavy_tailed"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "from arch import arch_model\nmodel = arch_model(returns, vol='GARCH', p=1, q=1, dist='t').fit()", + "documentation": "https://arch.readthedocs.io/" + }, + "r": { + "code": "library(rugarch)\nspec <- ugarchspec()\nmodel <- ugarchfit(spec, returns)", + "documentation": "https://cran.r-project.org/package=rugarch" + } + } + }, + "Value at Risk (VaR)": { + "description": "Estimates a loss threshold that should not be exceeded over a specified horizon at a selected confidence level.", + "when_to_use": "Use to summarize market or portfolio downside at a defined horizon and probability, accompanied by backtesting.", + "assumptions": ["Loss distribution or simulation process is appropriate", "Time horizon is explicit", "Dependence is represented", "Estimates are backtested"], + "use_cases": ["market risk", "portfolio limits", "capital planning"], + "analysis_goals": ["estimate financial risk"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous", "categorical"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["normal", "non_normal", "heavy_tailed"], + "relationship_type": ["linear", "non_linear"], + "implementation": { + "python": { + "code": "import numpy as np\nvar_99 = -np.quantile(portfolio_returns, 0.01)", + "documentation": "https://numpy.org/doc/stable/reference/generated/numpy.quantile.html" + }, + "r": { + "code": "library(PerformanceAnalytics)\nVaR(returns, p=0.99, method='historical')", + "documentation": "https://cran.r-project.org/package=PerformanceAnalytics" + } + } + }, + "Conditional Value at Risk (CVaR)": { + "description": "Estimates the expected loss conditional on losses exceeding a selected Value-at-Risk threshold.", + "when_to_use": "Use when the severity of tail losses matters, especially for skewed or heavy-tailed loss distributions.", + "assumptions": ["Tail sample is sufficiently informative", "Loss distribution or simulation process is appropriate", "Time horizon and confidence level are explicit"], + "use_cases": ["tail risk", "portfolio optimization", "stress assessment"], + "analysis_goals": ["measure tail risk", "estimate financial risk"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous", "categorical"], + "sample_size": ["large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["non_normal", "heavy_tailed"], + "relationship_type": ["linear", "non_linear"], + "implementation": { + "python": { + "code": "import numpy as np\ncutoff = np.quantile(losses, 0.99)\ncvar_99 = losses[losses >= cutoff].mean()", + "documentation": "https://numpy.org/doc/stable/reference/generated/numpy.quantile.html" + }, + "r": { + "code": "library(PerformanceAnalytics)\nES(returns, p=0.99, method='historical')", + "documentation": "https://cran.r-project.org/package=PerformanceAnalytics" + } + } + }, + "Monte Carlo Risk Simulation": { + "description": "Propagates uncertainty through a model by repeatedly sampling inputs and summarizing the resulting loss or outcome distribution.", + "when_to_use": "Use when risk depends on several uncertain inputs or nonlinear relationships that are difficult to solve analytically.", + "assumptions": ["Input distributions are defensible", "Dependencies among inputs are modeled", "Simulation count is sufficient", "Structural model reflects the process"], + "use_cases": ["project risk", "portfolio risk", "scenario analysis"], + "analysis_goals": ["simulate risk", "estimate financial risk", "predict"], + "dependent_variable": ["continuous", "count"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["small", "medium", "large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["any"], + "relationship_type": ["any"], + "implementation": { + "python": { + "code": "import numpy as np\nsamples = rng.multivariate_normal(mean, covariance, size=100000)\noutcomes = risk_function(samples)", + "documentation": "https://numpy.org/doc/stable/reference/random/generated/numpy.random.Generator.multivariate_normal.html" + }, + "r": { + "code": "library(MASS)\nsamples <- mvrnorm(100000, mu=mean, Sigma=covariance)\noutcomes <- risk_function(samples)", + "documentation": "https://stat.ethz.ch/R-manual/R-devel/library/MASS/html/mvrnorm.html" + } + } + }, + "Likelihood Ratio Evidence Model": { + "description": "Compares how probable observed evidence is under two competing propositions and reports their likelihood ratio.", + "when_to_use": "Use to evaluate forensic evidence when competing source or event propositions and validated probability models are available.", + "assumptions": ["Competing propositions are clearly defined", "Evidence probabilities are calibrated", "Conditional dependencies are represented", "Population data are relevant"], + "use_cases": ["trace evidence", "source attribution", "DNA evidence interpretation"], + "analysis_goals": ["evaluate forensic evidence"], + "dependent_variable": ["binary", "categorical"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "random"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["linear", "non_linear"], + "implementation": { + "python": { + "code": "import numpy as np\nlog_lr = model_hp.score_samples(evidence) - model_hd.score_samples(evidence)\nlr = np.exp(log_lr)", + "documentation": "https://scikit-learn.org/stable/modules/density.html" + }, + "r": { + "code": "log_lr <- log(p_evidence_given_hp) - log(p_evidence_given_hd)\nlr <- exp(log_lr)", + "documentation": "https://www.r-project.org/" + } + } + }, + "Benford Law Analysis": { + "description": "Screens naturally occurring numeric records for deviations from expected leading-digit frequencies.", + "when_to_use": "Use as an investigative screening tool for suitable financial or administrative data, never as proof of fraud by itself.", + "assumptions": ["Values span several orders of magnitude", "Data are not assigned identifiers or constrained values", "The generating process is expected to follow Benford patterns"], + "use_cases": ["financial screening", "audit triage", "record anomaly detection"], + "analysis_goals": ["screen financial records", "detect forensic anomalies"], + "dependent_variable": ["continuous", "count"], + "independent_variables": ["continuous", "categorical"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "random"], + "data_distribution": ["non_normal"], + "relationship_type": ["any"], + "implementation": { + "python": { + "code": "from scipy.stats import chisquare\nobserved = leading_digit_counts(values)\nresult = chisquare(observed, f_exp=benford_expected(len(values)))", + "documentation": "https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.chisquare.html" + }, + "r": { + "code": "library(benford.analysis)\nresult <- benford(values, number.of.digits=1)", + "documentation": "https://cran.r-project.org/package=benford.analysis" + } + } + }, + "Forensic Anomaly Detection": { + "description": "Uses Isolation Forest to rank unusual multidimensional records for expert review without requiring labeled anomalies.", + "when_to_use": "Use to prioritize atypical transactions, device records, or digital-forensic events for investigation.", + "assumptions": ["Features meaningfully represent abnormality", "Contamination is relatively low", "Preprocessing avoids data leakage", "Flags receive human validation"], + "use_cases": ["transaction screening", "digital forensics", "investigation triage"], + "analysis_goals": ["detect forensic anomalies", "classify"], + "dependent_variable": ["binary", "categorical"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["any"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "from sklearn.ensemble import IsolationForest\nmodel = IsolationForest(contamination='auto', random_state=42).fit(X)\nanomaly_score = -model.score_samples(X)", + "documentation": "https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.IsolationForest.html" + }, + "r": { + "code": "library(isotree)\nmodel <- isolation.forest(X)\nanomaly_score <- predict(model, X, type='score')", + "documentation": "https://cran.r-project.org/package=isotree" + } + } + } +} diff --git a/data/time_series_models.json b/data/time_series_models.json new file mode 100644 index 0000000..b3aaab3 --- /dev/null +++ b/data/time_series_models.json @@ -0,0 +1,485 @@ +{ + "Autoregressive (AR) Model": { + "description": "Models a stationary series as a linear function of its own previous values.", + "when_to_use": "Use when current values primarily depend on a short history of past observations and no moving-average component is needed.", + "assumptions": ["Weak stationarity", "Stable autoregressive roots", "Residuals are approximately white noise"], + "use_cases": ["short-memory forecasting", "lag-effect estimation"], + "analysis_goals": ["time_series", "predict"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["linear"], + "implementation": { + "python": { + "code": "from statsmodels.tsa.ar_model import AutoReg\nmodel = AutoReg(y, lags=3, old_names=False).fit()", + "documentation": "https://www.statsmodels.org/stable/generated/statsmodels.tsa.ar_model.AutoReg.html" + }, + "r": { + "code": "model <- arima(y, order=c(3, 0, 0))", + "documentation": "https://stat.ethz.ch/R-manual/R-devel/library/stats/html/arima.html" + } + } + }, + "Moving Average (MA) Model": { + "description": "Models a stationary series using present and past random shocks rather than lagged observations directly.", + "when_to_use": "Use when autocorrelation cuts off after a small number of lags and shock effects persist briefly.", + "assumptions": ["Weak stationarity", "Invertible moving-average process", "Innovations are uncorrelated"], + "use_cases": ["shock-response forecasting", "short-memory errors"], + "analysis_goals": ["time_series", "predict"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["linear"], + "implementation": { + "python": { + "code": "from statsmodels.tsa.arima.model import ARIMA\nmodel = ARIMA(y, order=(0, 0, 2)).fit()", + "documentation": "https://www.statsmodels.org/stable/generated/statsmodels.tsa.arima.model.ARIMA.html" + }, + "r": { + "code": "model <- arima(y, order=c(0, 0, 2))", + "documentation": "https://stat.ethz.ch/R-manual/R-devel/library/stats/html/arima.html" + } + } + }, + "Autoregressive Moving Average (ARMA) Model": { + "description": "Combines autoregressive lags and moving-average shocks for a stationary univariate series.", + "when_to_use": "Use for stationary series whose autocorrelation and partial autocorrelation both decay rather than cut off cleanly.", + "assumptions": ["Weak stationarity", "Causal and invertible process", "Residuals are approximately white noise"], + "use_cases": ["stationary forecasting", "compact lag and shock models"], + "analysis_goals": ["time_series", "predict"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["linear"], + "implementation": { + "python": { + "code": "from statsmodels.tsa.arima.model import ARIMA\nmodel = ARIMA(y, order=(2, 0, 1)).fit()", + "documentation": "https://www.statsmodels.org/stable/generated/statsmodels.tsa.arima.model.ARIMA.html" + }, + "r": { + "code": "model <- arima(y, order=c(2, 0, 1))", + "documentation": "https://stat.ethz.ch/R-manual/R-devel/library/stats/html/arima.html" + } + } + }, + "Seasonal ARIMA (SARIMA)": { + "description": "Extends ARIMA with seasonal autoregressive, differencing, and moving-average terms.", + "when_to_use": "Use for a univariate series with a stable repeating seasonal cycle after appropriate differencing.", + "assumptions": ["Regular seasonal frequency", "Stationarity after ordinary and seasonal differencing", "Residuals are approximately white noise"], + "use_cases": ["seasonal forecasting", "monthly and quarterly series"], + "analysis_goals": ["time_series", "predict"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["linear"], + "implementation": { + "python": { + "code": "from statsmodels.tsa.statespace.sarimax import SARIMAX\nmodel = SARIMAX(y, order=(1,1,1), seasonal_order=(1,1,1,12)).fit()", + "documentation": "https://www.statsmodels.org/stable/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html" + }, + "r": { + "code": "library(forecast)\nmodel <- Arima(y, order=c(1,1,1), seasonal=c(1,1,1))", + "documentation": "https://cran.r-project.org/package=forecast" + } + } + }, + "Holt-Winters Model": { + "description": "Uses exponentially weighted level, trend, and seasonal components with additive or multiplicative seasonality.", + "when_to_use": "Use for seasonal univariate data when level and trend evolve smoothly over time.", + "assumptions": ["Regular observation frequency", "Stable seasonal pattern", "Additive or multiplicative component form is appropriate"], + "use_cases": ["seasonal demand forecasting", "trend-seasonal series"], + "analysis_goals": ["time_series", "predict"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["small", "medium", "large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "from statsmodels.tsa.holtwinters import ExponentialSmoothing\nmodel = ExponentialSmoothing(y, trend='add', seasonal='mul', seasonal_periods=12).fit()", + "documentation": "https://www.statsmodels.org/stable/generated/statsmodels.tsa.holtwinters.ExponentialSmoothing.html" + }, + "r": { + "code": "model <- HoltWinters(y, seasonal='multiplicative')", + "documentation": "https://stat.ethz.ch/R-manual/R-devel/library/stats/html/HoltWinters.html" + } + } + }, + "Vector Autoregression (VAR)": { + "description": "Models several stationary time series jointly, with each variable depending on lags of every variable.", + "when_to_use": "Use when multiple endogenous series influence one another and joint forecasts or impulse responses are required.", + "assumptions": ["Joint stationarity", "Stable lag system", "Sufficient observations for the parameter count", "Residuals have limited serial correlation"], + "use_cases": ["multivariate forecasting", "impulse response analysis"], + "analysis_goals": ["time_series", "predict", "explore relationships"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["linear"], + "implementation": { + "python": { + "code": "from statsmodels.tsa.api import VAR\nmodel = VAR(data).fit(maxlags=4, ic='aic')", + "documentation": "https://www.statsmodels.org/stable/generated/statsmodels.tsa.vector_ar.var_model.VAR.html" + }, + "r": { + "code": "library(vars)\nmodel <- VAR(data, lag.max=4, ic='AIC')", + "documentation": "https://cran.r-project.org/package=vars" + } + } + }, + "Vector Error Correction Model (VECM)": { + "description": "Models short-run changes among nonstationary series while preserving one or more long-run cointegrating relationships.", + "when_to_use": "Use for multiple integrated series that are empirically cointegrated.", + "assumptions": ["Series share the same integration order", "Cointegration rank is correctly selected", "Lag structure is adequate", "Parameters are stable"], + "use_cases": ["cointegrated systems", "long-run equilibrium analysis"], + "analysis_goals": ["time_series", "predict", "explore relationships"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["linear"], + "implementation": { + "python": { + "code": "from statsmodels.tsa.vector_ar.vecm import VECM\nmodel = VECM(data, k_ar_diff=2, coint_rank=1).fit()", + "documentation": "https://www.statsmodels.org/stable/generated/statsmodels.tsa.vector_ar.vecm.VECM.html" + }, + "r": { + "code": "library(urca)\nmodel <- ca.jo(data, type='trace', K=3, ecdet='const')", + "documentation": "https://cran.r-project.org/package=urca" + } + } + }, + "State Space Model": { + "description": "Represents an observed series through evolving unobserved states and separate transition and observation equations.", + "when_to_use": "Use when latent level, trend, cycles, measurement error, missing observations, or time-varying components must be modeled explicitly.", + "assumptions": ["State transition and observation equations are appropriate", "Noise structure is specified correctly", "The system is identifiable"], + "use_cases": ["latent-state forecasting", "structural time series", "missing observations"], + "analysis_goals": ["time_series", "predict", "explore"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "random", "handled_automatically"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["linear", "non_linear"], + "implementation": { + "python": { + "code": "from statsmodels.tsa.statespace.structural import UnobservedComponents\nmodel = UnobservedComponents(y, level='local linear trend', seasonal=12).fit()", + "documentation": "https://www.statsmodels.org/stable/generated/statsmodels.tsa.statespace.structural.UnobservedComponents.html" + }, + "r": { + "code": "library(KFAS)\nmodel <- SSModel(y ~ SSMtrend(2) + SSMseasonal(12), H=NA)", + "documentation": "https://cran.r-project.org/package=KFAS" + } + } + }, + "Bayesian Structural Time Series (BSTS)": { + "description": "A Bayesian state-space model combining structural components, regression effects, and posterior predictive uncertainty.", + "when_to_use": "Use when trend, seasonality, interventions, variable selection, and full probabilistic uncertainty should be estimated together.", + "assumptions": ["Structural components are well specified", "Priors are defensible", "Posterior sampling converges", "Predictors are available over the forecast horizon"], + "use_cases": ["probabilistic forecasting", "intervention analysis", "Bayesian variable selection"], + "analysis_goals": ["time_series", "predict", "robust_prediction"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "random"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["linear"], + "implementation": { + "python": { + "code": "import pymc as pm\nwith pm.Model() as model:\n trend = pm.GaussianRandomWalk('trend', sigma=1, shape=len(y))\n observed = pm.Normal('observed', trend, sigma=1, observed=y)", + "documentation": "https://www.pymc.io/" + }, + "r": { + "code": "library(bsts)\nmodel <- bsts(y ~ x1 + x2, state.specification=AddLocalLinearTrend(list(), y), niter=2000)", + "documentation": "https://cran.r-project.org/package=bsts" + } + } + }, + "Dynamic Linear Model (DLM)": { + "description": "A linear Gaussian state-space model whose regression coefficients or latent states can evolve over time.", + "when_to_use": "Use when relationships change gradually and sequential updating or time-varying coefficients are important.", + "assumptions": ["Linear state and observation equations", "Gaussian disturbances unless extended", "Evolution variances are identifiable"], + "use_cases": ["time-varying regression", "online forecasting", "dynamic coefficients"], + "analysis_goals": ["time_series", "predict"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "random", "handled_automatically"], + "data_distribution": ["normal"], + "relationship_type": ["linear"], + "implementation": { + "python": { + "code": "from pydlm import dlm, trend, dynamic\nmodel = dlm(y) + trend(1, 0.1) + dynamic(features=X, discount=0.99)\nmodel.fit()", + "documentation": "https://pydlm.github.io/" + }, + "r": { + "code": "library(dlm)\nmodel <- dlmModPoly(order=2, dV=1, dW=c(0.1, 0.1))", + "documentation": "https://cran.r-project.org/package=dlm" + } + } + }, + "Stochastic Volatility Model": { + "description": "Treats latent log-volatility as its own stochastic time series rather than a deterministic function of past squared errors.", + "when_to_use": "Use for financial or economic series with persistent, uncertain volatility and a need for posterior volatility intervals.", + "assumptions": ["Return mean is adequately modeled", "Latent volatility dynamics are appropriate", "Posterior sampling converges"], + "use_cases": ["latent volatility estimation", "financial risk forecasting"], + "analysis_goals": ["time_series", "estimate financial risk", "measure tail risk"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["non_normal", "heavy_tailed"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "import pymc as pm\nwith pm.Model() as model:\n step = pm.GaussianRandomWalk('log_volatility', sigma=0.2, shape=len(returns))\n observed = pm.Normal('returns', 0, sigma=pm.math.exp(step / 2), observed=returns)", + "documentation": "https://www.pymc.io/" + }, + "r": { + "code": "library(stochvol)\nmodel <- svsample(returns, draws=5000, burnin=1000)", + "documentation": "https://cran.r-project.org/package=stochvol" + } + } + }, + "Threshold Autoregressive (TAR) Model": { + "description": "Uses different autoregressive equations when a threshold variable crosses estimated or specified regimes.", + "when_to_use": "Use when series dynamics change abruptly above and below meaningful thresholds.", + "assumptions": ["Regime-specific dynamics are stable", "Threshold variable and delay are appropriate", "Each regime has sufficient observations"], + "use_cases": ["asymmetric cycles", "threshold effects", "nonlinear autoregression"], + "analysis_goals": ["time_series", "predict"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "from statsmodels.tsa.tsatools import lagmat\n# Fit separate autoregressions above and below a validated threshold.", + "documentation": "https://www.statsmodels.org/stable/tsa.html" + }, + "r": { + "code": "library(tsDyn)\nmodel <- setar(y, m=3, thDelay=1)", + "documentation": "https://cran.r-project.org/package=tsDyn" + } + } + }, + "Markov Switching Model": { + "description": "Allows regression or autoregressive parameters to switch between latent regimes governed by a Markov process.", + "when_to_use": "Use when data appear to alternate between persistent unobserved states such as expansion and recession.", + "assumptions": ["A small number of latent regimes is adequate", "Transition probabilities are stable", "Regimes are identifiable"], + "use_cases": ["business-cycle regimes", "structural breaks", "switching volatility"], + "analysis_goals": ["time_series", "predict", "explore"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["normal", "non_normal"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "from statsmodels.tsa.regime_switching.markov_autoregression import MarkovAutoregression\nmodel = MarkovAutoregression(y, k_regimes=2, order=2, switching_variance=True).fit()", + "documentation": "https://www.statsmodels.org/stable/generated/statsmodels.tsa.regime_switching.markov_autoregression.MarkovAutoregression.html" + }, + "r": { + "code": "library(MSwM)\nmodel <- msmFit(lm(y ~ lag1), k=2, p=0)", + "documentation": "https://cran.r-project.org/package=MSwM" + } + } + }, + "Recurrent Neural Network for Time Series": { + "description": "A neural sequence model with recurrent hidden state for learning nonlinear temporal dependencies.", + "when_to_use": "Use for large datasets with nonlinear sequential patterns when simpler statistical models have been benchmarked.", + "assumptions": ["Training and deployment distributions are comparable", "Sequence windows prevent leakage", "Regularization and held-out validation are used"], + "use_cases": ["nonlinear sequence forecasting", "multistep prediction"], + "analysis_goals": ["time_series", "predict", "flexible_prediction"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["any"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "from tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import SimpleRNN, Dense\nmodel = Sequential([SimpleRNN(32, input_shape=(lookback, n_features)), Dense(1)])", + "documentation": "https://keras.io/api/layers/recurrent_layers/simple_rnn/" + }, + "r": { + "code": "library(keras)\nmodel <- keras_model_sequential() |> layer_simple_rnn(32, input_shape=c(lookback, n_features)) |> layer_dense(1)", + "documentation": "https://keras.posit.co/" + } + } + }, + "Long Short-Term Memory (LSTM)": { + "description": "A gated recurrent neural network designed to retain information over longer temporal spans.", + "when_to_use": "Use with substantial training data when long-range nonlinear dependencies outperform simpler baselines.", + "assumptions": ["Large representative training sample", "Leakage-safe temporal validation", "Architecture and regularization are tuned on validation data"], + "use_cases": ["long-dependency forecasting", "multivariate sequences"], + "analysis_goals": ["time_series", "predict", "flexible_prediction"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["any"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "from tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import LSTM, Dense\nmodel = Sequential([LSTM(64, input_shape=(lookback, n_features)), Dense(1)])", + "documentation": "https://keras.io/api/layers/recurrent_layers/lstm/" + }, + "r": { + "code": "library(keras)\nmodel <- keras_model_sequential() |> layer_lstm(64, input_shape=c(lookback, n_features)) |> layer_dense(1)", + "documentation": "https://keras.posit.co/" + } + } + }, + "Gated Recurrent Unit (GRU)": { + "description": "A gated recurrent neural network similar to LSTM but with fewer parameters and a simpler state update.", + "when_to_use": "Use as a computationally lighter recurrent alternative for large nonlinear sequence datasets.", + "assumptions": ["Representative training data", "Leakage-safe temporal validation", "Sequence length and regularization are validated"], + "use_cases": ["efficient sequence forecasting", "nonlinear temporal prediction"], + "analysis_goals": ["time_series", "predict", "flexible_prediction"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["any"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "from tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import GRU, Dense\nmodel = Sequential([GRU(64, input_shape=(lookback, n_features)), Dense(1)])", + "documentation": "https://keras.io/api/layers/recurrent_layers/gru/" + }, + "r": { + "code": "library(keras)\nmodel <- keras_model_sequential() |> layer_gru(64, input_shape=c(lookback, n_features)) |> layer_dense(1)", + "documentation": "https://keras.posit.co/" + } + } + }, + "Temporal Convolutional Network (TCN)": { + "description": "Uses causal dilated one-dimensional convolutions to model long sequence histories without recurrent connections.", + "when_to_use": "Use for large sequence datasets when parallel training and a long receptive field are useful.", + "assumptions": ["Causal padding prevents future leakage", "Receptive field covers relevant history", "Temporal validation is used"], + "use_cases": ["long-horizon sequences", "multichannel forecasting"], + "analysis_goals": ["time_series", "predict", "flexible_prediction"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["any"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "from tcn import TCN\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import Dense\nmodel = Sequential([TCN(input_shape=(lookback, n_features)), Dense(1)])", + "documentation": "https://github.com/philipperemy/keras-tcn" + }, + "r": { + "code": "library(keras)\n# Stack causal Conv1D layers with increasing dilation rates, then add a dense output.", + "documentation": "https://keras.posit.co/reference/layer_conv_1d.html" + } + } + }, + "Gradient Boosting for Time Series": { + "description": "Applies boosted decision trees to lagged, rolling, calendar, and external features derived without future leakage.", + "when_to_use": "Use for nonlinear forecasting with rich covariates and carefully engineered temporal features.", + "assumptions": ["Features are available at prediction time", "Lag construction prevents leakage", "Evaluation uses rolling or blocked temporal splits"], + "use_cases": ["feature-rich forecasting", "nonlinear demand prediction"], + "analysis_goals": ["time_series", "predict", "flexible_prediction"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["large"], + "missing_data": ["none", "imputed", "handled_automatically"], + "data_distribution": ["any"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "from xgboost import XGBRegressor\nmodel = XGBRegressor(n_estimators=500, max_depth=4).fit(X_lagged, y)", + "documentation": "https://xgboost.readthedocs.io/" + }, + "r": { + "code": "library(xgboost)\nmodel <- xgboost(data=as.matrix(X_lagged), label=y, nrounds=500, max_depth=4)", + "documentation": "https://xgboost.readthedocs.io/" + } + } + }, + "Random Forest for Time Series": { + "description": "Uses an ensemble of randomized regression trees on lagged and calendar features to capture nonlinear temporal relationships.", + "when_to_use": "Use for nonlinear forecasting with sufficient observations, useful lag features, and limited extrapolation needs.", + "assumptions": ["Features are available at forecast time", "Temporal feature construction avoids leakage", "Blocked or rolling validation is used"], + "use_cases": ["nonlinear lag forecasting", "feature-rich time series"], + "analysis_goals": ["time_series", "predict", "flexible_prediction"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous", "categorical", "binary"], + "sample_size": ["large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["any"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "from sklearn.ensemble import RandomForestRegressor\nmodel = RandomForestRegressor(n_estimators=500, random_state=42).fit(X_lagged, y)", + "documentation": "https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestRegressor.html" + }, + "r": { + "code": "library(ranger)\nmodel <- ranger(y ~ ., data=lagged_data, num.trees=500)", + "documentation": "https://cran.r-project.org/package=ranger" + } + } + }, + "Wavelet Transform Model": { + "description": "Decomposes a series into localized time-frequency components that can be denoised, analyzed, or forecast separately.", + "when_to_use": "Use for nonstationary signals with behavior occurring at several time scales or transient frequency patterns.", + "assumptions": ["Wavelet family and decomposition level are appropriate", "Boundary effects are handled", "Forecast reconstruction is validated"], + "use_cases": ["multiscale forecasting", "signal denoising", "time-frequency analysis"], + "analysis_goals": ["time_series", "predict", "explore"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["medium", "large"], + "missing_data": ["none", "imputed"], + "data_distribution": ["any"], + "relationship_type": ["non_linear"], + "implementation": { + "python": { + "code": "import pywt\ncoefficients = pywt.wavedec(y, 'db4', level=4)\nreconstructed = pywt.waverec(coefficients, 'db4')", + "documentation": "https://pywavelets.readthedocs.io/" + }, + "r": { + "code": "library(wavelets)\ncomponents <- dwt(y, filter='d4', n.levels=4)", + "documentation": "https://cran.r-project.org/package=wavelets" + } + } + }, + "Kalman Filter": { + "description": "Recursively estimates the hidden state of a linear Gaussian dynamic system as new observations arrive.", + "when_to_use": "Use for online state estimation, smoothing, sensor fusion, missing observations, or forecasting in a specified state-space system.", + "assumptions": ["Linear transition and observation equations", "Gaussian process and measurement noise", "Known or estimable covariance matrices"], + "use_cases": ["online filtering", "state estimation", "sensor fusion"], + "analysis_goals": ["time_series", "predict"], + "dependent_variable": ["continuous"], + "independent_variables": ["continuous"], + "sample_size": ["small", "medium", "large"], + "missing_data": ["none", "random", "handled_automatically"], + "data_distribution": ["normal"], + "relationship_type": ["linear"], + "implementation": { + "python": { + "code": "from filterpy.kalman import KalmanFilter\nfilter_model = KalmanFilter(dim_x=2, dim_z=1)\n# Set F, H, Q, R, P and then call predict() and update(z) sequentially.", + "documentation": "https://filterpy.readthedocs.io/" + }, + "r": { + "code": "library(KFAS)\nmodel <- SSModel(y ~ SSMtrend(2), H=NA)\nfiltered <- KFS(fitSSM(model, inits=c(0,0))$model)", + "documentation": "https://cran.r-project.org/package=KFAS" + } + } + } +} diff --git a/migrations/versions/8f4e3d2c1b0a_add_questionnaire_drafts.py b/migrations/versions/8f4e3d2c1b0a_add_questionnaire_drafts.py new file mode 100644 index 0000000..1d89123 --- /dev/null +++ b/migrations/versions/8f4e3d2c1b0a_add_questionnaire_drafts.py @@ -0,0 +1,55 @@ +"""Add server-side questionnaire drafts. + +Revision ID: 8f4e3d2c1b0a +Revises: 5c442376367c +""" + +from alembic import op +import sqlalchemy as sa + + +revision = "8f4e3d2c1b0a" +down_revision = "5c442376367c" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "questionnaire_drafts", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=True), + sa.Column("content", sa.JSON(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_questionnaire_drafts_user_id", + "questionnaire_drafts", + ["user_id"], + unique=False, + ) + op.create_index( + "ix_questionnaire_drafts_updated_at", + "questionnaire_drafts", + ["updated_at"], + unique=False, + ) + + +def downgrade(): + op.drop_index( + "ix_questionnaire_drafts_updated_at", + table_name="questionnaire_drafts", + ) + op.drop_index( + "ix_questionnaire_drafts_user_id", + table_name="questionnaire_drafts", + ) + op.drop_table("questionnaire_drafts") diff --git a/models.py b/models.py index 793fcc0..52cc9af 100644 --- a/models.py +++ b/models.py @@ -1,6 +1,6 @@ from flask_login import UserMixin from flask_sqlalchemy import SQLAlchemy -from datetime import datetime +from datetime import datetime import json import os from werkzeug.security import generate_password_hash, check_password_hash @@ -42,8 +42,26 @@ def is_admin(self): return self._is_admin @property - def is_expert(self): - return self._is_expert and self.is_approved_expert + def is_expert(self): + return self._is_expert and self.is_approved_expert + + @property + def role(self): + """Return the user's effective role for templates and older callers.""" + if self.is_admin: + return 'admin' + if self.is_expert: + return 'expert' + return 'user' + + @property + def expertise(self): + """Backward-compatible alias for the canonical expertise field.""" + return self.areas_of_expertise + + @expertise.setter + def expertise(self, value): + self.areas_of_expertise = value def __repr__(self): return f'' @@ -142,7 +160,7 @@ class Consultation(db.Model): def __repr__(self): return f'' -class Questionnaire(db.Model): +class Questionnaire(db.Model): __tablename__ = 'questionnaires' id = db.Column(db.Integer, primary_key=True) @@ -163,8 +181,31 @@ class Questionnaire(db.Model): postgresql_ops={'title': 'gin_trgm_ops', 'topic': 'gin_trgm_ops'}), ) - def __repr__(self): - return f'' + def __repr__(self): + return f'' + + +class QuestionnaireDraft(db.Model): + """Server-side working copy for guest and authenticated questionnaires.""" + + __tablename__ = 'questionnaire_drafts' + + id = db.Column(db.String(64), primary_key=True) + user_id = db.Column( + db.Integer, + db.ForeignKey('users.id', ondelete='CASCADE'), + nullable=True, + index=True, + ) + content = db.Column(db.JSON, nullable=False) + created_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) + updated_at = db.Column( + db.DateTime, + default=datetime.utcnow, + onupdate=datetime.utcnow, + nullable=False, + index=True, + ) # PostgreSQL-specific function to initialize extensions def initialize_postgres_extensions(app): @@ -188,14 +229,13 @@ def initialize_postgres_extensions(app): # Don't fail the entire app if extensions can't be created db.session.rollback() -def get_model_details(model_name): - try: - import os - model_db_path = os.path.join(os.path.dirname(__file__), 'data', 'model_database.json') - with open(model_db_path, 'r') as f: - models = json.load(f) - - # Get the model directly from the dictionary +def get_model_details(model_name): + try: + from utils.model_catalog import load_model_catalog + + models = load_model_catalog(os.path.dirname(__file__)) + + # Get the model directly from the dictionary if model_name in models: return models[model_name] return None diff --git a/public/static/js/chatbot.js b/public/static/js/chatbot.js index 2293b97..9b8fc56 100644 --- a/public/static/js/chatbot.js +++ b/public/static/js/chatbot.js @@ -125,9 +125,12 @@ class ChatBot { // Call API to get response const response = await fetch('/chatbot/ask', { method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, + headers: { + 'Content-Type': 'application/json', + 'X-CSRFToken': document.querySelector( + 'meta[name="csrf-token"]' + )?.content || '', + }, body: JSON.stringify({ question: userMessage, context: this.pageContext diff --git a/requirements.txt b/requirements.txt index 9dd9096..0e9b4c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ flask-login>=0.6.0,<1.0.0 flask-sqlalchemy>=3.1.0,<4.0.0 flask-mail>=0.9.0,<1.0.0 flask-migrate>=4.0.0,<5.0.0 +flask-wtf>=1.2.0,<2.0.0 SQLAlchemy>=2.0.0,<3.0.0 alembic>=1.16.0,<2.0.0 diff --git a/routes/admin_routes.py b/routes/admin_routes.py index e3f57f8..2844cea 100644 --- a/routes/admin_routes.py +++ b/routes/admin_routes.py @@ -17,6 +17,11 @@ ai_usage_storage_ready, initialize_ai_usage_storage, ) +from utils.validation import ( + is_valid_email, + is_valid_username, + normalize_email, +) from sqlalchemy.exc import SQLAlchemyError import logging import re @@ -109,16 +114,22 @@ def edit_user(user_id): user = User.query.get_or_404(user_id) if request.method == 'POST': # Update user information - user.username = request.form.get('username') - user.email = request.form.get('email') - # Update roles - is_admin = request.form.get('is_admin') == 'on' - is_expert = request.form.get('is_expert') == 'on' - user._is_admin = is_admin - if is_expert: + username = request.form.get('username', '').strip() + email = normalize_email(request.form.get('email')) + if not is_valid_username(username) or not is_valid_email(email): + flash('Provide a valid username and email address.', 'danger') + return render_template('admin/edit_user.html', user=user), 400 + user.username = username + user.email = email + role = request.form.get('role', 'user') + user._is_admin = role == 'admin' + if role == 'expert': user._is_expert = True user.is_approved_expert = request.form.get('is_approved_expert') == 'on' - user.areas_of_expertise = request.form.get('areas_of_expertise') + user.areas_of_expertise = request.form.get( + 'areas_of_expertise', + request.form.get('expertise', ''), + ) user.institution = request.form.get('institution') user.bio = request.form.get('bio') else: diff --git a/routes/auth_routes.py b/routes/auth_routes.py index 89aa333..24709e3 100644 --- a/routes/auth_routes.py +++ b/routes/auth_routes.py @@ -4,9 +4,33 @@ from werkzeug.security import generate_password_hash, check_password_hash from utils.email_service import get_email_provider, send_email from itsdangerous import URLSafeTimedSerializer, SignatureExpired, BadSignature +import hashlib import os +from urllib.parse import urljoin, urlsplit from flask import current_app +from utils.validation import ( + MIN_PASSWORD_LENGTH, + is_valid_email, + is_valid_password, + is_valid_username, + normalize_email, +) auth = Blueprint('auth', __name__) + +def _is_safe_redirect_target(target): + """Allow redirects only to local HTTP(S) paths.""" + if not target: + return False + host_url = urlsplit(request.host_url) + redirect_url = urlsplit(urljoin(request.host_url, target)) + return ( + redirect_url.scheme in {'http', 'https'} + and redirect_url.netloc == host_url.netloc + ) + + +def _password_fingerprint(user): + return hashlib.sha256(user.password_hash.encode()).hexdigest()[:16] @auth.route('/login', methods=['GET', 'POST']) def login(): """Handle user login""" @@ -23,6 +47,8 @@ def login(): if user and check_password_hash(user.password_hash, password): login_user(user, remember=remember) next_page = request.args.get('next') + if not _is_safe_redirect_target(next_page): + next_page = None flash('Login successful!', 'success') return redirect(next_page or url_for('main.home')) else: @@ -34,8 +60,8 @@ def register(): if current_user.is_authenticated: return redirect(url_for('main.home')) if request.method == 'POST': - username = request.form.get('username') - email = request.form.get('email') + username = request.form.get('username', '').strip() + email = normalize_email(request.form.get('email')) password = request.form.get('password') confirm_password = request.form.get('confirm_password') if not username or not email or not password or not confirm_password: @@ -44,6 +70,18 @@ def register(): if password != confirm_password: flash('Passwords do not match!', 'danger') return render_template('register.html') + if not is_valid_username(username): + flash('Username must be between 3 and 80 characters.', 'danger') + return render_template('register.html') + if not is_valid_email(email): + flash('Please provide a valid email address.', 'danger') + return render_template('register.html') + if not is_valid_password(password): + flash( + f'Password must be at least {MIN_PASSWORD_LENGTH} characters.', + 'danger', + ) + return render_template('register.html') user_exists = User.query.filter_by(username=username).first() email_exists = User.query.filter_by(email=email).first() if user_exists: @@ -61,7 +99,7 @@ def register(): flash('Account created successfully! Please log in.', 'success') return redirect(url_for('auth.login')) return render_template('register.html') -@auth.route('/logout') +@auth.route('/logout', methods=['POST']) @login_required def logout(): """Handle user logout""" @@ -82,7 +120,13 @@ def forgot_password(): if user: # Generate a secure token serializer = URLSafeTimedSerializer(current_app.config['SECRET_KEY']) - token = serializer.dumps(email, salt='password-reset-salt') + token = serializer.dumps( + { + 'email': email, + 'password_fingerprint': _password_fingerprint(user), + }, + salt='password-reset-salt', + ) # Build reset URL reset_url = url_for('auth.reset_password', token=token, _external=True) # Send reset email @@ -112,13 +156,24 @@ def reset_password(token): try: # Validate token (expires after 1 hour) serializer = URLSafeTimedSerializer(current_app.config['SECRET_KEY']) - email = serializer.loads(token, salt='password-reset-salt', max_age=3600) + token_data = serializer.loads( + token, + salt='password-reset-salt', + max_age=3600, + ) except (SignatureExpired, BadSignature): flash('The password reset link is invalid or has expired.', 'danger') return redirect(url_for('auth.forgot_password')) - user = User.query.filter_by(email=email).first() - if not user: - flash('User not found.', 'danger') + if not isinstance(token_data, dict): + flash('The password reset link is invalid or has expired.', 'danger') + return redirect(url_for('auth.forgot_password')) + user = User.query.filter_by(email=token_data.get('email')).first() + if ( + not user + or token_data.get('password_fingerprint') + != _password_fingerprint(user) + ): + flash('The password reset link is invalid or has expired.', 'danger') return redirect(url_for('auth.login')) if request.method == 'POST': password = request.form.get('password') @@ -129,6 +184,12 @@ def reset_password(token): if password != confirm_password: flash('Passwords do not match!', 'danger') return render_template('reset_password.html', token=token) + if not is_valid_password(password): + flash( + f'Password must be at least {MIN_PASSWORD_LENGTH} characters.', + 'danger', + ) + return render_template('reset_password.html', token=token) # Update password user.password_hash = generate_password_hash(password, method='pbkdf2:sha256') db.session.commit() diff --git a/routes/main_routes.py b/routes/main_routes.py index 0126f1b..e7f1449 100644 --- a/routes/main_routes.py +++ b/routes/main_routes.py @@ -12,11 +12,138 @@ from utils.ai_service import OpenAIServiceError, is_ai_enabled from utils.ai_usage import consume_user_ai_quota from utils.recommendation_ai import review_recommendation +from utils.validation import is_valid_email, normalize_email logger = logging.getLogger(__name__) main = Blueprint('main', __name__) +GOAL_COMPATIBILITY = { + 'predict': { + 'predict', + 'robust_prediction', + 'flexible_prediction', + 'predict survival probabilities', + }, + 'classify': { + 'classify', + 'separate groups', + 'estimate probabilities', + }, + 'explore': { + 'explore', + 'explain', + 'explore relationships', + 'identify latent variables', + 'reduce', + 'reduce dimensions', + 'dimensionality reduction', + 'visualize', + 'visualize relationships', + 'explore similarity structures', + }, + 'cluster': {'cluster'}, + 'hypothesis_test': { + 'test', + 'compare', + 'compare means', + 'test group differences', + 'evaluate treatment effects', + 'compare adjusted means', + 'compare multivariate means', + 'test overall group differences', + }, + 'non_parametric': {'test', 'compare', 'rank'}, + 'time_series': {'time_series'}, + 'risk_assessment': { + 'estimate financial risk', + 'measure tail risk', + 'simulate risk', + }, + 'forensic_analysis': { + 'evaluate forensic evidence', + 'detect forensic anomalies', + 'screen financial records', + }, +} + +OUTCOME_COMPATIBILITY = { + 'continuous': {'continuous'}, + 'categorical': {'categorical', 'categorical (limited)', 'multiclass'}, + 'binary': {'binary', 'binary (with link functions)'}, + 'count': {'count'}, + 'time_series': {'continuous'}, + 'time_to_event': {'time-to-event', 'censored'}, +} + +MISSING_DATA_COMPATIBILITY = { + 'none': {'none'}, + 'little': {'none', 'random', 'random (MAR)'}, + 'moderate': {'random', 'random (MAR)', 'imputed'}, + 'substantial': { + 'random', + 'random (MAR)', + 'systematic', + 'imputed', + 'handled_via_FIML', + 'handled_via_imputation', + 'handled_automatically', + }, +} + + +def _supports_goal(selected, supported): + return bool(GOAL_COMPATIBILITY.get(selected, {selected}) & set(supported)) + + +def _supports_outcome(selected, supported): + return bool( + OUTCOME_COMPATIBILITY.get(selected, {selected}) & set(supported) + ) + + +def _supports_missing_data(selected, supported): + return bool( + MISSING_DATA_COMPATIBILITY.get(selected, {selected}) & set(supported) + ) + + +def _supports_distribution(selected, supported): + supported = set(supported) + if 'any' in supported: + return True + if selected == 'unknown': + return False + if selected == 'normal': + return bool( + {'normal', 'gaussian', 'multivariate normal', 'multivariate_normal'} + & supported + ) + if selected == 'non_normal': + return bool( + { + 'non_normal', + 'nonparametric', + 'heavy_tailed', + 'asymmetric_laplace', + 'non_normal (with robust estimators)', + 'non_normal (with alternative likelihoods)', + } + & supported + ) + return selected in supported + + +def _supports_relationship(selected, supported): + supported = set(supported) + if 'any' in supported: + return True + if selected == 'unknown': + return False + if selected == 'non_linear': + return bool({'non_linear', 'non-linear', 'nonlinear'} & supported) + return selected in supported + def _decode_path_name(name): """Decode one path segment left encoded after Flask's URL decoding.""" @@ -41,34 +168,52 @@ def _decode_path_name(name): 'Kruskal-Wallis Test', 'Analysis of Variance (ANOVA)', 'Analysis of Covariance (ANCOVA)', - 'Repeated Measures ANOVA' - # Add other relevant test models here if they exist + 'Repeated Measures ANOVA', ]), ('Regression Models', [ 'Linear Regression', - 'Multiple Linear Regression', 'Logistic Regression', - 'Multinomial Logistic Regression', + 'Multinomial Regression', 'Ordinal Regression', 'Poisson Regression', 'Ridge Regression', 'Lasso Regression', 'Elastic Net Regression', - 'Quantile Regression', - 'Stepwise Regression', - 'Generalized Linear Model (GLM)', - 'Generalized Additive Model (GAM)', 'Kernel Regression', - 'Polynomial Regression', - 'Bayesian Linear Regression', # Consider if Bayesian models get their own group - 'Bayesian Quantile Regression' # Or are subtypes here + 'Negative Binomial Regression', + 'Zero-Inflated Poisson', + 'Quantile Regression', + 'Partial Least Squares', + 'Generalized Additive Model', + 'Bayesian Quantile Regression', ]), ('Time Series Models', [ + 'Autoregressive (AR) Model', + 'Moving Average (MA) Model', + 'Autoregressive Moving Average (ARMA) Model', 'ARIMA', + 'Seasonal ARIMA (SARIMA)', + 'ARIMAX', 'Exponential Smoothing', + 'Holt-Winters Model', + 'Vector Autoregression (VAR)', + 'Vector Error Correction Model (VECM)', + 'State Space Model', + 'Bayesian Structural Time Series (BSTS)', + 'Dynamic Linear Model (DLM)', 'Prophet', - 'Vector Autoregression (VAR)' - # Add other time series models + 'GARCH', + 'Stochastic Volatility Model', + 'Threshold Autoregressive (TAR) Model', + 'Markov Switching Model', + 'Recurrent Neural Network for Time Series', + 'Long Short-Term Memory (LSTM)', + 'Gated Recurrent Unit (GRU)', + 'Temporal Convolutional Network (TCN)', + 'Gradient Boosting for Time Series', + 'Random Forest for Time Series', + 'Wavelet Transform Model', + 'Kalman Filter', ]), ('Multivariate Analysis', [ 'Principal Component Analysis (PCA)', @@ -77,11 +222,9 @@ def _decode_path_name(name): 'Discriminant Analysis', 'Canonical Correlation', 'Multidimensional Scaling', + 'Partial Least Squares', 'Multivariate Analysis of Covariance (MANCOVA)', 'Multivariate Analysis of Variance (MANOVA)', - 'Analysis of Covariance (ANCOVA)', - 'Analysis of Variance (ANOVA)' - # Add other multivariate models (K-Means, DBSCAN etc. could fit here or ML) ]), ('Machine Learning Models', [ 'Decision Trees', @@ -94,15 +237,9 @@ def _decode_path_name(name): 'K-Nearest Neighbors (KNN)', 'Naive Bayes classifier', 'Neural Networks', - 'K-Means', - 'Hierarchical Clustering', - 'DBSCAN' - # Add other ML models ]), ('Mixed and Hierarchical Models', [ 'Mixed Effects Model', - 'Hierarchical Linear Model', - 'Multilevel Model', 'Bayesian Hierarchical Regression' ]), ('Structural Models', [ @@ -119,14 +256,18 @@ def _decode_path_name(name): 'Bayesian Model Averaging', 'Bayesian Quantile Regression', 'Bayesian Additive Regression Trees (BART)' - ]), - #(optional) 'Deep Learning Models', [ - # 'Convolutional Neural Networks (CNN)', - # 'Recurrent Neural Networks (RNN)', - # 'Long Short-Term Memory (LSTM)', - # 'Gated Recurrent Units (GRU)', - # 'Transformer Models' - #]) + ]), + ('Risk Models', [ + 'Value at Risk (VaR)', + 'Conditional Value at Risk (CVaR)', + 'Monte Carlo Risk Simulation', + 'GARCH', + ]), + ('Forensic Models', [ + 'Likelihood Ratio Evidence Model', + 'Benford Law Analysis', + 'Forensic Anomaly Detection', + ]), ]) # Make model groups available to all templates @main.context_processor @@ -157,15 +298,47 @@ def get_model_recommendation(analysis_goal, dependent_variable, independent_vari 'classical': ['Linear Regression', 'Logistic Regression', 'Poisson Regression', 'Ridge Regression', 'Lasso Regression', 'Elastic Net Regression'], 'tree_based': ['Decision Trees', 'Random Forest', 'Gradient Boosting', 'XGBoost', 'LightGBM', 'CatBoost'], - 'bayesian': ['Bayesian Linear Regression', 'Bayesian Hierarchical Regression', 'Bayesian Model Averaging', - 'Bayesian Quantile Regression', 'Bayesian Additive Regression Trees'], - 'hierarchical': ['Mixed Effects Model', 'Hierarchical Linear Model', 'Multilevel Model'], + 'bayesian': ['Bayesian Hierarchical Regression', 'Bayesian Model Averaging', + 'Bayesian Quantile Regression', 'Bayesian Linear Regression', + 'Bayesian Additive Regression Trees (BART)'], + 'hierarchical': ['Mixed Effects Model', 'Bayesian Hierarchical Regression'], 'neural_network': ['Neural Networks'], - 'nonparametric': ['Support Vector Machines', 'K-Nearest Neighbors', 'Kernel_Regression'], - 'dimensionality_reduction': ['Principal Component Analysis', 'Factor Analysis', 'Multidimensional Scaling'], - 'clustering': ['Cluster Analysis', 'K-Means', 'Hierarchical Clustering', 'DBSCAN', 'Gaussian Mixture Models'], - 'time_series': ['ARIMA', 'Exponential Smoothing', 'Prophet'], - 'hypothesis_testing': ['T_test', 'Chi_Square_Test', 'Mann_Whitney_U_Test', 'Kruskal_Wallis_Test', + 'nonparametric': ['Support Vector Machines (SVM)', 'K-Nearest Neighbors (KNN)', 'Kernel Regression'], + 'dimensionality_reduction': ['Principal Component Analysis (PCA)', 'Factor Analysis', 'Multidimensional Scaling'], + 'clustering': ['K-Means clustering'], + 'time_series_classical': [ + 'Autoregressive (AR) Model', 'Moving Average (MA) Model', + 'Autoregressive Moving Average (ARMA) Model', 'ARIMA', + 'Seasonal ARIMA (SARIMA)', 'ARIMAX', 'Exponential Smoothing', + 'Holt-Winters Model', 'Prophet', + ], + 'time_series_multivariate': [ + 'Vector Autoregression (VAR)', + 'Vector Error Correction Model (VECM)', + ], + 'time_series_state_space': [ + 'State Space Model', 'Bayesian Structural Time Series (BSTS)', + 'Dynamic Linear Model (DLM)', 'Kalman Filter', + ], + 'time_series_volatility': ['GARCH', 'Stochastic Volatility Model'], + 'time_series_regime': [ + 'Threshold Autoregressive (TAR) Model', + 'Markov Switching Model', + ], + 'time_series_deep': [ + 'Recurrent Neural Network for Time Series', + 'Long Short-Term Memory (LSTM)', 'Gated Recurrent Unit (GRU)', + 'Temporal Convolutional Network (TCN)', + ], + 'time_series_ensemble': [ + 'Gradient Boosting for Time Series', 'Random Forest for Time Series', + ], + 'time_series_signal': ['Wavelet Transform Model'], + 'risk': ['Value at Risk (VaR)', 'Conditional Value at Risk (CVaR)', + 'Monte Carlo Risk Simulation'], + 'forensic': ['Likelihood Ratio Evidence Model', 'Benford Law Analysis', + 'Forensic Anomaly Detection'], + 'hypothesis_testing': ['T test', 'Chi-Square Test', 'Mann-Whitney U Test', 'Kruskal-Wallis Test', 'Analysis of Variance (ANOVA)', 'Analysis of Covariance (ANCOVA)'] } # Build a reverse lookup of model to family @@ -173,9 +346,14 @@ def get_model_recommendation(analysis_goal, dependent_variable, independent_vari for family, models in model_families.items(): for model in models: model_to_family[model] = family + time_series_models = { + model + for family, models in model_families.items() + if family.startswith('time_series_') + for model in models + } # Define clustering models (these don't require a dependent variable) - clustering_models = ['Cluster Analysis', 'K-Means', 'Hierarchical Clustering', 'DBSCAN', - 'Gaussian Mixture Models', 'Principal Component Analysis', 'Factor Analysis'] + clustering_models = ['K-Means clustering'] # For clustering analysis, ensure we have a default dependent variable if not provided if analysis_goal == 'cluster' and not dependent_variable: dependent_variable = 'continuous' # A sensible default for clustering @@ -184,8 +362,17 @@ def get_model_recommendation(analysis_goal, dependent_variable, independent_vari for model_name, model in MODEL_DATABASE.items(): score = 0 current_app.logger.debug(f"SCORING {model_name}: Starting score = {score}") + if ( + model_name in time_series_models + and analysis_goal != 'time_series' + and not ( + model_name in {'GARCH', 'Stochastic Volatility Model'} + and analysis_goal == 'risk_assessment' + ) + ): + continue # Check analysis goal compatibility - heavily weighted - if analysis_goal in model.get('analysis_goals', []): + if _supports_goal(analysis_goal, model.get('analysis_goals', [])): score += 3 current_app.logger.debug(f" + Analysis goal match: +3 → {score}") else: @@ -193,14 +380,17 @@ def get_model_recommendation(analysis_goal, dependent_variable, independent_vari current_app.logger.debug(f" × Skipping {model_name}: analysis goal mismatch") continue # Skip models that don't match the primary analysis goal # Special handling for clustering models when the goal is 'cluster' - is_clustering_model = model_name in clustering_models or 'cluster' in analysis_goal.lower() + is_clustering_model = model_name in clustering_models # Check dependent variable compatibility - heavily weighted # Skip this check for clustering models when the goal is 'cluster' if is_clustering_model and analysis_goal == 'cluster': # Clustering models get a bonus instead of being checked for dependent variable score += 3 current_app.logger.debug(f" + Clustering model bonus: +3 → {score}") - elif dependent_variable in model.get('dependent_variable', []): + elif _supports_outcome( + dependent_variable, + model.get('dependent_variable', []), + ): score += 3 current_app.logger.debug(f" + Dependent variable match: +3 → {score}") else: @@ -208,7 +398,10 @@ def get_model_recommendation(analysis_goal, dependent_variable, independent_vari current_app.logger.debug(f" × Skipping {model_name}: dependent variable mismatch") continue # Skip models that don't match the dependent variable type # Check relationship type compatibility - important factor - if relationship_type in model.get('relationship_type', []): + if _supports_relationship( + relationship_type, + model.get('relationship_type', []), + ): score += 2 current_app.logger.debug(f" + Relationship type match: +2 → {score}") elif relationship_type == 'linear' and 'non_linear' in model.get('relationship_type', []): @@ -240,14 +433,14 @@ def get_model_recommendation(analysis_goal, dependent_variable, independent_vari current_app.logger.debug(f" + Regularization compatibility: +0.75 → {score}") # No bonus when variables are explicitly not correlated # Boost other models that work well with correlated variables - if variables_correlated == 'yes' and model_name in ['Principal Component Analysis', + if variables_correlated == 'yes' and model_name in ['Principal Component Analysis (PCA)', 'Factor Analysis', 'Partial Least Squares', 'Random Forest', 'Gradient Boosting', 'XGBoost', 'CatBoost', 'LightGBM']: score += 1.5 current_app.logger.debug(f" + Correlated variable compatibility: +1.5 → {score}") # Models that assume independent variables - slight penalty for correlated data - if variables_correlated == 'yes' and model_name in ['Linear Regression', 'Stepwise Regression']: + if variables_correlated == 'yes' and model_name == 'Linear Regression': score -= 0.5 current_app.logger.debug(f" - Correlated variable penalty: -0.5 → {score}") # Boost for Linear Regression in standard prediction scenarios with normal data and linear relationships @@ -256,14 +449,21 @@ def get_model_recommendation(analysis_goal, dependent_variable, independent_vari missing_data in ['none', 'little'] and dependent_variable == 'continuous': score += 5.0 current_app.logger.debug(f" + Linear Regression boost: +5.0 → {score}") - # Strong boost for Cluster Analysis in exploratory or clustering scenarios - if model_name == 'Cluster Analysis' and (analysis_goal == 'explore' or analysis_goal == 'cluster'): - score += 15.0 # Massively increased to ensure Cluster Analysis wins for clustering - current_app.logger.debug(f"CLUSTER BONUS: {model_name} +15.0 for {analysis_goal} analysis") + # Strong boost for K-Means in clustering scenarios. + if model_name == 'K-Means clustering' and analysis_goal == 'cluster': + score += 5.0 + current_app.logger.debug( + f"CLUSTER BONUS: {model_name} +5.0 for {analysis_goal} analysis" + ) + if model_name == 'ARIMA' and analysis_goal == 'time_series': + score += 3.0 + current_app.logger.debug( + f"TIME SERIES BASELINE BONUS: {model_name} +3.0" + ) # Penalty for non-clustering models in exploratory or clustering scenarios - if (analysis_goal == 'explore' or analysis_goal == 'cluster') and model_name not in ['Cluster Analysis', 'Factor Analysis', 'Principal Component Analysis', - 'Multidimensional Scaling', 'UMAP', 'K-Means', 'Hierarchical Clustering', 'DBSCAN', 'Gaussian Mixture Models']: - score -= 5.0 # Significant penalty for non-exploratory models + if (analysis_goal == 'explore' or analysis_goal == 'cluster') and model_name not in ['K-Means clustering', 'Factor Analysis', 'Principal Component Analysis (PCA)', + 'Multidimensional Scaling']: + score -= 5.0 current_app.logger.debug(f"EXPLORE/CLUSTER PENALTY: {model_name} -5.0 for being non-{analysis_goal}") # Extra boost for Elastic Net which combines benefits of Lasso and Ridge if model_name == 'Elastic Net Regression' and missing_data in ['none', 'little']: @@ -276,11 +476,14 @@ def get_model_recommendation(analysis_goal, dependent_variable, independent_vari score += 1 current_app.logger.debug(f" + Sample size match: +1 → {score}") # Check missing data handling - if missing_data in model.get('missing_data', []): + if _supports_missing_data(missing_data, model.get('missing_data', [])): score += 1.5 current_app.logger.debug(f" + Missing data compatibility: +1.5 → {score}") # Check data distribution compatibility - if data_distribution in model.get('data_distribution', []): + if _supports_distribution( + data_distribution, + model.get('data_distribution', []), + ): score += 1.5 current_app.logger.debug(f" + Data distribution match: +1.5 → {score}") elif data_distribution == 'normal' and 'non_normal' in model.get('data_distribution', []): @@ -290,8 +493,8 @@ def get_model_recommendation(analysis_goal, dependent_variable, independent_vari # Add a bonus for advanced/specialized models # This helps counter the bias towards simpler models like linear regression if model_name in ['Random Forest', 'XGBoost', 'Gradient Boosting', 'Neural Networks', - 'Bayesian Linear Regression', 'Generalized Additive Model', - 'Hierarchical Linear Model', 'Quantile Regression']: + 'Bayesian Linear Regression', 'Generalized Additive Model', + 'Quantile Regression']: score += 0.5 current_app.logger.debug(f" + Advanced model bonus: +0.5 → {score}") # Add extra bonus for Bayesian models - they're often underrepresented @@ -305,8 +508,8 @@ def get_model_recommendation(analysis_goal, dependent_variable, independent_vari current_app.logger.debug(f" + Hierarchical model bonus: +2.0 → {score}") # Bonus for advanced models that handle complex relationships if relationship_type == 'non_linear' and model_name in ['Random Forest', 'XGBoost', 'Neural Networks', - 'Gradient Boosting', 'Support Vector Machines', - 'Bayesian Additive Regression Trees']: + 'Gradient Boosting', 'Support Vector Machines (SVM)', + 'Bayesian Additive Regression Trees (BART)']: score += 1.5 current_app.logger.debug(f" + Non-linear model bonus: +1.5 → {score}") # Add a penalty for overused models to promote diversity @@ -321,10 +524,6 @@ def get_model_recommendation(analysis_goal, dependent_variable, independent_vari if (analysis_goal == 'explore' or analysis_goal == 'cluster') and model_name == 'Neural Networks': score -= 5.0 # Significant penalty for neural networks in clustering tasks current_app.logger.debug(f" - Neural Networks penalty for clustering: -5.0 → {score:.4f}") - # Fix for exploratory analysis with continuous dependent variable - ensure clustering models win - if (analysis_goal == 'explore' or analysis_goal == 'cluster') and dependent_variable == 'continuous' and model_name == 'Cluster Analysis': - score += 5.0 # Extra boost to ensure Cluster Analysis wins for exploratory analysis - current_app.logger.debug(f" + Exploratory/Cluster continuous fix: +5.0 → {score:.4f}") model_scores[model_name] = score # Get top models # First, identify the best matching model @@ -380,33 +579,44 @@ def get_default_alternatives(analysis_goal, dependent_variable): alternatives = [] if analysis_goal == 'predict': if dependent_variable == 'continuous': - alternatives = ['Ridge Regression', 'Random Forest', 'XGBoost', 'Bayesian Linear Regression', 'Gradient Boosting'] + alternatives = ['Ridge Regression', 'Random Forest', 'XGBoost', 'Gradient Boosting'] elif dependent_variable == 'binary': - alternatives = ['Random Forest', 'Support Vector Machine', 'XGBoost', 'Neural Network'] + alternatives = ['Random Forest', 'Support Vector Machines (SVM)', 'XGBoost', 'Neural Networks'] elif dependent_variable == 'count': alternatives = ['Negative Binomial Regression', 'Zero-Inflated Poisson', 'Quantile Regression'] elif dependent_variable == 'ordinal': - alternatives = ['Multinomial Logistic Regression', 'Neural Network', 'Ordinal Regression'] + alternatives = ['Multinomial Regression', 'Neural Networks', 'Ordinal Regression'] elif dependent_variable == 'time_to_event': - alternatives = ['Kaplan-Meier', 'Weibull Model', 'Cox Proportional Hazards'] + alternatives = ['Kaplan-Meier Curve', 'Cox Proportional Hazards Model'] elif analysis_goal == 'classify': if dependent_variable == 'binary': - alternatives = ['Random Forest', 'Support Vector Machine', 'XGBoost', 'Neural Network', 'Gradient Boosting'] + alternatives = ['Random Forest', 'Support Vector Machines (SVM)', 'XGBoost', 'Neural Networks', 'Gradient Boosting'] elif dependent_variable == 'categorical': - alternatives = ['Random Forest', 'Neural Network', 'Support Vector Machine', 'XGBoost'] + alternatives = ['Random Forest', 'Neural Networks', 'Support Vector Machines (SVM)', 'XGBoost'] elif analysis_goal == 'explore': - alternatives = ['Cluster Analysis', 'Factor Analysis', 'Multidimensional Scaling', 'Principal Component Analysis', 'UMAP'] + alternatives = ['Factor Analysis', 'Multidimensional Scaling', 'Principal Component Analysis (PCA)'] elif analysis_goal == 'cluster': - alternatives = ['Cluster Analysis', 'K-Means', 'Hierarchical Clustering', 'DBSCAN', 'Gaussian Mixture Models'] + alternatives = ['K-Means clustering'] elif analysis_goal == 'hypothesis_test': if dependent_variable == 'continuous': - alternatives = ['Analysis of Variance (ANOVA)', 'Mann_Whitney_U_Test', 'Wilcoxon Signed-Rank Test', 'T_test'] + alternatives = ['Analysis of Variance (ANOVA)', 'Mann-Whitney U Test', 'T test'] elif dependent_variable == 'categorical': - alternatives = ['Fisher\'s Exact Test', 'G-Test', 'McNemar\'s Test', 'Chi_Square_Test'] + alternatives = ['Chi-Square Test'] elif analysis_goal == 'non_parametric': - alternatives = ['Wilcoxon Signed-Rank Test', 'Kruskal_Wallis_Test', 'Spearman Correlation', 'Mann_Whitney_U_Test'] + alternatives = ['Kruskal-Wallis Test', 'Mann-Whitney U Test'] elif analysis_goal == 'time_series': - alternatives = ['Exponential Smoothing', 'Prophet', 'ARIMA', 'ARIMAX', 'GARCH'] + alternatives = [ + 'ARIMA', + 'Seasonal ARIMA (SARIMA)', + 'Exponential Smoothing', + 'State Space Model', + ] + elif analysis_goal == 'risk_assessment': + alternatives = ['Value at Risk (VaR)', 'Conditional Value at Risk (CVaR)', + 'Monte Carlo Risk Simulation', 'GARCH'] + elif analysis_goal == 'forensic_analysis': + alternatives = ['Likelihood Ratio Evidence Model', 'Benford Law Analysis', + 'Forensic Anomaly Detection'] # Remove alternatives that might not exist in the database MODEL_DATABASE = current_app.config.get('MODEL_DATABASE', {}) return [alt for alt in alternatives if alt in MODEL_DATABASE][:4] # Increased from 3 to 4 alternatives @@ -417,7 +627,7 @@ def generate_explanation(model_name, analysis_goal, dependent_variable, independ model_info = get_model_details(model_name) or {} explanation = f"\n Based on your data characteristics, a {model_name} is recommended because:\n \n" reasons = [] - if analysis_goal in model_info.get('analysis_goals', []): + if _supports_goal(analysis_goal, model_info.get('analysis_goals', [])): reasons.append(f"It is suitable for {analysis_goal} analysis with {dependent_variable} dependent variables") if independent_variables and all(var in model_info.get('independent_variables', []) for var in independent_variables): reasons.append(f"It can handle {', '.join(independent_variables)} independent variables") @@ -432,16 +642,21 @@ def generate_explanation(model_name, analysis_goal, dependent_variable, independ reasons.append("It works well with medium sample sizes") elif sample_size_int >= 100 and 'large' in model_info.get('sample_size', []): reasons.append("It is optimized for large datasets") - if missing_data in model_info.get('missing_data', []): + if _supports_missing_data(missing_data, model_info.get('missing_data', [])): reasons.append(f"It can handle {missing_data} missing data patterns") - if data_distribution in model_info.get('data_distribution', []): + if _supports_distribution( + data_distribution, + model_info.get('data_distribution', []), + ): reasons.append(f"It is appropriate for {data_distribution} data distribution") - if relationship_type in model_info.get('relationship_type', []): + if _supports_relationship( + relationship_type, + model_info.get('relationship_type', []), + ): reasons.append(f"It can model {relationship_type} relationships") # Add reason related to correlated variables if specified if variables_correlated == 'yes' and model_name in ['Elastic Net Regression', 'Ridge Regression', 'Lasso Regression', - 'Principal Component Analysis', 'Factor Analysis', - 'Partial Least Squares']: + 'Principal Component Analysis (PCA)', 'Factor Analysis']: reasons.append("It excels at handling correlated predictors") # Add numbered reasons for i, reason in enumerate(reasons, 1): @@ -467,31 +682,37 @@ def get_default_model(analysis_goal, dependent_variable): if dependent_variable == 'continuous': target_models = ['Linear Regression', 'Ridge Regression', 'Lasso Regression'] elif dependent_variable == 'binary': - target_models = ['Logistic Regression', 'Support Vector Machines'] + target_models = ['Logistic Regression', 'Support Vector Machines (SVM)'] elif dependent_variable == 'count': target_models = ['Poisson Regression', 'Negative Binomial Regression'] elif dependent_variable == 'ordinal': target_models = ['Ordinal Regression', 'Multinomial Regression'] elif dependent_variable == 'time_to_event': - target_models = ['Cox Proportional Hazards', 'Kaplan Meier'] + target_models = ['Cox Proportional Hazards Model', 'Kaplan-Meier Curve'] elif analysis_goal == 'classify': if dependent_variable == 'binary': - target_models = ['Logistic Regression', 'Support Vector Machines'] + target_models = ['Logistic Regression', 'Support Vector Machines (SVM)'] elif dependent_variable == 'categorical': target_models = ['Multinomial Regression', 'Random Forest'] elif analysis_goal == 'explore': - target_models = ['Principal Component Analysis', 'Factor Analysis', 'Cluster Analysis'] + target_models = ['Principal Component Analysis (PCA)', 'Factor Analysis'] elif analysis_goal == 'cluster': - target_models = ['Cluster Analysis', 'Principal Component Analysis'] + target_models = ['K-Means clustering'] elif analysis_goal == 'hypothesis_test': if dependent_variable == 'continuous': - target_models = ['T_test', 'Analysis of Variance (ANOVA)'] + target_models = ['T test', 'Analysis of Variance (ANOVA)'] elif dependent_variable == 'categorical': - target_models = ['Chi_Square_Test', 'Fisher\'s Exact Test'] + target_models = ['Chi-Square Test'] elif analysis_goal == 'non_parametric': - target_models = ['Mann_Whitney_U_Test', 'Kruskal_Wallis_Test'] + target_models = ['Mann-Whitney U Test', 'Kruskal-Wallis Test'] elif analysis_goal == 'time_series': target_models = ['ARIMA', 'Exponential Smoothing'] + elif analysis_goal == 'risk_assessment': + target_models = ['Value at Risk (VaR)', 'Conditional Value at Risk (CVaR)', + 'Monte Carlo Risk Simulation'] + elif analysis_goal == 'forensic_analysis': + target_models = ['Likelihood Ratio Evidence Model', 'Forensic Anomaly Detection', + 'Benford Law Analysis'] else: target_models = ['Linear Regression', 'Logistic Regression'] # Find the first matching model that exists in the database @@ -519,7 +740,10 @@ def profile(): """View and edit user profile""" if request.method == 'POST': # Update basic profile information - email = request.form.get('email') + email = normalize_email(request.form.get('email')) + if not is_valid_email(email): + flash('Please provide a valid email address.', 'danger') + return redirect(url_for('main.profile')) # Check if email already exists for another user if email != current_user.email: existing_user = User.query.filter_by(email=email).first() @@ -546,6 +770,8 @@ def results(): dependent_variable_type = request.form.get('dependent_variable_type', '') # Get independent variables (multiply selected) independent_variables = request.form.getlist('independent_variables') + if independent_variables == ['mixed']: + independent_variables = ['continuous', 'categorical'] # Get other attributes sample_size = request.form.get('sample_size', '') missing_data = request.form.get('missing_data', '') @@ -555,6 +781,17 @@ def results(): use_ai_review = request.form.get('use_ai_review') == 'on' # Get model database from app config MODEL_DATABASE = current_app.config.get('MODEL_DATABASE', {}) + allowed_goals = set(GOAL_COMPATIBILITY) + allowed_outcomes = set(OUTCOME_COMPATIBILITY) + allowed_missing_data = set(MISSING_DATA_COMPATIBILITY) + allowed_distributions = {'unknown', 'normal', 'non_normal'} + allowed_relationships = { + 'unknown', + 'linear', + 'non_linear', + 'hierarchical', + } + allowed_predictors = {'continuous', 'categorical', 'binary'} # For clustering analysis, dependent variable can be empty # If it's empty, set it to 'continuous' which works well with clustering models if analysis_goal == 'cluster' and not dependent_variable_type: @@ -563,10 +800,32 @@ def results(): if not (research_question and analysis_goal): flash('Please provide all required information to get a recommendation.', 'warning') return redirect(url_for('main.analysis_form')) + if len(research_question) > 500 or analysis_goal not in allowed_goals: + flash('Please provide a valid research question and analysis goal.', 'warning') + return redirect(url_for('main.analysis_form')) # For non-clustering analysis, require dependent variable if analysis_goal != 'cluster' and not dependent_variable_type: flash('Please select what type of outcome you are measuring.', 'warning') return redirect(url_for('main.analysis_form')) + if ( + dependent_variable_type not in allowed_outcomes + or missing_data not in allowed_missing_data + or data_distribution not in allowed_distributions + or relationship_type not in allowed_relationships + or not set(independent_variables).issubset(allowed_predictors) + ): + flash( + 'Some study-design fields were missing or invalid. Please review the form.', + 'warning', + ) + return redirect(url_for('main.analysis_form')) + if sample_size: + try: + if int(sample_size) < 1: + raise ValueError + except (TypeError, ValueError): + flash('Sample size must be a positive whole number.', 'warning') + return redirect(url_for('main.analysis_form')) # Get model recommendation recommended_model, explanation, alternative_models = get_model_recommendation( analysis_goal, dependent_variable_type, independent_variables, @@ -585,8 +844,17 @@ def results(): similar_models = { model_name: model for model_name, model in MODEL_DATABASE.items() if (model_name != recommended_model and - analysis_goal in model.get('analysis_goals', []) and - (not dependent_variable_type or dependent_variable_type in model.get('dependent_variable', []))) + _supports_goal( + analysis_goal, + model.get('analysis_goals', []), + ) and + ( + not dependent_variable_type + or _supports_outcome( + dependent_variable_type, + model.get('dependent_variable', []), + ) + )) } # If we have alternative models from the recommendation engine, use those # Otherwise, fall back to similar models based on metadata diff --git a/routes/questionnaire_routes.py b/routes/questionnaire_routes.py index b937a0f..563a286 100644 --- a/routes/questionnaire_routes.py +++ b/routes/questionnaire_routes.py @@ -5,7 +5,8 @@ """ import hashlib import logging -from datetime import datetime, timezone +import secrets +from datetime import datetime, timedelta, timezone from flask import ( Blueprint, @@ -20,8 +21,9 @@ ) from flask_login import login_required, current_user from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm.attributes import flag_modified -from models import db, Questionnaire +from models import db, Questionnaire, QuestionnaireDraft from utils.ai_service import is_ai_enabled from utils.ai_usage import consume_user_ai_quota from utils.export_utils import export_to_word @@ -36,6 +38,67 @@ logger = logging.getLogger(__name__) questionnaire_bp = Blueprint('questionnaire', __name__, url_prefix='/questionnaire') + +DRAFT_SESSION_KEY = 'questionnaire_draft_id' +DRAFT_MAX_AGE = timedelta(days=7) + + +def _load_draft(): + """Return the current server-side draft when it belongs to this session.""" + draft_id = session.get(DRAFT_SESSION_KEY) + if not draft_id: + return None + draft = db.session.get(QuestionnaireDraft, draft_id) + if draft is None: + session.pop(DRAFT_SESSION_KEY, None) + return None + if draft.updated_at < datetime.utcnow() - DRAFT_MAX_AGE: + db.session.delete(draft) + db.session.commit() + session.pop(DRAFT_SESSION_KEY, None) + return None + if ( + draft.user_id is not None + and ( + not current_user.is_authenticated + or draft.user_id != current_user.id + ) + ): + session.pop(DRAFT_SESSION_KEY, None) + return None + return draft + + +def _save_draft(content): + """Persist a questionnaire working copy and keep only its ID in session.""" + draft = _load_draft() + if draft is None: + draft = QuestionnaireDraft( + id=secrets.token_urlsafe(32), + user_id=( + current_user.id if current_user.is_authenticated else None + ), + content=content, + ) + db.session.add(draft) + else: + draft.content = content + flag_modified(draft, 'content') + if draft.user_id is None and current_user.is_authenticated: + draft.user_id = current_user.id + db.session.commit() + session[DRAFT_SESSION_KEY] = draft.id + return draft + + +def _draft_content_or_redirect(): + draft = _load_draft() + if draft is None: + flash('Please design a questionnaire first.', 'warning') + return None + return draft.content + + @questionnaire_bp.route('/') def index(): """Landing page for the questionnaire design service.""" @@ -124,29 +187,29 @@ def design(): 'questionnaire was generated instead.', 'warning', ) - # Store questionnaire data in session - session['questionnaire'] = questionnaire - session['research_topic'] = research_topic - session['research_description'] = research_description - session['target_audience'] = target_audience - session['questionnaire_purpose'] = questionnaire_purpose - session['used_ai_enhancement'] = ai_applied + _save_draft({ + 'questionnaire': questionnaire, + 'research_topic': research_topic, + 'research_description': research_description, + 'target_audience': target_audience, + 'questionnaire_purpose': questionnaire_purpose, + 'used_ai_enhancement': ai_applied, + }) return redirect(url_for('questionnaire.preview')) return render_template('questionnaire/design.html') @questionnaire_bp.route('/preview') def preview(): """Preview the generated questionnaire.""" - # Check if questionnaire data exists in session - if 'questionnaire' not in session: - flash('Please design a questionnaire first.', 'error') + draft = _draft_content_or_redirect() + if draft is None: return redirect(url_for('questionnaire.design')) return render_template( 'questionnaire/preview.html', - questionnaire=session['questionnaire'], - research_topic=session.get('research_topic', ''), - research_description=session.get('research_description', ''), - target_audience=session.get('target_audience', ''), - questionnaire_purpose=session.get('questionnaire_purpose', '') + questionnaire=draft['questionnaire'], + research_topic=draft.get('research_topic', ''), + research_description=draft.get('research_description', ''), + target_audience=draft.get('target_audience', ''), + questionnaire_purpose=draft.get('questionnaire_purpose', '') ) @questionnaire_bp.route('/edit', methods=['GET', 'POST']) def edit(): @@ -154,9 +217,8 @@ def edit(): GET: Show form to edit questionnaire POST: Process edits and update the questionnaire """ - # Check if questionnaire data exists in session - if 'questionnaire' not in session: - flash('Please design a questionnaire first.', 'error') + draft = _draft_content_or_redirect() + if draft is None: return redirect(url_for('questionnaire.design')) if request.method == 'POST': # Process the form data @@ -213,7 +275,7 @@ def edit(): ai_enhanced = False ai_created = False # Check if this question was in the original questionnaire - original_questionnaire = session.get('questionnaire', []) + original_questionnaire = draft.get('questionnaire', []) if int(section_index) < len(original_questionnaire): original_section = original_questionnaire[int(section_index)] original_questions = original_section.get('questions', []) @@ -239,37 +301,37 @@ def edit(): 'description': section_description, 'questions': questions }) - # Update session with edited data - session['questionnaire'] = sections_data - session['research_topic'] = research_topic - session['target_audience'] = target_audience - session['questionnaire_purpose'] = questionnaire_purpose - session['research_description'] = research_description + draft.update({ + 'questionnaire': sections_data, + 'research_topic': research_topic, + 'target_audience': target_audience, + 'questionnaire_purpose': questionnaire_purpose, + 'research_description': research_description, + }) + _save_draft(draft) flash('Questionnaire updated successfully.', 'success') return redirect(url_for('questionnaire.preview')) return render_template( 'questionnaire/edit.html', - questionnaire=session['questionnaire'], - research_topic=session.get('research_topic', ''), - research_description=session.get('research_description', ''), - target_audience=session.get('target_audience', ''), - questionnaire_purpose=session.get('questionnaire_purpose', '') + questionnaire=draft['questionnaire'], + research_topic=draft.get('research_topic', ''), + research_description=draft.get('research_description', ''), + target_audience=draft.get('target_audience', ''), + questionnaire_purpose=draft.get('questionnaire_purpose', '') ) @questionnaire_bp.route('/save', methods=['POST']) @login_required def save_questionnaire(): """Save the current questionnaire to the database.""" - # Check if questionnaire data exists in session - if 'questionnaire' not in session: - flash('Please design a questionnaire first.', 'error') + draft = _draft_content_or_redirect() + if draft is None: return redirect(url_for('questionnaire.design')) - # Get questionnaire data from session - questionnaire_data = session['questionnaire'] - research_topic = session.get('research_topic', 'Untitled Questionnaire') - research_description = session.get('research_description', '') - target_audience = session.get('target_audience', '') - questionnaire_purpose = session.get('questionnaire_purpose', '') - is_ai_enhanced = session.get('used_ai_enhancement', False) + questionnaire_data = draft['questionnaire'] + research_topic = draft.get('research_topic', 'Untitled Questionnaire') + research_description = draft.get('research_description', '') + target_audience = draft.get('target_audience', '') + questionnaire_purpose = draft.get('questionnaire_purpose', '') + is_ai_enhanced = draft.get('used_ai_enhancement', False) try: # Check if we're updating an existing questionnaire questionnaire_id = request.form.get('questionnaire_id') @@ -328,13 +390,14 @@ def load_questionnaire(questionnaire_id): if not questionnaire: flash('Questionnaire not found or you do not have permission to view it.', 'error') return redirect(url_for('questionnaire.my_questionnaires')) - # Store questionnaire data in session - session['questionnaire'] = questionnaire.content - session['research_topic'] = questionnaire.title - session['research_description'] = questionnaire.description - session['target_audience'] = questionnaire.target_audience - session['questionnaire_purpose'] = questionnaire.purpose - session['used_ai_enhancement'] = questionnaire.is_ai_enhanced + _save_draft({ + 'questionnaire': questionnaire.content, + 'research_topic': questionnaire.title, + 'research_description': questionnaire.description, + 'target_audience': questionnaire.target_audience, + 'questionnaire_purpose': questionnaire.purpose, + 'used_ai_enhancement': questionnaire.is_ai_enhanced, + }) session['saved_questionnaire_id'] = questionnaire.id return redirect(url_for('questionnaire.preview')) @questionnaire_bp.route('/delete/', methods=['POST']) @@ -357,16 +420,14 @@ def delete_questionnaire(questionnaire_id): @questionnaire_bp.route('/export/word') def export_word(): """Export questionnaire to Word document.""" - # Check if questionnaire data exists in session - if 'questionnaire' not in session: - flash('Please design a questionnaire first.', 'error') + draft = _draft_content_or_redirect() + if draft is None: return redirect(url_for('questionnaire.design')) - # Get questionnaire data from session - questionnaire = session['questionnaire'] - research_topic = session.get('research_topic', 'Questionnaire') - research_description = session.get('research_description', '') - target_audience = session.get('target_audience', '') - questionnaire_purpose = session.get('questionnaire_purpose', '') + questionnaire = draft['questionnaire'] + research_topic = draft.get('research_topic', 'Questionnaire') + research_description = draft.get('research_description', '') + target_audience = draft.get('target_audience', '') + questionnaire_purpose = draft.get('questionnaire_purpose', '') # Generate a filename filename = f"{research_topic.replace(' ', '_')}_questionnaire.docx" # Create the Word document @@ -392,16 +453,14 @@ def export_pdf(): flash('PDF export is not available. Please install reportlab package.', 'error') return redirect(url_for('questionnaire.preview')) - # Check if questionnaire data exists in session - if 'questionnaire' not in session: - flash('Please design a questionnaire first.', 'error') + draft = _draft_content_or_redirect() + if draft is None: return redirect(url_for('questionnaire.design')) - # Get questionnaire data from session - questionnaire = session['questionnaire'] - research_topic = session.get('research_topic', 'Questionnaire') - research_description = session.get('research_description', '') - target_audience = session.get('target_audience', '') - questionnaire_purpose = session.get('questionnaire_purpose', '') + questionnaire = draft['questionnaire'] + research_topic = draft.get('research_topic', 'Questionnaire') + research_description = draft.get('research_description', '') + target_audience = draft.get('target_audience', '') + questionnaire_purpose = draft.get('questionnaire_purpose', '') # Generate a filename filename = f"{research_topic.replace(' ', '_')}_questionnaire.pdf" # Create the PDF document diff --git a/routes/user_routes.py b/routes/user_routes.py index 51ac9f3..73177e7 100644 --- a/routes/user_routes.py +++ b/routes/user_routes.py @@ -1,59 +1,36 @@ -from flask import Blueprint, render_template, request, redirect, url_for, flash +from flask import Blueprint, request, redirect, url_for, flash from flask_login import login_required, current_user -from models import db, User, Analysis -from werkzeug.security import generate_password_hash +from models import db + user = Blueprint('user', __name__) + + @user.route('/profile') @login_required def profile(): - """Display user profile with analyses history""" - analyses = Analysis.query.filter_by(user_id=current_user.id).order_by(Analysis.timestamp.desc()).all() - return render_template('profile.html', user=current_user, analyses=analyses) + """Preserve the former URL while using the canonical profile route.""" + return redirect(url_for('main.profile')) + + @user.route('/edit-profile', methods=['GET', 'POST']) @login_required def edit_profile(): - """Edit user profile""" + """Preserve the former URL without maintaining a second profile editor.""" if request.method == 'POST': - # Update basic profile information - email = request.form.get('email') - # Check if email already exists for another user - if email != current_user.email: - existing_user = User.query.filter_by(email=email).first() - if existing_user: - flash('Email already in use.', 'danger') - return redirect(url_for('user.edit_profile')) - # Update password if provided - password = request.form.get('password') - confirm_password = request.form.get('confirm_password') - if password: - if password != confirm_password: - flash('Passwords do not match.', 'danger') - return redirect(url_for('main.profile')) - current_user.password_hash = generate_password_hash(password, method='pbkdf2:sha256') - # Update other fields - current_user.email = email - # If user is an expert, also update expertise fields - if current_user.role == 'expert': - current_user.institution = request.form.get('institution', '') - current_user.expertise = request.form.get('expertise', '') - current_user.bio = request.form.get('bio', '') - db.session.commit() - flash('Profile updated successfully.', 'success') - return redirect(url_for('main.profile')) - return render_template('edit_profile.html', user=current_user) + flash('Please update your details from the profile page.', 'info') + return redirect(url_for('main.profile')) + + @user.route('/delete-account', methods=['POST']) @login_required def delete_account(): """Delete user account""" # Confirm with password password = request.form.get('password') - if not current_user.check_password(password): + if not password or not current_user.check_password(password): flash('Incorrect password. Account deletion cancelled.', 'danger') return redirect(url_for('main.profile')) - # Delete user's analyses - Analysis.query.filter_by(user_id=current_user.id).delete() - # Delete user db.session.delete(current_user) db.session.commit() flash('Your account has been permanently deleted.', 'info') - return redirect(url_for('main.index')) \ No newline at end of file + return redirect(url_for('main.index')) diff --git a/templates/admin/ai_settings.html b/templates/admin/ai_settings.html index ef2a887..0add673 100644 --- a/templates/admin/ai_settings.html +++ b/templates/admin/ai_settings.html @@ -122,7 +122,12 @@

Connection test

try { const response = await fetch('{{ url_for("admin.initialize_ai_storage") }}', { method: 'POST', - headers: {'Content-Type': 'application/json'}, + headers: { + 'Content-Type': 'application/json', + 'X-CSRFToken': document.querySelector( + 'meta[name="csrf-token"]' + ).content + }, body: JSON.stringify({confirm: true}) }); const data = await response.json(); @@ -150,7 +155,12 @@

Connection test

try { const response = await fetch('{{ url_for("admin.test_ai_integration") }}', { method: 'POST', - headers: {'Content-Type': 'application/json'}, + headers: { + 'Content-Type': 'application/json', + 'X-CSRFToken': document.querySelector( + 'meta[name="csrf-token"]' + ).content + }, body: JSON.stringify({prompt: value}) }); const data = await response.json(); diff --git a/templates/admin/application_details.html b/templates/admin/application_details.html index 37474ee..e0691c2 100644 --- a/templates/admin/application_details.html +++ b/templates/admin/application_details.html @@ -275,26 +275,31 @@

Admin Actions

{% endif %} {% elif application.status == 'pending_review' %}
+
+
{% else %}
+
+
{% endif %} {% if not application.resume_url and application.status != 'approved' and application.status != 'rejected' %}
+
{% endif %} @@ -315,6 +320,7 @@

Communication History

Request Additional Information
+
@@ -362,6 +368,7 @@
Request Additional Information
Submit Additional Information
+
diff --git a/templates/admin/consultations_list.html b/templates/admin/consultations_list.html index b269f5a..b8d5e07 100644 --- a/templates/admin/consultations_list.html +++ b/templates/admin/consultations_list.html @@ -63,13 +63,14 @@
+
-{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/templates/admin/edit_user.html b/templates/admin/edit_user.html index e130505..a4ef9d4 100644 --- a/templates/admin/edit_user.html +++ b/templates/admin/edit_user.html @@ -16,6 +16,7 @@

Edit User

+
@@ -29,13 +30,13 @@

Edit User

-
+
@@ -44,8 +45,8 @@

Edit User

- - + +
@@ -76,7 +77,7 @@
User Information

Created: {{ user.created_at.strftime('%Y-%m-%d %H:%M') }}

Current Role: {{ user.role }}

- {% if user.role == 'expert' %} + {% if user._is_expert %}

Expert Status: {% if user.is_approved_expert %} Approved @@ -103,4 +104,4 @@

User Information
}); {% endblock %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/templates/admin/email_config.html b/templates/admin/email_config.html index 2c61606..b54cea0 100644 --- a/templates/admin/email_config.html +++ b/templates/admin/email_config.html @@ -52,6 +52,7 @@

Transactional Email

+ Back to Dashboard diff --git a/templates/admin/expert_applications.html b/templates/admin/expert_applications.html index 3bde433..1f389d1 100644 --- a/templates/admin/expert_applications.html +++ b/templates/admin/expert_applications.html @@ -346,6 +346,7 @@
Professional Bio
+
@@ -368,11 +369,13 @@
Professional Bio
{% else %} {% if not application.resume_url %} + {% endif %}
+
{% endif %} @@ -383,22 +386,27 @@
Professional Bio
{% elif application.status == 'pending_review' %}
+
+
+
{% else %}
+
+
{% endif %} diff --git a/templates/admin/manage_experts.html b/templates/admin/manage_experts.html index dd5051b..4381b42 100644 --- a/templates/admin/manage_experts.html +++ b/templates/admin/manage_experts.html @@ -252,6 +252,7 @@

Manage Users

{{ user.username }} {{ user.email }} - + {{ user.role }} - {% if user.role == 'expert' %} + {% if user._is_expert %} {% if user.is_approved_expert %} Approved {% else %} @@ -73,6 +73,7 @@ @@ -87,4 +88,4 @@
-{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/templates/admin_expert_applications.html b/templates/admin_expert_applications.html index d1005b9..bf56dab 100644 --- a/templates/admin_expert_applications.html +++ b/templates/admin_expert_applications.html @@ -78,6 +78,7 @@
Professional Bio
diff --git a/templates/analysis_form.html b/templates/analysis_form.html index 1e7fc7c..04ae1f6 100644 --- a/templates/analysis_form.html +++ b/templates/analysis_form.html @@ -177,7 +177,8 @@

Describe your research design

data-generation-progress data-progress-target="model-selection-progress" data-progress-mode="model" - data-progress-ai-field="use_ai_review"> + data-progress-ai-field="use_ai_review"> +
Describe your research design + +
Think about what you want to achieve with your analysis. @@ -221,7 +224,8 @@

Describe your research design

- + +
This is what you're trying to understand or predict in your study. diff --git a/templates/apply_expert.html b/templates/apply_expert.html index 70882ca..cf8a9e7 100644 --- a/templates/apply_expert.html +++ b/templates/apply_expert.html @@ -159,6 +159,7 @@

Expert Application Form

+
+ {% block title %}Statistical Model Suggester{% endblock %} @@ -96,7 +97,14 @@
  • AI integration
  • {% endif %}
  • -
  • Log out
  • +
  • + + + +
  • + {% else %} diff --git a/templates/expert/application_details.html b/templates/expert/application_details.html index aabdcc1..3f9704b 100644 --- a/templates/expert/application_details.html +++ b/templates/expert/application_details.html @@ -289,6 +289,7 @@
    Bio:
    {% if application.status == 'needs_info' %}
    +
    @@ -348,6 +349,7 @@

    Communication History

    Submit Additional Information
    +
    diff --git a/templates/expert/my_profile.html b/templates/expert/my_profile.html index 51462dd..6305755 100644 --- a/templates/expert/my_profile.html +++ b/templates/expert/my_profile.html @@ -12,6 +12,7 @@

    My Expert Profile

    +
    diff --git a/templates/expert_application_status.html b/templates/expert_application_status.html index f4675c7..1d29a0c 100644 --- a/templates/expert_application_status.html +++ b/templates/expert_application_status.html @@ -77,6 +77,7 @@
    Resume/CV

    +
    @@ -107,6 +108,7 @@
    Additional Information Requested
    +
    diff --git a/templates/expert_profile.html b/templates/expert_profile.html index 841eed9..d8eb4d0 100644 --- a/templates/expert_profile.html +++ b/templates/expert_profile.html @@ -25,7 +25,7 @@

    {{ expert.username }}

    Areas of Expertise
    -

    {{ expert.expertise }}

    +

    {{ expert.areas_of_expertise }}

    {% if expert.bio %} @@ -74,7 +74,7 @@

    Question:
    -

    {{ consultation.question }}

    +

    {{ consultation.description }}

    Expert Response:
    @@ -94,4 +94,4 @@
    Expert Response:

    -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/templates/experts_list.html b/templates/experts_list.html index b551f24..6e89196 100644 --- a/templates/experts_list.html +++ b/templates/experts_list.html @@ -200,7 +200,7 @@

    Statistical Analysis Experts

    - {% if current_user.is_authenticated and current_user.role != 'expert' %} + {% if current_user.is_authenticated and not current_user.is_expert %} Apply to Become an Expert @@ -257,7 +257,7 @@
    {{ expert.username }}
    Expertise:
    -

    {{ expert.expertise }}

    +

    {{ expert.areas_of_expertise }}

    View Profile @@ -305,4 +305,4 @@
    Expertise:
    }; }); -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/templates/forgot_password.html b/templates/forgot_password.html index 9544c6b..5b8fdcf 100644 --- a/templates/forgot_password.html +++ b/templates/forgot_password.html @@ -13,6 +13,7 @@

    Forgot Password

    Enter your email address below and we'll send you a link to reset your password.

    +
    diff --git a/templates/index.html b/templates/index.html index e6c1dcd..4c5f95f 100644 --- a/templates/index.html +++ b/templates/index.html @@ -13,6 +13,7 @@

    Model Selection Form

    +
    Model Selection Form - + + +
    What is the main goal of your analysis?
    diff --git a/templates/login.html b/templates/login.html index 466ee66..978f763 100644 --- a/templates/login.html +++ b/templates/login.html @@ -12,6 +12,7 @@

    Login

    +
    diff --git a/templates/profile.html b/templates/profile.html index 9189a24..7678408 100644 --- a/templates/profile.html +++ b/templates/profile.html @@ -12,6 +12,7 @@

    Profile

    +
    {{ user.username[0].upper() }} diff --git a/templates/questionnaire/design.html b/templates/questionnaire/design.html index caebb65..bd21339 100644 --- a/templates/questionnaire/design.html +++ b/templates/questionnaire/design.html @@ -82,7 +82,8 @@

    Design Your Questionnaire

    data-generation-progress data-progress-target="questionnaire-progress" data-progress-mode="questionnaire" - data-progress-ai-field="use_ai_enhancement"> + data-progress-ai-field="use_ai_enhancement"> +

    Basic Information

    diff --git a/templates/questionnaire/edit.html b/templates/questionnaire/edit.html index 2daae65..86ab774 100644 --- a/templates/questionnaire/edit.html +++ b/templates/questionnaire/edit.html @@ -94,6 +94,7 @@

    Edit Your Questionnaire

    +