From cdcfc0c713ddaea35af7509b323e344198511842 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Tue, 9 Aug 2016 23:10:00 +0100 Subject: [PATCH 01/78] Send verification emails from the settings page. --- newparp/__init__.py | 21 +++++++++++++----- .../templates/settings/log_in_details.html | 2 +- newparp/views/settings.py | 22 +++++++++++++++++-- requirements.txt | 1 + 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/newparp/__init__.py b/newparp/__init__.py index 376bacfb..7edbbc61 100644 --- a/newparp/__init__.py +++ b/newparp/__init__.py @@ -2,6 +2,7 @@ import logging from flask import Flask, abort, redirect, request, send_from_directory +from flask_mail import Mail from werkzeug.routing import BaseConverter from newparp.helpers import check_csrf_token @@ -12,17 +13,12 @@ redis_disconnect, set_cookie, ) -from newparp import views -from newparp.views import ( - account, admin, characters, chat, chat_api, chat_list, errors, guides, - roulette, search, search_characters, settings, -) -from newparp.views.admin import spamless, spamless2 app = Flask(__name__) app.url_map.strict_slashes = False + # Config app.config["SERVER_NAME"] = os.environ["BASE_DOMAIN"] @@ -52,6 +48,19 @@ app.teardown_request(redis_disconnect) +mail = Mail(app) + + +# Views/routes + +from newparp import views +from newparp.views import ( + account, admin, characters, chat, chat_api, chat_list, errors, guides, + roulette, search, search_characters, settings, +) +from newparp.views.admin import spamless, spamless2 + + class RegexConverter(BaseConverter): def __init__(self, url_map, *items): super(RegexConverter, self).__init__(url_map) diff --git a/newparp/templates/settings/log_in_details.html b/newparp/templates/settings/log_in_details.html index 5ff8efc5..85950b4b 100644 --- a/newparp/templates/settings/log_in_details.html +++ b/newparp/templates/settings/log_in_details.html @@ -6,7 +6,7 @@

E-mail address

{% if request.args["saved"] == "email_address": %} -

Your e-mail address has been changed.

+

We've sent a message to the address you just entered. Click the link in the message to finish changing your address.

{% endif %}
diff --git a/newparp/views/settings.py b/newparp/views/settings.py index 803892e4..690de61f 100644 --- a/newparp/views/settings.py +++ b/newparp/views/settings.py @@ -1,7 +1,10 @@ -from flask import g, jsonify, redirect, render_template, request, url_for +from flask import current_app, g, jsonify, redirect, render_template, request, url_for +from flask_mail import Message as EmailMessage from sqlalchemy import and_ from sqlalchemy.orm import joinedload +from uuid import uuid4 +from newparp import mail from newparp.helpers import alt_formats, themes from newparp.helpers.auth import log_in_required from newparp.model import Block @@ -85,13 +88,28 @@ def log_in_details(): return render_template("settings/log_in_details.html") +def send_email(action, email_address): + email_token = str(uuid4()) + g.redis.setex(":".join([action, str(g.user.id), email_address]), 86400 if action == "verify" else 600, email_token) + + message = EmailMessage( + subject="Verify your email address", + sender="admin@msparp.com", + recipients=[email_address], + body="https://msparp.com/verify_email?user_id=%s&email_address=%s&token=%s" % ( # TODO url_for + g.user.id, email_address, email_token, + ), + ) + mail.send(message) + + @use_db @log_in_required def change_email(): email_address = request.form.get("email_address").strip()[:100] if not email_address or email_validator.match(email_address) is None: return render_template("settings/log_in_details.html", error="invalid_email") - g.user.email_address = email_address + send_email("verify", email_address) return redirect(url_for("settings_log_in_details", saved="email_address")) diff --git a/requirements.txt b/requirements.txt index 6f5232d6..a44269a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ certifi==2016.2.28 cffi==1.7.0 classtools==0.1 Flask==0.11.1 +flask-mail==0.9.1 flower==0.9.1 futures==3.0.5 gevent==1.1.1 From c66a95ff618a3d74ea826b161c5d7503da267d7e Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Thu, 11 Aug 2016 19:51:57 +0100 Subject: [PATCH 02/78] Email verification. --- newparp/__init__.py | 1 + newparp/helpers/email.py | 19 +++++++ .../templates/settings/log_in_details.html | 4 ++ newparp/views/settings.py | 57 ++++++++++++------- 4 files changed, 61 insertions(+), 20 deletions(-) create mode 100644 newparp/helpers/email.py diff --git a/newparp/__init__.py b/newparp/__init__.py index 7edbbc61..6ad45115 100644 --- a/newparp/__init__.py +++ b/newparp/__init__.py @@ -103,6 +103,7 @@ def make_rules(subdomain, path, func, formats=False, paging=False): app.add_url_rule("/settings/theme", "settings_theme", settings.theme, methods=("POST",)) app.add_url_rule("/settings/log_in_details", "settings_log_in_details", settings.log_in_details, methods=("GET",)) app.add_url_rule("/settings/change_email", "settings_change_email", settings.change_email, methods=("POST",)) +app.add_url_rule("/settings/verify_email", "settings_verify_email", settings.verify_email, methods=("GET",)) app.add_url_rule("/settings/change_password", "settings_change_password", settings.change_password, methods=("POST",)) make_rules("settings", "/settings/blocks", settings.blocks, formats=True) app.add_url_rule("/settings/unblock", "settings_unblock", settings.unblock, methods=("POST",)) diff --git a/newparp/helpers/email.py b/newparp/helpers/email.py new file mode 100644 index 00000000..686bc24c --- /dev/null +++ b/newparp/helpers/email.py @@ -0,0 +1,19 @@ +from flask import g, url_for +from flask_mail import Message as EmailMessage +from uuid import uuid4 + +from newparp import mail + + +def send_email(action, email_address): + email_token = str(uuid4()) + g.redis.setex(":".join([action, str(g.user.id), email_address]), 86400 if action == "verify" else 600, email_token) + + message = EmailMessage( + subject="Verify your email address", + sender="admin@msparp.com", + recipients=[email_address], + body=url_for("settings_verify_email", user_id=g.user.id, email_address=email_address, token=email_token, _external=True), + ) + mail.send(message) + diff --git a/newparp/templates/settings/log_in_details.html b/newparp/templates/settings/log_in_details.html index 85950b4b..1e987063 100644 --- a/newparp/templates/settings/log_in_details.html +++ b/newparp/templates/settings/log_in_details.html @@ -7,6 +7,10 @@

E-mail address

{% if request.args["saved"] == "email_address": %}

We've sent a message to the address you just entered. Click the link in the message to finish changing your address.

+ {% elif request.args["saved"] == "email_verified": %} +

Your e-mail address has been verified.

+ {% elif request.args["saved"] == "email_address": %} +

Your e-mail address has been changed.

{% endif %} diff --git a/newparp/views/settings.py b/newparp/views/settings.py index 690de61f..14aca4b1 100644 --- a/newparp/views/settings.py +++ b/newparp/views/settings.py @@ -1,16 +1,16 @@ -from flask import current_app, g, jsonify, redirect, render_template, request, url_for -from flask_mail import Message as EmailMessage +from flask import abort, current_app, g, jsonify, redirect, render_template, request, url_for from sqlalchemy import and_ from sqlalchemy.orm import joinedload -from uuid import uuid4 +from sqlalchemy.orm.exc import NoResultFound -from newparp import mail from newparp.helpers import alt_formats, themes from newparp.helpers.auth import log_in_required -from newparp.model import Block +from newparp.helpers.email import send_email +from newparp.model import Block, User from newparp.model.connections import use_db from newparp.model.validators import email_validator + @use_db @log_in_required def home_get(): @@ -20,6 +20,7 @@ def home_get(): themes=themes, ) + @use_db @log_in_required def home_post(): @@ -88,21 +89,6 @@ def log_in_details(): return render_template("settings/log_in_details.html") -def send_email(action, email_address): - email_token = str(uuid4()) - g.redis.setex(":".join([action, str(g.user.id), email_address]), 86400 if action == "verify" else 600, email_token) - - message = EmailMessage( - subject="Verify your email address", - sender="admin@msparp.com", - recipients=[email_address], - body="https://msparp.com/verify_email?user_id=%s&email_address=%s&token=%s" % ( # TODO url_for - g.user.id, email_address, email_token, - ), - ) - mail.send(message) - - @use_db @log_in_required def change_email(): @@ -113,6 +99,37 @@ def change_email(): return redirect(url_for("settings_log_in_details", saved="email_address")) +@use_db +def verify_email(): + try: + user_id = int(request.args["user_id"].strip()) + email_address = request.args["email_address"].strip() + token = request.args["token"].strip() + except (KeyError, ValueError): + abort(404) + stored_token = g.redis.get("verify:%s:%s" % (user_id, email_address)) + if not user_id or not email_address or not token or not stored_token: + abort(404) + + if not stored_token == token: + abort(404) + + try: + user = g.db.query(User).filter(User.id == user_id).one() + except NoResultFound: + abort(404) + + g.redis.delete("verify:%s:%s" % (user_id, email_address)) + + next_message = "email_verified" if user.email_address == email_address else "email_changed" + + user.email_address = email_address + user.email_verified = True + + g.redis.set("session:" + g.session_id, user.id, 2592000) + return redirect(url_for("settings_log_in_details", saved=next_message)) + + @use_db @log_in_required def change_password(): From 587672940d278e770324ee3721977869d01b8674 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Thu, 11 Aug 2016 20:05:30 +0100 Subject: [PATCH 03/78] Send a verification email on registration. --- newparp/views/account.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/newparp/views/account.py b/newparp/views/account.py index c0ffe26d..55e8cc9c 100644 --- a/newparp/views/account.py +++ b/newparp/views/account.py @@ -9,6 +9,7 @@ from newparp.helpers import alt_formats from newparp.helpers.auth import not_logged_in_required +from newparp.helpers.email import send_email from newparp.model import User from newparp.model.connections import use_db from newparp.model.validators import username_validator, email_validator, reserved_usernames @@ -109,7 +110,7 @@ def register_post(): new_user = User( username=username, - email_address=email_address if email_address != "" else None, + email_address=email_address, group="active", last_ip=request.headers.get("X-Forwarded-For", request.remote_addr), ) @@ -118,6 +119,10 @@ def register_post(): g.db.flush() g.redis.set("session:" + g.session_id, new_user.id, 2592000) g.redis.setex("register:" + request.headers.get("X-Forwarded-For", request.remote_addr), 86400, 1) + + g.user = new_user + send_email("verify", email_address) + g.db.commit() redirect_url = referer_or_home() From 276b8550cfe03aed396a3f0fd910211b57dfd1be Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Thu, 11 Aug 2016 20:54:11 +0100 Subject: [PATCH 04/78] Get mail server settings from the environ. --- newparp/__init__.py | 5 +++++ newparp/templates/settings/log_in_details.html | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/newparp/__init__.py b/newparp/__init__.py index 6ad45115..f6681929 100644 --- a/newparp/__init__.py +++ b/newparp/__init__.py @@ -48,6 +48,11 @@ app.teardown_request(redis_disconnect) +app.config["MAIL_SERVER"] = os.environ.get("MAIL_SERVER", "localhost") +app.config["MAIL_PORT"] = int(os.environ.get("MAIL_PORT", 25)) +app.config["MAIL_USE_TLS"] = "MAIL_USE_TLS" in os.environ +app.config["MAIL_USERNAME"] = os.environ.get("MAIL_USERNAME") +app.config["MAIL_PASSWORD"] = os.environ.get("MAIL_PASSWORD") mail = Mail(app) diff --git a/newparp/templates/settings/log_in_details.html b/newparp/templates/settings/log_in_details.html index 1e987063..27cfc6d6 100644 --- a/newparp/templates/settings/log_in_details.html +++ b/newparp/templates/settings/log_in_details.html @@ -9,7 +9,7 @@

E-mail address

We've sent a message to the address you just entered. Click the link in the message to finish changing your address.

{% elif request.args["saved"] == "email_verified": %}

Your e-mail address has been verified.

- {% elif request.args["saved"] == "email_address": %} + {% elif request.args["saved"] == "email_changed": %}

Your e-mail address has been changed.

{% endif %} From 9adc796cebb0940186590cb69c8fe9315d32ba96 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Thu, 11 Aug 2016 21:25:21 +0100 Subject: [PATCH 05/78] Email templates. --- newparp/helpers/email.py | 9 +++++++-- newparp/templates/email/base.html | 22 ++++++++++++++++++++++ newparp/templates/email/verify.html | 9 +++++++++ newparp/templates/email/verify_plain.html | 5 +++++ 4 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 newparp/templates/email/base.html create mode 100644 newparp/templates/email/verify.html create mode 100644 newparp/templates/email/verify_plain.html diff --git a/newparp/helpers/email.py b/newparp/helpers/email.py index 686bc24c..0c7ec9c2 100644 --- a/newparp/helpers/email.py +++ b/newparp/helpers/email.py @@ -1,4 +1,4 @@ -from flask import g, url_for +from flask import g, render_template, url_for from flask_mail import Message as EmailMessage from uuid import uuid4 @@ -13,7 +13,12 @@ def send_email(action, email_address): subject="Verify your email address", sender="admin@msparp.com", recipients=[email_address], - body=url_for("settings_verify_email", user_id=g.user.id, email_address=email_address, token=email_token, _external=True), + + body=render_template("email/%s_plain.html" % action, user=g.user, email_address=email_address, email_token=email_token), + html=render_template("email/%s.html" % action, user=g.user, email_address=email_address, email_token=email_token), + + +# body=url_for("settings_verify_email", user_id=g.user.id, email_address=email_address, token=email_token, _external=True), ) mail.send(message) diff --git a/newparp/templates/email/base.html b/newparp/templates/email/base.html new file mode 100644 index 00000000..2d1224ff --- /dev/null +++ b/newparp/templates/email/base.html @@ -0,0 +1,22 @@ + + + +Cherubplay + + + +
+

+ + MSPARP + +

+
+ +
+{% block content: %}{% endblock %} +
+ + + + diff --git a/newparp/templates/email/verify.html b/newparp/templates/email/verify.html new file mode 100644 index 00000000..ea7515d1 --- /dev/null +++ b/newparp/templates/email/verify.html @@ -0,0 +1,9 @@ +{% extends "email/base.html" %} +{% block content: %} +

Verify your e-mail address

+
+

Before we can change your e-mail address, we need to verify that it's really you.

+

Please click here to verify your e-mail address.

+

This link will expire in 24 hours.

+
+{% endblock %} diff --git a/newparp/templates/email/verify_plain.html b/newparp/templates/email/verify_plain.html new file mode 100644 index 00000000..f2bb7e68 --- /dev/null +++ b/newparp/templates/email/verify_plain.html @@ -0,0 +1,5 @@ +Before we can change your e-mail address, we need to verify that it's really you. + +Please visit the following URL to verify your e-mail address: {{url_for("settings_verify_email", _external=True, _scheme="https", user_id=user.id, email_address=email_address, token=email_token)}} + +This link will expire in 24 hours. From 485921ee5b9f5faed470e76fb343ea7282194a79 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Fri, 12 Aug 2016 23:45:51 +0100 Subject: [PATCH 06/78] Don't bother sending an email when people change their email to itself. --- newparp/views/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/newparp/views/settings.py b/newparp/views/settings.py index 14aca4b1..dc80832b 100644 --- a/newparp/views/settings.py +++ b/newparp/views/settings.py @@ -95,6 +95,9 @@ def change_email(): email_address = request.form.get("email_address").strip()[:100] if not email_address or email_validator.match(email_address) is None: return render_template("settings/log_in_details.html", error="invalid_email") + # This is pointless. + if g.user.email_verified and email_address == g.user.email_address: + return redirect(url_for("settings_log_in_details", saved="email_changed")) send_email("verify", email_address) return redirect(url_for("settings_log_in_details", saved="email_address")) From 9183f1caa7d07e1ddceeb78d2b15315cadd0c6c5 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Fri, 12 Aug 2016 23:58:14 +0100 Subject: [PATCH 07/78] Use different templates for the welcome message. --- newparp/helpers/email.py | 26 ++++++++++++++++------ newparp/templates/email/welcome.html | 8 +++++++ newparp/templates/email/welcome_plain.html | 5 +++++ newparp/views/account.py | 2 +- 4 files changed, 33 insertions(+), 8 deletions(-) create mode 100644 newparp/templates/email/welcome.html create mode 100644 newparp/templates/email/welcome_plain.html diff --git a/newparp/helpers/email.py b/newparp/helpers/email.py index 0c7ec9c2..d0cac59c 100644 --- a/newparp/helpers/email.py +++ b/newparp/helpers/email.py @@ -5,20 +5,32 @@ from newparp import mail +expiry_times = { + "welcome": 86400, + "verify": 86400, + "reset": 600, +} + +subjects = { + "welcome": "Welcome to MSPARP", + "verify": "Verify your e-mail address", + "reset": "Reset your password", +} + + def send_email(action, email_address): email_token = str(uuid4()) - g.redis.setex(":".join([action, str(g.user.id), email_address]), 86400 if action == "verify" else 600, email_token) - + g.redis.setex( + ":".join([action, str(g.user.id), email_address]), + expiry_times[action], + email_token, + ) message = EmailMessage( - subject="Verify your email address", + subject=subjects[action], sender="admin@msparp.com", recipients=[email_address], - body=render_template("email/%s_plain.html" % action, user=g.user, email_address=email_address, email_token=email_token), html=render_template("email/%s.html" % action, user=g.user, email_address=email_address, email_token=email_token), - - -# body=url_for("settings_verify_email", user_id=g.user.id, email_address=email_address, token=email_token, _external=True), ) mail.send(message) diff --git a/newparp/templates/email/welcome.html b/newparp/templates/email/welcome.html new file mode 100644 index 00000000..d90777c7 --- /dev/null +++ b/newparp/templates/email/welcome.html @@ -0,0 +1,8 @@ +{% extends "email/base.html" %} +{% block content: %} +

Welcome to MSPARP

+
+

Please click here to verify your e-mail address.

+

This link will expire in 24 hours.

+
+{% endblock %} diff --git a/newparp/templates/email/welcome_plain.html b/newparp/templates/email/welcome_plain.html new file mode 100644 index 00000000..fe3dc09c --- /dev/null +++ b/newparp/templates/email/welcome_plain.html @@ -0,0 +1,5 @@ +Welcome to MSPARP. + +Please visit the following URL to verify your e-mail address: {{url_for("settings_verify_email", _external=True, _scheme="https", user_id=user.id, email_address=email_address, token=email_token)}} + +This link will expire in 24 hours. diff --git a/newparp/views/account.py b/newparp/views/account.py index 55e8cc9c..7af7b146 100644 --- a/newparp/views/account.py +++ b/newparp/views/account.py @@ -121,7 +121,7 @@ def register_post(): g.redis.setex("register:" + request.headers.get("X-Forwarded-For", request.remote_addr), 86400, 1) g.user = new_user - send_email("verify", email_address) + send_email("welcome", email_address) g.db.commit() From c20f997fc7386d27714e79b07658f8eb8cac7535 Mon Sep 17 00:00:00 2001 From: Raleigh Date: Thu, 18 Aug 2016 19:30:29 -0500 Subject: [PATCH 08/78] Update welcome.html --- newparp/templates/email/welcome.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/newparp/templates/email/welcome.html b/newparp/templates/email/welcome.html index d90777c7..283b26b6 100644 --- a/newparp/templates/email/welcome.html +++ b/newparp/templates/email/welcome.html @@ -2,7 +2,8 @@ {% block content: %}

Welcome to MSPARP

-

Please click here to verify your e-mail address.

-

This link will expire in 24 hours.

+

Thank you for signing up for MSPARP. Your new account is almost ready. Please click here to verify your e-mail address and complete your registration.

+

This link will expire in 24 hours. If you are having trouble with the registration process, please contact us through our Ticket System.

+

Cordially,
MSPARP Administration

{% endblock %} From 182f01658285a900b7581a1326af15370aec0b00 Mon Sep 17 00:00:00 2001 From: Raleigh Date: Thu, 18 Aug 2016 19:45:07 -0500 Subject: [PATCH 09/78] Update welcome_plain.html --- newparp/templates/email/welcome_plain.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/newparp/templates/email/welcome_plain.html b/newparp/templates/email/welcome_plain.html index fe3dc09c..8d733057 100644 --- a/newparp/templates/email/welcome_plain.html +++ b/newparp/templates/email/welcome_plain.html @@ -1,5 +1,8 @@ Welcome to MSPARP. -Please visit the following URL to verify your e-mail address: {{url_for("settings_verify_email", _external=True, _scheme="https", user_id=user.id, email_address=email_address, token=email_token)}} +Thank you for signing up for MSPARP. Your new account is almost ready. Please visit the following URL to verify your e-mail address: {{url_for("settings_verify_email", _external=True, _scheme="https", user_id=user.id, email_address=email_address, token=email_token)}} -This link will expire in 24 hours. +This link will expire in 24 hours. If you are having trouble with the registration process, please contact us at http://help.msparp.com/. + +Cordially, +MSPARP Administration From 83ffd91d4377f98e4a8eb287e8edc92a65a66b5d Mon Sep 17 00:00:00 2001 From: Raleigh Date: Thu, 18 Aug 2016 19:51:57 -0500 Subject: [PATCH 10/78] Update verify.html --- newparp/templates/email/verify.html | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/newparp/templates/email/verify.html b/newparp/templates/email/verify.html index ea7515d1..314da29e 100644 --- a/newparp/templates/email/verify.html +++ b/newparp/templates/email/verify.html @@ -2,8 +2,9 @@ {% block content: %}

Verify your e-mail address

-

Before we can change your e-mail address, we need to verify that it's really you.

-

Please click here to verify your e-mail address.

-

This link will expire in 24 hours.

+

We recently recieved a request to change your registered email address at msparp.com. Before we can change your e-mail address, we need to verify that it's really you.

+

Please click here to verify your identity. If you did not make this request, please ignore this message.

+

This link will expire in 24 hours. If you are having trouble, please contact us through our Ticket System.

+

Cordially,
MSPARP Staff

{% endblock %} From 7298addbc4cc066dc8e65141a5cb20d7ca63dc61 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Fri, 19 Aug 2016 01:54:26 +0100 Subject: [PATCH 11/78] Update base.html --- newparp/templates/email/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newparp/templates/email/base.html b/newparp/templates/email/base.html index 2d1224ff..5a287eb0 100644 --- a/newparp/templates/email/base.html +++ b/newparp/templates/email/base.html @@ -1,7 +1,7 @@ -Cherubplay +MSPARP From 65caaea7c36e8d6f3efc6b4564951d27aea8e745 Mon Sep 17 00:00:00 2001 From: Raleigh Date: Thu, 18 Aug 2016 19:55:47 -0500 Subject: [PATCH 12/78] Update verify_plain.html --- newparp/templates/email/verify_plain.html | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/newparp/templates/email/verify_plain.html b/newparp/templates/email/verify_plain.html index f2bb7e68..1919d7ed 100644 --- a/newparp/templates/email/verify_plain.html +++ b/newparp/templates/email/verify_plain.html @@ -1,5 +1,8 @@ -Before we can change your e-mail address, we need to verify that it's really you. +We recently received a request to change your registered email address at msparp.com. Before we can change your e-mail address, we need to verify that it's really you. -Please visit the following URL to verify your e-mail address: {{url_for("settings_verify_email", _external=True, _scheme="https", user_id=user.id, email_address=email_address, token=email_token)}} +Please visit the following URL to verify your e-mail address: {{url_for("settings_verify_email", _external=True, _scheme="https", user_id=user.id, email_address=email_address, token=email_token)}} If you did not make this request, please ignore this message. -This link will expire in 24 hours. +This link will expire in 24 hours. If you are having trouble, please contact us at http://help.msparp.com + +Cordially, +MSPARP Administration From bd48fb3d365667854adcc8527a18e06ae4b6fd3a Mon Sep 17 00:00:00 2001 From: Raleigh Date: Thu, 18 Aug 2016 19:55:56 -0500 Subject: [PATCH 13/78] Update verify.html --- newparp/templates/email/verify.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/newparp/templates/email/verify.html b/newparp/templates/email/verify.html index 314da29e..b61bdbe8 100644 --- a/newparp/templates/email/verify.html +++ b/newparp/templates/email/verify.html @@ -2,9 +2,9 @@ {% block content: %}

Verify your e-mail address

-

We recently recieved a request to change your registered email address at msparp.com. Before we can change your e-mail address, we need to verify that it's really you.

+

We recently received a request to change your registered email address at msparp.com. Before we can change your e-mail address, we need to verify that it's really you.

Please click here to verify your identity. If you did not make this request, please ignore this message.

This link will expire in 24 hours. If you are having trouble, please contact us through our Ticket System.

-

Cordially,
MSPARP Staff

+

Cordially,
MSPARP Administration

{% endblock %} From 219eddef46d17486324240856005dc2be40083a4 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Sat, 20 Aug 2016 20:44:08 +0100 Subject: [PATCH 14/78] Add test tasks to the celery includes. --- newparp/tasks/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/newparp/tasks/__init__.py b/newparp/tasks/__init__.py index 15a26bb0..0c644fdb 100644 --- a/newparp/tasks/__init__.py +++ b/newparp/tasks/__init__.py @@ -15,6 +15,7 @@ "newparp.tasks.reaper", "newparp.tasks.roulette_matchmaker", "newparp.tasks.chat", + "newparp.tasks.test", ]) # Sentry exception logging if there is a sentry object. From 8d026336b9b2458415606e9f7b27d759c75a2289 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Wed, 24 Aug 2016 19:27:27 +0100 Subject: [PATCH 15/78] Fix the redis key for welcome emails. --- newparp/helpers/email.py | 8 +++++++- newparp/views/settings.py | 5 ++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/newparp/helpers/email.py b/newparp/helpers/email.py index d0cac59c..c8297de8 100644 --- a/newparp/helpers/email.py +++ b/newparp/helpers/email.py @@ -17,11 +17,17 @@ "reset": "Reset your password", } +keys = { + "welcome": "verify", + "verify": "verify", + "reset": "reset", +} + def send_email(action, email_address): email_token = str(uuid4()) g.redis.setex( - ":".join([action, str(g.user.id), email_address]), + ":".join([keys[action], str(g.user.id), email_address]), expiry_times[action], email_token, ) diff --git a/newparp/views/settings.py b/newparp/views/settings.py index dc80832b..e48c8a91 100644 --- a/newparp/views/settings.py +++ b/newparp/views/settings.py @@ -110,7 +110,10 @@ def verify_email(): token = request.args["token"].strip() except (KeyError, ValueError): abort(404) - stored_token = g.redis.get("verify:%s:%s" % (user_id, email_address)) + stored_token = ( + g.redis.get("verify:%s:%s" % (user_id, email_address)) + or g.redis.get("welcome:%s:%s" % (user_id, email_address)) + ) if not user_id or not email_address or not token or not stored_token: abort(404) From ec9e5bc364a625b587c45f5a4d251258614ddc43 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Thu, 25 Aug 2016 20:42:35 +0100 Subject: [PATCH 16/78] Remove link colour in emails. --- newparp/templates/email/verify.html | 2 +- newparp/templates/email/welcome.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/newparp/templates/email/verify.html b/newparp/templates/email/verify.html index b61bdbe8..f5604474 100644 --- a/newparp/templates/email/verify.html +++ b/newparp/templates/email/verify.html @@ -3,7 +3,7 @@

Verify your e-mail address

We recently received a request to change your registered email address at msparp.com. Before we can change your e-mail address, we need to verify that it's really you.

-

Please click here to verify your identity. If you did not make this request, please ignore this message.

+

Please click here to verify your identity. If you did not make this request, please ignore this message.

This link will expire in 24 hours. If you are having trouble, please contact us through our Ticket System.

Cordially,
MSPARP Administration

diff --git a/newparp/templates/email/welcome.html b/newparp/templates/email/welcome.html index 283b26b6..3200bfe6 100644 --- a/newparp/templates/email/welcome.html +++ b/newparp/templates/email/welcome.html @@ -2,7 +2,7 @@ {% block content: %}

Welcome to MSPARP

-

Thank you for signing up for MSPARP. Your new account is almost ready. Please click here to verify your e-mail address and complete your registration.

+

Thank you for signing up for MSPARP. Your new account is almost ready. Please click here to verify your e-mail address and complete your registration.

This link will expire in 24 hours. If you are having trouble with the registration process, please contact us through our Ticket System.

Cordially,
MSPARP Administration

From b26a8d8ebe3a354ce97ff401208532c2565fe75b Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Sat, 27 Aug 2016 23:17:16 +0100 Subject: [PATCH 17/78] Move search initialisation to a pipeline. --- newparp/views/search.py | 56 +++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/newparp/views/search.py b/newparp/views/search.py index a2c7148a..122b26b5 100644 --- a/newparp/views/search.py +++ b/newparp/views/search.py @@ -70,24 +70,17 @@ def search_save(): return redirect(url_for("rp_search")) -@use_db -@log_in_required -def search_get(): - return render_template("search.html") - - -@use_db -@log_in_required -def search_post(): +def _create_searcher(): + pipe = g.redis.pipeline() searcher_id = str(uuid4()) - g.redis.set("searcher:%s:session_id" % searcher_id, g.session_id) - g.redis.expire("searcher:%s:session_id" % searcher_id, 30) + pipe.set("searcher:%s:session_id" % searcher_id, g.session_id) + pipe.expire("searcher:%s:session_id" % searcher_id, 30) - g.redis.set("searcher:%s:search_character_id" % searcher_id, g.user.search_character_id) - g.redis.expire("searcher:%s:search_character_id" % searcher_id, 30) + pipe.set("searcher:%s:search_character_id" % searcher_id, g.user.search_character_id) + pipe.expire("searcher:%s:search_character_id" % searcher_id, 30) - g.redis.hmset("searcher:%s:character" % searcher_id, { + pipe.hmset("searcher:%s:character" % searcher_id, { "name": g.user.name, "acronym": g.user.acronym, "color": g.user.color, @@ -97,29 +90,42 @@ def search_post(): "replacements": g.user.replacements, "regexes": g.user.regexes, }) - g.redis.expire("searcher:%s:character" % searcher_id, 30) + pipe.expire("searcher:%s:character" % searcher_id, 30) - g.redis.set("searcher:%s:style" % searcher_id, g.user.search_style) - g.redis.expire("searcher:%s:style" % searcher_id, 30) + pipe.set("searcher:%s:style" % searcher_id, g.user.search_style) + pipe.expire("searcher:%s:style" % searcher_id, 30) - g.redis.sadd("searcher:%s:levels" % searcher_id, *g.user.search_levels) - g.redis.expire("searcher:%s:levels" % searcher_id, 30) + pipe.sadd("searcher:%s:levels" % searcher_id, *g.user.search_levels) + pipe.expire("searcher:%s:levels" % searcher_id, 30) if g.user.search_filters: - g.redis.rpush("searcher:%s:filters" % searcher_id, *g.user.search_filters) # XXX why is this a list and not a set? - g.redis.expire("searcher:%s:filters" % searcher_id, 30) + pipe.rpush("searcher:%s:filters" % searcher_id, *g.user.search_filters) # XXX why is this a list and not a set? + pipe.expire("searcher:%s:filters" % searcher_id, 30) - g.redis.delete("searcher:%s:choices" % searcher_id) choices = [_[0] for _ in g.db.query( SearchCharacterChoice.search_character_id, ).filter( SearchCharacterChoice.user_id == g.user.id, ).all()] if choices: - g.redis.sadd("searcher:%s:choices" % searcher_id, *choices) - g.redis.expire("searcher:%s:choices" % searcher_id, 30) + pipe.sadd("searcher:%s:choices" % searcher_id, *choices) + pipe.expire("searcher:%s:choices" % searcher_id, 30) + + pipe.execute() + + return searcher_id + + +@use_db +@log_in_required +def search_get(): + return render_template("search.html") + - return jsonify({ "id": searcher_id }) +@use_db +@log_in_required +def search_post(): + return jsonify({ "id": _create_searcher() }) def search_continue(): From bc1c18096cae5b3c4bd4a1d03139f7902dcd091b Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Sun, 28 Aug 2016 20:12:15 +0100 Subject: [PATCH 18/78] New handler class for searching. --- newparp/workers/live.py | 95 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/newparp/workers/live.py b/newparp/workers/live.py index c9435a21..d0033c01 100644 --- a/newparp/workers/live.py +++ b/newparp/workers/live.py @@ -23,7 +23,7 @@ from tornado.platform.asyncio import AsyncIOMainLoop from tornado.web import Application, RequestHandler from tornado.websocket import WebSocketHandler, WebSocketClosedError -from uuid import uuid4 +from uuid import UUID, uuid4 from newparp.helpers.chat import ( UnauthorizedException, @@ -41,6 +41,7 @@ from newparp.helpers.users import queue_user_meta from newparp.model import sm, AnyChat, Ban, ChatUser, Message, User, SearchCharacter from newparp.model.connections import redis_pool +from newparp.tasks.matchmaker import new_searcher redis = StrictRedis(connection_pool=redis_pool) thread_pool = ThreadPoolExecutor() @@ -261,6 +262,97 @@ async def on_redis_message(self, message): self.close() +class SearchHandler(WebSocketHandler): + + def check_origin(self, origin): + if "localhost" in os.environ["BASE_DOMAIN"].lower(): + return True + + return origin_regex.match(origin) is not None + + def prepare(self): + if "newparp" not in self.cookies: + self.send_error(401) + return + self.searcher_id = searcher_id = self.path_args[0] + pipe = redis.pipeline() + pipe.get("searcher:%s:session_id" % searcher_id) # TODO check user id + pipe.get("searcher:%s:search_character_id" % searcher_id) + pipe.hlen("searcher:%s:character" % searcher_id) + pipe.get("searcher:%s:style" % searcher_id) + pipe.scard("searcher:%s:levels" % searcher_id) + result = pipe.execute() + if not all(result) or result[0] != self.cookies["newparp"].value: + self.send_error(404) + return + + @coroutine + def open(self, searcher_id): + self.redis_task = asyncio.ensure_future(self.redis_listen()) + redis.sadd("searchers", searcher_id) + new_searcher.delay(searcher_id) + + def on_message(self, message): + pipe = redis.pipeline() + pipe.sismember("searchers", self.searcher_id) + pipe.expire("searcher:%s:session_id" % self.searcher_id, 30) + pipe.expire("searcher:%s:search_character_id" % self.searcher_id, 30) + pipe.expire("searcher:%s:character" % self.searcher_id, 30) + pipe.expire("searcher:%s:style" % self.searcher_id, 30) + pipe.expire("searcher:%s:levels" % self.searcher_id, 30) + pipe.expire("searcher:%s:filters" % self.searcher_id, 30) + pipe.expire("searcher:%s:choices" % self.searcher_id, 30) + if not all(pipe.execute()) + self.close() + + async def redis_listen(self): + self.redis_client = await asyncio_redis.Connection.create( + host=os.environ["REDIS_HOST"], + port=int(os.environ["REDIS_PORT"]), + db=int(os.environ["REDIS_DB"]), + ) + # Set the connection name, subscribe, and listen. + await self.redis_client.client_setname("searcher:%s" % self.searcher_id) + try: + subscriber = await self.redis_client.start_subscribe() + await subscriber.subscribe(["searcher:%s" % self.searcher_id]) + while self.ws_connection: + message = await subscriber.next_published() + asyncio.ensure_future(self.on_redis_message(message)) + finally: + self.redis_client.close() + + async def on_redis_message(self, message): + if DEBUG: + print("redis message: %s" % str(message)) + self.write_message(message.value) + + def on_close(self): + # Unsubscribe here and let the exit callback handle disconnecting. + if hasattr(self, "redis_task"): + self.redis_task.cancel() + + if hasattr(self, "redis_client"): + self.redis_client.close() + + pipe = redis.pipeline() + pipe.srem("searchers", self.searcher_id) + pipe.delete("searcher:%s:session_id" % self.searcher_id) + pipe.delete("searcher:%s:search_character_id" % self.searcher_id) + pipe.delete("searcher:%s:character" % self.searcher_id) + pipe.delete("searcher:%s:style" % self.searcher_id) + pipe.delete("searcher:%s:levels" % self.searcher_id) + pipe.execute() + + if DEBUG: + print("socket closed: %s" % (self.searcher_id)) + + try: + sockets.remove(self) + except KeyError: + pass + + class HealthHandler(RequestHandler): @property def loop(self): @@ -320,6 +412,7 @@ def stop_loop(): application = Application([ (r"/(\d+)", ChatHandler), + (r"/search/([0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})", SearchHandler), (r"/health", HealthHandler) ]) From 701fea872b1757adc9defcc0e3f1e83669f33177 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Sun, 28 Aug 2016 20:12:49 +0100 Subject: [PATCH 19/78] New tasks for searching. --- newparp/static/js/newparp.js | 19 +++++++++++++++-- newparp/tasks/matchmaker.py | 41 ++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/newparp/static/js/newparp.js b/newparp/static/js/newparp.js index 1704964b..27719d23 100644 --- a/newparp/static/js/newparp.js +++ b/newparp/static/js/newparp.js @@ -588,9 +588,24 @@ var msparp = (function() { }, // Character search "search": function(token) { + var ws; $.ajaxSetup({data: {"token": token}}); - $(window).unload(function () { if (searching) { stop_search(); }}); - start_search(); + + + + + $.post("/" + search_type, {}, function(data) { + searcher_id = data.id; + ws = new WebSocket("wss://live." + location.host + "/search/" + searcher_id); + window.ws = ws; + }).error(function() { + searching = false; + body.removeClass("searching").addClass("search_error"); + }); + + + + }, // Roulette "roulette": function(token) { diff --git a/newparp/tasks/matchmaker.py b/newparp/tasks/matchmaker.py index 7f49b628..5f3859c8 100644 --- a/newparp/tasks/matchmaker.py +++ b/newparp/tasks/matchmaker.py @@ -1,3 +1,4 @@ +from celery import chord from celery.utils.log import get_task_logger from newparp.helpers.matchmaker import run_matchmaker @@ -97,6 +98,7 @@ def check_compatibility(redis, s1, s2): def get_character_info(db, searcher): return searcher["character"] + @celery.task(base=WorkerTask, queue="worker") def run(): db = run.db @@ -107,3 +109,42 @@ def run(): check_compatibility, SearchedChat, get_character_info, ) + +@celery.task(base=WorkerTask, queue="matchmaker") +def new_searcher(searcher_id): + # TODO lock + logger.debug("new searcher: %s") + searchers = new_searcher.redis.smembers("searchers") + try: + searchers.remove(searcher_id) + except KeyError: + logger.debug("no longer searching") + return + if not searchers: + logger.debug("not enough searchers, skipping") + return + chord( + (compare.s(searcher_id, _) for _ in searchers if _ != searcher_id), + comparison_callback.s(searcher_id), + ).delay() + + +@celery.task(base=WorkerTask, queue="matchmaker") +def compare(searcher_id_1, searcher_id_2): + logger.debug("comparing %s and %s" % (searcher_id_1, searcher_id_2)) + + +@celery.task(base=WorkerTask, queue="matchmaker") +def comparison_callback(results, searcher_id): + logger.debug("match results: %s" % results) + matched_searchers = [_ for _ in results if _ is not None] + if not matched_searchers: + logger.debug("no results") + return + logger.debug("results: %s" % matched_searchers) + # TODO something + + + + + From b24425448d401e9a17d085f09d7f1f90058a6314 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Sun, 28 Aug 2016 20:16:26 +0100 Subject: [PATCH 20/78] Unset roulette character if necessary. --- newparp/views/characters.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/newparp/views/characters.py b/newparp/views/characters.py index 238d8099..ae440ba8 100644 --- a/newparp/views/characters.py +++ b/newparp/views/characters.py @@ -108,7 +108,9 @@ def delete_character_post(character_id): character_id = character.id if g.user.default_character_id == character_id: g.user.default_character_id = None - g.db.flush() + if g.user.roulette_character_id == character_id: + g.user.roulette_character_id = None + g.db.flush() g.db.query(CharacterTag).filter(CharacterTag.character_id == character_id).delete() # Don't use g.db.delete(character) because it does a load of extra queries # for foreign keys and stuff. From 5feebc2b2529bdac705e4a6f1e143fbcb2a220d5 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Mon, 29 Aug 2016 19:32:51 +0100 Subject: [PATCH 21/78] Set up ping loop. --- newparp/static/js/newparp.js | 19 ++++++++++--------- newparp/workers/live.py | 4 +++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/newparp/static/js/newparp.js b/newparp/static/js/newparp.js index 27719d23..b3b4bb14 100644 --- a/newparp/static/js/newparp.js +++ b/newparp/static/js/newparp.js @@ -588,24 +588,25 @@ var msparp = (function() { }, // Character search "search": function(token) { - var ws; + var ws, ws_interval; $.ajaxSetup({data: {"token": token}}); - - - - $.post("/" + search_type, {}, function(data) { + body.addClass("searching"); searcher_id = data.id; ws = new WebSocket("wss://live." + location.host + "/search/" + searcher_id); + ws.onopen = function() { + console.log("ready"); + ws_interval = window.setInterval(function() { console.log("ping"); ws.send("ping"); }, 10000) + } + ws.onclose = function() { + body.removeClass("searching").addClass("search_error"); + window.clearInterval(ws_interval); + } window.ws = ws; }).error(function() { searching = false; body.removeClass("searching").addClass("search_error"); }); - - - - }, // Roulette "roulette": function(token) { diff --git a/newparp/workers/live.py b/newparp/workers/live.py index d0033c01..526c53e2 100644 --- a/newparp/workers/live.py +++ b/newparp/workers/live.py @@ -302,7 +302,7 @@ def on_message(self, message): pipe.expire("searcher:%s:levels" % self.searcher_id, 30) pipe.expire("searcher:%s:filters" % self.searcher_id, 30) pipe.expire("searcher:%s:choices" % self.searcher_id, 30) - if not all(pipe.execute()) + if not all(pipe.execute()[:-2]): # filters and choices are optional self.close() async def redis_listen(self): @@ -342,6 +342,8 @@ def on_close(self): pipe.delete("searcher:%s:character" % self.searcher_id) pipe.delete("searcher:%s:style" % self.searcher_id) pipe.delete("searcher:%s:levels" % self.searcher_id) + pipe.delete("searcher:%s:filters" % self.searcher_id) + pipe.delete("searcher:%s:choices" % self.searcher_id) pipe.execute() if DEBUG: From 0efbb6bab1188975eb66d8939ecec133d0ffc36f Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Sat, 3 Sep 2016 19:14:58 +0100 Subject: [PATCH 22/78] Use Lua scripts to get searcher information. --- newparp/helpers/matchmaker.py | 60 +++++++++++++++++++++++++++++++++++ newparp/workers/live.py | 33 +++++++++---------- 2 files changed, 75 insertions(+), 18 deletions(-) diff --git a/newparp/helpers/matchmaker.py b/newparp/helpers/matchmaker.py index 4c259883..1b51d811 100644 --- a/newparp/helpers/matchmaker.py +++ b/newparp/helpers/matchmaker.py @@ -8,6 +8,66 @@ from newparp.model import Block, ChatUser, Message, User + +def validate_searcher_exists(redis, searcher_id): + """Check whether a searcher's mandatory keys are present.""" + return redis.eval("""local session_id = redis.call("get", "searcher:"..ARGV[1]..":session_id") or "-" + return { + session_id, + redis.call("get", "session:"..session_id), + redis.call("get", "searcher:"..ARGV[1]..":search_character_id"), + redis.call("hlen", "searcher:"..ARGV[1]..":character"), + redis.call("get", "searcher:"..ARGV[1]..":style"), + redis.call("scard", "searcher:"..ARGV[1]..":levels"), + }""", 0, searcher_id) + + +def validate_searcher_is_searching(redis, searcher_id): + """Check whether a searcher's mandatory keys are present and they're in the searchers set.""" + return redis.eval("""local session_id = redis.call("get", "searcher:"..ARGV[1]..":session_id") or "-" + return { + redis.call("sismember", "searchers", ARGV[1]), + session_id, + redis.call("get", "session:"..session_id), + redis.call("get", "searcher:"..ARGV[1]..":search_character_id"), + redis.call("hlen", "searcher:"..ARGV[1]..":character"), + redis.call("get", "searcher:"..ARGV[1]..":style"), + redis.call("scard", "searcher:"..ARGV[1]..":levels"), + }""", 0, searcher_id) + + +def refresh_searcher(redis, searcher_id): + """Reset the expiry times on a searcher's keys.""" + return redis.eval("""local session_id = redis.call("get", "searcher:"..ARGV[1]..":session_id") or "-" + return { + redis.call("get", "session:"..session_id), + redis.call("sismember", "searchers", ARGV[1]), + redis.call("expire", "searcher:"..ARGV[1]..":session_id", 30), + redis.call("expire", "searcher:"..ARGV[1]..":search_character_id", 30), + redis.call("expire", "searcher:"..ARGV[1]..":character", 30), + redis.call("expire", "searcher:"..ARGV[1]..":style", 30), + redis.call("expire", "searcher:"..ARGV[1]..":levels", 30), + redis.call("expire", "searcher:"..ARGV[1]..":filters", 30), + redis.call("expire", "searcher:"..ARGV[1]..":choices", 30), + }""", 0, searcher_id) + + +def fetch_searcher(redis, searcher_id): + """Fetch searcher keys for matching.""" + return redis.eval("""local session_id = redis.call("get", "searcher:"..ARGV[1]..":session_id") or "-" + return { + redis.call("sismember", "searchers", ARGV[1]), + session_id, + redis.call("get", "session:"..session_id), + redis.call("get", "searcher:"..ARGV[1]..":search_character_id"), + redis.call("hget", "searcher:"..ARGV[1]..":character", "name"), + redis.call("get", "searcher:"..ARGV[1]..":style"), + redis.call("smembers", "searcher:"..ARGV[1]..":levels"), + redis.call("lrange", "searcher:"..ARGV[1]..":filters", 30), + redis.call("expire", "searcher:"..ARGV[1]..":choices", 30), + }""", 0, searcher_id) + + option_messages = { "script": "This is a script style chat.", "paragraph": "This is a paragraph style chat.", diff --git a/newparp/workers/live.py b/newparp/workers/live.py index 526c53e2..e5b324e1 100644 --- a/newparp/workers/live.py +++ b/newparp/workers/live.py @@ -38,6 +38,7 @@ disconnect, send_quit_message, ) +from newparp.helpers.matchmaker import validate_searcher_exists, refresh_searcher from newparp.helpers.users import queue_user_meta from newparp.model import sm, AnyChat, Ban, ChatUser, Message, User, SearchCharacter from newparp.model.connections import redis_pool @@ -274,15 +275,19 @@ def prepare(self): if "newparp" not in self.cookies: self.send_error(401) return + self.searcher_id = searcher_id = self.path_args[0] - pipe = redis.pipeline() - pipe.get("searcher:%s:session_id" % searcher_id) # TODO check user id - pipe.get("searcher:%s:search_character_id" % searcher_id) - pipe.hlen("searcher:%s:character" % searcher_id) - pipe.get("searcher:%s:style" % searcher_id) - pipe.scard("searcher:%s:levels" % searcher_id) - result = pipe.execute() - if not all(result) or result[0] != self.cookies["newparp"].value: + try: + UUID(self.path_args[0]) + except ValueError: + self.send_error(404) + return + + result = validate_searcher_exists(redis, self.searcher_id) + if ( + not all(result) + or result[0] != self.cookies["newparp"].value + ): self.send_error(404) return @@ -293,16 +298,8 @@ def open(self, searcher_id): new_searcher.delay(searcher_id) def on_message(self, message): - pipe = redis.pipeline() - pipe.sismember("searchers", self.searcher_id) - pipe.expire("searcher:%s:session_id" % self.searcher_id, 30) - pipe.expire("searcher:%s:search_character_id" % self.searcher_id, 30) - pipe.expire("searcher:%s:character" % self.searcher_id, 30) - pipe.expire("searcher:%s:style" % self.searcher_id, 30) - pipe.expire("searcher:%s:levels" % self.searcher_id, 30) - pipe.expire("searcher:%s:filters" % self.searcher_id, 30) - pipe.expire("searcher:%s:choices" % self.searcher_id, 30) - if not all(pipe.execute()[:-2]): # filters and choices are optional + result = refresh_searcher(redis, self.searcher_id) + if not all(result[:-2]): # -2 because filters and choices are optional self.close() async def redis_listen(self): From 9c41731aa360c1c74045b2de71bac9395a6b2b1d Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Sat, 3 Sep 2016 22:47:30 +0100 Subject: [PATCH 23/78] Search character comparison. --- newparp/helpers/matchmaker.py | 18 +++++++++++------- newparp/tasks/matchmaker.py | 35 ++++++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/newparp/helpers/matchmaker.py b/newparp/helpers/matchmaker.py index 1b51d811..4bfe3636 100644 --- a/newparp/helpers/matchmaker.py +++ b/newparp/helpers/matchmaker.py @@ -2,6 +2,7 @@ import json import logging +from collections import namedtuple from random import shuffle from sqlalchemy import and_, func from uuid import uuid4 @@ -11,7 +12,7 @@ def validate_searcher_exists(redis, searcher_id): """Check whether a searcher's mandatory keys are present.""" - return redis.eval("""local session_id = redis.call("get", "searcher:"..ARGV[1]..":session_id") or "-" + return redis.eval("""local session_id = redis.call("get", "searcher:"..ARGV[1]..":session_id") or "" return { session_id, redis.call("get", "session:"..session_id), @@ -24,7 +25,7 @@ def validate_searcher_exists(redis, searcher_id): def validate_searcher_is_searching(redis, searcher_id): """Check whether a searcher's mandatory keys are present and they're in the searchers set.""" - return redis.eval("""local session_id = redis.call("get", "searcher:"..ARGV[1]..":session_id") or "-" + return redis.eval("""local session_id = redis.call("get", "searcher:"..ARGV[1]..":session_id") or "" return { redis.call("sismember", "searchers", ARGV[1]), session_id, @@ -38,7 +39,7 @@ def validate_searcher_is_searching(redis, searcher_id): def refresh_searcher(redis, searcher_id): """Reset the expiry times on a searcher's keys.""" - return redis.eval("""local session_id = redis.call("get", "searcher:"..ARGV[1]..":session_id") or "-" + return redis.eval("""local session_id = redis.call("get", "searcher:"..ARGV[1]..":session_id") or "" return { redis.call("get", "session:"..session_id), redis.call("sismember", "searchers", ARGV[1]), @@ -52,9 +53,12 @@ def refresh_searcher(redis, searcher_id): }""", 0, searcher_id) +searcher = namedtuple("searcher", ("id", "searching", "session_id", "user_id", "search_character_id", "name", "style", "levels", "filters", "choices")) + + def fetch_searcher(redis, searcher_id): """Fetch searcher keys for matching.""" - return redis.eval("""local session_id = redis.call("get", "searcher:"..ARGV[1]..":session_id") or "-" + return searcher(searcher_id, *redis.eval("""local session_id = redis.call("get", "searcher:"..ARGV[1]..":session_id") or "" return { redis.call("sismember", "searchers", ARGV[1]), session_id, @@ -63,9 +67,9 @@ def fetch_searcher(redis, searcher_id): redis.call("hget", "searcher:"..ARGV[1]..":character", "name"), redis.call("get", "searcher:"..ARGV[1]..":style"), redis.call("smembers", "searcher:"..ARGV[1]..":levels"), - redis.call("lrange", "searcher:"..ARGV[1]..":filters", 30), - redis.call("expire", "searcher:"..ARGV[1]..":choices", 30), - }""", 0, searcher_id) + redis.call("lrange", "searcher:"..ARGV[1]..":filters", 0, -1), + redis.call("smembers", "searcher:"..ARGV[1]..":choices"), + }""", 0, searcher_id)) option_messages = { diff --git a/newparp/tasks/matchmaker.py b/newparp/tasks/matchmaker.py index 5f3859c8..e404df8c 100644 --- a/newparp/tasks/matchmaker.py +++ b/newparp/tasks/matchmaker.py @@ -1,7 +1,7 @@ from celery import chord from celery.utils.log import get_task_logger -from newparp.helpers.matchmaker import run_matchmaker +from newparp.helpers.matchmaker import run_matchmaker, fetch_searcher from newparp.model import SearchedChat from newparp.tasks import celery, WorkerTask @@ -131,18 +131,47 @@ def new_searcher(searcher_id): @celery.task(base=WorkerTask, queue="matchmaker") def compare(searcher_id_1, searcher_id_2): + redis = compare.redis logger.debug("comparing %s and %s" % (searcher_id_1, searcher_id_2)) + s1 = fetch_searcher(compare.redis, searcher_id_1) + logger.debug(s1) + s2 = fetch_searcher(compare.redis, searcher_id_2) + logger.debug(s2) + + alive = True + for searcher in (s1, s2): + if not all(searcher[:-2]): + logger.debug("%s not alive" % searcher.id) + redis.srem("searchers", searcher.id) + alive = False + if not alive: + return False + + options = {} + + if ( + # Match if either person has wildcard, or if they're otherwise compatible. + (len(s2.choices) == 0 or s1.search_character_id in s2.choices) + and (len(s1.choices) == 0 or s2.search_character_id in s1.choices) + ): + # don't do this until comparison_callback + #redis.set(match_key, 1) + #redis.expire(match_key, 1800) + return True, options + + return False, None + @celery.task(base=WorkerTask, queue="matchmaker") def comparison_callback(results, searcher_id): logger.debug("match results: %s" % results) - matched_searchers = [_ for _ in results if _ is not None] + matched_searchers = [_ for _ in results if _[0] is True] if not matched_searchers: logger.debug("no results") return logger.debug("results: %s" % matched_searchers) - # TODO something + From 41ede35adb128d0aeb37734c75300ec52006b8f5 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Sun, 4 Sep 2016 19:41:04 +0100 Subject: [PATCH 24/78] Send people to chats when they're matched. --- newparp/helpers/matchmaker.py | 13 +++++-- newparp/static/js/newparp.js | 13 ++++++- newparp/tasks/matchmaker.py | 69 +++++++++++++++++++++++++++++++---- 3 files changed, 82 insertions(+), 13 deletions(-) diff --git a/newparp/helpers/matchmaker.py b/newparp/helpers/matchmaker.py index 4bfe3636..113a517b 100644 --- a/newparp/helpers/matchmaker.py +++ b/newparp/helpers/matchmaker.py @@ -53,23 +53,28 @@ def refresh_searcher(redis, searcher_id): }""", 0, searcher_id) -searcher = namedtuple("searcher", ("id", "searching", "session_id", "user_id", "search_character_id", "name", "style", "levels", "filters", "choices")) +searcher = namedtuple("searcher", ("id", "searching", "session_id", "user_id", "search_character_id", "character", "style", "levels", "filters", "choices")) def fetch_searcher(redis, searcher_id): """Fetch searcher keys for matching.""" - return searcher(searcher_id, *redis.eval("""local session_id = redis.call("get", "searcher:"..ARGV[1]..":session_id") or "" + searcher_keys = redis.eval("""local session_id = redis.call("get", "searcher:"..ARGV[1]..":session_id") or "" return { redis.call("sismember", "searchers", ARGV[1]), session_id, redis.call("get", "session:"..session_id), redis.call("get", "searcher:"..ARGV[1]..":search_character_id"), - redis.call("hget", "searcher:"..ARGV[1]..":character", "name"), + redis.call("hgetall", "searcher:"..ARGV[1]..":character"), redis.call("get", "searcher:"..ARGV[1]..":style"), redis.call("smembers", "searcher:"..ARGV[1]..":levels"), redis.call("lrange", "searcher:"..ARGV[1]..":filters", 0, -1), redis.call("smembers", "searcher:"..ARGV[1]..":choices"), - }""", 0, searcher_id)) + }""", 0, searcher_id) + # Hashes get returned as lists so we need to convert them manually. + print(searcher_keys[4]) + if searcher_keys[4]: + searcher_keys[4] = {k: v for k, v in zip(*(iter(searcher_keys[4]),) * 2)} + return searcher(searcher_id, *searcher_keys) option_messages = { diff --git a/newparp/static/js/newparp.js b/newparp/static/js/newparp.js index b3b4bb14..18ebe4de 100644 --- a/newparp/static/js/newparp.js +++ b/newparp/static/js/newparp.js @@ -591,6 +591,7 @@ var msparp = (function() { var ws, ws_interval; $.ajaxSetup({data: {"token": token}}); $.post("/" + search_type, {}, function(data) { + matched = false; body.addClass("searching"); searcher_id = data.id; ws = new WebSocket("wss://live." + location.host + "/search/" + searcher_id); @@ -598,11 +599,21 @@ var msparp = (function() { console.log("ready"); ws_interval = window.setInterval(function() { console.log("ping"); ws.send("ping"); }, 10000) } + ws.onmessage = function(e) { + var data = JSON.parse(e.data); + console.log(data); + if (data.status == "matched") { + matched = true; + window.location.href = "/" + data.url; + } else if (data.status == "quit") { + ws.close(); + } + } ws.onclose = function() { + if (matched) { return; } body.removeClass("searching").addClass("search_error"); window.clearInterval(ws_interval); } - window.ws = ws; }).error(function() { searching = false; body.removeClass("searching").addClass("search_error"); diff --git a/newparp/tasks/matchmaker.py b/newparp/tasks/matchmaker.py index e404df8c..56bed968 100644 --- a/newparp/tasks/matchmaker.py +++ b/newparp/tasks/matchmaker.py @@ -1,8 +1,13 @@ +import json + from celery import chord from celery.utils.log import get_task_logger +from random import shuffle +from sqlalchemy import and_, func, or_ +from uuid import uuid4 from newparp.helpers.matchmaker import run_matchmaker, fetch_searcher -from newparp.model import SearchedChat +from newparp.model import Block, ChatUser, SearchedChat, User from newparp.tasks import celery, WorkerTask logger = get_task_logger(__name__) @@ -134,9 +139,9 @@ def compare(searcher_id_1, searcher_id_2): redis = compare.redis logger.debug("comparing %s and %s" % (searcher_id_1, searcher_id_2)) - s1 = fetch_searcher(compare.redis, searcher_id_1) + s1 = fetch_searcher(redis, searcher_id_1) logger.debug(s1) - s2 = fetch_searcher(compare.redis, searcher_id_2) + s2 = fetch_searcher(redis, searcher_id_2) logger.debug(s2) alive = True @@ -146,7 +151,7 @@ def compare(searcher_id_1, searcher_id_2): redis.srem("searchers", searcher.id) alive = False if not alive: - return False + return None, None options = {} @@ -158,22 +163,70 @@ def compare(searcher_id_1, searcher_id_2): # don't do this until comparison_callback #redis.set(match_key, 1) #redis.expire(match_key, 1800) - return True, options + return s2.id, options - return False, None + return None, None @celery.task(base=WorkerTask, queue="matchmaker") -def comparison_callback(results, searcher_id): +def comparison_callback(results, searcher_id_1): + redis = comparison_callback.redis + db = comparison_callback.db + + # Check if there's a match. logger.debug("match results: %s" % results) - matched_searchers = [_ for _ in results if _[0] is True] + matched_searchers = [_ for _ in results if _[0] is not None] if not matched_searchers: logger.debug("no results") return logger.debug("results: %s" % matched_searchers) + shuffle(matched_searchers) + + # Fetch searcher 1. + s1 = fetch_searcher(redis, searcher_id_1) + logger.debug(s1) + if not all(s1[:-2]): + logger.debug("%s has expired" % searcher_id_1) + return + + # Pick a second searcher from the matches. + for searcher_id_2, options in matched_searchers: + s2 = fetch_searcher(redis, searcher_id_2) + logger.debug(s2) + if all(s2[:-2]) and db.query(func.count("*")).select_from(Block).filter(or_( + and_(Block.blocking_user_id == s1.user_id, Block.blocked_user_id == s2.user_id), + and_(Block.blocking_user_id == s2.user_id, Block.blocked_user_id == s1.user_id), + )).scalar() == 0: + logger.debug("matched %s" % searcher_id_2) + break + else: + logger.debug("all matches have expired") + return + + new_url = str(uuid4()).replace("-", "") + logger.info("matched %s and %s, sending to %s." % (s1.id, s2.id, new_url)) + new_chat = SearchedChat(url=new_url) + db.add(new_chat) + db.flush() + s1_user = db.query(User).filter(User.id == s1.user_id).one() + s2_user = db.query(User).filter(User.id == s2.user_id).one() + db.add(ChatUser.from_user(s1_user, chat_id=new_chat.id, number=1, search_character_id=s1.search_character_id, **s1.character)) + if s1_user != s2_user: + db.add(ChatUser.from_user(s2_user, chat_id=new_chat.id, number=2, search_character_id=s2.search_character_id, **s2.character)) + if options: + db.add(Message( + chat_id=new_chat.id, + type="search_info", + text=" ".join(option_messages[_] for _ in options), + )) + db.commit() + match_message = json.dumps({ "status": "matched", "url": new_url }) + redis.publish("searcher:%s" % s1.id, match_message) + redis.publish("searcher:%s" % s2.id, match_message) + redis.srem("searchers", s1.id, s2.id) From 0a08ac13fe4c87980a6984ff3f743e73e95210d4 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Tue, 6 Sep 2016 22:58:24 +0100 Subject: [PATCH 25/78] Copy the rest of the comparisons. --- newparp/helpers/matchmaker.py | 5 ++- newparp/tasks/matchmaker.py | 70 ++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/newparp/helpers/matchmaker.py b/newparp/helpers/matchmaker.py index 113a517b..99127f89 100644 --- a/newparp/helpers/matchmaker.py +++ b/newparp/helpers/matchmaker.py @@ -70,10 +70,11 @@ def fetch_searcher(redis, searcher_id): redis.call("lrange", "searcher:"..ARGV[1]..":filters", 0, -1), redis.call("smembers", "searcher:"..ARGV[1]..":choices"), }""", 0, searcher_id) - # Hashes get returned as lists so we need to convert them manually. - print(searcher_keys[4]) + # Hashes and sets get returned as lists so we need to convert them manually. if searcher_keys[4]: searcher_keys[4] = {k: v for k, v in zip(*(iter(searcher_keys[4]),) * 2)} + searcher_keys[6] = set(searcher_keys[6]) + searcher_keys[8] = set(searcher_keys[8]) return searcher(searcher_id, *searcher_keys) diff --git a/newparp/tasks/matchmaker.py b/newparp/tasks/matchmaker.py index 56bed968..dc783859 100644 --- a/newparp/tasks/matchmaker.py +++ b/newparp/tasks/matchmaker.py @@ -6,8 +6,8 @@ from sqlalchemy import and_, func, or_ from uuid import uuid4 -from newparp.helpers.matchmaker import run_matchmaker, fetch_searcher -from newparp.model import Block, ChatUser, SearchedChat, User +from newparp.helpers.matchmaker import option_messages, run_matchmaker, fetch_searcher +from newparp.model import Block, ChatUser, Message, SearchedChat, User from newparp.tasks import celery, WorkerTask logger = get_task_logger(__name__) @@ -140,9 +140,7 @@ def compare(searcher_id_1, searcher_id_2): logger.debug("comparing %s and %s" % (searcher_id_1, searcher_id_2)) s1 = fetch_searcher(redis, searcher_id_1) - logger.debug(s1) s2 = fetch_searcher(redis, searcher_id_2) - logger.debug(s2) alive = True for searcher in (s1, s2): @@ -153,7 +151,52 @@ def compare(searcher_id_1, searcher_id_2): if not alive: return None, None - options = {} + # Don't pair people with themselves. + if s1.user_id == s2.user_id: + return None, None + + # Don't match if they've already been paired up recently. + match_key = "matched:%s:%s" % tuple(sorted([s1.user_id, s2.user_id])) + if redis.exists(match_key): + return None, None + + options = [] + + # Style options should be matched with themselves or "either". + if s1.style != "either" and s2.style != "either" and s1.style != s2.style: + return None, None + if s1.style != "either": + options.append(s1.style) + elif s2.style != "either": + options.append(s2.style) + + # Levels have to overlap. + levels_in_common = s1.levels & s2.levels + logger.debug("Levels in common: %s" % levels_in_common) + if levels_in_common: + options.append( + "nsfw-extreme" if "nsfw-extreme" in levels_in_common + else "nsfw" if "nsfw" in levels_in_common + else "sfw" + ) + else: + return None, None + + # Check filters. + s1_name = s1.character["name"].lower().encode("utf8") + for search_filter in s2.filters: + search_filter = search_filter.encode("utf8") + logger.debug("comparing %s and %s" % (s1_name, search_filter)) + if search_filter in s1_name: + logger.debug("FILTER %s MATCHED" % search_filter) + return None, None + s2_name = s2.character["name"].lower().encode("utf8") + for search_filter in s1.filters: + search_filter = search_filter.encode("utf8") + logger.debug("comparing %s and %s" % (s2_name, search_filter)) + if search_filter in s2_name: + logger.debug("FILTER %s MATCHED" % search_filter) + return None, None if ( # Match if either person has wildcard, or if they're otherwise compatible. @@ -161,8 +204,6 @@ def compare(searcher_id_1, searcher_id_2): and (len(s1.choices) == 0 or s2.search_character_id in s1.choices) ): # don't do this until comparison_callback - #redis.set(match_key, 1) - #redis.expire(match_key, 1800) return s2.id, options return None, None @@ -174,7 +215,6 @@ def comparison_callback(results, searcher_id_1): db = comparison_callback.db # Check if there's a match. - logger.debug("match results: %s" % results) matched_searchers = [_ for _ in results if _[0] is not None] if not matched_searchers: logger.debug("no results") @@ -184,7 +224,6 @@ def comparison_callback(results, searcher_id_1): # Fetch searcher 1. s1 = fetch_searcher(redis, searcher_id_1) - logger.debug(s1) if not all(s1[:-2]): logger.debug("%s has expired" % searcher_id_1) return @@ -192,7 +231,6 @@ def comparison_callback(results, searcher_id_1): # Pick a second searcher from the matches. for searcher_id_2, options in matched_searchers: s2 = fetch_searcher(redis, searcher_id_2) - logger.debug(s2) if all(s2[:-2]) and db.query(func.count("*")).select_from(Block).filter(or_( and_(Block.blocking_user_id == s1.user_id, Block.blocked_user_id == s2.user_id), and_(Block.blocking_user_id == s2.user_id, Block.blocked_user_id == s1.user_id), @@ -224,9 +262,13 @@ def comparison_callback(results, searcher_id_1): db.commit() + pipe = redis.pipeline() + match_key = "matched:%s:%s" % tuple(sorted([s1.user_id, s2.user_id])) + pipe.set(match_key, 1) + pipe.expire(match_key, 1800) + pipe.srem("searchers", s1.id, s2.id) match_message = json.dumps({ "status": "matched", "url": new_url }) - redis.publish("searcher:%s" % s1.id, match_message) - redis.publish("searcher:%s" % s2.id, match_message) - - redis.srem("searchers", s1.id, s2.id) + pipe.publish("searcher:%s" % s1.id, match_message) + pipe.publish("searcher:%s" % s2.id, match_message) + pipe.execute() From 935a61152e877e47eeb21a1298b18c9deffc60d2 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Tue, 6 Sep 2016 23:21:00 +0100 Subject: [PATCH 26/78] Remove the old matchmaker tasks. --- newparp/tasks/config.py | 15 +++-- newparp/tasks/matchmaker.py | 108 +----------------------------------- 2 files changed, 9 insertions(+), 114 deletions(-) diff --git a/newparp/tasks/config.py b/newparp/tasks/config.py index cecc7a59..a5eba225 100644 --- a/newparp/tasks/config.py +++ b/newparp/tasks/config.py @@ -4,7 +4,7 @@ from kombu import Exchange, Queue # Debug -if 'DEBUG' in os.environ: +if "DEBUG" in os.environ: CELERY_REDIRECT_STDOUTS_LEVEL = "DEBUG" # Broker and Result backends @@ -28,13 +28,16 @@ CELERY_DISABLE_RATE_LIMITS = True # Queue config -CELERY_DEFAULT_QUEUE = 'default' +CELERY_DEFAULT_QUEUE = "default" CELERY_QUEUES = ( # Default queue - Queue('default', Exchange('default'), routing_key='default'), + Queue("default", Exchange("default"), routing_key="default"), # Worker queue - Queue('worker', Exchange('worker'), routing_key='worker', delivery_mode=1), + Queue("worker", Exchange("worker"), routing_key="worker", delivery_mode=1), + + # Matchmaker queue + Queue("matchmaker", Exchange("matchmaker"), routing_key="matchmaker", delivery_mode=1), ) # Beats config @@ -55,10 +58,6 @@ "task": "newparp.tasks.background.update_user_meta", "schedule": timedelta(seconds=5), }, - "matchmaker": { - "task": "newparp.tasks.matchmaker.run", - "schedule": timedelta(seconds=10), - }, "roulette_matchmaker": { "task": "newparp.tasks.roulette_matchmaker.run", "schedule": timedelta(seconds=10), diff --git a/newparp/tasks/matchmaker.py b/newparp/tasks/matchmaker.py index dc783859..b75c621b 100644 --- a/newparp/tasks/matchmaker.py +++ b/newparp/tasks/matchmaker.py @@ -1,119 +1,15 @@ -import json - from celery import chord from celery.utils.log import get_task_logger from random import shuffle from sqlalchemy import and_, func, or_ from uuid import uuid4 -from newparp.helpers.matchmaker import option_messages, run_matchmaker, fetch_searcher +from newparp.helpers.matchmaker import fetch_searcher, option_messages from newparp.model import Block, ChatUser, Message, SearchedChat, User from newparp.tasks import celery, WorkerTask logger = get_task_logger(__name__) -def get_searcher_info(redis, searcher_ids): - searchers = [] - for searcher_id in searcher_ids: - session_id = redis.get("searcher:%s:session_id" % searcher_id) - # This will fail if they've logged out since sending the request. - try: - user_id = int(redis.get("session:%s" % session_id)) - search_character_id = int(redis.get("searcher:%s:search_character_id" % searcher_id)) - except (TypeError, ValueError): - continue - searchers.append({ - "id": searcher_id, - "user_id": user_id, - "search_character_id": search_character_id, - "character": redis.hgetall("searcher:%s:character" % searcher_id), - "style": redis.get("searcher:%s:style" % searcher_id), - "levels": redis.smembers("searcher:%s:levels" % searcher_id), - "filters": redis.lrange("searcher:%s:filters" % searcher_id, 0, -1), - "choices": {int(_) for _ in redis.smembers("searcher:%s:choices" % searcher_id)}, - }) - return searchers - - -def check_compatibility(redis, s1, s2): - - # Don't pair people with themselves. - if s1["user_id"] == s2["user_id"]: - return False, None - - # Don't match if they've already been paired up recently. - match_key = "matched:%s:%s" % tuple(sorted([s1["user_id"], s2["user_id"]])) - if redis.exists(match_key): - return False, None - - options = [] - - # Style options should be matched with themselves or "either". - if ( - s1["style"] != "either" - and s2["style"] != "either" - and s1["style"] != s2["style"] - ): - return False, None - if s1["style"] != "either": - options.append(s1["style"]) - elif s2["style"] != "either": - options.append(s2["style"]) - - # Levels have to overlap. - levels_in_common = s1["levels"] & s2["levels"] - logger.debug("Levels in common: %s" % levels_in_common) - if levels_in_common: - options.append( - "nsfw-extreme" if "nsfw-extreme" in levels_in_common - else "nsfw" if "nsfw" in levels_in_common - else "sfw" - ) - else: - return False, None - - # Check filters. - s1_name = s1["character"]["name"].lower().encode("utf8") - for search_filter in s2["filters"]: - search_filter = search_filter.encode("utf8") - logger.debug("comparing %s and %s" % (s1_name, search_filter)) - if search_filter in s1_name: - logger.debug("FILTER %s MATCHED" % search_filter) - return False, None - s2_name = s2["character"]["name"].lower().encode("utf8") - for search_filter in s1["filters"]: - search_filter = search_filter.encode("utf8") - logger.debug("comparing %s and %s" % (s2_name, search_filter)) - if search_filter in s2_name: - logger.debug("FILTER %s MATCHED" % search_filter) - return False, None - - if ( - # Match if either person has wildcard, or if they're otherwise compatible. - (len(s2["choices"]) == 0 or s1["search_character_id"] in s2["choices"]) - and (len(s1["choices"]) == 0 or s2["search_character_id"] in s1["choices"]) - ): - redis.set(match_key, 1) - redis.expire(match_key, 1800) - return True, options - - return False, None - - -def get_character_info(db, searcher): - return searcher["character"] - - -@celery.task(base=WorkerTask, queue="worker") -def run(): - db = run.db - redis = run.redis - - run_matchmaker( - db, redis, 2, "searchers", "searcher", get_searcher_info, - check_compatibility, SearchedChat, get_character_info, - ) - @celery.task(base=WorkerTask, queue="matchmaker") def new_searcher(searcher_id): @@ -267,7 +163,7 @@ def comparison_callback(results, searcher_id_1): pipe.set(match_key, 1) pipe.expire(match_key, 1800) pipe.srem("searchers", s1.id, s2.id) - match_message = json.dumps({ "status": "matched", "url": new_url }) + match_message = """{"status":"matched","url":"%s"}""" % new_url pipe.publish("searcher:%s" % s1.id, match_message) pipe.publish("searcher:%s" % s2.id, match_message) pipe.execute() From 8b1323c5b0b092d2c3ef837855b5723410dff10f Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Tue, 6 Sep 2016 23:53:17 +0100 Subject: [PATCH 27/78] Lock the matchmaker. --- newparp/tasks/matchmaker.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/newparp/tasks/matchmaker.py b/newparp/tasks/matchmaker.py index b75c621b..65a5b82e 100644 --- a/newparp/tasks/matchmaker.py +++ b/newparp/tasks/matchmaker.py @@ -13,9 +13,14 @@ @celery.task(base=WorkerTask, queue="matchmaker") def new_searcher(searcher_id): - # TODO lock + redis = new_searcher.redis + if redis.exists("lock:matchmaker"): + new_searcher.apply_async((searcher_id,), countdown=2) + return + logger.debug("new searcher: %s") - searchers = new_searcher.redis.smembers("searchers") + searchers = redis.smembers("searchers") + try: searchers.remove(searcher_id) except KeyError: @@ -24,11 +29,14 @@ def new_searcher(searcher_id): if not searchers: logger.debug("not enough searchers, skipping") return + chord( (compare.s(searcher_id, _) for _ in searchers if _ != searcher_id), comparison_callback.s(searcher_id), ).delay() + redis.setex("lock:matchmaker", 60, 1) + @celery.task(base=WorkerTask, queue="matchmaker") def compare(searcher_id_1, searcher_id_2): @@ -114,6 +122,7 @@ def comparison_callback(results, searcher_id_1): matched_searchers = [_ for _ in results if _[0] is not None] if not matched_searchers: logger.debug("no results") + redis.delete("lock:matchmaker") return logger.debug("results: %s" % matched_searchers) shuffle(matched_searchers) @@ -122,6 +131,7 @@ def comparison_callback(results, searcher_id_1): s1 = fetch_searcher(redis, searcher_id_1) if not all(s1[:-2]): logger.debug("%s has expired" % searcher_id_1) + redis.delete("lock:matchmaker") return # Pick a second searcher from the matches. @@ -135,6 +145,7 @@ def comparison_callback(results, searcher_id_1): break else: logger.debug("all matches have expired") + redis.delete("lock:matchmaker") return new_url = str(uuid4()).replace("-", "") @@ -166,5 +177,6 @@ def comparison_callback(results, searcher_id_1): match_message = """{"status":"matched","url":"%s"}""" % new_url pipe.publish("searcher:%s" % s1.id, match_message) pipe.publish("searcher:%s" % s2.id, match_message) + pipe.delete("lock:matchmaker") pipe.execute() From 2274775324ba29485c8d2897e09d139af95baaaa Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Tue, 6 Sep 2016 23:59:41 +0100 Subject: [PATCH 28/78] Remove old search views. --- newparp/__init__.py | 2 -- newparp/views/search.py | 49 ----------------------------------------- 2 files changed, 51 deletions(-) diff --git a/newparp/__init__.py b/newparp/__init__.py index f6681929..76c30972 100644 --- a/newparp/__init__.py +++ b/newparp/__init__.py @@ -151,8 +151,6 @@ def make_rules(subdomain, path, func, formats=False, paging=False): app.add_url_rule("/search/save", "rp_search_save", search.search_save, methods=("POST",)) app.add_url_rule("/search", "rp_search", search.search_get, methods=("GET",)) app.add_url_rule("/search", "rp_search_post", search.search_post, methods=("POST",)) -app.add_url_rule("/search/continue", "rp_search_continue", search.search_continue, methods=("POST",)) -app.add_url_rule("/search/stop", "rp_search_stop", search.search_stop, methods=("POST",)) app.add_url_rule("/roulette/save", "rp_roulette_save", roulette.roulette_save, methods=("POST",)) app.add_url_rule("/roulette", "rp_roulette", roulette.roulette_get, methods=("GET",)) diff --git a/newparp/views/search.py b/newparp/views/search.py index 122b26b5..c717c66e 100644 --- a/newparp/views/search.py +++ b/newparp/views/search.py @@ -127,52 +127,3 @@ def search_get(): def search_post(): return jsonify({ "id": _create_searcher() }) - -def search_continue(): - - searcher_id = request.form["id"][:36] - cached_session_id = g.redis.get("searcher:%s:session_id" % searcher_id) - - # Send people back to /search if we don't have their data cached. - if g.user_id is None or cached_session_id != g.session_id: - abort(404) - - g.redis.expire("searcher:%s:session_id" % searcher_id, 30) - g.redis.expire("searcher:%s:search_character_id" % searcher_id, 30) - g.redis.expire("searcher:%s:character" % searcher_id, 30) - g.redis.expire("searcher:%s:style" % searcher_id, 30) - g.redis.expire("searcher:%s:levels" % searcher_id, 30) - g.redis.expire("searcher:%s:filters" % searcher_id, 30) - g.redis.expire("searcher:%s:choices" % searcher_id, 30) - - g.pubsub = g.redis.pubsub() - g.pubsub.subscribe("searcher:%s" % searcher_id) - - g.redis.sadd("searchers", searcher_id) - - try: - while True: - msg = g.pubsub.get_message(timeout=30) - if not msg or msg["type"] == "message": - break - - if msg: - # The pubsub channel sends us a JSON string, so we return that - # instead of using jsonify. - resp = make_response(msg["data"]) - resp.headers["Content-type"] = "application/json" - return resp - else: - return jsonify({"status": "unmatched"}) - finally: - g.pubsub.close() - -@use_db -@log_in_required -def search_stop(): - searcher_id = request.form["id"][:36] - g.redis.srem("searchers", searcher_id) - # Kill the long poll request. - g.redis.publish("searcher:%s" % searcher_id, "{\"status\":\"quit\"}") - return "", 204 - From 5132939c462316e25077643d4196521bf71b153f Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Thu, 8 Sep 2016 21:54:40 +0100 Subject: [PATCH 29/78] Remove tasks, views and templates for roulette. --- newparp/__init__.py | 8 +-- newparp/tasks/__init__.py | 1 - newparp/tasks/config.py | 4 -- newparp/tasks/roulette_matchmaker.py | 87 ----------------------- newparp/templates/home_roulette.html | 25 ------- newparp/templates/roulette.html | 10 --- newparp/views/__init__.py | 43 ++++-------- newparp/views/roulette.py | 100 --------------------------- 8 files changed, 16 insertions(+), 262 deletions(-) delete mode 100644 newparp/tasks/roulette_matchmaker.py delete mode 100644 newparp/templates/home_roulette.html delete mode 100644 newparp/templates/roulette.html delete mode 100644 newparp/views/roulette.py diff --git a/newparp/__init__.py b/newparp/__init__.py index 76c30972..c2c3e196 100644 --- a/newparp/__init__.py +++ b/newparp/__init__.py @@ -61,7 +61,7 @@ from newparp import views from newparp.views import ( account, admin, characters, chat, chat_api, chat_list, errors, guides, - roulette, search, search_characters, settings, + search, search_characters, settings, ) from newparp.views.admin import spamless, spamless2 @@ -152,12 +152,6 @@ def make_rules(subdomain, path, func, formats=False, paging=False): app.add_url_rule("/search", "rp_search", search.search_get, methods=("GET",)) app.add_url_rule("/search", "rp_search_post", search.search_post, methods=("POST",)) -app.add_url_rule("/roulette/save", "rp_roulette_save", roulette.roulette_save, methods=("POST",)) -app.add_url_rule("/roulette", "rp_roulette", roulette.roulette_get, methods=("GET",)) -app.add_url_rule("/roulette", "rp_roulette_post", roulette.roulette_post, methods=("POST",)) -app.add_url_rule("/roulette/continue", "rp_roulette_continue", roulette.roulette_continue, methods=("POST",)) -app.add_url_rule("/roulette/stop", "rp_roulette_stop", roulette.roulette_stop, methods=("POST",)) - # 6. Groups make_rules("rp", "/groups", views.groups, formats=True) diff --git a/newparp/tasks/__init__.py b/newparp/tasks/__init__.py index 0c644fdb..72a490b6 100644 --- a/newparp/tasks/__init__.py +++ b/newparp/tasks/__init__.py @@ -13,7 +13,6 @@ "newparp.tasks.background", "newparp.tasks.matchmaker", "newparp.tasks.reaper", - "newparp.tasks.roulette_matchmaker", "newparp.tasks.chat", "newparp.tasks.test", ]) diff --git a/newparp/tasks/config.py b/newparp/tasks/config.py index a5eba225..eb1392bf 100644 --- a/newparp/tasks/config.py +++ b/newparp/tasks/config.py @@ -58,10 +58,6 @@ "task": "newparp.tasks.background.update_user_meta", "schedule": timedelta(seconds=5), }, - "roulette_matchmaker": { - "task": "newparp.tasks.roulette_matchmaker.run", - "schedule": timedelta(seconds=10), - }, "ping_longpolls": { "task": "newparp.tasks.reaper.ping_longpolls", "schedule": timedelta(seconds=5), diff --git a/newparp/tasks/roulette_matchmaker.py b/newparp/tasks/roulette_matchmaker.py deleted file mode 100644 index 62a56cd6..00000000 --- a/newparp/tasks/roulette_matchmaker.py +++ /dev/null @@ -1,87 +0,0 @@ -from sqlalchemy.orm.exc import NoResultFound - -from newparp.helpers.matchmaker import run_matchmaker -from newparp.model import Character, RouletteChat, SearchCharacter -from newparp.tasks import celery, WorkerTask - -def get_searcher_info(redis, searcher_ids): - searchers = [] - for searcher_id in searcher_ids: - session_id = redis.get("roulette:%s:session_id" % searcher_id) - # This will fail if they've logged out since sending the request. - try: - searcher = { - "id": searcher_id, - "user_id": int(redis.get("session:%s" % session_id)), - "search_character_id": int(redis.get("roulette:%s:search_character_id" % searcher_id)) - } - character_id = redis.get("roulette:%s:character_id" % searcher_id) - if character_id is not None: - searcher["character_id"] = int(character_id) - except (TypeError, ValueError): - continue - searchers.append(searcher) - return searchers - - -def check_compatibility(redis, s1, s2): - # Don't pair people with themselves. - if s1["user_id"] == s2["user_id"]: - return False, ("roulette",) - # Don't match if they've already been paired up recently. - match_key = "matched:%s:%s" % tuple(sorted([s1["user_id"], s2["user_id"]])) - if redis.exists(match_key): - return False, ("roulette",) - redis.set(match_key, 1) - redis.expire(match_key, 1800) - return True, ("roulette",) - - -def get_character_info(db, searcher): - # Use character if it exists. - if "character_id" in searcher: - try: - character = db.query(Character).filter( - Character.id == searcher["character_id"], - Character.user_id == searcher["user_id"], - ).one() - except NoResultFound: - return {} - return { - "name": character.name, - "acronym": character.acronym, - "color": character.color, - "quirk_prefix": character.quirk_prefix, - "quirk_suffix": character.quirk_suffix, - "case": character.case, - "replacements": character.replacements, - "regexes": character.regexes, - } - # Otherwise use search character. - try: - search_character = db.query(SearchCharacter).filter( - SearchCharacter.id == searcher["search_character_id"], - ).one() - except NoResultFound: - return {} - return { - "name": search_character.name, - "acronym": search_character.acronym, - "color": search_character.color, - "quirk_prefix": search_character.quirk_prefix, - "quirk_suffix": search_character.quirk_suffix, - "case": search_character.case, - "replacements": search_character.replacements, - "regexes": search_character.regexes, - } - -@celery.task(base=WorkerTask, queue="worker") -def run(): - db = run.db - redis = run.redis - - run_matchmaker( - db, redis, 3, "roulette_searchers", "roulette", get_searcher_info, - check_compatibility, RouletteChat, get_character_info, - ) - diff --git a/newparp/templates/home_roulette.html b/newparp/templates/home_roulette.html deleted file mode 100644 index 78c1522b..00000000 --- a/newparp/templates/home_roulette.html +++ /dev/null @@ -1,25 +0,0 @@ -{% extends "base.html" %} -{% block content: %} - - -

Roulette

-

This is MSPARP's new roulette feature. If you want to use filters to select which characters you connect to, you can still do so with character search. Otherwise, you can select your character (it's optional) and click Enter. You'll be randomly matched with another user but both of you will appear anonymously to each other.

-
- - -{% endblock %} diff --git a/newparp/templates/roulette.html b/newparp/templates/roulette.html deleted file mode 100644 index 8d90aa39..00000000 --- a/newparp/templates/roulette.html +++ /dev/null @@ -1,10 +0,0 @@ -{% extends "base.html" %} -{% block title: %}Roulette - {% endblock %} -{% block content: %} -
Connecting...
-
-
Sorry, there seems to be a problem with searching. Please refresh or try again later. :(
-{% endblock %} -{% block scripts: %} - -{% endblock %} diff --git a/newparp/views/__init__.py b/newparp/views/__init__.py index 0444dadd..7aab8363 100644 --- a/newparp/views/__init__.py +++ b/newparp/views/__init__.py @@ -17,40 +17,27 @@ def home(): if g.user is None: return render_template("home_guest.html") - mode = request.args.get("mode", g.user.last_search_mode) - g.user.last_search_mode = mode - characters = g.db.query(Character).filter(Character.user_id == g.user.id).order_by(Character.title).all() search_character_groups = g.db.query(SearchCharacterGroup).order_by( SearchCharacterGroup.order, ).options(joinedload(SearchCharacterGroup.characters)).all() - if mode == "roulette": - return render_template( - "home_roulette.html", - characters=characters, - search_character_groups=search_character_groups, - ) - - elif mode == "search": - picky = set(_[0] for _ in g.db.query( - SearchCharacterChoice.search_character_id, - ).filter( - SearchCharacterChoice.user_id == g.user.id, - ).all()) - return render_template( - "home_search.html", - characters=characters, - search_character_groups=search_character_groups, - case_options=case_options, - replacements=json.loads(g.user.replacements), - regexes=json.loads(g.user.regexes), - User=User, - picky=picky, - ) - - abort(404) + picky = set(_[0] for _ in g.db.query( + SearchCharacterChoice.search_character_id, + ).filter( + SearchCharacterChoice.user_id == g.user.id, + ).all()) + return render_template( + "home_search.html", + characters=characters, + search_character_groups=search_character_groups, + case_options=case_options, + replacements=json.loads(g.user.replacements), + regexes=json.loads(g.user.regexes), + User=User, + picky=picky, + ) @alt_formats({"json"}) diff --git a/newparp/views/roulette.py b/newparp/views/roulette.py deleted file mode 100644 index 782e0bd1..00000000 --- a/newparp/views/roulette.py +++ /dev/null @@ -1,100 +0,0 @@ -from flask import abort, g, jsonify, make_response, redirect, render_template, request, url_for -from sqlalchemy.orm.exc import NoResultFound -from uuid import uuid4 - -from newparp.helpers.auth import log_in_required -from newparp.model import Character, SearchCharacter -from newparp.model.connections import use_db - - -@use_db -@log_in_required -def roulette_save(): - try: - form_id = int(request.form["id"][2:]) - except ValueError: - abort(404) - # Character - if request.form["id"][0] == "c": - try: - character = g.db.query(Character).filter( - Character.id == form_id, - ).one() - except NoResultFound: - abort(404) - g.user.roulette_search_character_id = 1 - g.user.roulette_character = character - # Search character - elif request.form["id"][0] == "s": - try: - search_character = g.db.query(SearchCharacter).filter( - SearchCharacter.id == form_id, - ).one() - except NoResultFound: - abort(404) - g.user.roulette_search_character = search_character - g.user.roulette_character = None - else: - abort(400) - return redirect(url_for("rp_roulette")) - - -@use_db -@log_in_required -def roulette_get(): - return render_template("roulette.html") - - -@use_db -@log_in_required -def roulette_post(): - searcher_id = str(uuid4()) - g.redis.set("roulette:%s:session_id" % searcher_id, g.session_id) - g.redis.set("roulette:%s:search_character_id" % searcher_id, g.user.roulette_search_character_id) - if g.user.roulette_character_id is not None: - g.redis.set("roulette:%s:character_id" % searcher_id, g.user.roulette_character_id) - g.redis.expire("roulette:%s:session_id" % searcher_id, 30) - g.redis.expire("roulette:%s:search_character_id" % searcher_id, 30) - g.redis.expire("roulette:%s:character_id" % searcher_id, 30) - return jsonify({ "id": searcher_id }) - - -def roulette_continue(): - searcher_id = request.form["id"][:36] - cached_session_id = g.redis.get("roulette:%s:session_id" % searcher_id) - # Send people back to /roulette if we don't have their data cached. - if g.user_id is None or cached_session_id != g.session_id: - abort(404) - g.redis.expire("roulette:%s:session_id" % searcher_id, 30) - g.redis.expire("roulette:%s:search_character_id" % searcher_id, 30) - g.redis.expire("roulette:%s:character_id" % searcher_id, 30) - g.pubsub = g.redis.pubsub() - g.pubsub.subscribe("roulette:%s" % searcher_id) - g.redis.sadd("roulette_searchers", searcher_id) - try: - while True: - msg = g.pubsub.get_message(timeout=30) - if not msg or msg["type"] == "message": - break - - if msg: - # The pubsub channel sends us a JSON string, so we return that - # instead of using jsonify. - resp = make_response(msg["data"]) - resp.headers["Content-type"] = "application/json" - return resp - else: - return jsonify({"status": "unmatched"}) - finally: - g.pubsub.close() - -@use_db -@log_in_required -def roulette_stop(): - searcher_id = request.form["id"][:36] - g.redis.srem("roulette_searchers", searcher_id) - # Kill the long poll request. - g.redis.publish("roulette:%s" % searcher_id, "{\"status\":\"quit\"}") - return "", 204 - - From 635998e32b578d45cd36bb49e0ef7483d0aa9926 Mon Sep 17 00:00:00 2001 From: Olly Parton Date: Fri, 9 Sep 2016 22:56:39 +0100 Subject: [PATCH 30/78] Remove roulette from the navigation. --- newparp/__init__.py | 2 +- newparp/templates/base.html | 3 +-- newparp/templates/chat/chat.html | 9 +++------ newparp/templates/chat/log.html | 3 +-- newparp/templates/chat/log_day.html | 3 +-- newparp/views/__init__.py | 5 ++++- 6 files changed, 11 insertions(+), 14 deletions(-) diff --git a/newparp/__init__.py b/newparp/__init__.py index c2c3e196..ea5cae05 100644 --- a/newparp/__init__.py +++ b/newparp/__init__.py @@ -177,7 +177,7 @@ def make_rules(subdomain, path, func, formats=False, paging=False): app.add_url_rule("//subscribe", "rp_chat_subscribe", chat.subscribe, methods=("POST",)) app.add_url_rule("//unsubscribe", "rp_chat_unsubscribe", chat.unsubscribe, methods=("POST",)) -app.add_url_rule("/redirect", "redirect", views.redirect, methods=("GET",)) +app.add_url_rule("/redirect", "redirect", views.redirect_view, methods=("GET",)) # 8. Chat API diff --git a/newparp/templates/base.html b/newparp/templates/base.html index f961cf80..e43af798 100644 --- a/newparp/templates/base.html +++ b/newparp/templates/base.html @@ -64,8 +64,7 @@