From 40969f0b6f3b5cf17546eb4953badeb8bba5718f Mon Sep 17 00:00:00 2001 From: rjdang Date: Sun, 27 Jan 2019 15:13:23 -0800 Subject: [PATCH 1/4] fix for hangouts auth, attempted fix for slack error --- README.rst | 38 ++++++++++++++++++++++++++++++- chat_archive/backends/hangouts.py | 7 +++++- chat_archive/backends/slack.py | 2 +- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index bad1d9c..85ac47f 100644 --- a/README.rst +++ b/README.rst @@ -121,7 +121,7 @@ Supported commands: errors during every synchronization run :-)." "``-c``, ``--color=CHOICE,`` ``--colour=CHOICE``","Specify whether ANSI escape sequences for text and background colors and text styles are to be used or not, depending on the value of ``CHOICE``: - + - The values 'always', 'true', 'yes' and '1' enable colors. - The values 'never', 'false', 'no' and '0' disable colors. - When the value is 'auto' (this is the default) then colors will @@ -405,6 +405,42 @@ In the future more backends may be added: generated ten years ago (circa 2008) because I have megabytes of such chat logs stored in backups 🙂. + +Google Hangouts Notes ++++++++++++++++++++++ +Google Hangouts has recently changed their authorization flow and it cannot easily be automated. Here are the steps to manually authorize Google Hangouts: +1. Install hangups (https://github.com/tdryer/hangups). I recommend using the latest git version due to the fragile nature of its reverse engineered protocol: + +.. code-block:: bash +pip install 'git+https://github.com/tdryer/hangups' + +---- + +2. To authorize hangups, you must include an oauth cookie. Run the following python script, open the link in a browser, log in to Google, open the Developer Tools (Ctrl+Shift+i in Firefox), go to the "Storage" tab, and copy the cookie labeled "oauth token". Paste this token string into the terminal prompt. Now hangups will be authorized with a token in `$HOME/.cache/hangups/refresh_token.txt`, and chat_archive will read the token from there. + +.. code-block:: python +#!/usr/bin/env python3 + +import os, hangups, requests, appdirs + +print('Open this URL:') +print(hangups.auth.OAUTH2_LOGIN_URL) + +authorization_code = input('Enter oauth_code cookie value: ') + +with requests.Session() as session: + session.headers = {'user-agent': hangups.auth.USER_AGENT} + access_token, refresh_token = hangups.auth._auth_with_code( + session, authorization_code + ) + +dirs = appdirs.AppDirs('hangups', 'hangups') +token_path = os.path.join(dirs.user_cache_dir, 'refresh_token.txt') +hangups.auth.RefreshTokenCache(token_path).set(refresh_token) + +---- + + History ------- diff --git a/chat_archive/backends/hangouts.py b/chat_archive/backends/hangouts.py index ab8b591..d4bc7f4 100644 --- a/chat_archive/backends/hangouts.py +++ b/chat_archive/backends/hangouts.py @@ -12,6 +12,7 @@ import html import os import time +import appdirs # External dependencies. import hangups @@ -70,6 +71,10 @@ def client(self): """The hangups client object.""" # Make sure the directory with cached credentials exists. ensure_directory_exists(os.path.dirname(self.cookie_file)) + + dirs = appdirs.AppDirs('hangups', 'hangups') + token_path = os.path.join(dirs.user_cache_dir, 'refresh_token.txt') + return Client( get_auth( GoogleAccountCredentials( @@ -81,7 +86,7 @@ def client(self): description="Google account password", ), ), - RefreshTokenCache(self.cookie_file), + RefreshTokenCache(token_path) ) ) diff --git a/chat_archive/backends/slack.py b/chat_archive/backends/slack.py index c322e4d..8c75428 100644 --- a/chat_archive/backends/slack.py +++ b/chat_archive/backends/slack.py @@ -135,7 +135,7 @@ def import_messages(self, source, conversation_in_db): external_id=message["ts"], html=html, raw=message["text"], - sender=self.get_or_create_contact(external_id=message["user"]), + sender=self.get_or_create_contact(external_id=message.get("user")), text=html_to_text(html), timestamp=datetime.datetime.utcfromtimestamp(float(message["ts"])), ) From f1382cd153fba6e23a5961840b04f93999e7236c Mon Sep 17 00:00:00 2001 From: rjdang Date: Sun, 27 Jan 2019 15:18:07 -0800 Subject: [PATCH 2/4] rst lint fix --- README.rst | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/README.rst b/README.rst index 85ac47f..66db59b 100644 --- a/README.rst +++ b/README.rst @@ -407,36 +407,36 @@ In the future more backends may be added: Google Hangouts Notes -+++++++++++++++++++++ +--------------------- Google Hangouts has recently changed their authorization flow and it cannot easily be automated. Here are the steps to manually authorize Google Hangouts: -1. Install hangups (https://github.com/tdryer/hangups). I recommend using the latest git version due to the fragile nature of its reverse engineered protocol: +1) Install hangups (https://github.com/tdryer/hangups). I recommend using the latest git version due to the fragile nature of its reverse engineered protocol: .. code-block:: bash -pip install 'git+https://github.com/tdryer/hangups' + pip install 'git+https://github.com/tdryer/hangups' ---- -2. To authorize hangups, you must include an oauth cookie. Run the following python script, open the link in a browser, log in to Google, open the Developer Tools (Ctrl+Shift+i in Firefox), go to the "Storage" tab, and copy the cookie labeled "oauth token". Paste this token string into the terminal prompt. Now hangups will be authorized with a token in `$HOME/.cache/hangups/refresh_token.txt`, and chat_archive will read the token from there. +2) To authorize hangups, you must include an oauth cookie. Run the following python script, open the link in a browser, log in to Google, open the Developer Tools (Ctrl+Shift+i in Firefox), go to the "Storage" tab, and copy the cookie labeled "oauth token". Paste this token string into the terminal prompt. Now hangups will be authorized with a token in `$HOME/.cache/hangups/refresh_token.txt`, and chat_archive will read the token from there. .. code-block:: python -#!/usr/bin/env python3 + #!/usr/bin/env python3 -import os, hangups, requests, appdirs + import os, hangups, requests, appdirs -print('Open this URL:') -print(hangups.auth.OAUTH2_LOGIN_URL) + print('Open this URL:') + print(hangups.auth.OAUTH2_LOGIN_URL) -authorization_code = input('Enter oauth_code cookie value: ') + authorization_code = input('Enter oauth_code cookie value: ') -with requests.Session() as session: - session.headers = {'user-agent': hangups.auth.USER_AGENT} - access_token, refresh_token = hangups.auth._auth_with_code( - session, authorization_code - ) + with requests.Session() as session: + session.headers = {'user-agent': hangups.auth.USER_AGENT} + access_token, refresh_token = hangups.auth._auth_with_code( + session, authorization_code + ) -dirs = appdirs.AppDirs('hangups', 'hangups') -token_path = os.path.join(dirs.user_cache_dir, 'refresh_token.txt') -hangups.auth.RefreshTokenCache(token_path).set(refresh_token) + dirs = appdirs.AppDirs('hangups', 'hangups') + token_path = os.path.join(dirs.user_cache_dir, 'refresh_token.txt') + hangups.auth.RefreshTokenCache(token_path).set(refresh_token) ---- From fb6fef6b3b5c0832a22313c35db8fdf4b118074c Mon Sep 17 00:00:00 2001 From: rjdang Date: Sun, 27 Jan 2019 15:19:43 -0800 Subject: [PATCH 3/4] rst lint fix --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 66db59b..f77c72e 100644 --- a/README.rst +++ b/README.rst @@ -409,9 +409,13 @@ In the future more backends may be added: Google Hangouts Notes --------------------- Google Hangouts has recently changed their authorization flow and it cannot easily be automated. Here are the steps to manually authorize Google Hangouts: + +---- + 1) Install hangups (https://github.com/tdryer/hangups). I recommend using the latest git version due to the fragile nature of its reverse engineered protocol: .. code-block:: bash + pip install 'git+https://github.com/tdryer/hangups' ---- @@ -419,6 +423,7 @@ Google Hangouts has recently changed their authorization flow and it cannot easi 2) To authorize hangups, you must include an oauth cookie. Run the following python script, open the link in a browser, log in to Google, open the Developer Tools (Ctrl+Shift+i in Firefox), go to the "Storage" tab, and copy the cookie labeled "oauth token". Paste this token string into the terminal prompt. Now hangups will be authorized with a token in `$HOME/.cache/hangups/refresh_token.txt`, and chat_archive will read the token from there. .. code-block:: python + #!/usr/bin/env python3 import os, hangups, requests, appdirs From b8d8fc204bbb9c066d36b975571d3954442b6f81 Mon Sep 17 00:00:00 2001 From: rjdang Date: Sun, 27 Jan 2019 16:16:57 -0800 Subject: [PATCH 4/4] telegram and slack fixes --- chat_archive/backends/slack.py | 3 +++ chat_archive/backends/telegram.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/chat_archive/backends/slack.py b/chat_archive/backends/slack.py index 8c75428..f946c23 100644 --- a/chat_archive/backends/slack.py +++ b/chat_archive/backends/slack.py @@ -10,6 +10,7 @@ import datetime import decimal import html +import time # External dependencies. from humanfriendly import Spinner @@ -139,6 +140,8 @@ def import_messages(self, source, conversation_in_db): text=html_to_text(html), timestamp=datetime.datetime.utcfromtimestamp(float(message["ts"])), ) + + time.sleep(0.01) if not conversation_in_db.import_complete: conversation_in_db.import_complete = True diff --git a/chat_archive/backends/telegram.py b/chat_archive/backends/telegram.py index 91c48cc..248b0ac 100644 --- a/chat_archive/backends/telegram.py +++ b/chat_archive/backends/telegram.py @@ -15,6 +15,7 @@ # Standard library modules. import asyncio import os +import pytz # External dependencies. from property_manager import lazy_property, mutable_property, required_property @@ -121,7 +122,8 @@ async def connect_then_sync(self): ) if not conversation_in_db.import_complete: await self.perform_initial_sync(dialog, conversation_in_db) - elif dialog.date > conversation_in_db.last_modified: + + elif dialog.date > conversation_in_db.last_modified.replace(tzinfo=pytz.utc): logger.info("Conversation was updated (%s) ..", dialog.id) await self.update_conversation(dialog, conversation_in_db) conversation_in_db.last_modified = dialog.date