From e6ec5c519459bcbd7fcfd537b0836110f5191ad6 Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Wed, 11 Mar 2020 11:14:57 +0100 Subject: [PATCH] Redo jsmods loading and fb_dtsg retrieving --- fbchat/_client.py | 9 ++- fbchat/_session.py | 14 ++-- fbchat/_util.py | 32 +++++---- tests/test_util.py | 160 +++++++++++++++++++++++++++++++++++---------- 4 files changed, 163 insertions(+), 52 deletions(-) diff --git a/fbchat/_client.py b/fbchat/_client.py index bca4428..a2672cb 100644 --- a/fbchat/_client.py +++ b/fbchat/_client.py @@ -474,10 +474,13 @@ class Client: j = self.session._post("/mercury/attachments/photo/", data) _exception.handle_payload_error(j) - url = _util.get_jsmods_require(j, 3) - if url is None: + if "jsmods" not in j: + raise _exception.ParseError("No jsmods when fetching image URL", data=j) + require = _util.get_jsmods_require(j["jsmods"]["require"]) + if "ServerRedirect.redirectPageTo" not in require: raise _exception.ParseError("Could not fetch image URL", data=j) - return url + # Return the first argument + return require["ServerRedirect.redirectPageTo"][0] def _get_private_data(self): (j,) = self.session._graphql_requests( diff --git a/fbchat/_session.py b/fbchat/_session.py index 2bedefc..03f062e 100644 --- a/fbchat/_session.py +++ b/fbchat/_session.py @@ -373,6 +373,15 @@ class Session: def _payload_post(self, url, data, files=None): j = self._post(url, data, files=files) _exception.handle_payload_error(j) + + # update fb_dtsg token if received in response + if "jsmods" in j: + define = _util.get_jsmods_define(j["jsmods"]["define"]) + if "DTSGInitData" in define: + self._fb_dtsg = define["DTSGInitData"]["token"] + elif "DTSGInitialData" in define: + self._fb_dtsg = define["DTSGInitialData"]["token"] + try: return j["payload"] except (KeyError, TypeError) as e: @@ -436,11 +445,6 @@ class Session: _exception.handle_payload_error(j) - # update JS token if received in response - fb_dtsg = _util.get_jsmods_require(j, 2) - if fb_dtsg is not None: - self._fb_dtsg = fb_dtsg - try: message_ids = [ (action["message_id"], action["thread_fbid"]) diff --git a/fbchat/_util.py b/fbchat/_util.py index bd96a3b..f5b1420 100644 --- a/fbchat/_util.py +++ b/fbchat/_util.py @@ -7,7 +7,7 @@ import urllib.parse from ._common import log from . import _exception -from typing import Iterable, Optional, Any +from typing import Iterable, Optional, Any, Mapping, Sequence #: Default list of user agents USER_AGENTS = [ @@ -66,16 +66,26 @@ def generate_offline_threading_id(): return str(int(msgs, 2)) -def get_jsmods_require(j, index): - if j.get("jsmods") and j["jsmods"].get("require"): - try: - return j["jsmods"]["require"][0][index][0] - except (KeyError, IndexError) as e: - log.warning( - "Error when getting jsmods_require: " - "{}. Facebook might have changed protocol".format(j) - ) - return None +def get_jsmods_require(require) -> Mapping[str, Sequence[Any]]: + rtn = {} + for item in require: + if len(item) == 1: + (module,) = item + rtn[module] = [] + continue + method = "{}.{}".format(item[0], item[1]) + requirements = item[2] + arguments = item[3] + rtn[method] = arguments + return rtn + + +def get_jsmods_define(define) -> Mapping[str, Mapping[str, Any]]: + rtn = {} + for item in define: + module, requirements, data, _ = item + rtn[module] = data + return rtn def mimetype_to_key(mimetype: str) -> str: diff --git a/tests/test_util.py b/tests/test_util.py index aae1343..8214d3a 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -5,6 +5,7 @@ from fbchat._util import ( strip_json_cruft, parse_json, get_jsmods_require, + get_jsmods_define, mimetype_to_key, get_url_parameter, seconds_to_datetime, @@ -38,41 +39,134 @@ def test_parse_json_invalid(): parse_json("No JSON object here!") -def test_get_jsmods_require_get_image_url(): - data = { - "__ar": 1, - "payload": None, - "jsmods": { - "require": [ - [ - "ServerRedirect", - "redirectPageTo", - [], - [ - "https://scontent-arn2-1.xx.fbcdn.net/v/image.png&dl=1", - False, - False, - ], - ], - ["TuringClientSignalCollectionTrigger", ..., [], ...], - ["TuringClientSignalCollectionTrigger", "retrieveSignals", [], ...], - ["BanzaiODS"], - ["BanzaiScuba"], - ], - "define": ..., - }, - "js": ..., - "css": ..., - "bootloadable": ..., - "resource_map": ..., - "ixData": {}, - "bxData": {}, - "gkxData": ..., - "qexData": {}, - "lid": "123", +def test_get_jsmods_require(): + argument = { + "signalsToCollect": [ + 30000, + 30001, + 30003, + 30004, + 30005, + 30002, + 30007, + 30008, + 30009, + ] } + data = [ + ["BanzaiODS"], + [ + "TuringClientSignalCollectionTrigger", + "startStaticSignalCollection", + [], + [argument], + ], + ] + assert get_jsmods_require(data) == { + "BanzaiODS": [], + "TuringClientSignalCollectionTrigger.startStaticSignalCollection": [argument], + } + + +def test_get_jsmods_require_get_image_url(): + data = [ + [ + "ServerRedirect", + "redirectPageTo", + [], + ["https://scontent-arn2-1.xx.fbcdn.net/v/image.png&dl=1", False, False], + ], + ["TuringClientSignalCollectionTrigger", "...", [], [...]], + ["TuringClientSignalCollectionTrigger", "retrieveSignals", [], [...]], + ["BanzaiODS"], + ["BanzaiScuba"], + ] url = "https://scontent-arn2-1.xx.fbcdn.net/v/image.png&dl=1" - assert get_jsmods_require(data, 3) == url + assert get_jsmods_require(data)["ServerRedirect.redirectPageTo"][0] == url + + +def test_get_jsmods_define(): + data = [ + [ + "BootloaderConfig", + [], + { + "jsRetries": [200, 500], + "jsRetryAbortNum": 2, + "jsRetryAbortTime": 5, + "payloadEndpointURI": "https://www.facebook.com/ajax/bootloader-endpoint/", + "preloadBE": False, + "assumeNotNonblocking": True, + "shouldCoalesceModuleRequestsMadeInSameTick": True, + "staggerJsDownloads": False, + "preloader_num_preloads": 0, + "preloader_preload_after_dd": False, + "preloader_num_loads": 1, + "preloader_enabled": False, + "retryQueuedBootloads": False, + "silentDups": False, + "asyncPreloadBoost": True, + }, + 123, + ], + [ + "CSSLoaderConfig", + [], + {"timeout": 5000, "modulePrefix": "BLCSS:", "loadEventSupported": True}, + 456, + ], + ["CurrentCommunityInitialData", [], {}, 789], + [ + "CurrentEnvironment", + [], + {"facebookdotcom": True, "messengerdotcom": False}, + 987, + ], + ] + assert get_jsmods_define(data) == { + "BootloaderConfig": { + "jsRetries": [200, 500], + "jsRetryAbortNum": 2, + "jsRetryAbortTime": 5, + "payloadEndpointURI": "https://www.facebook.com/ajax/bootloader-endpoint/", + "preloadBE": False, + "assumeNotNonblocking": True, + "shouldCoalesceModuleRequestsMadeInSameTick": True, + "staggerJsDownloads": False, + "preloader_num_preloads": 0, + "preloader_preload_after_dd": False, + "preloader_num_loads": 1, + "preloader_enabled": False, + "retryQueuedBootloads": False, + "silentDups": False, + "asyncPreloadBoost": True, + }, + "CSSLoaderConfig": { + "timeout": 5000, + "modulePrefix": "BLCSS:", + "loadEventSupported": True, + }, + "CurrentCommunityInitialData": {}, + "CurrentEnvironment": {"facebookdotcom": True, "messengerdotcom": False}, + } + + +def test_get_jsmods_define_get_fb_dtsg(): + data = [ + ["DTSGInitialData", [], {"token": "AQG-abcdefgh:AQGijklmnopq"}, 258], + [ + "DTSGInitData", + [], + {"token": "AQG-abcdefgh:AQGijklmnopq", "async_get_token": "ABC123:DEF456"}, + 3515, + ], + ] + jsmods = get_jsmods_define(data) + assert ( + jsmods["DTSGInitData"]["token"] + == jsmods["DTSGInitialData"]["token"] + == "AQG-abcdefgh:AQGijklmnopq" + ) def test_mimetype_to_key():