Redo jsmods loading and fb_dtsg retrieving

fbchat/_client.py

@@ -474,10 +474,13 @@ class Client:
         j = self.session._post("/mercury/attachments/photo/", data)
         _exception.handle_payload_error(j)
 
-        url = _util.get_jsmods_require(j, 3)
+        if "jsmods" not in j:
-        if url is None:
+            raise _exception.ParseError("No jsmods when fetching image URL", data=j)
+        require = _util.get_jsmods_require(j["jsmods"]["require"])
+        if "ServerRedirect.redirectPageTo" not in require:
             raise _exception.ParseError("Could not fetch image URL", data=j)
-        return url
+        # Return the first argument
+        return require["ServerRedirect.redirectPageTo"][0]
 
     def _get_private_data(self):
         (j,) = self.session._graphql_requests(

fbchat/_session.py

@@ -373,6 +373,15 @@ class Session:
     def _payload_post(self, url, data, files=None):
         j = self._post(url, data, files=files)
         _exception.handle_payload_error(j)
+
+        # update fb_dtsg token if received in response
+        if "jsmods" in j:
+            define = _util.get_jsmods_define(j["jsmods"]["define"])
+            if "DTSGInitData" in define:
+                self._fb_dtsg = define["DTSGInitData"]["token"]
+            elif "DTSGInitialData" in define:
+                self._fb_dtsg = define["DTSGInitialData"]["token"]
+
         try:
             return j["payload"]
         except (KeyError, TypeError) as e:
@@ -436,11 +445,6 @@ class Session:
 
         _exception.handle_payload_error(j)
 
-        # update JS token if received in response
-        fb_dtsg = _util.get_jsmods_require(j, 2)
-        if fb_dtsg is not None:
-            self._fb_dtsg = fb_dtsg
-
         try:
             message_ids = [
                 (action["message_id"], action["thread_fbid"])

fbchat/_util.py

@@ -7,7 +7,7 @@ import urllib.parse
 from ._common import log
 from . import _exception
 
-from typing import Iterable, Optional, Any
+from typing import Iterable, Optional, Any, Mapping, Sequence
 
 #: Default list of user agents
 USER_AGENTS = [
@@ -66,16 +66,26 @@ def generate_offline_threading_id():
     return str(int(msgs, 2))
 
 
-def get_jsmods_require(j, index):
+def get_jsmods_require(require) -> Mapping[str, Sequence[Any]]:
-    if j.get("jsmods") and j["jsmods"].get("require"):
+    rtn = {}
-        try:
+    for item in require:
-            return j["jsmods"]["require"][0][index][0]
+        if len(item) == 1:
-        except (KeyError, IndexError) as e:
+            (module,) = item
-            log.warning(
+            rtn[module] = []
-                "Error when getting jsmods_require: "
+            continue
-                "{}. Facebook might have changed protocol".format(j)
+        method = "{}.{}".format(item[0], item[1])
-            )
+        requirements = item[2]
-    return None
+        arguments = item[3]
+        rtn[method] = arguments
+    return rtn
+
+
+def get_jsmods_define(define) -> Mapping[str, Mapping[str, Any]]:
+    rtn = {}
+    for item in define:
+        module, requirements, data, _ = item
+        rtn[module] = data
+    return rtn
 
 
 def mimetype_to_key(mimetype: str) -> str:

tests/test_util.py

@@ -5,6 +5,7 @@ from fbchat._util import (
     strip_json_cruft,
     parse_json,
     get_jsmods_require,
+    get_jsmods_define,
     mimetype_to_key,
     get_url_parameter,
     seconds_to_datetime,
@@ -38,41 +39,134 @@ def test_parse_json_invalid():
         parse_json("No JSON object here!")
 
 
-def test_get_jsmods_require_get_image_url():
+def test_get_jsmods_require():
-    data = {
+    argument = {
-        "__ar": 1,
+        "signalsToCollect": [
-        "payload": None,
+            30000,
-        "jsmods": {
+            30001,
-            "require": [
+            30003,
-                [
+            30004,
-                    "ServerRedirect",
+            30005,
-                    "redirectPageTo",
+            30002,
-                    [],
+            30007,
-                    [
+            30008,
-                        "https://scontent-arn2-1.xx.fbcdn.net/v/image.png&dl=1",
+            30009,
-                        False,
+        ]
-                        False,
-                    ],
-                ],
-                ["TuringClientSignalCollectionTrigger", ..., [], ...],
-                ["TuringClientSignalCollectionTrigger", "retrieveSignals", [], ...],
-                ["BanzaiODS"],
-                ["BanzaiScuba"],
-            ],
-            "define": ...,
-        },
-        "js": ...,
-        "css": ...,
-        "bootloadable": ...,
-        "resource_map": ...,
-        "ixData": {},
-        "bxData": {},
-        "gkxData": ...,
-        "qexData": {},
-        "lid": "123",
     }
+    data = [
+        ["BanzaiODS"],
+        [
+            "TuringClientSignalCollectionTrigger",
+            "startStaticSignalCollection",
+            [],
+            [argument],
+        ],
+    ]
+    assert get_jsmods_require(data) == {
+        "BanzaiODS": [],
+        "TuringClientSignalCollectionTrigger.startStaticSignalCollection": [argument],
+    }
+
+
+def test_get_jsmods_require_get_image_url():
+    data = [
+        [
+            "ServerRedirect",
+            "redirectPageTo",
+            [],
+            ["https://scontent-arn2-1.xx.fbcdn.net/v/image.png&dl=1", False, False],
+        ],
+        ["TuringClientSignalCollectionTrigger", "...", [], [...]],
+        ["TuringClientSignalCollectionTrigger", "retrieveSignals", [], [...]],
+        ["BanzaiODS"],
+        ["BanzaiScuba"],
+    ]
     url = "https://scontent-arn2-1.xx.fbcdn.net/v/image.png&dl=1"
-    assert get_jsmods_require(data, 3) == url
+    assert get_jsmods_require(data)["ServerRedirect.redirectPageTo"][0] == url
+
+
+def test_get_jsmods_define():
+    data = [
+        [
+            "BootloaderConfig",
+            [],
+            {
+                "jsRetries": [200, 500],
+                "jsRetryAbortNum": 2,
+                "jsRetryAbortTime": 5,
+                "payloadEndpointURI": "https://www.facebook.com/ajax/bootloader-endpoint/",
+                "preloadBE": False,
+                "assumeNotNonblocking": True,
+                "shouldCoalesceModuleRequestsMadeInSameTick": True,
+                "staggerJsDownloads": False,
+                "preloader_num_preloads": 0,
+                "preloader_preload_after_dd": False,
+                "preloader_num_loads": 1,
+                "preloader_enabled": False,
+                "retryQueuedBootloads": False,
+                "silentDups": False,
+                "asyncPreloadBoost": True,
+            },
+            123,
+        ],
+        [
+            "CSSLoaderConfig",
+            [],
+            {"timeout": 5000, "modulePrefix": "BLCSS:", "loadEventSupported": True},
+            456,
+        ],
+        ["CurrentCommunityInitialData", [], {}, 789],
+        [
+            "CurrentEnvironment",
+            [],
+            {"facebookdotcom": True, "messengerdotcom": False},
+            987,
+        ],
+    ]
+    assert get_jsmods_define(data) == {
+        "BootloaderConfig": {
+            "jsRetries": [200, 500],
+            "jsRetryAbortNum": 2,
+            "jsRetryAbortTime": 5,
+            "payloadEndpointURI": "https://www.facebook.com/ajax/bootloader-endpoint/",
+            "preloadBE": False,
+            "assumeNotNonblocking": True,
+            "shouldCoalesceModuleRequestsMadeInSameTick": True,
+            "staggerJsDownloads": False,
+            "preloader_num_preloads": 0,
+            "preloader_preload_after_dd": False,
+            "preloader_num_loads": 1,
+            "preloader_enabled": False,
+            "retryQueuedBootloads": False,
+            "silentDups": False,
+            "asyncPreloadBoost": True,
+        },
+        "CSSLoaderConfig": {
+            "timeout": 5000,
+            "modulePrefix": "BLCSS:",
+            "loadEventSupported": True,
+        },
+        "CurrentCommunityInitialData": {},
+        "CurrentEnvironment": {"facebookdotcom": True, "messengerdotcom": False},
+    }
+
+
+def test_get_jsmods_define_get_fb_dtsg():
+    data = [
+        ["DTSGInitialData", [], {"token": "AQG-abcdefgh:AQGijklmnopq"}, 258],
+        [
+            "DTSGInitData",
+            [],
+            {"token": "AQG-abcdefgh:AQGijklmnopq", "async_get_token": "ABC123:DEF456"},
+            3515,
+        ],
+    ]
+    jsmods = get_jsmods_define(data)
+    assert (
+        jsmods["DTSGInitData"]["token"]
+        == jsmods["DTSGInitialData"]["token"]
+        == "AQG-abcdefgh:AQGijklmnopq"
+    )
 
 
 def test_mimetype_to_key():