Bump version: 1.8.2 → 1.8.3

`mimetypes.guess_type` fails if the url is something like `http://example.com/file.zip?u=10`.

Backported from 6bffb66

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.8.0
+current_version = 1.8.3
 commit = True
 tag = True
 

fbchat/__init__.py

@@ -13,7 +13,7 @@ from ._client import Client
 from ._util import log  # TODO: Remove this (from examples too)
 
 __title__ = "fbchat"
-__version__ = "1.8.0"
+__version__ = "1.8.3"
 __description__ = "Facebook Chat (Messenger) for Python"
 
 __copyright__ = "Copyright 2015 - 2019 by Taehoon Kim"

fbchat/_client.py

@@ -87,7 +87,6 @@ class Client(object):
         """
         self._sticky, self._pool = (None, None)
         self._seq = "0"
-        self._client_id = hex(int(random() * 2 ** 31))[2:]
         self._default_thread_id = None
         self._default_thread_type = None
         self._pull_channel = 0
@@ -108,57 +107,14 @@ class Client(object):
     INTERNAL REQUEST METHODS
     """
 
-    def _do_refresh(self):
-        # TODO: Raise the error instead, and make the user do the refresh manually
-        # It may be a bad idea to do this in an exception handler, if you have a better method, please suggest it!
-        log.warning("Refreshing state and resending request")
-        self._state = State.from_session(session=self._state._session)
+    def _get(self, url, params):
+        return self._state._get(url, params)
 
-    def _get(self, url, params, error_retries=3):
-        params.update(self._state.get_params())
-        r = self._state._session.get(prefix_url(url), params=params)
-        content = check_request(r)
-        j = to_json(content)
-        try:
-            handle_payload_error(j)
-        except FBchatPleaseRefresh:
-            if error_retries > 0:
-                self._do_refresh()
-                return self._get(url, params, error_retries=error_retries - 1)
-            raise
-        return j
-
-    def _post(self, url, data, files=None, as_graphql=False, error_retries=3):
-        data.update(self._state.get_params())
-        r = self._state._session.post(prefix_url(url), data=data, files=files)
-        content = check_request(r)
-        try:
-            if as_graphql:
-                return _graphql.response_to_json(content)
-            else:
-                j = to_json(content)
-                # TODO: Remove this, and move it to _payload_post instead
-                # We can't yet, since errors raised in here need to be caught below
-                handle_payload_error(j)
-                return j
-        except FBchatPleaseRefresh:
-            if error_retries > 0:
-                self._do_refresh()
-                return self._post(
-                    url,
-                    data,
-                    files=files,
-                    as_graphql=as_graphql,
-                    error_retries=error_retries - 1,
-                )
-            raise
+    def _post(self, url, params, files=None):
+        return self._state._post(url, params, files=files)
 
     def _payload_post(self, url, data, files=None):
-        j = self._post(url, data, files=files)
-        try:
-            return j["payload"]
-        except (KeyError, TypeError):
-            raise FBchatException("Missing payload: {}".format(j))
+        return self._state._payload_post(url, data, files=files)
 
     def graphql_requests(self, *queries):
         """Execute GraphQL queries.
@@ -172,12 +128,7 @@ class Client(object):
         Raises:
             FBchatException: If request failed
         """
-        data = {
-            "method": "GET",
-            "response_format": "json",
-            "queries": _graphql.queries_to_json(*queries),
-        }
-        return tuple(self._post("/api/graphqlbatch/", data, as_graphql=True))
+        return tuple(self._state._graphql_requests(*queries))
 
     def graphql_request(self, query):
         """Shorthand for ``graphql_requests(query)[0]``.
@@ -222,16 +173,11 @@ class Client(object):
         """
         try:
             # Load cookies into current session
-            state = State.from_cookies(session_cookies, user_agent=user_agent)
+            self._state = State.from_cookies(session_cookies, user_agent=user_agent)
+            self._uid = self._state.user_id
         except Exception as e:
             log.exception("Failed loading session")
             return False
-        uid = state.get_user_id()
-        if uid is None:
-            log.warning("Could not find c_user cookie")
-            return False
-        self._state = state
-        self._uid = uid
         return True
 
     def login(self, email, password, max_tries=5, user_agent=None):
@@ -257,23 +203,19 @@ class Client(object):
 
         for i in range(1, max_tries + 1):
             try:
-                state = State.login(
+                self._state = State.login(
                     email,
                     password,
                     on_2fa_callback=self.on2FACode,
                     user_agent=user_agent,
                 )
-                uid = state.get_user_id()
-                if uid is None:
-                    raise FBchatException("Could not find user id")
+                self._uid = self._state.user_id
             except Exception:
                 if i >= max_tries:
                     raise
                 log.exception("Attempt #{} failed, retrying".format(i))
                 time.sleep(1)
             else:
-                self._state = state
-                self._uid = uid
                 self.onLoggedIn(email=email)
                 break
 
@@ -1089,101 +1031,13 @@ class Client(object):
     def _oldMessage(self, message):
         return message if isinstance(message, Message) else Message(text=message)
 
-    def _getSendData(self, message=None, thread_id=None, thread_type=ThreadType.USER):
-        """Return the data needed to send a request to `SendURL`."""
-        messageAndOTID = generateOfflineThreadingID()
-        timestamp = now()
-        data = {
-            "client": "mercury",
-            "author": "fbid:{}".format(self._uid),
-            "timestamp": timestamp,
-            "source": "source:chat:web",
-            "offline_threading_id": messageAndOTID,
-            "message_id": messageAndOTID,
-            "threading_id": generateMessageID(self._client_id),
-            "ephemeral_ttl_mode:": "0",
-        }
-
-        # Set recipient
-        if thread_type in [ThreadType.USER, ThreadType.PAGE]:
-            data["other_user_fbid"] = thread_id
-        elif thread_type == ThreadType.GROUP:
-            data["thread_fbid"] = thread_id
-
-        if message is None:
-            message = Message()
-
-        if message.text or message.sticker or message.emoji_size:
-            data["action_type"] = "ma-type:user-generated-message"
-
-        if message.text:
-            data["body"] = message.text
-
-        for i, mention in enumerate(message.mentions):
-            data["profile_xmd[{}][id]".format(i)] = mention.thread_id
-            data["profile_xmd[{}][offset]".format(i)] = mention.offset
-            data["profile_xmd[{}][length]".format(i)] = mention.length
-            data["profile_xmd[{}][type]".format(i)] = "p"
-
-        if message.emoji_size:
-            if message.text:
-                data["tags[0]"] = "hot_emoji_size:" + message.emoji_size.name.lower()
-            else:
-                data["sticker_id"] = message.emoji_size.value
-
-        if message.sticker:
-            data["sticker_id"] = message.sticker.uid
-
-        if message.quick_replies:
-            xmd = {"quick_replies": []}
-            for quick_reply in message.quick_replies:
-                q = dict()
-                q["content_type"] = quick_reply._type
-                q["payload"] = quick_reply.payload
-                q["external_payload"] = quick_reply.external_payload
-                q["data"] = quick_reply.data
-                if quick_reply.is_response:
-                    q["ignore_for_webhook"] = False
-                if isinstance(quick_reply, QuickReplyText):
-                    q["title"] = quick_reply.title
-                if not isinstance(quick_reply, QuickReplyLocation):
-                    q["image_url"] = quick_reply.image_url
-                xmd["quick_replies"].append(q)
-            if len(message.quick_replies) == 1 and message.quick_replies[0].is_response:
-                xmd["quick_replies"] = xmd["quick_replies"][0]
-            data["platform_xmd"] = json.dumps(xmd)
-
-        if message.reply_to_id:
-            data["replied_to_message_id"] = message.reply_to_id
-
-        return data
-
     def _doSendRequest(self, data, get_thread_id=False):
         """Send the data to `SendURL`, and returns the message ID or None on failure."""
-        j = self._post("/messaging/send/", data)
-
-        # update JS token if received in response
-        fb_dtsg = get_jsmods_require(j, 2)
-        if fb_dtsg is not None:
-            self._state.fb_dtsg = fb_dtsg
-
-        try:
-            message_ids = [
-                (action["message_id"], action["thread_fbid"])
-                for action in j["payload"]["actions"]
-                if "message_id" in action
-            ]
-            if len(message_ids) != 1:
-                log.warning("Got multiple message ids' back: {}".format(message_ids))
-            if get_thread_id:
-                return message_ids[0]
-            else:
-                return message_ids[0][0]
-        except (KeyError, IndexError, TypeError) as e:
-            raise FBchatException(
-                "Error when sending message: "
-                "No message IDs could be found: {}".format(j)
-            )
+        mid, thread_id = self._state._do_send_request(data)
+        if get_thread_id:
+            return mid, thread_id
+        else:
+            return mid
 
     def send(self, message, thread_id=None, thread_type=ThreadType.USER):
         """Send message to a thread.
@@ -1200,9 +1054,9 @@ class Client(object):
             FBchatException: If request failed
         """
         thread_id, thread_type = self._getThread(thread_id, thread_type)
-        data = self._getSendData(
-            message=message, thread_id=thread_id, thread_type=thread_type
-        )
+        thread = thread_type._to_class()(thread_id)
+        data = thread._to_send_data()
+        data.update(message._to_send_data())
         return self._doSendRequest(data)
 
     def sendMessage(self, message, thread_id=None, thread_type=ThreadType.USER):
@@ -1240,7 +1094,8 @@ class Client(object):
             FBchatException: If request failed
         """
         thread_id, thread_type = self._getThread(thread_id, thread_type)
-        data = self._getSendData(thread_id=thread_id, thread_type=thread_type)
+        thread = thread_type._to_class()(thread_id)
+        data = thread._to_send_data()
         data["action_type"] = "ma-type:user-generated-message"
         data["lightweight_action_attachment[lwa_state]"] = (
             "INITIATED" if wave_first else "RECIPROCATED"
@@ -1304,9 +1159,10 @@ class Client(object):
         self, location, current=True, message=None, thread_id=None, thread_type=None
     ):
         thread_id, thread_type = self._getThread(thread_id, thread_type)
-        data = self._getSendData(
-            message=message, thread_id=thread_id, thread_type=thread_type
-        )
+        thread = thread_type._to_class()(thread_id)
+        data = thread._to_send_data()
+        if message is not None:
+            data.update(message._to_send_data())
         data["action_type"] = "ma-type:user-generated-message"
         data["location_attachment[coordinates][latitude]"] = location.latitude
         data["location_attachment[coordinates][longitude]"] = location.longitude
@@ -1362,30 +1218,7 @@ class Client(object):
         )
 
     def _upload(self, files, voice_clip=False):
-        """Upload files to Facebook.
-
-        `files` should be a list of files that requests can upload, see
-        `requests.request <https://docs.python-requests.org/en/master/api/#requests.request>`_.
-
-        Return a list of tuples with a file's ID and mimetype.
-        """
-        file_dict = {"upload_{}".format(i): f for i, f in enumerate(files)}
-
-        data = {"voice_clip": voice_clip}
-
-        j = self._payload_post(
-            "https://upload.facebook.com/ajax/mercury/upload.php", data, files=file_dict
-        )
-
-        if len(j["metadata"]) != len(files):
-            raise FBchatException(
-                "Some files could not be uploaded: {}, {}".format(j, files)
-            )
-
-        return [
-            (data[mimetype_to_key(data["filetype"])], data["filetype"])
-            for data in j["metadata"]
-        ]
+        return self._state._upload(files, voice_clip=voice_clip)
 
     def _sendFiles(
         self, files, message=None, thread_id=None, thread_type=ThreadType.USER
@@ -1395,12 +1228,9 @@ class Client(object):
         `files` should be a list of tuples, with a file's ID and mimetype.
         """
         thread_id, thread_type = self._getThread(thread_id, thread_type)
-        data = self._getSendData(
-            message=self._oldMessage(message),
-            thread_id=thread_id,
-            thread_type=thread_type,
-        )
-
+        thread = thread_type._to_class()(thread_id)
+        data = thread._to_send_data()
+        data.update(self._oldMessage(message)._to_send_data())
         data["action_type"] = "ma-type:user-generated-message"
         data["has_attachment"] = True
 
@@ -1580,7 +1410,7 @@ class Client(object):
         Raises:
             FBchatException: If request failed
         """
-        data = self._getSendData(message=self._oldMessage(message))
+        data = self._oldMessage(message)._to_send_data()
 
         if len(user_ids) < 2:
             raise FBchatUserError("Error when creating group: Not enough participants")
@@ -1606,7 +1436,7 @@ class Client(object):
             FBchatException: If request failed
         """
         thread_id, thread_type = self._getThread(thread_id, None)
-        data = self._getSendData(thread_id=thread_id, thread_type=ThreadType.GROUP)
+        data = Group(thread_id)._to_send_data()
 
         data["action_type"] = "ma-type:log-message"
         data["log_message_type"] = "log:subscribe"
@@ -2341,7 +2171,7 @@ class Client(object):
         data = {
             "seq": self._seq,
             "channel": "p_" + self._uid,
-            "clientid": self._client_id,
+            "clientid": self._state._client_id,
             "partition": -2,
             "cap": 0,
             "uid": self._uid,
@@ -2362,7 +2192,7 @@ class Client(object):
             "msgs_recv": 0,
             "sticky_token": self._sticky,
             "sticky_pool": self._pool,
-            "clientid": self._client_id,
+            "clientid": self._state._client_id,
             "state": "active" if self._markAlive else "offline",
         }
         return self._get(

fbchat/_group.py

@@ -104,6 +104,9 @@ class Group(Thread):
             plan=plan,
         )
 
+    def _to_send_data(self):
+        return {"thread_fbid": self.uid}
+
 
 @attr.s(cmp=False, init=False)
 class Room(Group):

fbchat/_message.py

@@ -26,7 +26,7 @@ class EmojiSize(Enum):
             "s": cls.SMALL,
         }
         for tag in tags or ():
-            data = tag.split(":", maxsplit=1)
+            data = tag.split(":", 1)
             if len(data) > 1 and data[0] == "hot_emoji_size":
                 return string_to_emojisize.get(data[1])
         return None
@@ -151,6 +151,55 @@ class Message(object):
             return False
         return any(map(lambda tag: "forward" in tag or "copy" in tag, tags))
 
+    def _to_send_data(self):
+        data = {}
+
+        if self.text or self.sticker or self.emoji_size:
+            data["action_type"] = "ma-type:user-generated-message"
+
+        if self.text:
+            data["body"] = self.text
+
+        for i, mention in enumerate(self.mentions):
+            data["profile_xmd[{}][id]".format(i)] = mention.thread_id
+            data["profile_xmd[{}][offset]".format(i)] = mention.offset
+            data["profile_xmd[{}][length]".format(i)] = mention.length
+            data["profile_xmd[{}][type]".format(i)] = "p"
+
+        if self.emoji_size:
+            if self.text:
+                data["tags[0]"] = "hot_emoji_size:" + self.emoji_size.name.lower()
+            else:
+                data["sticker_id"] = self.emoji_size.value
+
+        if self.sticker:
+            data["sticker_id"] = self.sticker.uid
+
+        if self.quick_replies:
+            xmd = {"quick_replies": []}
+            for quick_reply in self.quick_replies:
+                # TODO: Move this to `_quick_reply.py`
+                q = dict()
+                q["content_type"] = quick_reply._type
+                q["payload"] = quick_reply.payload
+                q["external_payload"] = quick_reply.external_payload
+                q["data"] = quick_reply.data
+                if quick_reply.is_response:
+                    q["ignore_for_webhook"] = False
+                if isinstance(quick_reply, _quick_reply.QuickReplyText):
+                    q["title"] = quick_reply.title
+                if not isinstance(quick_reply, _quick_reply.QuickReplyLocation):
+                    q["image_url"] = quick_reply.image_url
+                xmd["quick_replies"].append(q)
+            if len(self.quick_replies) == 1 and self.quick_replies[0].is_response:
+                xmd["quick_replies"] = xmd["quick_replies"][0]
+            data["platform_xmd"] = json.dumps(xmd)
+
+        if self.reply_to_id:
+            data["replied_to_message_id"] = self.reply_to_id
+
+        return data
+
     @classmethod
     def _from_graphql(cls, data):
         if data.get("message_sender") is None:

fbchat/_state.py

@@ -7,11 +7,19 @@ import re
 import requests
 import random
 
-from . import _util, _exception
+from . import _graphql, _util, _exception
 
 FB_DTSG_REGEX = re.compile(r'name="fb_dtsg" value="(.*?)"')
 
 
+def get_user_id(session):
+    # TODO: Optimize this `.get_dict()` call!
+    rtn = session.cookies.get_dict().get("c_user")
+    if rtn is None:
+        raise _exception.FBchatException("Could not find user id")
+    return str(rtn)
+
+
 def find_input_fields(html):
     return bs4.BeautifulSoup(html, "html.parser", parse_only=bs4.SoupStrainer("input"))
 
@@ -24,6 +32,10 @@ def session_factory(user_agent=None):
     return session
 
 
+def client_id_factory():
+    return hex(int(random.random() * 2 ** 31))[2:]
+
+
 def is_home(url):
     parts = _util.urlparse(url)
     # Check the urls `/home.php` and `/`
@@ -91,25 +103,21 @@ def _2fa_helper(session, code, r):
 class State(object):
     """Stores and manages state required for most Facebook requests."""
 
-    fb_dtsg = attr.ib()
+    user_id = attr.ib()
+    _fb_dtsg = attr.ib()
     _revision = attr.ib()
     _session = attr.ib(factory=session_factory)
     _counter = attr.ib(0)
+    _client_id = attr.ib(factory=client_id_factory)
     _logout_h = attr.ib(None)
 
-    def get_user_id(self):
-        rtn = self.get_cookies().get("c_user")
-        if rtn is None:
-            return None
-        return str(rtn)
-
     def get_params(self):
         self._counter += 1  # TODO: Make this operation atomic / thread-safe
         return {
             "__a": 1,
             "__req": _util.str_base(self._counter, 36),
             "__rev": self._revision,
-            "fb_dtsg": self.fb_dtsg,
+            "fb_dtsg": self._fb_dtsg,
         }
 
     @classmethod
@@ -163,6 +171,9 @@ class State(object):
 
     @classmethod
     def from_session(cls, session):
+        # TODO: Automatically set user_id when the cookie changes in the session
+        user_id = get_user_id(session)
+
         r = session.get(_util.prefix_url("/"))
 
         soup = find_input_fields(r.text)
@@ -180,7 +191,11 @@ class State(object):
         logout_h = logout_h_element["value"] if logout_h_element else None
 
         return cls(
-            fb_dtsg=fb_dtsg, revision=revision, session=session, logout_h=logout_h
+            user_id=user_id,
+            fb_dtsg=fb_dtsg,
+            revision=revision,
+            session=session,
+            logout_h=logout_h,
         )
 
     def get_cookies(self):
@@ -191,3 +206,126 @@ class State(object):
         session = session_factory(user_agent=user_agent)
         session.cookies = requests.cookies.merge_cookies(session.cookies, cookies)
         return cls.from_session(session=session)
+
+    def _do_refresh(self):
+        # TODO: Raise the error instead, and make the user do the refresh manually
+        # It may be a bad idea to do this in an exception handler, if you have a better method, please suggest it!
+        _util.log.warning("Refreshing state and resending request")
+        new = State.from_session(session=self._session)
+        self.user_id = new.user_id
+        self._fb_dtsg = new._fb_dtsg
+        self._revision = new._revision
+        self._counter = new._counter
+        self._logout_h = new._logout_h or self._logout_h
+
+    def _get(self, url, params, error_retries=3):
+        params.update(self.get_params())
+        r = self._session.get(_util.prefix_url(url), params=params)
+        content = _util.check_request(r)
+        j = _util.to_json(content)
+        try:
+            _util.handle_payload_error(j)
+        except _exception.FBchatPleaseRefresh:
+            if error_retries > 0:
+                self._do_refresh()
+                return self._get(url, params, error_retries=error_retries - 1)
+            raise
+        return j
+
+    def _post(self, url, data, files=None, as_graphql=False, error_retries=3):
+        data.update(self.get_params())
+        r = self._session.post(_util.prefix_url(url), data=data, files=files)
+        content = _util.check_request(r)
+        try:
+            if as_graphql:
+                return _graphql.response_to_json(content)
+            else:
+                j = _util.to_json(content)
+                # TODO: Remove this, and move it to _payload_post instead
+                # We can't yet, since errors raised in here need to be caught below
+                _util.handle_payload_error(j)
+                return j
+        except _exception.FBchatPleaseRefresh:
+            if error_retries > 0:
+                self._do_refresh()
+                return self._post(
+                    url,
+                    data,
+                    files=files,
+                    as_graphql=as_graphql,
+                    error_retries=error_retries - 1,
+                )
+            raise
+
+    def _payload_post(self, url, data, files=None):
+        j = self._post(url, data, files=files)
+        try:
+            return j["payload"]
+        except (KeyError, TypeError):
+            raise _exception.FBchatException("Missing payload: {}".format(j))
+
+    def _graphql_requests(self, *queries):
+        data = {
+            "method": "GET",
+            "response_format": "json",
+            "queries": _graphql.queries_to_json(*queries),
+        }
+        return self._post("/api/graphqlbatch/", data, as_graphql=True)
+
+    def _upload(self, files, voice_clip=False):
+        """Upload files to Facebook.
+
+        `files` should be a list of files that requests can upload, see
+        `requests.request <https://docs.python-requests.org/en/master/api/#requests.request>`_.
+
+        Return a list of tuples with a file's ID and mimetype.
+        """
+        file_dict = {"upload_{}".format(i): f for i, f in enumerate(files)}
+
+        data = {"voice_clip": voice_clip}
+
+        j = self._payload_post(
+            "https://upload.facebook.com/ajax/mercury/upload.php", data, files=file_dict
+        )
+
+        if len(j["metadata"]) != len(files):
+            raise _exception.FBchatException(
+                "Some files could not be uploaded: {}, {}".format(j, files)
+            )
+
+        return [
+            (data[_util.mimetype_to_key(data["filetype"])], data["filetype"])
+            for data in j["metadata"]
+        ]
+
+    def _do_send_request(self, data):
+        offline_threading_id = _util.generateOfflineThreadingID()
+        data["client"] = "mercury"
+        data["author"] = "fbid:{}".format(self.user_id)
+        data["timestamp"] = _util.now()
+        data["source"] = "source:chat:web"
+        data["offline_threading_id"] = offline_threading_id
+        data["message_id"] = offline_threading_id
+        data["threading_id"] = _util.generateMessageID(self._client_id)
+        data["ephemeral_ttl_mode:"] = "0"
+        j = self._post("/messaging/send/", data)
+
+        # update JS token if received in response
+        fb_dtsg = _util.get_jsmods_require(j, 2)
+        if fb_dtsg is not None:
+            self._fb_dtsg = fb_dtsg
+
+        try:
+            message_ids = [
+                (action["message_id"], action["thread_fbid"])
+                for action in j["payload"]["actions"]
+                if "message_id" in action
+            ]
+            if len(message_ids) != 1:
+                log.warning("Got multiple message ids' back: {}".format(message_ids))
+            return message_ids[0]
+        except (KeyError, IndexError, TypeError) as e:
+            raise _exception.FBchatException(
+                "Error when sending message: "
+                "No message IDs could be found: {}".format(j)
+            )

fbchat/_thread.py

@@ -16,6 +16,17 @@ class ThreadType(Enum):
     ROOM = 2
     PAGE = 3
 
+    def _to_class(self):
+        """Convert this enum value to the corresponding class."""
+        from . import _user, _group, _page
+
+        return {
+            ThreadType.USER: _user.User,
+            ThreadType.GROUP: _group.Group,
+            ThreadType.ROOM: _group.Room,
+            ThreadType.PAGE: _page.Page,
+        }[self]
+
 
 class ThreadLocation(Enum):
     """Used to specify where a thread is located (inbox, pending, archived, other)."""
@@ -130,3 +141,7 @@ class Thread(object):
                 else:
                     rtn["own_nickname"] = pc[1].get("nickname")
         return rtn
+
+    def _to_send_data(self):
+        # TODO: Only implement this in subclasses
+        return {"other_user_fbid": self.uid}

fbchat/_util.py

@@ -219,11 +219,12 @@ def get_files_from_urls(file_urls):
         r = requests.get(file_url)
         # We could possibly use r.headers.get('Content-Disposition'), see
         # https://stackoverflow.com/a/37060758
+        file_name = basename(file_url).split("?")[0].split("#")[0]
         files.append(
             (
-                basename(file_url).split("?")[0].split("#")[0],
+                file_name,
                 r.content,
-                r.headers.get("Content-Type") or guess_type(file_url)[0],
+                r.headers.get("Content-Type") or guess_type(file_name)[0],
             )
         )
     return files