Version up, thanks to @ekohilas

Fixes 2FA bug and updates pip requirements

fbchat/__init__.py

@@ -17,7 +17,7 @@ from .client import *
 
 
 __copyright__ = 'Copyright 2015 - {} by Taehoon Kim'.format(datetime.now().year)
-__version__ = '1.0.11'
+__version__ = '1.0.21'
 __license__ = 'BSD'
 __author__ = 'Taehoon Kim; Moreels Pieter-Jan; Mads Marquart'
 __email__ = 'carpedm20@gmail.com'

fbchat/client.py

@@ -43,7 +43,7 @@ class Client(object):
         :type max_tries: int
         :type session_cookies: dict
         :type logging_level: int
-        :raises: Exception on failed login
+        :raises: FBchatException on failed login
         """
 
         self.sticky, self.pool = (None, None)
@@ -54,14 +54,15 @@ class Client(object):
         self.client = 'mercury'
         self.default_thread_id = None
         self.default_thread_type = None
+        self.req_url = ReqUrl()
 
         if not user_agent:
             user_agent = choice(USER_AGENTS)
 
         self._header = {
             'Content-Type' : 'application/x-www-form-urlencoded',
-            'Referer' : ReqUrl.BASE,
-            'Origin' : ReqUrl.BASE,
+            'Referer' : self.req_url.BASE,
+            'Origin' : self.req_url.BASE,
             'User-Agent' : user_agent,
             'Connection' : 'keep-alive',
         }
@@ -91,13 +92,50 @@ class Client(object):
         self.req_counter += 1
         return payload
 
-    def _get(self, url, query=None, timeout=30):
-        payload = self._generatePayload(query)
-        return self._session.get(url, headers=self._header, params=payload, timeout=timeout)
+    def _fix_fb_errors(self, error_code):
+        """
+        This fixes "Please try closing and re-opening your browser window" errors (1357004)
+        This error usually happens after 1-2 days of inactivity
+        It may be a bad idea to do this in an exception handler, if you have a better method, please suggest it!
+        """
+        if error_code == '1357004':
+            log.warning('Got error #1357004. Doing a _postLogin, and resending request')
+            self._postLogin()
+            return True
+        return False
 
-    def _post(self, url, query=None, timeout=30):
+    def _get(self, url, query=None, timeout=30, fix_request=False, as_json=False, error_retries=3):
         payload = self._generatePayload(query)
-        return self._session.post(url, headers=self._header, data=payload, timeout=timeout)
+        r = self._session.get(url, headers=self._header, params=payload, timeout=timeout)
+        if not fix_request:
+            return r
+        try:
+            return check_request(r, as_json=as_json)
+        except FBchatFacebookError as e:
+            if error_retries > 0 and self._fix_fb_errors(e.fb_error_code):
+                return self._get(url, query=query, timeout=timeout, fix_request=fix_request, as_json=as_json, error_retries=error_retries-1)
+            raise e
+
+    def _post(self, url, query=None, timeout=30, fix_request=False, as_json=False, error_retries=3):
+        payload = self._generatePayload(query)
+        r = self._session.post(url, headers=self._header, data=payload, timeout=timeout)
+        if not fix_request:
+            return r
+        try:
+            return check_request(r, as_json=as_json)
+        except FBchatFacebookError as e:
+            if error_retries > 0 and self._fix_fb_errors(e.fb_error_code):
+                return self._post(url, query=query, timeout=timeout, fix_request=fix_request, as_json=as_json, error_retries=error_retries-1)
+            raise e
+
+    def _graphql(self, payload, error_retries=3):
+        content = self._post(self.req_url.GRAPHQL, payload, fix_request=True, as_json=False)
+        try:
+            return graphql_response_to_json(content)
+        except FBchatFacebookError as e:
+            if error_retries > 0 and self._fix_fb_errors(e.fb_error_code):
+                return self._graphql(payload, error_retries=error_retries-1)
+            raise e
 
     def _cleanGet(self, url, query=None, timeout=30):
         return self._session.get(url, headers=self._header, params=query, timeout=timeout)
@@ -106,38 +144,42 @@ class Client(object):
         self.req_counter += 1
         return self._session.post(url, headers=self._header, data=query, timeout=timeout)
 
-    def _postFile(self, url, files=None, query=None, timeout=30):
+    def _postFile(self, url, files=None, query=None, timeout=30, fix_request=False, as_json=False, error_retries=3):
         payload=self._generatePayload(query)
         # Removes 'Content-Type' from the header
         headers = dict((i, self._header[i]) for i in self._header if i != 'Content-Type')
-        return self._session.post(url, headers=headers, data=payload, timeout=timeout, files=files)
+        r = self._session.post(url, headers=headers, data=payload, timeout=timeout, files=files)
+        if not fix_request:
+            return r
+        try:
+            return check_request(r, as_json=as_json)
+        except FBchatFacebookError as e:
+            if error_retries > 0 and self._fix_fb_errors(e.fb_error_code):
+                return self._postFile(url, files=files, query=query, timeout=timeout, fix_request=fix_request, as_json=as_json, error_retries=error_retries-1)
+            raise e
 
     def graphql_requests(self, *queries):
         """
         .. todo::
             Documenting this
 
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
-        payload = {
+
+        return tuple(self._graphql({
             'method': 'GET',
             'response_format': 'json',
             'queries': graphql_queries_to_json(*queries)
-        }
-
-        j = graphql_response_to_json(checkRequest(self._post(ReqUrl.GRAPHQL, payload), do_json_check=False))
-
-        return tuple(j)
+        }))
 
     def graphql_request(self, query):
         """
         Shorthand for `graphql_requests(query)[0]`
 
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
         return self.graphql_requests(query)[0]
 
-
     """
     END INTERNAL REQUEST METHODS
     """
@@ -157,11 +199,14 @@ class Client(object):
         self.payloadDefault = {}
         self.client_id = hex(int(random()*2147483648))[2:]
         self.start_time = now()
-        self.uid = str(self._session.cookies['c_user'])
+        self.uid = self._session.cookies.get_dict().get('c_user')
+        if self.uid is None:
+            raise FBchatException('Could not find c_user cookie')
+        self.uid = str(self.uid)
         self.user_channel = "p_" + self.uid
         self.ttstamp = ''
 
-        r = self._get(ReqUrl.BASE)
+        r = self._get(self.req_url.BASE)
         soup = bs(r.text, "lxml")
         self.fb_dtsg = soup.find("input", {'name':'fb_dtsg'})['value']
         self.fb_h = soup.find("input", {'name':'h'})['value']
@@ -193,23 +238,25 @@ class Client(object):
 
     def _login(self):
         if not (self.email and self.password):
-            raise Exception("Email and password not found.")
+            raise FBchatUserError("Email and password not found.")
 
-        soup = bs(self._get(ReqUrl.MOBILE).text, "lxml")
+        soup = bs(self._get(self.req_url.MOBILE).text, "lxml")
         data = dict((elem['name'], elem['value']) for elem in soup.findAll("input") if elem.has_attr('value') and elem.has_attr('name'))
         data['email'] = self.email
         data['pass'] = self.password
         data['login'] = 'Log In'
 
-        r = self._cleanPost(ReqUrl.LOGIN, data)
+        r = self._cleanPost(self.req_url.LOGIN, data)
 
         # Usually, 'Checkpoint' will refer to 2FA
-        if 'checkpoint' in r.url and 'Enter Security Code to Continue' in r.text:
+        if ('checkpoint' in r.url
+                and ('enter security code to continue' in r.text.lower()
+                    or 'enter login code to continue' in r.text.lower())):
             r = self._2FA(r)
 
         # Sometimes Facebook tries to show the user a "Save Device" dialog
         if 'save-device' in r.url:
-            r = self._cleanGet(ReqUrl.SAVE_DEVICE)
+            r = self._cleanGet(self.req_url.SAVE_DEVICE)
 
         if 'home' in r.url:
             self._postLogin()
@@ -230,7 +277,7 @@ class Client(object):
         data['codes_submitted'] = 0
         log.info('Submitting 2FA code.')
 
-        r = self._cleanPost(ReqUrl.CHECKPOINT, data)
+        r = self._cleanPost(self.req_url.CHECKPOINT, data)
 
         if 'home' in r.url:
             return r
@@ -242,14 +289,14 @@ class Client(object):
         data['name_action_selected'] = 'save_device'
         data['submit[Continue]'] = 'Continue'
         log.info('Saving browser.')  # At this stage, we have dtsg, nh, name_action_selected, submit[Continue]
-        r = self._cleanPost(ReqUrl.CHECKPOINT, data)
+        r = self._cleanPost(self.req_url.CHECKPOINT, data)
 
         if 'home' in r.url:
             return r
 
         del(data['name_action_selected'])
         log.info('Starting Facebook checkup flow.')  # At this stage, we have dtsg, nh, submit[Continue]
-        r = self._cleanPost(ReqUrl.CHECKPOINT, data)
+        r = self._cleanPost(self.req_url.CHECKPOINT, data)
 
         if 'home' in r.url:
             return r
@@ -257,7 +304,7 @@ class Client(object):
         del(data['submit[Continue]'])
         data['submit[This was me]'] = 'This Was Me'
         log.info('Verifying login attempt.')  # At this stage, we have dtsg, nh, submit[This was me]
-        r = self._cleanPost(ReqUrl.CHECKPOINT, data)
+        r = self._cleanPost(self.req_url.CHECKPOINT, data)
 
         if 'home' in r.url:
             return r
@@ -266,7 +313,7 @@ class Client(object):
         data['submit[Continue]'] = 'Continue'
         data['name_action_selected'] = 'save_device'
         log.info('Saving device again.')  # At this stage, we have dtsg, nh, submit[Continue], name_action_selected
-        r = self._cleanPost(ReqUrl.CHECKPOINT, data)
+        r = self._cleanPost(self.req_url.CHECKPOINT, data)
         return r
 
     def isLoggedIn(self):
@@ -277,7 +324,7 @@ class Client(object):
         :rtype: bool
         """
         # Send a request to the login url, to see if we're directed to the home page
-        r = self._cleanGet(ReqUrl.LOGIN)
+        r = self._cleanGet(self.req_url.LOGIN)
         return 'home' in r.url
 
     def getSession(self):
@@ -305,7 +352,8 @@ class Client(object):
             # Load cookies into current session
             self._session.cookies = requests.cookies.merge_cookies(self._session.cookies, session_cookies)
             self._postLogin()
-        except Exception:
+        except Exception as e:
+            log.exception('Failed loading session')
             self._resetValues()
             return False
         return True
@@ -318,15 +366,15 @@ class Client(object):
         :param password: Facebook account password
         :param max_tries: Maximum number of times to try logging in
         :type max_tries: int
-        :raises: Exception on failed login
+        :raises: FBchatException on failed login
         """
         self.onLoggingIn(email=email)
 
         if max_tries < 1:
-            raise Exception('Cannot login: max_tries should be at least one')
+            raise FBchatUserError('Cannot login: max_tries should be at least one')
 
         if not (email and password):
-            raise Exception('Email and password not set')
+            raise FBchatUserError('Email and password not set')
 
         self.email = email
         self.password = password
@@ -341,7 +389,7 @@ class Client(object):
                 self.onLoggedIn(email=email)
                 break
         else:
-            raise Exception('Login failed. Check email/password. (Failed on url: {})'.format(login_url))
+            raise FBchatUserError('Login failed. Check email/password. (Failed on url: {})'.format(login_url))
 
     def logout(self):
         """
@@ -356,7 +404,7 @@ class Client(object):
             'h': self.fb_h
         }
 
-        r = self._get(ReqUrl.LOGOUT, data)
+        r = self._get(self.req_url.LOGOUT, data)
 
         self._resetValues()
 
@@ -414,15 +462,15 @@ class Client(object):
 
         :return: :class:`models.User` objects
         :rtype: list
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
 
         data = {
             'viewer': self.uid,
         }
-        j = checkRequest(self._post(ReqUrl.ALL_USERS, query=data))
-        if not j['payload']:
-            raise Exception('Missing payload')
+        j = self._post(self.req_url.ALL_USERS, query=data, fix_request=True, as_json=True)
+        if j.get('payload') is None:
+            raise FBchatException('Missing payload while fetching users: {}'.format(j))
 
         users = []
 
@@ -444,7 +492,7 @@ class Client(object):
         :param limit: The max. amount of users to fetch
         :return: :class:`models.User` objects, ordered by relevance
         :rtype: list
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
 
         j = self.graphql_request(GraphQL(query=GraphQL.SEARCH_USER, params={'search': name, 'limit': limit}))
@@ -458,7 +506,7 @@ class Client(object):
         :param name: Name of the page
         :return: :class:`models.Page` objects, ordered by relevance
         :rtype: list
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
 
         j = self.graphql_request(GraphQL(query=GraphQL.SEARCH_PAGE, params={'search': name, 'limit': limit}))
@@ -473,7 +521,7 @@ class Client(object):
         :param limit: The max. amount of groups to fetch
         :return: :class:`models.Group` objects, ordered by relevance
         :rtype: list
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
 
         j = self.graphql_request(GraphQL(query=GraphQL.SEARCH_GROUP, params={'search': name, 'limit': limit}))
@@ -488,7 +536,7 @@ class Client(object):
         :param limit: The max. amount of groups to fetch
         :return: :class:`models.User`, :class:`models.Group` and :class:`models.Page` objects, ordered by relevance
         :rtype: list
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
 
         j = self.graphql_request(GraphQL(query=GraphQL.SEARCH_THREAD, params={'search': name, 'limit': limit}))
@@ -514,10 +562,10 @@ class Client(object):
         data = {
             "ids[{}]".format(i): _id for i, _id in enumerate(ids)
         }
-        j = checkRequest(self._post(ReqUrl.INFO, data))
+        j = self._post(self.req_url.INFO, data, fix_request=True, as_json=True)
 
-        if not j['payload']['profiles']:
-            raise Exception('No users/pages returned')
+        if j.get('payload') is None or j['payload'].get('profiles') is None:
+            raise FBchatException('No users/pages returned: {}'.format(j))
 
         entries = {}
         for _id in j['payload']['profiles']:
@@ -542,7 +590,7 @@ class Client(object):
                     'name': k.get('name')
                 }
             else:
-                raise Exception('{} had an unknown thread type: {}'.format(_id, k))
+                raise FBchatException('{} had an unknown thread type: {}'.format(_id, k))
 
         log.debug(entries)
         return entries
@@ -557,7 +605,7 @@ class Client(object):
         :param user_ids: One or more user ID(s) to query
         :return: :class:`models.User` objects, labeled by their ID
         :rtype: dict
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
 
         threads = self.fetchThreadInfo(*user_ids)
@@ -566,7 +614,7 @@ class Client(object):
             if threads[k].type == ThreadType.USER:
                 users[k] = threads[k]
             else:
-                raise Exception('Thread {} was not a user'.format(threads[k]))
+                raise FBchatUserError('Thread {} was not a user'.format(threads[k]))
 
         return users
 
@@ -580,7 +628,7 @@ class Client(object):
         :param page_ids: One or more page ID(s) to query
         :return: :class:`models.Page` objects, labeled by their ID
         :rtype: dict
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
 
         threads = self.fetchThreadInfo(*page_ids)
@@ -589,7 +637,7 @@ class Client(object):
             if threads[k].type == ThreadType.PAGE:
                 pages[k] = threads[k]
             else:
-                raise Exception('Thread {} was not a page'.format(threads[k]))
+                raise FBchatUserError('Thread {} was not a page'.format(threads[k]))
 
         return pages
 
@@ -600,7 +648,7 @@ class Client(object):
         :param group_ids: One or more group ID(s) to query
         :return: :class:`models.Group` objects, labeled by their ID
         :rtype: dict
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
 
         threads = self.fetchThreadInfo(*group_ids)
@@ -609,7 +657,7 @@ class Client(object):
             if threads[k].type == ThreadType.GROUP:
                 groups[k] = threads[k]
             else:
-                raise Exception('Thread {} was not a group'.format(threads[k]))
+                raise FBchatUserError('Thread {} was not a group'.format(threads[k]))
 
         return groups
 
@@ -623,7 +671,7 @@ class Client(object):
         :param thread_ids: One or more thread ID(s) to query
         :return: :class:`models.Thread` objects, labeled by their ID
         :rtype: dict
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
 
         queries = []
@@ -662,14 +710,14 @@ class Client(object):
             elif entry.get('thread_type') == 'ONE_TO_ONE':
                 _id = entry['thread_key']['other_user_id']
                 if pages_and_users.get(_id) is None:
-                    raise Exception('Could not fetch thread {}'.format(_id))
+                    raise FBchatException('Could not fetch thread {}'.format(_id))
                 entry.update(pages_and_users[_id])
                 if entry['type'] == ThreadType.USER:
                     rtn[_id] = graphql_to_user(entry)
                 else:
                     rtn[_id] = graphql_to_page(entry)
             else:
-                raise Exception('{} had an unknown thread type: {}'.format(thread_ids[i], entry))
+                raise FBchatException('{} had an unknown thread type: {}'.format(thread_ids[i], entry))
 
         return rtn
 
@@ -684,7 +732,7 @@ class Client(object):
         :type before: int
         :return: :class:`models.Message` objects
         :rtype: list
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
 
         j = self.graphql_request(GraphQL(doc_id='1386147188135407', params={
@@ -695,8 +743,8 @@ class Client(object):
             'before': before
         }))
 
-        if j['message_thread'] is None:
-            raise Exception('Could not fetch thread {}'.format(thread_id))
+        if j.get('message_thread') is None:
+            raise FBchatException('Could not fetch thread {}: {}'.format(thread_id, j))
 
         return list(reversed([graphql_to_message(message) for message in j['message_thread']['messages']['nodes']]))
 
@@ -709,11 +757,11 @@ class Client(object):
         :type limit: int
         :return: :class:`models.Thread` objects
         :rtype: list
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
 
         if limit > 20 or limit < 1:
-            raise Exception('`limit` should be between 1 and 20')
+            raise FBchatUserError('`limit` should be between 1 and 20')
 
         data = {
             'client' : self.client,
@@ -721,9 +769,9 @@ class Client(object):
             'inbox[limit]' : limit,
         }
 
-        j = checkRequest(self._post(ReqUrl.THREADS, data))
+        j = self._post(self.req_url.THREADS, data, fix_request=True, as_json=True)
         if j.get('payload') is None:
-            raise Exception('Missing payload: {}, with data: {}'.format(j, data))
+            raise FBchatException('Missing payload: {}, with data: {}'.format(j, data))
 
         participants = {}
         for p in j['payload']['participants']:
@@ -732,18 +780,19 @@ class Client(object):
             elif p['type'] == 'user':
                 participants[p['fbid']] = User(p['fbid'], url=p['href'], first_name=p['short_name'], is_friend=p['is_friend'], gender=GENDERS[p['gender']], photo=p['image_src'], name=p['name'])
             else:
-                raise Exception('A participant had an unknown type {}: {}'.format(p['type'], p))
+                raise FBchatException('A participant had an unknown type {}: {}'.format(p['type'], p))
 
         entries = []
         for k in j['payload']['threads']:
             if k['thread_type'] == 1:
                 if k['other_user_fbid'] not in participants:
-                    raise Exception('A thread was not in participants: {}'.format(j['payload']))
+                    raise FBchatException('The thread {} was not in participants: {}'.format(k, j['payload']))
+                participants[k['other_user_fbid']].message_count = k['message_count']
                 entries.append(participants[k['other_user_fbid']])
             elif k['thread_type'] == 2:
-                entries.append(Group(k['thread_fbid'], participants=set([p.strip('fbid:') for p in k['participants']]), photo=k['image_src'], name=k['name']))
+                entries.append(Group(k['thread_fbid'], participants=set([p.strip('fbid:') for p in k['participants']]), photo=k['image_src'], name=k['name'], message_count=k['message_count']))
             else:
-                raise Exception('A thread had an unknown thread type: {}'.format(k))
+                raise FBchatException('A thread had an unknown thread type: {}'.format(k))
 
         return entries
 
@@ -752,7 +801,7 @@ class Client(object):
         .. todo::
             Documenting this
 
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
         form = {
             'client': 'mercury_sync',
@@ -761,7 +810,7 @@ class Client(object):
             # 'last_action_timestamp': 0
         }
 
-        j = checkRequest(self._post(ReqUrl.THREAD_SYNC, form))
+        j = self._post(self.req_url.THREAD_SYNC, form, fix_request=True, as_json=True)
 
         return {
             "message_counts": j['payload']['message_counts'],
@@ -820,7 +869,7 @@ class Client(object):
 
     def _doSendRequest(self, data):
         """Sends the data to `SendURL`, and returns the message ID or None on failure"""
-        j = checkRequest(self._post(ReqUrl.SEND, data))
+        j = self._post(self.req_url.SEND, data, fix_request=True, as_json=True)
 
         try:
             message_ids = [action['message_id'] for action in j['payload']['actions'] if 'message_id' in action]
@@ -828,7 +877,14 @@ class Client(object):
                 log.warning("Got multiple message ids' back: {}".format(message_ids))
             message_id = message_ids[0]
         except (KeyError, IndexError) as e:
-            raise Exception('Error when sending message: No message IDs could be found: {}'.format(j))
+            raise FBchatException('Error when sending message: No message IDs could be found: {}'.format(j))
+
+        # update JS token if received in response
+        if j.get('jsmods') is not None and j['jsmods'].get('require') is not None:
+            try:
+                self.payloadDefault['fb_dtsg'] = j['jsmods']['require'][0][3][0]
+            except (KeyError, IndexError) as e:
+                log.warning('Error when updating fb_dtsg. Facebook might have changed protocol!')
 
         return message_id
 
@@ -841,7 +897,7 @@ class Client(object):
         :param thread_type: See :ref:`intro_threads`
         :type thread_type: models.ThreadType
         :return: :ref:`Message ID <intro_message_ids>` of the sent message
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
         thread_id, thread_type = self._getThread(thread_id, thread_type)
         data = self._getSendData(thread_id, thread_type)
@@ -865,7 +921,7 @@ class Client(object):
         :type size: models.EmojiSize
         :type thread_type: models.ThreadType
         :return: :ref:`Message ID <intro_message_ids>` of the sent emoji
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
         thread_id, thread_type = self._getThread(thread_id, thread_type)
         data = self._getSendData(thread_id, thread_type)
@@ -885,13 +941,13 @@ class Client(object):
     def _uploadImage(self, image_path, data, mimetype):
         """Upload an image and get the image_id for sending in a message"""
 
-        j = checkRequest(self._postFile(ReqUrl.UPLOAD, {
+        j = self._postFile(self.req_url.UPLOAD, {
             'file': (
                 image_path,
                 data,
                 mimetype
             )
-        }))
+        }, fix_request=True, as_json=True)
         # Return the image_id
         return j['payload']['metadata'][0]['image_id']
 
@@ -905,7 +961,7 @@ class Client(object):
         :param thread_type: See :ref:`intro_threads`
         :type thread_type: models.ThreadType
         :return: :ref:`Message ID <intro_message_ids>` of the sent image
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
         thread_id, thread_type = self._getThread(thread_id, thread_type)
         data = self._getSendData(thread_id, thread_type)
@@ -930,7 +986,7 @@ class Client(object):
         :param thread_type: See :ref:`intro_threads`
         :type thread_type: models.ThreadType
         :return: :ref:`Message ID <intro_message_ids>` of the sent image
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
         thread_id, thread_type = self._getThread(thread_id, thread_type)
         mimetype = guess_type(image_url)[0]
@@ -942,13 +998,13 @@ class Client(object):
         """
         Sends a local image to a thread
 
-        :param image_path: URL of an image to upload and send
+        :param image_path: Path of an image to upload and send
         :param message: Additional message
         :param thread_id: User/Group ID to send to. See :ref:`intro_threads`
         :param thread_type: See :ref:`intro_threads`
         :type thread_type: models.ThreadType
         :return: :ref:`Message ID <intro_message_ids>` of the sent image
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
         thread_id, thread_type = self._getThread(thread_id, thread_type)
         mimetype = guess_type(image_path)[0]
@@ -963,7 +1019,7 @@ class Client(object):
         :param thread_id: Group ID to add people to. See :ref:`intro_threads`
         :type user_ids: list
         :return: :ref:`Message ID <intro_message_ids>` of the executed action
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
         thread_id, thread_type = self._getThread(thread_id, None)
         data = self._getSendData(thread_id, ThreadType.GROUP)
@@ -979,7 +1035,7 @@ class Client(object):
 
         for i, user_id in enumerate(user_ids):
             if user_id == self.uid:
-                raise Exception('Error when adding users: Cannot add self to group thread')
+                raise FBchatUserError('Error when adding users: Cannot add self to group thread')
             else:
                 data['log_message_data[added_participants][' + str(i) + ']'] = "fbid:" + str(user_id)
 
@@ -991,7 +1047,7 @@ class Client(object):
 
         :param user_id: User ID to remove
         :param thread_id: Group ID to remove people from. See :ref:`intro_threads`
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
 
         thread_id, thread_type = self._getThread(thread_id, None)
@@ -1001,7 +1057,7 @@ class Client(object):
             "tid": thread_id
         }
 
-        j = checkRequest(self._post(ReqUrl.REMOVE_USER, data))
+        j = self._post(self.req_url.REMOVE_USER, data, fix_request=True, as_json=True)
 
     def changeThreadTitle(self, title, thread_id=None, thread_type=ThreadType.USER):
         """
@@ -1012,7 +1068,7 @@ class Client(object):
         :param thread_id: Group ID to change title of. See :ref:`intro_threads`
         :param thread_type: See :ref:`intro_threads`
         :type thread_type: models.ThreadType
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
 
         thread_id, thread_type = self._getThread(thread_id, thread_type)
@@ -1038,7 +1094,7 @@ class Client(object):
         :param thread_id: User/Group ID to change color of. See :ref:`intro_threads`
         :param thread_type: See :ref:`intro_threads`
         :type thread_type: models.ThreadType
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
         thread_id, thread_type = self._getThread(thread_id, thread_type)
 
@@ -1048,7 +1104,7 @@ class Client(object):
             'thread_or_other_fbid': thread_id
         }
 
-        j = checkRequest(self._post(ReqUrl.THREAD_NICKNAME, data))
+        j = self._post(self.req_url.THREAD_NICKNAME, data, fix_request=True, as_json=True)
 
     def changeThreadColor(self, color, thread_id=None):
         """
@@ -1057,7 +1113,7 @@ class Client(object):
         :param color: New thread color
         :param thread_id: User/Group ID to change color of. See :ref:`intro_threads`
         :type color: models.ThreadColor
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
         thread_id, thread_type = self._getThread(thread_id, None)
 
@@ -1066,7 +1122,7 @@ class Client(object):
             'thread_or_other_fbid': thread_id
         }
 
-        j = checkRequest(self._post(ReqUrl.THREAD_COLOR, data))
+        j = self._post(self.req_url.THREAD_COLOR, data, fix_request=True, as_json=True)
 
     def changeThreadEmoji(self, emoji, thread_id=None):
         """
@@ -1076,7 +1132,7 @@ class Client(object):
 
         :param color: New thread emoji
         :param thread_id: User/Group ID to change emoji of. See :ref:`intro_threads`
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
         thread_id, thread_type = self._getThread(thread_id, None)
 
@@ -1085,7 +1141,7 @@ class Client(object):
             'thread_or_other_fbid': thread_id
         }
 
-        j = checkRequest(self._post(ReqUrl.THREAD_EMOJI, data))
+        j = self._post(self.req_url.THREAD_EMOJI, data, fix_request=True, as_json=True)
 
     def reactToMessage(self, message_id, reaction):
         """
@@ -1094,7 +1150,7 @@ class Client(object):
         :param message_id: :ref:`Message ID <intro_message_ids>` to react to
         :param reaction: Reaction emoji to use
         :type reaction: models.MessageReaction
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
         full_data = {
             "doc_id": 1491398900900362,
@@ -1117,7 +1173,7 @@ class Client(object):
                 .replace('u%27', '%27')\
                 .replace('%5CU{}'.format(MessageReactionFix[reaction.value][0]), MessageReactionFix[reaction.value][1])
 
-        j = checkRequest(self._post('{}/?{}'.format(ReqUrl.MESSAGE_REACTION, url_part)))
+        j = self._post('{}/?{}'.format(self.req_url.MESSAGE_REACTION, url_part), fix_request=True, as_json=True)
 
     def setTypingStatus(self, status, thread_id=None, thread_type=None):
         """
@@ -1128,7 +1184,7 @@ class Client(object):
         :param thread_type: See :ref:`intro_threads`
         :type status: models.TypingStatus
         :type thread_type: models.ThreadType
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
         thread_id, thread_type = self._getThread(thread_id, None)
 
@@ -1139,7 +1195,7 @@ class Client(object):
             "source": "mercury-chat"
         }
 
-        j = checkRequest(self._post(ReqUrl.TYPING, data))
+        j = self._post(self.req_url.TYPING, data, fix_request=True, as_json=True)
 
     """
     END SEND METHODS
@@ -1155,7 +1211,7 @@ class Client(object):
             "thread_ids[%s][0]" % userID: threadID
         }
 
-        r = self._post(ReqUrl.DELIVERED, data)
+        r = self._post(self.req_url.DELIVERED, data)
         return r.ok
 
     def markAsRead(self, userID):
@@ -1169,7 +1225,7 @@ class Client(object):
             "ids[%s]" % userID: True
         }
 
-        r = self._post(ReqUrl.READ_STATUS, data)
+        r = self._post(self.req_url.READ_STATUS, data)
         return r.ok
 
     def markAsSeen(self):
@@ -1177,7 +1233,7 @@ class Client(object):
         .. todo::
             Documenting this
         """
-        r = self._post(ReqUrl.MARK_SEEN, {"seen_timestamp": 0})
+        r = self._post(self.req_url.MARK_SEEN, {"seen_timestamp": 0})
         return r.ok
 
     def friendConnect(self, friend_id):
@@ -1190,7 +1246,7 @@ class Client(object):
             "action": "confirm"
         }
 
-        r = self._post(ReqUrl.CONNECT, data)
+        r = self._post(self.req_url.CONNECT, data)
         return r.ok
 
 
@@ -1210,7 +1266,7 @@ class Client(object):
             'viewer_uid': self.uid,
             'state': 'active'
         }
-        checkRequest(self._get(ReqUrl.PING, data), do_json_check=False)
+        self._get(self.req_url.PING, data, fix_request=True, as_json=False)
 
     def _fetchSticky(self):
         """Call pull api to get sticky and pool parameter, newer api needs these parameters to work"""
@@ -1221,10 +1277,10 @@ class Client(object):
             "clientid": self.client_id
         }
 
-        j = checkRequest(self._get(ReqUrl.STICKY, data))
+        j = self._get(self.req_url.STICKY, data, fix_request=True, as_json=True)
 
-        if 'lb_info' not in j:
-            raise Exception('Missing lb_info')
+        if j.get('lb_info') is None:
+            raise FBchatException('Missing lb_info: {}'.format(j))
 
         return j['lb_info']['sticky'], j['lb_info']['pool']
 
@@ -1238,7 +1294,7 @@ class Client(object):
             "clientid": self.client_id,
         }
 
-        j = checkRequest(self._get(ReqUrl.STICKY, data))
+        j = self._get(ReqUrl.STICKY, data, fix_request=True, as_json=True)
 
         self.seq = j.get('seq', '0')
         return j
@@ -1389,6 +1445,14 @@ class Client(object):
                 elif mtype == "deltaflow":
                     pass
 
+                # Chat timestamp
+                elif mtype == "chatproxy-presence":
+                    buddylist = {}
+                    for _id in m.get('buddyList', {}):
+                        payload = m['buddyList'][_id]
+                        buddylist[_id] = payload.get('lat')
+                    self.onChatTimestamp(buddylist=buddylist, msg=m)
+
                 # Unknown message type
                 else:
                     self.onUnknownMesssageType(msg=m)
@@ -1400,7 +1464,7 @@ class Client(object):
         """
         Start listening from an external event loop
 
-        :raises: Exception if request failed
+        :raises: FBchatException if request failed
         """
         self.listening = True
         self.sticky, self.pool = self._fetchSticky()
@@ -1416,18 +1480,27 @@ class Client(object):
         :rtype: bool
         """
         try:
-            if markAlive: self._ping(self.sticky, self.pool)
-            try:
-                content = self._pullMessage(self.sticky, self.pool)
-                if content: self._parseMessage(content)
-            except requests.exceptions.RequestException:
-                pass
-            except Exception as e:
-                return self.onListenError(exception=e)
+            if markAlive:
+                self._ping(self.sticky, self.pool)
+            content = self._pullMessage(self.sticky, self.pool)
+            if content:
+                self._parseMessage(content)
         except KeyboardInterrupt:
             return False
-        except requests.exceptions.Timeout:
+        except requests.Timeout:
             pass
+        except requests.ConnectionError:
+            # If the client has lost their internet connection, keep trying every 30 seconds
+            time.sleep(30)
+        except FBchatFacebookError as e:
+            # Fix 502 and 503 pull errors
+            if e.request_status_code in [502, 503]:
+                self.req_url.change_pull_channel()
+                self.startListening()
+            else:
+                raise e
+        except Exception as e:
+            return self.onListenError(exception=e)
 
         return True
 
@@ -1469,7 +1542,7 @@ class Client(object):
 
     def on2FACode(self):
         """Called when a 2FA code is needed to progress"""
-        input('Please enter your 2FA code --> ')
+        return input('Please enter your 2FA code --> ')
 
     def onLoggedIn(self, email=None):
         """
@@ -1488,8 +1561,10 @@ class Client(object):
         Called when an error was encountered while listening
 
         :param exception: The exception that was encountered
+        :return: Whether the loop should keep running
         """
         log.exception('Got exception while listening')
+        return True
 
 
     def onMessage(self, mid=None, author_id=None, message=None, thread_id=None, thread_type=ThreadType.USER, ts=None, metadata=None, msg={}):
@@ -1676,6 +1751,14 @@ class Client(object):
         """
         pass
 
+    def onChatTimestamp(self, buddylist={}, msg={}):
+        """
+        Called when the client receives chat online presence update
+
+        :param buddylist: A list of dicts with friend id and last seen timestamp
+        :param msg: A full set of the data recieved
+        """
+        log.debug('Chat Timestamps received: {}'.format(buddylist))
 
     def onUnknownMesssageType(self, msg={}):
         """

fbchat/graphql.py

@@ -31,7 +31,7 @@ def graphql_color_to_enum(color):
     try:
         return ThreadColor('#{}'.format(color[2:].lower()))
     except ValueError:
-        raise Exception('Could not get ThreadColor from color: {}'.format(color))
+        raise FBchatException('Could not get ThreadColor from color: {}'.format(color))
 
 def get_customization_info(thread):
     if thread is None or thread.get('customization_info') is None:
@@ -47,17 +47,18 @@ def get_customization_info(thread):
         for k in info.get('participant_customizations', []):
             rtn['nicknames'][k['participant_id']] = k.get('nickname')
     elif info.get('participant_customizations'):
-        _id = thread.get('thread_key', {}).get('other_user_id') or thread.get('id')
-        if info['participant_customizations'][0]['participant_id'] == _id:
-            rtn['nickname'] = info['participant_customizations'][0]
-            if len(info['participant_customizations']) > 1:
-                rtn['own_nickname'] = info['participant_customizations'][1]
-        elif info['participant_customizations'][1]['participant_id'] == _id:
-            rtn['nickname'] = info['participant_customizations'][1]
-            if len(info['participant_customizations']) > 1:
-                rtn['own_nickname'] = info['participant_customizations'][0]
-        else:
-            raise Exception('No participant matching the user {} found: {}'.format(_id, info['participant_customizations']))
+        uid = thread.get('thread_key', {}).get('other_user_id') or thread.get('id')
+        pc = info['participant_customizations']
+        if len(pc) > 0:
+            if pc[0].get('participant_id') == uid:
+                rtn['nickname'] = pc[0].get('nickname')
+            else:
+                rtn['own_nickname'] = pc[0].get('nickname')
+        if len(pc) > 1:
+            if pc[1].get('participant_id') == uid:
+                rtn['nickname'] = pc[1].get('nickname')
+            else:
+                rtn['own_nickname'] = pc[1].get('nickname')
     return rtn
 
 def graphql_to_message(message):
@@ -98,7 +99,8 @@ def graphql_to_user(user):
         emoji=c_info.get('emoji'),
         own_nickname=c_info.get('own_nickname'),
         photo=user['profile_picture'].get('uri'),
-        name=user.get('name')
+        name=user.get('name'),
+        message_count=user.get('messages_count')
     )
 
 def graphql_to_group(group):
@@ -112,7 +114,8 @@ def graphql_to_group(group):
         color=c_info.get('color'),
         emoji=c_info.get('emoji'),
         photo=group['image'].get('uri'),
-        name=group.get('name')
+        name=group.get('name'),
+        message_count=group.get('messages_count')
     )
 
 def graphql_to_page(page):
@@ -126,7 +129,8 @@ def graphql_to_page(page):
         city=page.get('city').get('name'),
         category=page.get('category_type'),
         photo=page['profile_picture'].get('uri'),
-        name=page.get('name')
+        name=page.get('name'),
+        message_count=page.get('messages_count')
     )
 
 def graphql_queries_to_json(*queries):
@@ -139,7 +143,11 @@ def graphql_queries_to_json(*queries):
     return json.dumps(rtn)
 
 def graphql_response_to_json(content):
-    j = json.loads(content, cls=ConcatJSONDecoder)
+    content = strip_to_json(content) # Usually only needed in some error cases
+    try:
+        j = json.loads(content, cls=ConcatJSONDecoder)
+    except Exception:
+        raise FBchatException('Error while parsing JSON: {}'.format(repr(content)))
 
     rtn = [None]*(len(j))
     for x in j:
@@ -172,7 +180,7 @@ class GraphQL(object):
                 'query_params': params
             }
         else:
-            raise Exception('A query or doc_id must be specified')
+            raise FBchatUserError('A query or doc_id must be specified')
 
 
     FRAGMENT_USER = """

fbchat/models.py

@@ -4,6 +4,26 @@ from __future__ import unicode_literals
 import enum
 
 
+class FBchatException(Exception):
+    """Custom exception thrown by fbchat. All exceptions in the fbchat module inherits this"""
+
+class FBchatFacebookError(FBchatException):
+    #: The error code that Facebook returned
+    fb_error_code = str
+    #: The error message that Facebook returned (In the user's own language)
+    fb_error_message = str
+    #: The status code that was sent in the http response (eg. 404) (Usually only set if not successful, aka. not 200)
+    request_status_code = int
+    def __init__(self, message, fb_error_code=None, fb_error_message=None, request_status_code=None):
+        super(FBchatFacebookError, self).__init__(message)
+        """Thrown by fbchat when Facebook returns an error"""
+        self.fb_error_code = str(fb_error_code)
+        self.fb_error_message = fb_error_message
+        self.request_status_code = request_status_code
+
+class FBchatUserError(FBchatException):
+    """Thrown by fbchat when wrong values are entered"""
+
 class Thread(object):
     #: The unique identifier of the thread. Can be used a `thread_id`. See :ref:`intro_threads` for more info
     uid = str
@@ -15,14 +35,16 @@ class Thread(object):
     name = str
     #: Timestamp of last message
     last_message_timestamp = str
-
-    def __init__(self, _type, uid, photo=None, name=None, last_message_timestamp=None):
+    #: Number of messages in the thread
+    message_count = int
+    def __init__(self, _type, uid, photo=None, name=None, last_message_timestamp=None, message_count=None):
         """Represents a Facebook thread"""
         self.uid = str(uid)
         self.type = _type
         self.photo = photo
         self.name = name
         self.last_message_timestamp = last_message_timestamp
+        self.message_count = message_count
 
     def __repr__(self):
         return self.__unicode__()

fbchat/utils.py

@@ -48,7 +48,7 @@ GENDERS = {
     11: 'unknown_plural',
 
     # For graphql requests
-    #'': 'unknown',
+    'UNKNOWN': 'unknown',
     'FEMALE': 'female_singular',
     'MALE': 'male_singular',
     #'': 'female_singular_guess',
@@ -93,6 +93,16 @@ class ReqUrl(object):
     TYPING = "https://www.facebook.com/ajax/messaging/typ.php"
     GRAPHQL = "https://www.facebook.com/api/graphqlbatch/"
 
+    pull_channel = 0
+
+    def change_pull_channel(self, channel=None):
+        if channel is None:
+            self.pull_channel = (self.pull_channel + 1) % 5 # Pull channel will be 0-4
+        else:
+            self.pull_channel = channel
+        self.STICKY = "https://{}-edge-chat.facebook.com/pull".format(self.pull_channel)
+        self.PING = "https://{}-edge-chat.facebook.com/active_ping".format(self.pull_channel)
+
 
 facebookEncoding = 'UTF-8'
 
@@ -103,7 +113,7 @@ def strip_to_json(text):
     try:
         return text[text.index('{'):]
     except ValueError:
-        raise Exception('No JSON object found: {}, {}'.format(repr(text), text.index('{')))
+        raise FBchatException('No JSON object found: {}, {}'.format(repr(text), text.index('{')))
 
 def get_decoded_r(r):
     return get_decoded(r._content)
@@ -143,30 +153,31 @@ def generateOfflineThreadingID():
     return str(int(msgs, 2))
 
 def check_json(j):
-    if 'error' in j and j['error'] is not None:
-        if 'errorDescription' in j:
-            # 'errorDescription' is in the users own language!
-            raise Exception('Error #{} when sending request: {}'.format(j['error'], j['errorDescription']))
-        elif 'debug_info' in j['error']:
-            raise Exception('Error #{} when sending request: {}'.format(j['error']['code'], repr(j['error']['debug_info'])))
-        else:
-            raise Exception('Error {} when sending request'.format(j['error']))
+    if j.get('error') is None:
+        return
+    if 'errorDescription' in j:
+        # 'errorDescription' is in the users own language!
+        raise FBchatFacebookError('Error #{} when sending request: {}'.format(j['error'], j['errorDescription']), fb_error_code=j['error'], fb_error_message=j['errorDescription'])
+    elif 'debug_info' in j['error'] and 'code' in j['error']:
+        raise FBchatFacebookError('Error #{} when sending request: {}'.format(j['error']['code'], repr(j['error']['debug_info'])), fb_error_code=j['error']['code'], fb_error_message=j['error']['debug_info'])
+    else:
+        raise FBchatFacebookError('Error {} when sending request'.format(j['error']), fb_error_code=j['error'])
 
-def checkRequest(r, do_json_check=True):
+def check_request(r, as_json=True):
     if not r.ok:
-        raise Exception('Error when sending request: Got {} response'.format(r.status_code))
+        raise FBchatFacebookError('Error when sending request: Got {} response'.format(r.status_code), request_status_code=r.status_code)
 
     content = get_decoded_r(r)
 
     if content is None or len(content) == 0:
-        raise Exception('Error when sending request: Got empty response')
+        raise FBchatFacebookError('Error when sending request: Got empty response')
 
-    if do_json_check:
+    if as_json:
         content = strip_to_json(content)
         try:
             j = json.loads(content)
-        except Exception as e:
-            raise Exception('Error while parsing JSON: {}'.format(repr(content)), e)
+        except ValueError:
+            raise FBchatFacebookError('Error while parsing JSON: {}'.format(repr(content)))
         check_json(j)
         return j
     else:

requirements.txt

@@ -1,3 +1,4 @@
 requests
 lxml
 beautifulsoup4
+enum34; python_version == '2.7'

setup.py

@@ -18,7 +18,7 @@ with open('README.rst') as f:
 
 try:
     requirements = [line.rstrip('\n') for line in open(os.path.join('fbchat.egg-info', 'requires.txt'))]
-except FileNotFoundError:
+except IOError:
     requirements = [line.rstrip('\n') for line in open('requirements.txt')]
 
 version = None