From 59ba418faa65d5aa95691a4d7a176155bcda1260 Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Tue, 19 Sep 2017 16:31:53 +0200 Subject: [PATCH 1/7] Added custom exceptions Added `FBchatException`, `FBchatFacebookError` and `FBchatUserError`, which can help in differentiating between errors --- fbchat/client.py | 115 +++++++++++++++++++++++----------------------- fbchat/graphql.py | 4 +- fbchat/models.py | 20 ++++++++ fbchat/utils.py | 27 +++++------ 4 files changed, 94 insertions(+), 72 deletions(-) diff --git a/fbchat/client.py b/fbchat/client.py index 3c40cc7..53dc847 100644 --- a/fbchat/client.py +++ b/fbchat/client.py @@ -193,7 +193,7 @@ class Client(object): def _login(self): if not (self.email and self.password): - raise Exception("Email and password not found.") + raise FBchatUserError("Email and password not found.") soup = bs(self._get(ReqUrl.MOBILE).text, "lxml") data = dict((elem['name'], elem['value']) for elem in soup.findAll("input") if elem.has_attr('value') and elem.has_attr('name')) @@ -306,7 +306,8 @@ class Client(object): # Load cookies into current session self._session.cookies = requests.cookies.merge_cookies(self._session.cookies, session_cookies) self._postLogin() - except Exception: + except Exception as e: + self.exception('Failed loading session') self._resetValues() return False return True @@ -319,15 +320,15 @@ class Client(object): :param password: Facebook account password :param max_tries: Maximum number of times to try logging in :type max_tries: int - :raises: Exception on failed login + :raises: FBchatException on failed login """ self.onLoggingIn(email=email) if max_tries < 1: - raise Exception('Cannot login: max_tries should be at least one') + raise FBchatUserError('Cannot login: max_tries should be at least one') if not (email and password): - raise Exception('Email and password not set') + raise FBchatUserError('Email and password not set') self.email = email self.password = password @@ -342,7 +343,7 @@ class Client(object): self.onLoggedIn(email=email) break else: - raise Exception('Login failed. Check email/password. (Failed on url: {})'.format(login_url)) + raise FBchatUserError('Login failed. Check email/password. (Failed on url: {})'.format(login_url)) def logout(self): """ @@ -415,15 +416,15 @@ class Client(object): :return: :class:`models.User` objects :rtype: list - :raises: Exception if request failed + :raises: FBchatException if request failed """ data = { 'viewer': self.uid, } j = checkRequest(self._post(ReqUrl.ALL_USERS, query=data)) - if not j['payload']: - raise Exception('Missing payload') + if j.get('payload') is None: + raise FBchatException('Missing payload while fetching users: {}'.format(j)) users = [] @@ -445,7 +446,7 @@ class Client(object): :param limit: The max. amount of users to fetch :return: :class:`models.User` objects, ordered by relevance :rtype: list - :raises: Exception if request failed + :raises: FBchatException if request failed """ j = self.graphql_request(GraphQL(query=GraphQL.SEARCH_USER, params={'search': name, 'limit': limit})) @@ -459,7 +460,7 @@ class Client(object): :param name: Name of the page :return: :class:`models.Page` objects, ordered by relevance :rtype: list - :raises: Exception if request failed + :raises: FBchatException if request failed """ j = self.graphql_request(GraphQL(query=GraphQL.SEARCH_PAGE, params={'search': name, 'limit': limit})) @@ -474,7 +475,7 @@ class Client(object): :param limit: The max. amount of groups to fetch :return: :class:`models.Group` objects, ordered by relevance :rtype: list - :raises: Exception if request failed + :raises: FBchatException if request failed """ j = self.graphql_request(GraphQL(query=GraphQL.SEARCH_GROUP, params={'search': name, 'limit': limit})) @@ -489,7 +490,7 @@ class Client(object): :param limit: The max. amount of groups to fetch :return: :class:`models.User`, :class:`models.Group` and :class:`models.Page` objects, ordered by relevance :rtype: list - :raises: Exception if request failed + :raises: FBchatException if request failed """ j = self.graphql_request(GraphQL(query=GraphQL.SEARCH_THREAD, params={'search': name, 'limit': limit})) @@ -517,8 +518,8 @@ class Client(object): } j = checkRequest(self._post(ReqUrl.INFO, data)) - if not j['payload']['profiles']: - raise Exception('No users/pages returned') + if j.get('payload') is None or j['payload'].get('profiles') is None: + raise FBchatException('No users/pages returned: {}'.format(j)) entries = {} for _id in j['payload']['profiles']: @@ -543,7 +544,7 @@ class Client(object): 'name': k.get('name') } else: - raise Exception('{} had an unknown thread type: {}'.format(_id, k)) + raise FBchatException('{} had an unknown thread type: {}'.format(_id, k)) log.debug(entries) return entries @@ -558,7 +559,7 @@ class Client(object): :param user_ids: One or more user ID(s) to query :return: :class:`models.User` objects, labeled by their ID :rtype: dict - :raises: Exception if request failed + :raises: FBchatException if request failed """ threads = self.fetchThreadInfo(*user_ids) @@ -567,7 +568,7 @@ class Client(object): if threads[k].type == ThreadType.USER: users[k] = threads[k] else: - raise Exception('Thread {} was not a user'.format(threads[k])) + raise FBchatUserError('Thread {} was not a user'.format(threads[k])) return users @@ -581,7 +582,7 @@ class Client(object): :param page_ids: One or more page ID(s) to query :return: :class:`models.Page` objects, labeled by their ID :rtype: dict - :raises: Exception if request failed + :raises: FBchatException if request failed """ threads = self.fetchThreadInfo(*page_ids) @@ -590,7 +591,7 @@ class Client(object): if threads[k].type == ThreadType.PAGE: pages[k] = threads[k] else: - raise Exception('Thread {} was not a page'.format(threads[k])) + raise FBchatUserError('Thread {} was not a page'.format(threads[k])) return pages @@ -601,7 +602,7 @@ class Client(object): :param group_ids: One or more group ID(s) to query :return: :class:`models.Group` objects, labeled by their ID :rtype: dict - :raises: Exception if request failed + :raises: FBchatException if request failed """ threads = self.fetchThreadInfo(*group_ids) @@ -610,7 +611,7 @@ class Client(object): if threads[k].type == ThreadType.GROUP: groups[k] = threads[k] else: - raise Exception('Thread {} was not a group'.format(threads[k])) + raise FBchatUserError('Thread {} was not a group'.format(threads[k])) return groups @@ -624,7 +625,7 @@ class Client(object): :param thread_ids: One or more thread ID(s) to query :return: :class:`models.Thread` objects, labeled by their ID :rtype: dict - :raises: Exception if request failed + :raises: FBchatException if request failed """ queries = [] @@ -638,7 +639,7 @@ class Client(object): })) j = self.graphql_requests(*queries) - + for i, entry in enumerate(j): if entry.get('message_thread') is None: # If you don't have an existing thread with this person, attempt to retrieve user data anyways @@ -663,14 +664,14 @@ class Client(object): elif entry.get('thread_type') == 'ONE_TO_ONE': _id = entry['thread_key']['other_user_id'] if pages_and_users.get(_id) is None: - raise Exception('Could not fetch thread {}'.format(_id)) + raise FBchatException('Could not fetch thread {}'.format(_id)) entry.update(pages_and_users[_id]) if entry['type'] == ThreadType.USER: rtn[_id] = graphql_to_user(entry) else: rtn[_id] = graphql_to_page(entry) else: - raise Exception('{} had an unknown thread type: {}'.format(thread_ids[i], entry)) + raise FBchatException('{} had an unknown thread type: {}'.format(thread_ids[i], entry)) return rtn @@ -685,7 +686,7 @@ class Client(object): :type before: int :return: :class:`models.Message` objects :rtype: list - :raises: Exception if request failed + :raises: FBchatException if request failed """ j = self.graphql_request(GraphQL(doc_id='1386147188135407', params={ @@ -696,8 +697,8 @@ class Client(object): 'before': before })) - if j['message_thread'] is None: - raise Exception('Could not fetch thread {}'.format(thread_id)) + if j.get('message_thread') is None: + raise FBchatException('Could not fetch thread {}: {}'.format(thread_id, j)) return list(reversed([graphql_to_message(message) for message in j['message_thread']['messages']['nodes']])) @@ -710,11 +711,11 @@ class Client(object): :type limit: int :return: :class:`models.Thread` objects :rtype: list - :raises: Exception if request failed + :raises: FBchatException if request failed """ if limit > 20 or limit < 1: - raise Exception('`limit` should be between 1 and 20') + raise FBchatUserError('`limit` should be between 1 and 20') data = { 'client' : self.client, @@ -724,7 +725,7 @@ class Client(object): j = checkRequest(self._post(ReqUrl.THREADS, data)) if j.get('payload') is None: - raise Exception('Missing payload: {}, with data: {}'.format(j, data)) + raise FBchatException('Missing payload: {}, with data: {}'.format(j, data)) participants = {} for p in j['payload']['participants']: @@ -733,19 +734,19 @@ class Client(object): elif p['type'] == 'user': participants[p['fbid']] = User(p['fbid'], url=p['href'], first_name=p['short_name'], is_friend=p['is_friend'], gender=GENDERS[p['gender']], photo=p['image_src'], name=p['name']) else: - raise Exception('A participant had an unknown type {}: {}'.format(p['type'], p)) + raise FBchatException('A participant had an unknown type {}: {}'.format(p['type'], p)) entries = [] for k in j['payload']['threads']: if k['thread_type'] == 1: if k['other_user_fbid'] not in participants: - raise Exception('A thread was not in participants: {}'.format(j['payload'])) + raise FBchatException('The thread {} was not in participants: {}'.format(k, j['payload'])) participants[k['other_user_fbid']].message_count = k['message_count'] entries.append(participants[k['other_user_fbid']]) elif k['thread_type'] == 2: entries.append(Group(k['thread_fbid'], participants=set([p.strip('fbid:') for p in k['participants']]), photo=k['image_src'], name=k['name'], message_count=k['message_count'])) else: - raise Exception('A thread had an unknown thread type: {}'.format(k)) + raise FBchatException('A thread had an unknown thread type: {}'.format(k)) return entries @@ -754,7 +755,7 @@ class Client(object): .. todo:: Documenting this - :raises: Exception if request failed + :raises: FBchatException if request failed """ form = { 'client': 'mercury_sync', @@ -830,14 +831,14 @@ class Client(object): log.warning("Got multiple message ids' back: {}".format(message_ids)) message_id = message_ids[0] except (KeyError, IndexError) as e: - raise Exception('Error when sending message: No message IDs could be found: {}'.format(j)) + raise FBchatException('Error when sending message: No message IDs could be found: {}'.format(j)) - # update JS token if receive from response - if ('jsmods' in j) and ('require' in j['jsmods']): + # update JS token if received in response + if j.get('jsmods') is not None and j['jsmods'].get('require') is not None: try: self.payloadDefault['fb_dtsg'] = j['jsmods']['require'][0][3][0] except (KeyError, IndexError) as e: - log.warning("Error when update fb_dtsg. Facebook might have changed protocol.") + log.warning('Error when updating fb_dtsg. Facebook might have changed protocol!') return message_id @@ -850,7 +851,7 @@ class Client(object): :param thread_type: See :ref:`intro_threads` :type thread_type: models.ThreadType :return: :ref:`Message ID ` of the sent message - :raises: Exception if request failed + :raises: FBchatException if request failed """ thread_id, thread_type = self._getThread(thread_id, thread_type) data = self._getSendData(thread_id, thread_type) @@ -874,7 +875,7 @@ class Client(object): :type size: models.EmojiSize :type thread_type: models.ThreadType :return: :ref:`Message ID ` of the sent emoji - :raises: Exception if request failed + :raises: FBchatException if request failed """ thread_id, thread_type = self._getThread(thread_id, thread_type) data = self._getSendData(thread_id, thread_type) @@ -914,7 +915,7 @@ class Client(object): :param thread_type: See :ref:`intro_threads` :type thread_type: models.ThreadType :return: :ref:`Message ID ` of the sent image - :raises: Exception if request failed + :raises: FBchatException if request failed """ thread_id, thread_type = self._getThread(thread_id, thread_type) data = self._getSendData(thread_id, thread_type) @@ -939,7 +940,7 @@ class Client(object): :param thread_type: See :ref:`intro_threads` :type thread_type: models.ThreadType :return: :ref:`Message ID ` of the sent image - :raises: Exception if request failed + :raises: FBchatException if request failed """ thread_id, thread_type = self._getThread(thread_id, thread_type) mimetype = guess_type(image_url)[0] @@ -957,7 +958,7 @@ class Client(object): :param thread_type: See :ref:`intro_threads` :type thread_type: models.ThreadType :return: :ref:`Message ID ` of the sent image - :raises: Exception if request failed + :raises: FBchatException if request failed """ thread_id, thread_type = self._getThread(thread_id, thread_type) mimetype = guess_type(image_path)[0] @@ -972,7 +973,7 @@ class Client(object): :param thread_id: Group ID to add people to. See :ref:`intro_threads` :type user_ids: list :return: :ref:`Message ID ` of the executed action - :raises: Exception if request failed + :raises: FBchatException if request failed """ thread_id, thread_type = self._getThread(thread_id, None) data = self._getSendData(thread_id, ThreadType.GROUP) @@ -988,7 +989,7 @@ class Client(object): for i, user_id in enumerate(user_ids): if user_id == self.uid: - raise Exception('Error when adding users: Cannot add self to group thread') + raise FBchatUserError('Error when adding users: Cannot add self to group thread') else: data['log_message_data[added_participants][' + str(i) + ']'] = "fbid:" + str(user_id) @@ -1000,7 +1001,7 @@ class Client(object): :param user_id: User ID to remove :param thread_id: Group ID to remove people from. See :ref:`intro_threads` - :raises: Exception if request failed + :raises: FBchatException if request failed """ thread_id, thread_type = self._getThread(thread_id, None) @@ -1021,7 +1022,7 @@ class Client(object): :param thread_id: Group ID to change title of. See :ref:`intro_threads` :param thread_type: See :ref:`intro_threads` :type thread_type: models.ThreadType - :raises: Exception if request failed + :raises: FBchatException if request failed """ thread_id, thread_type = self._getThread(thread_id, thread_type) @@ -1047,7 +1048,7 @@ class Client(object): :param thread_id: User/Group ID to change color of. See :ref:`intro_threads` :param thread_type: See :ref:`intro_threads` :type thread_type: models.ThreadType - :raises: Exception if request failed + :raises: FBchatException if request failed """ thread_id, thread_type = self._getThread(thread_id, thread_type) @@ -1066,7 +1067,7 @@ class Client(object): :param color: New thread color :param thread_id: User/Group ID to change color of. See :ref:`intro_threads` :type color: models.ThreadColor - :raises: Exception if request failed + :raises: FBchatException if request failed """ thread_id, thread_type = self._getThread(thread_id, None) @@ -1085,7 +1086,7 @@ class Client(object): :param color: New thread emoji :param thread_id: User/Group ID to change emoji of. See :ref:`intro_threads` - :raises: Exception if request failed + :raises: FBchatException if request failed """ thread_id, thread_type = self._getThread(thread_id, None) @@ -1103,7 +1104,7 @@ class Client(object): :param message_id: :ref:`Message ID ` to react to :param reaction: Reaction emoji to use :type reaction: models.MessageReaction - :raises: Exception if request failed + :raises: FBchatException if request failed """ full_data = { "doc_id": 1491398900900362, @@ -1137,7 +1138,7 @@ class Client(object): :param thread_type: See :ref:`intro_threads` :type status: models.TypingStatus :type thread_type: models.ThreadType - :raises: Exception if request failed + :raises: FBchatException if request failed """ thread_id, thread_type = self._getThread(thread_id, None) @@ -1232,8 +1233,8 @@ class Client(object): j = checkRequest(self._get(ReqUrl.STICKY, data)) - if 'lb_info' not in j: - raise Exception('Missing lb_info') + if j.get('lb_info') is None: + raise FBchatException('Missing lb_info: {}'.format(j)) return j['lb_info']['sticky'], j['lb_info']['pool'] @@ -1417,7 +1418,7 @@ class Client(object): """ Start listening from an external event loop - :raises: Exception if request failed + :raises: FBchatException if request failed """ self.listening = True self.sticky, self.pool = self._fetchSticky() diff --git a/fbchat/graphql.py b/fbchat/graphql.py index 70679cf..ecfca3b 100644 --- a/fbchat/graphql.py +++ b/fbchat/graphql.py @@ -31,7 +31,7 @@ def graphql_color_to_enum(color): try: return ThreadColor('#{}'.format(color[2:].lower())) except ValueError: - raise Exception('Could not get ThreadColor from color: {}'.format(color)) + raise FBchatException('Could not get ThreadColor from color: {}'.format(color)) def get_customization_info(thread): if thread is None or thread.get('customization_info') is None: @@ -176,7 +176,7 @@ class GraphQL(object): 'query_params': params } else: - raise Exception('A query or doc_id must be specified') + raise FBchatUserError('A query or doc_id must be specified') FRAGMENT_USER = """ diff --git a/fbchat/models.py b/fbchat/models.py index 3f9fd77..c61ceb1 100644 --- a/fbchat/models.py +++ b/fbchat/models.py @@ -4,6 +4,26 @@ from __future__ import unicode_literals import enum +class FBchatException(Exception): + """Custom exception thrown by fbchat. All exceptions in the fbchat module inherits this""" + +class FBchatFacebookError(FBchatException): + #: The error code that Facebook returned + fb_error_code = str + #: The error message that Facebook returned (In the user's own language) + fb_error_message = str + #: The status code that was sent in the http response (eg. 404) (Usually only set if not successful, aka. not 200) + request_status_code = int + def __init__(self, message, fb_error_code=None, fb_error_message=None, request_status_code=None): + super(FBchatFacebookError, self).__init__(message) + """Thrown by fbchat when Facebook returns an error""" + self.fb_error_code = str(fb_error_code) + self.fb_error_message = fb_error_message + self.request_status_code = request_status_code + +class FBchatUserError(FBchatException): + """Thrown by fbchat when wrong values are entered""" + class Thread(object): #: The unique identifier of the thread. Can be used a `thread_id`. See :ref:`intro_threads` for more info uid = str diff --git a/fbchat/utils.py b/fbchat/utils.py index f18baf8..4f38183 100644 --- a/fbchat/utils.py +++ b/fbchat/utils.py @@ -103,7 +103,7 @@ def strip_to_json(text): try: return text[text.index('{'):] except ValueError: - raise Exception('No JSON object found: {}, {}'.format(repr(text), text.index('{'))) + raise FBchatException('No JSON object found: {}, {}'.format(repr(text), text.index('{'))) def get_decoded_r(r): return get_decoded(r._content) @@ -143,30 +143,31 @@ def generateOfflineThreadingID(): return str(int(msgs, 2)) def check_json(j): - if 'error' in j and j['error'] is not None: - if 'errorDescription' in j: - # 'errorDescription' is in the users own language! - raise Exception('Error #{} when sending request: {}'.format(j['error'], j['errorDescription'])) - elif 'debug_info' in j['error']: - raise Exception('Error #{} when sending request: {}'.format(j['error']['code'], repr(j['error']['debug_info']))) - else: - raise Exception('Error {} when sending request'.format(j['error'])) + if j.get('error') is None: + return + if 'errorDescription' in j: + # 'errorDescription' is in the users own language! + raise FBchatFacebookError('Error #{} when sending request: {}'.format(j['error'], j['errorDescription']), fb_error_code=j['error'], fb_error_message=j['errorDescription']) + elif 'debug_info' in j['error'] and 'code' in j['error']: + raise FBchatFacebookError('Error #{} when sending request: {}'.format(j['error']['code'], repr(j['error']['debug_info'])), fb_error_code=j['error']['code'], fb_error_message=j['error']['debug_info']) + else: + raise FBchatFacebookError('Error {} when sending request'.format(j['error']), fb_error_code=j['error']) def checkRequest(r, do_json_check=True): if not r.ok: - raise Exception('Error when sending request: Got {} response'.format(r.status_code)) + raise FBchatFacebookError('Error when sending request: Got {} response'.format(r.status_code), request_error_code=r.status_code) content = get_decoded_r(r) if content is None or len(content) == 0: - raise Exception('Error when sending request: Got empty response') + raise FBchatFacebookError('Error when sending request: Got empty response') if do_json_check: content = strip_to_json(content) try: j = json.loads(content) - except Exception as e: - raise Exception('Error while parsing JSON: {}'.format(repr(content)), e) + except ValueError: + raise FBchatFacebookError('Error while parsing JSON: {}'.format(repr(content))) check_json(j) return j else: From ead9a3c0e93ad74dc9c3503bbc350d3722c8fc36 Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Tue, 19 Sep 2017 16:36:08 +0200 Subject: [PATCH 2/7] Improved error handling, and improved uid-loading Requests would sometimes throw an error while retrieving the c_user cookie (If there were multiple cookies with this name) --- fbchat/client.py | 5 ++++- fbchat/graphql.py | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/fbchat/client.py b/fbchat/client.py index 53dc847..2abb7fa 100644 --- a/fbchat/client.py +++ b/fbchat/client.py @@ -157,7 +157,10 @@ class Client(object): self.payloadDefault = {} self.client_id = hex(int(random()*2147483648))[2:] self.start_time = now() - self.uid = str(self._session.cookies['c_user']) + self.uid = self._session.cookies.get_dict().get('c_user') + if self.uid is None: + raise Exception('Could not find c_user cookie') + self.uid = str(self.uid) self.user_channel = "p_" + self.uid self.ttstamp = '' diff --git a/fbchat/graphql.py b/fbchat/graphql.py index ecfca3b..83676f4 100644 --- a/fbchat/graphql.py +++ b/fbchat/graphql.py @@ -143,7 +143,11 @@ def graphql_queries_to_json(*queries): return json.dumps(rtn) def graphql_response_to_json(content): - j = json.loads(content, cls=ConcatJSONDecoder) + content = strip_to_json(content) # Usually only needed in some error cases + try: + j = json.loads(content, cls=ConcatJSONDecoder) + except Exception as e: + raise Exception('Error while parsing JSON: {}'.format(repr(content)), e) rtn = [None]*(len(j)) for x in j: From 1d2576b06d0ea9ba901a1716a495193c9004e623 Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Tue, 19 Sep 2017 16:36:24 +0200 Subject: [PATCH 3/7] More custom exceptions --- fbchat/graphql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fbchat/graphql.py b/fbchat/graphql.py index 83676f4..857d19e 100644 --- a/fbchat/graphql.py +++ b/fbchat/graphql.py @@ -146,8 +146,8 @@ def graphql_response_to_json(content): content = strip_to_json(content) # Usually only needed in some error cases try: j = json.loads(content, cls=ConcatJSONDecoder) - except Exception as e: - raise Exception('Error while parsing JSON: {}'.format(repr(content)), e) + except Exception: + raise FBchatException('Error while parsing JSON: {}'.format(repr(content))) rtn = [None]*(len(j)) for x in j: From cb8b0915de3529246d8c648d1c083216b6bddbaf Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Tue, 19 Sep 2017 16:42:03 +0200 Subject: [PATCH 4/7] Improved default `doOneListen` loop --- fbchat/client.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/fbchat/client.py b/fbchat/client.py index 2abb7fa..2f9d818 100644 --- a/fbchat/client.py +++ b/fbchat/client.py @@ -1438,17 +1438,17 @@ class Client(object): """ try: if markAlive: self._ping(self.sticky, self.pool) - try: - content = self._pullMessage(self.sticky, self.pool) - if content: self._parseMessage(content) - except requests.exceptions.RequestException: - pass - except Exception as e: - return self.onListenError(exception=e) + content = self._pullMessage(self.sticky, self.pool) + if content: + self._parseMessage(content) except KeyboardInterrupt: return False - except requests.exceptions.Timeout: + except requests.Timeout: pass + except requests.ConnectionError: + time.sleep(30) + except Exception as e: + return self.onListenError(exception=e) return True @@ -1509,8 +1509,10 @@ class Client(object): Called when an error was encountered while listening :param exception: The exception that was encountered + :return: Whether the loop should keep running """ log.exception('Got exception while listening') + return True def onMessage(self, mid=None, author_id=None, message=None, thread_id=None, thread_type=ThreadType.USER, ts=None, metadata=None, msg={}): From a64982583b2c71cba31c073bae6ed07863e24133 Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Tue, 19 Sep 2017 23:08:48 +0200 Subject: [PATCH 5/7] Fixes 502/503 errors and a the `1357004` error Thereby also moving ReqUrl to self.req_url --- fbchat/client.py | 157 +++++++++++++++++++++++++++++++---------------- fbchat/utils.py | 14 ++++- 2 files changed, 116 insertions(+), 55 deletions(-) diff --git a/fbchat/client.py b/fbchat/client.py index 2f9d818..fd40153 100644 --- a/fbchat/client.py +++ b/fbchat/client.py @@ -43,7 +43,7 @@ class Client(object): :type max_tries: int :type session_cookies: dict :type logging_level: int - :raises: Exception on failed login + :raises: FBchatException on failed login """ self.sticky, self.pool = (None, None) @@ -54,14 +54,15 @@ class Client(object): self.client = 'mercury' self.default_thread_id = None self.default_thread_type = None + self.req_url = ReqUrl() if not user_agent: user_agent = choice(USER_AGENTS) self._header = { 'Content-Type' : 'application/x-www-form-urlencoded', - 'Referer' : ReqUrl.BASE, - 'Origin' : ReqUrl.BASE, + 'Referer' : self.req_url.BASE, + 'Origin' : self.req_url.BASE, 'User-Agent' : user_agent, 'Connection' : 'keep-alive', } @@ -91,13 +92,50 @@ class Client(object): self.req_counter += 1 return payload - def _get(self, url, query=None, timeout=30): - payload = self._generatePayload(query) - return self._session.get(url, headers=self._header, params=payload, timeout=timeout) + def _fix_1357004(self, error_code): + """ + This fixes "Please try closing and re-opening your browser window" errors (1357004) + This error usually happens after 1-2 days of inactivity + It may be a bad idea to do this in an exception handler, if you have a better method, please suggest it! + """ + if error_code == '1357004': + log.warning('Got error #1357004. Doing a _postLogin, and resending request') + self._postLogin() + return True + return False - def _post(self, url, query=None, timeout=30): + def _get(self, url, query=None, timeout=30, fix_request=False, as_json=False, error_retries=3): payload = self._generatePayload(query) - return self._session.post(url, headers=self._header, data=payload, timeout=timeout) + r = self._session.get(url, headers=self._header, params=payload, timeout=timeout) + if not fix_request: + return r + try: + return check_request(r, as_json=as_json) + except FBchatFacebookError as e: + if error_retries > 0 and self._fix_1357004(e.fb_error_code): + return self._get(url, query=query, timeout=timeout, fix_request=fix_request, as_json=as_json, error_retries=error_retries-1) + raise e + + def _post(self, url, query=None, timeout=30, fix_request=False, as_json=False, error_retries=3): + payload = self._generatePayload(query) + r = self._session.post(url, headers=self._header, data=payload, timeout=timeout) + if not fix_request: + return r + try: + return check_request(r, as_json=as_json) + except FBchatFacebookError as e: + if error_retries > 0 and self._fix_1357004(e.fb_error_code): + return self._post(url, query=query, timeout=timeout, fix_request=fix_request, as_json=as_json, error_retries=error_retries-1) + raise e + + def _graphql(self, payload, error_retries=3): + content = self._post(self.req_url.GRAPHQL, payload, fix_request=True, as_json=False) + try: + return graphql_response_to_json(content) + except FBchatFacebookError as e: + if error_retries > 0 and self._fix_1357004(e.fb_error_code): + return self._graphql(payload, error_retries=error_retries-1) + raise e def _cleanGet(self, url, query=None, timeout=30): return self._session.get(url, headers=self._header, params=query, timeout=timeout) @@ -106,38 +144,42 @@ class Client(object): self.req_counter += 1 return self._session.post(url, headers=self._header, data=query, timeout=timeout) - def _postFile(self, url, files=None, query=None, timeout=30): + def _postFile(self, url, files=None, query=None, timeout=30, fix_request=False, as_json=False, error_retries=3): payload=self._generatePayload(query) # Removes 'Content-Type' from the header headers = dict((i, self._header[i]) for i in self._header if i != 'Content-Type') - return self._session.post(url, headers=headers, data=payload, timeout=timeout, files=files) + r = self._session.post(url, headers=headers, data=payload, timeout=timeout, files=files) + if not fix_request: + return r + try: + return check_request(r, as_json=as_json) + except FBchatFacebookError as e: + if error_retries > 0 and self._fix_1357004(e.fb_error_code): + return self._postFile(url, files=files, query=query, timeout=timeout, fix_request=fix_request, as_json=as_json, error_retries=error_retries-1) + raise e def graphql_requests(self, *queries): """ .. todo:: Documenting this - :raises: Exception if request failed + :raises: FBchatException if request failed """ - payload = { + + return tuple(self._graphql({ 'method': 'GET', 'response_format': 'json', 'queries': graphql_queries_to_json(*queries) - } - - j = graphql_response_to_json(checkRequest(self._post(ReqUrl.GRAPHQL, payload), do_json_check=False)) - - return tuple(j) + })) def graphql_request(self, query): """ Shorthand for `graphql_requests(query)[0]` - :raises: Exception if request failed + :raises: FBchatException if request failed """ return self.graphql_requests(query)[0] - """ END INTERNAL REQUEST METHODS """ @@ -159,12 +201,12 @@ class Client(object): self.start_time = now() self.uid = self._session.cookies.get_dict().get('c_user') if self.uid is None: - raise Exception('Could not find c_user cookie') + raise FBchatException('Could not find c_user cookie') self.uid = str(self.uid) self.user_channel = "p_" + self.uid self.ttstamp = '' - r = self._get(ReqUrl.BASE) + r = self._get(self.req_url.BASE) soup = bs(r.text, "lxml") self.fb_dtsg = soup.find("input", {'name':'fb_dtsg'})['value'] self.fb_h = soup.find("input", {'name':'h'})['value'] @@ -198,13 +240,13 @@ class Client(object): if not (self.email and self.password): raise FBchatUserError("Email and password not found.") - soup = bs(self._get(ReqUrl.MOBILE).text, "lxml") + soup = bs(self._get(self.req_url.MOBILE).text, "lxml") data = dict((elem['name'], elem['value']) for elem in soup.findAll("input") if elem.has_attr('value') and elem.has_attr('name')) data['email'] = self.email data['pass'] = self.password data['login'] = 'Log In' - r = self._cleanPost(ReqUrl.LOGIN, data) + r = self._cleanPost(self.req_url.LOGIN, data) # Usually, 'Checkpoint' will refer to 2FA if ('checkpoint' in r.url and @@ -213,7 +255,7 @@ class Client(object): # Sometimes Facebook tries to show the user a "Save Device" dialog if 'save-device' in r.url: - r = self._cleanGet(ReqUrl.SAVE_DEVICE) + r = self._cleanGet(self.req_url.SAVE_DEVICE) if 'home' in r.url: self._postLogin() @@ -234,7 +276,7 @@ class Client(object): data['codes_submitted'] = 0 log.info('Submitting 2FA code.') - r = self._cleanPost(ReqUrl.CHECKPOINT, data) + r = self._cleanPost(self.req_url.CHECKPOINT, data) if 'home' in r.url: return r @@ -246,14 +288,14 @@ class Client(object): data['name_action_selected'] = 'save_device' data['submit[Continue]'] = 'Continue' log.info('Saving browser.') # At this stage, we have dtsg, nh, name_action_selected, submit[Continue] - r = self._cleanPost(ReqUrl.CHECKPOINT, data) + r = self._cleanPost(self.req_url.CHECKPOINT, data) if 'home' in r.url: return r del(data['name_action_selected']) log.info('Starting Facebook checkup flow.') # At this stage, we have dtsg, nh, submit[Continue] - r = self._cleanPost(ReqUrl.CHECKPOINT, data) + r = self._cleanPost(self.req_url.CHECKPOINT, data) if 'home' in r.url: return r @@ -261,7 +303,7 @@ class Client(object): del(data['submit[Continue]']) data['submit[This was me]'] = 'This Was Me' log.info('Verifying login attempt.') # At this stage, we have dtsg, nh, submit[This was me] - r = self._cleanPost(ReqUrl.CHECKPOINT, data) + r = self._cleanPost(self.req_url.CHECKPOINT, data) if 'home' in r.url: return r @@ -270,7 +312,7 @@ class Client(object): data['submit[Continue]'] = 'Continue' data['name_action_selected'] = 'save_device' log.info('Saving device again.') # At this stage, we have dtsg, nh, submit[Continue], name_action_selected - r = self._cleanPost(ReqUrl.CHECKPOINT, data) + r = self._cleanPost(self.req_url.CHECKPOINT, data) return r def isLoggedIn(self): @@ -281,7 +323,7 @@ class Client(object): :rtype: bool """ # Send a request to the login url, to see if we're directed to the home page - r = self._cleanGet(ReqUrl.LOGIN) + r = self._cleanGet(self.req_url.LOGIN) return 'home' in r.url def getSession(self): @@ -309,7 +351,7 @@ class Client(object): # Load cookies into current session self._session.cookies = requests.cookies.merge_cookies(self._session.cookies, session_cookies) self._postLogin() - except Exception as e: + except FBchatException as e: self.exception('Failed loading session') self._resetValues() return False @@ -361,7 +403,7 @@ class Client(object): 'h': self.fb_h } - r = self._get(ReqUrl.LOGOUT, data) + r = self._get(self.req_url.LOGOUT, data) self._resetValues() @@ -425,7 +467,7 @@ class Client(object): data = { 'viewer': self.uid, } - j = checkRequest(self._post(ReqUrl.ALL_USERS, query=data)) + j = self._post(self.req_url.ALL_USERS, query=data, fix_request=True, as_json=True) if j.get('payload') is None: raise FBchatException('Missing payload while fetching users: {}'.format(j)) @@ -519,7 +561,7 @@ class Client(object): data = { "ids[{}]".format(i): _id for i, _id in enumerate(ids) } - j = checkRequest(self._post(ReqUrl.INFO, data)) + j = self._post(self.req_url.INFO, data, fix_request=True, as_json=True) if j.get('payload') is None or j['payload'].get('profiles') is None: raise FBchatException('No users/pages returned: {}'.format(j)) @@ -726,7 +768,7 @@ class Client(object): 'inbox[limit]' : limit, } - j = checkRequest(self._post(ReqUrl.THREADS, data)) + j = self._post(self.req_url.THREADS, data, fix_request=True, as_json=True) if j.get('payload') is None: raise FBchatException('Missing payload: {}, with data: {}'.format(j, data)) @@ -767,7 +809,7 @@ class Client(object): # 'last_action_timestamp': 0 } - j = checkRequest(self._post(ReqUrl.THREAD_SYNC, form)) + j = self._post(self.req_url.THREAD_SYNC, form, fix_request=True, as_json=True) return { "message_counts": j['payload']['message_counts'], @@ -826,7 +868,7 @@ class Client(object): def _doSendRequest(self, data): """Sends the data to `SendURL`, and returns the message ID or None on failure""" - j = checkRequest(self._post(ReqUrl.SEND, data)) + j = self._post(self.req_url.SEND, data, fix_request=True, as_json=True) try: message_ids = [action['message_id'] for action in j['payload']['actions'] if 'message_id' in action] @@ -898,13 +940,13 @@ class Client(object): def _uploadImage(self, image_path, data, mimetype): """Upload an image and get the image_id for sending in a message""" - j = checkRequest(self._postFile(ReqUrl.UPLOAD, { + j = self._postFile(self.req_url.UPLOAD, { 'file': ( image_path, data, mimetype ) - })) + }, fix_request=True, as_json=True) # Return the image_id return j['payload']['metadata'][0]['image_id'] @@ -1014,7 +1056,7 @@ class Client(object): "tid": thread_id } - j = checkRequest(self._post(ReqUrl.REMOVE_USER, data)) + j = self._post(self.req_url.REMOVE_USER, data, fix_request=True, as_json=True) def changeThreadTitle(self, title, thread_id=None, thread_type=ThreadType.USER): """ @@ -1061,7 +1103,7 @@ class Client(object): 'thread_or_other_fbid': thread_id } - j = checkRequest(self._post(ReqUrl.THREAD_NICKNAME, data)) + j = self._post(self.req_url.THREAD_NICKNAME, data, fix_request=True, as_json=True) def changeThreadColor(self, color, thread_id=None): """ @@ -1079,7 +1121,7 @@ class Client(object): 'thread_or_other_fbid': thread_id } - j = checkRequest(self._post(ReqUrl.THREAD_COLOR, data)) + j = self._post(self.req_url.THREAD_COLOR, data, fix_request=True, as_json=True) def changeThreadEmoji(self, emoji, thread_id=None): """ @@ -1098,7 +1140,7 @@ class Client(object): 'thread_or_other_fbid': thread_id } - j = checkRequest(self._post(ReqUrl.THREAD_EMOJI, data)) + j = self._post(self.req_url.THREAD_EMOJI, data, fix_request=True, as_json=True) def reactToMessage(self, message_id, reaction): """ @@ -1130,7 +1172,7 @@ class Client(object): .replace('u%27', '%27')\ .replace('%5CU{}'.format(MessageReactionFix[reaction.value][0]), MessageReactionFix[reaction.value][1]) - j = checkRequest(self._post('{}/?{}'.format(ReqUrl.MESSAGE_REACTION, url_part))) + j = self._post('{}/?{}'.format(self.req_url.MESSAGE_REACTION, url_part), fix_request=True, as_json=True) def setTypingStatus(self, status, thread_id=None, thread_type=None): """ @@ -1152,7 +1194,7 @@ class Client(object): "source": "mercury-chat" } - j = checkRequest(self._post(ReqUrl.TYPING, data)) + j = self._post(self.req_url.TYPING, data, fix_request=True, as_json=True) """ END SEND METHODS @@ -1168,7 +1210,7 @@ class Client(object): "thread_ids[%s][0]" % userID: threadID } - r = self._post(ReqUrl.DELIVERED, data) + r = self._post(self.req_url.DELIVERED, data) return r.ok def markAsRead(self, userID): @@ -1182,7 +1224,7 @@ class Client(object): "ids[%s]" % userID: True } - r = self._post(ReqUrl.READ_STATUS, data) + r = self._post(self.req_url.READ_STATUS, data) return r.ok def markAsSeen(self): @@ -1190,7 +1232,7 @@ class Client(object): .. todo:: Documenting this """ - r = self._post(ReqUrl.MARK_SEEN, {"seen_timestamp": 0}) + r = self._post(self.req_url.MARK_SEEN, {"seen_timestamp": 0}) return r.ok def friendConnect(self, friend_id): @@ -1203,7 +1245,7 @@ class Client(object): "action": "confirm" } - r = self._post(ReqUrl.CONNECT, data) + r = self._post(self.req_url.CONNECT, data) return r.ok @@ -1223,7 +1265,7 @@ class Client(object): 'viewer_uid': self.uid, 'state': 'active' } - checkRequest(self._get(ReqUrl.PING, data), do_json_check=False) + self._get(self.req_url.PING, data, fix_request=True, as_json=False) def _fetchSticky(self): """Call pull api to get sticky and pool parameter, newer api needs these parameters to work""" @@ -1234,7 +1276,7 @@ class Client(object): "clientid": self.client_id } - j = checkRequest(self._get(ReqUrl.STICKY, data)) + j = self._get(self.req_url.STICKY, data, fix_request=True, as_json=True) if j.get('lb_info') is None: raise FBchatException('Missing lb_info: {}'.format(j)) @@ -1251,7 +1293,7 @@ class Client(object): "clientid": self.client_id, } - j = checkRequest(self._get(ReqUrl.STICKY, data)) + j = self._get(ReqUrl.STICKY, data, fix_request=True, as_json=True) self.seq = j.get('seq', '0') return j @@ -1437,7 +1479,8 @@ class Client(object): :rtype: bool """ try: - if markAlive: self._ping(self.sticky, self.pool) + if markAlive: + self._ping(self.sticky, self.pool) content = self._pullMessage(self.sticky, self.pool) if content: self._parseMessage(content) @@ -1446,7 +1489,15 @@ class Client(object): except requests.Timeout: pass except requests.ConnectionError: + # If the client has lost their internet connection, keep trying every 30 seconds time.sleep(30) + except FBchatFacebookError as e: + # Fix 502 and 503 pull errors + if e.request_status_code in [502, 503]: + self.req_url.change_pull_channel() + self.startListening() + else: + raise e except Exception as e: return self.onListenError(exception=e) diff --git a/fbchat/utils.py b/fbchat/utils.py index 4f38183..8f49f26 100644 --- a/fbchat/utils.py +++ b/fbchat/utils.py @@ -93,6 +93,16 @@ class ReqUrl(object): TYPING = "https://www.facebook.com/ajax/messaging/typ.php" GRAPHQL = "https://www.facebook.com/api/graphqlbatch/" + pull_channel = 0 + + def change_pull_channel(self, channel=None): + if channel is None: + self.pull_channel = (self.pull_channel + 1) % 5 # Pull channel will be 0-4 + else: + self.pull_channel = channel + self.STICKY = "https://{}-edge-chat.facebook.com/pull".format(self.pull_channel) + self.PING = "https://{}-edge-chat.facebook.com/active_ping".format(self.pull_channel) + facebookEncoding = 'UTF-8' @@ -153,7 +163,7 @@ def check_json(j): else: raise FBchatFacebookError('Error {} when sending request'.format(j['error']), fb_error_code=j['error']) -def checkRequest(r, do_json_check=True): +def check_request(r, as_json=True): if not r.ok: raise FBchatFacebookError('Error when sending request: Got {} response'.format(r.status_code), request_error_code=r.status_code) @@ -162,7 +172,7 @@ def checkRequest(r, do_json_check=True): if content is None or len(content) == 0: raise FBchatFacebookError('Error when sending request: Got empty response') - if do_json_check: + if as_json: content = strip_to_json(content) try: j = json.loads(content) From 1cebbf92e6682c16b6900bca5037fd5602be83ac Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Wed, 20 Sep 2017 11:31:44 +0200 Subject: [PATCH 6/7] Fixed loading sessions --- fbchat/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fbchat/client.py b/fbchat/client.py index fd40153..6416b88 100644 --- a/fbchat/client.py +++ b/fbchat/client.py @@ -351,8 +351,8 @@ class Client(object): # Load cookies into current session self._session.cookies = requests.cookies.merge_cookies(self._session.cookies, session_cookies) self._postLogin() - except FBchatException as e: - self.exception('Failed loading session') + except Exception as e: + log.exception('Failed loading session') self._resetValues() return False return True From f0c6e8612f8f05aa47aa681483bb4c51b037264f Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Thu, 21 Sep 2017 10:09:48 +0200 Subject: [PATCH 7/7] Fixed typo and made name more generic --- fbchat/client.py | 10 +++++----- fbchat/utils.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fbchat/client.py b/fbchat/client.py index 6416b88..6552287 100644 --- a/fbchat/client.py +++ b/fbchat/client.py @@ -92,7 +92,7 @@ class Client(object): self.req_counter += 1 return payload - def _fix_1357004(self, error_code): + def _fix_fb_errors(self, error_code): """ This fixes "Please try closing and re-opening your browser window" errors (1357004) This error usually happens after 1-2 days of inactivity @@ -112,7 +112,7 @@ class Client(object): try: return check_request(r, as_json=as_json) except FBchatFacebookError as e: - if error_retries > 0 and self._fix_1357004(e.fb_error_code): + if error_retries > 0 and self._fix_fb_errors(e.fb_error_code): return self._get(url, query=query, timeout=timeout, fix_request=fix_request, as_json=as_json, error_retries=error_retries-1) raise e @@ -124,7 +124,7 @@ class Client(object): try: return check_request(r, as_json=as_json) except FBchatFacebookError as e: - if error_retries > 0 and self._fix_1357004(e.fb_error_code): + if error_retries > 0 and self._fix_fb_errors(e.fb_error_code): return self._post(url, query=query, timeout=timeout, fix_request=fix_request, as_json=as_json, error_retries=error_retries-1) raise e @@ -133,7 +133,7 @@ class Client(object): try: return graphql_response_to_json(content) except FBchatFacebookError as e: - if error_retries > 0 and self._fix_1357004(e.fb_error_code): + if error_retries > 0 and self._fix_fb_errors(e.fb_error_code): return self._graphql(payload, error_retries=error_retries-1) raise e @@ -154,7 +154,7 @@ class Client(object): try: return check_request(r, as_json=as_json) except FBchatFacebookError as e: - if error_retries > 0 and self._fix_1357004(e.fb_error_code): + if error_retries > 0 and self._fix_fb_errors(e.fb_error_code): return self._postFile(url, files=files, query=query, timeout=timeout, fix_request=fix_request, as_json=as_json, error_retries=error_retries-1) raise e diff --git a/fbchat/utils.py b/fbchat/utils.py index 8f49f26..7d3434b 100644 --- a/fbchat/utils.py +++ b/fbchat/utils.py @@ -165,7 +165,7 @@ def check_json(j): def check_request(r, as_json=True): if not r.ok: - raise FBchatFacebookError('Error when sending request: Got {} response'.format(r.status_code), request_error_code=r.status_code) + raise FBchatFacebookError('Error when sending request: Got {} response'.format(r.status_code), request_status_code=r.status_code) content = get_decoded_r(r)