From f07122d44634cf32b2505c3378a1ee984043c5e4 Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Mon, 1 Jul 2019 13:30:22 +0200 Subject: [PATCH 01/14] Move request payload into State model --- fbchat/_client.py | 30 +++++++++--------------------- fbchat/_state.py | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 21 deletions(-) create mode 100644 fbchat/_state.py diff --git a/fbchat/_client.py b/fbchat/_client.py index daebf3f..4ea7646 100644 --- a/fbchat/_client.py +++ b/fbchat/_client.py @@ -11,6 +11,7 @@ from collections import OrderedDict from ._util import * from .models import * from ._graphql import graphql_queries_to_json, graphql_response_to_json, GraphQL +from ._state import State import time import json @@ -69,11 +70,7 @@ class Client(object): :raises: FBchatException on failed login """ self._sticky, self._pool = (None, None) - self._session = requests.session() - self._req_counter = 1 - self._seq = "0" - # See `createPoll` for the reason for using `OrderedDict` here - self._payload_default = OrderedDict() + self._resetValues() self._default_thread_id = None self._default_thread_type = None self._pull_channel = 0 @@ -106,14 +103,11 @@ class Client(object): """ def _generatePayload(self, query): - """Adds the following defaults to the payload: - __rev, __user, __a, ttstamp, fb_dtsg, __req - """ if not query: query = {} - query.update(self._payload_default) query["__req"] = str_base(self._req_counter, 36) self._req_counter += 1 + query.update(self._state.get_params()) return query def _fix_fb_errors(self, error_code): @@ -271,14 +265,14 @@ class Client(object): """ def _resetValues(self): - self._payload_default = OrderedDict() + self._state = State() self._session = requests.session() self._req_counter = 1 self._seq = "0" self._uid = None def _postLogin(self): - self._payload_default = OrderedDict() + self._state = State() self._client_id = hex(int(random() * 2147483648))[2:] self._uid = self._session.cookies.get_dict().get("c_user") if self._uid is None: @@ -298,12 +292,9 @@ class Client(object): if fb_h_element: self._fb_h = fb_h_element["value"] - # Set default payload - self._payload_default["__rev"] = int( - r.text.split('"client_revision":', 1)[1].split(",", 1)[0] - ) - self._payload_default["__a"] = "1" - self._payload_default["fb_dtsg"] = fb_dtsg + revision = int(r.text.split('"client_revision":', 1)[1].split(",", 1)[0]) + + self._state = State(fb_dtsg=fb_dtsg, revision=revision) def _login(self, email, password): soup = bs(self._get("https://m.facebook.com/").text, "html.parser") @@ -1299,7 +1290,7 @@ class Client(object): # update JS token if received in response fb_dtsg = get_jsmods_require(j, 2) if fb_dtsg is not None: - self._payload_default["fb_dtsg"] = fb_dtsg + self._state.fb_dtsg = fb_dtsg try: message_ids = [ @@ -2092,9 +2083,6 @@ class Client(object): # We're using ordered dicts, because the Facebook endpoint that parses the POST # parameters is badly implemented, and deals with ordering the options wrongly. - # This also means we had to change `client._payload_default` to an ordered dict, - # since that's being copied in between this point and the `requests` call - # # If you can find a way to fix this for the endpoint, or if you find another # endpoint, please do suggest it ;) data = OrderedDict([("question_text", poll.title), ("target_id", thread_id)]) diff --git a/fbchat/_state.py b/fbchat/_state.py new file mode 100644 index 0000000..a72f558 --- /dev/null +++ b/fbchat/_state.py @@ -0,0 +1,23 @@ +# -*- coding: UTF-8 -*- +from __future__ import unicode_literals + +import attr + +from . import _util + + +@attr.s(slots=True, kw_only=True) +class State(object): + """Stores and manages state required for most Facebook requests.""" + + fb_dtsg = attr.ib(None) + _revision = attr.ib(None) + + def get_params(self): + if self.fb_dtsg is None: + return {} + return { + "__a": 1, + "__rev": self._revision, + "fb_dtsg": self.fb_dtsg, + } From 8e7afa2edf4e2747eed1fcff0bb3325a09226a0c Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Mon, 1 Jul 2019 13:31:30 +0200 Subject: [PATCH 02/14] Move request counter into State model --- fbchat/_client.py | 4 ---- fbchat/_state.py | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/fbchat/_client.py b/fbchat/_client.py index 4ea7646..05404fc 100644 --- a/fbchat/_client.py +++ b/fbchat/_client.py @@ -105,8 +105,6 @@ class Client(object): def _generatePayload(self, query): if not query: query = {} - query["__req"] = str_base(self._req_counter, 36) - self._req_counter += 1 query.update(self._state.get_params()) return query @@ -188,7 +186,6 @@ class Client(object): ) def _cleanPost(self, url, query=None): - self._req_counter += 1 return self._session.post( prefix_url(url), headers=self._header, data=query, verify=self.ssl_verify ) @@ -267,7 +264,6 @@ class Client(object): def _resetValues(self): self._state = State() self._session = requests.session() - self._req_counter = 1 self._seq = "0" self._uid = None diff --git a/fbchat/_state.py b/fbchat/_state.py index a72f558..a5892e9 100644 --- a/fbchat/_state.py +++ b/fbchat/_state.py @@ -12,12 +12,15 @@ class State(object): fb_dtsg = attr.ib(None) _revision = attr.ib(None) + _counter = attr.ib(0) def get_params(self): if self.fb_dtsg is None: return {} + self._counter += 1 # TODO: Make this operation atomic / thread-safe return { "__a": 1, + "__req": _util.str_base(self._counter, 36), "__rev": self._revision, "fb_dtsg": self.fb_dtsg, } From a4268f36cf4a955a9fdac91ed4a9e67219fe826b Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Thu, 2 May 2019 20:50:25 +0200 Subject: [PATCH 03/14] Move logout `h` into the State model --- fbchat/_client.py | 18 +++++++++--------- fbchat/_state.py | 5 +++++ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/fbchat/_client.py b/fbchat/_client.py index 05404fc..0bcb96b 100644 --- a/fbchat/_client.py +++ b/fbchat/_client.py @@ -284,13 +284,14 @@ class Client(object): else: fb_dtsg = re.search(r'name="fb_dtsg" value="(.*?)"', r.text).group(1) - fb_h_element = soup.find("input", {"name": "h"}) - if fb_h_element: - self._fb_h = fb_h_element["value"] + logout_h = None + logout_h_element = soup.find("input", {"name": "h"}) + if logout_h_element: + logout_h = logout_h_element["value"] revision = int(r.text.split('"client_revision":', 1)[1].split(",", 1)[0]) - self._state = State(fb_dtsg=fb_dtsg, revision=revision) + self._state = State(fb_dtsg=fb_dtsg, revision=revision, logout_h=logout_h) def _login(self, email, password): soup = bs(self._get("https://m.facebook.com/").text, "html.parser") @@ -468,16 +469,15 @@ class Client(object): :return: True if the action was successful :rtype: bool """ - if not hasattr(self, "_fb_h"): + logout_h = self._state.logout_h + if not logout_h: h_r = self._post("/bluebar/modern_settings_menu/", {"pmid": "4"}) - self._fb_h = re.search(r'name=\\"h\\" value=\\"(.*?)\\"', h_r.text).group(1) + logout_h = re.search(r'name=\\"h\\" value=\\"(.*?)\\"', h_r.text).group(1) - data = {"ref": "mb", "h": self._fb_h} + data = {"ref": "mb", "h": logout_h} r = self._get("/logout.php", data) - self._resetValues() - return r.ok """ diff --git a/fbchat/_state.py b/fbchat/_state.py index a5892e9..a0542cc 100644 --- a/fbchat/_state.py +++ b/fbchat/_state.py @@ -13,6 +13,11 @@ class State(object): fb_dtsg = attr.ib(None) _revision = attr.ib(None) _counter = attr.ib(0) + _logout_h = attr.ib(None) + + @property + def logout_h(self): + return self._logout_h def get_params(self): if self.fb_dtsg is None: From 56786406ecb46a8b76c42fff6694df0244a0d217 Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Thu, 2 May 2019 21:04:00 +0200 Subject: [PATCH 04/14] Refactor most of _postLogin into the State model --- fbchat/_client.py | 20 ++------------------ fbchat/_state.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/fbchat/_client.py b/fbchat/_client.py index 0bcb96b..aed367f 100644 --- a/fbchat/_client.py +++ b/fbchat/_client.py @@ -266,32 +266,16 @@ class Client(object): self._session = requests.session() self._seq = "0" self._uid = None + self._client_id = hex(int(random() * 2147483648))[2:] def _postLogin(self): - self._state = State() - self._client_id = hex(int(random() * 2147483648))[2:] self._uid = self._session.cookies.get_dict().get("c_user") if self._uid is None: raise FBchatException("Could not find c_user cookie") self._uid = str(self._uid) r = self._get("/") - soup = bs(r.text, "html.parser") - - fb_dtsg_element = soup.find("input", {"name": "fb_dtsg"}) - if fb_dtsg_element: - fb_dtsg = fb_dtsg_element["value"] - else: - fb_dtsg = re.search(r'name="fb_dtsg" value="(.*?)"', r.text).group(1) - - logout_h = None - logout_h_element = soup.find("input", {"name": "h"}) - if logout_h_element: - logout_h = logout_h_element["value"] - - revision = int(r.text.split('"client_revision":', 1)[1].split(",", 1)[0]) - - self._state = State(fb_dtsg=fb_dtsg, revision=revision, logout_h=logout_h) + self._state = State.from_base_request(r.text) def _login(self, email, password): soup = bs(self._get("https://m.facebook.com/").text, "html.parser") diff --git a/fbchat/_state.py b/fbchat/_state.py index a0542cc..7b734a6 100644 --- a/fbchat/_state.py +++ b/fbchat/_state.py @@ -2,9 +2,13 @@ from __future__ import unicode_literals import attr +import bs4 +import re from . import _util +FB_DTSG_REGEX = re.compile(r'name="fb_dtsg" value="(.*?)"') + @attr.s(slots=True, kw_only=True) class State(object): @@ -29,3 +33,21 @@ class State(object): "__rev": self._revision, "fb_dtsg": self.fb_dtsg, } + + @classmethod + def from_base_request(cls, content): + soup = bs4.BeautifulSoup(content, "html.parser") + + fb_dtsg_element = soup.find("input", {"name": "fb_dtsg"}) + if fb_dtsg_element: + fb_dtsg = fb_dtsg_element["value"] + else: + # Fall back to searching with a regex + fb_dtsg = FB_DTSG_REGEX.search(content).group(1) + + revision = int(content.split('"client_revision":', 1)[1].split(",", 1)[0]) + + logout_h_element = soup.find("input", {"name": "h"}) + logout_h = logout_h_element["value"] if logout_h_element else None + + return cls(fb_dtsg=fb_dtsg, revision=revision, logout_h=logout_h) From 5df10ecc31b2a719cf60fcf503e1eea382a953aa Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Thu, 23 May 2019 19:38:52 +0200 Subject: [PATCH 05/14] Remove _cleanGet and _cleanPost Client methods --- fbchat/_client.py | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/fbchat/_client.py b/fbchat/_client.py index aed367f..a6718cb 100644 --- a/fbchat/_client.py +++ b/fbchat/_client.py @@ -120,13 +120,22 @@ class Client(object): return True return False - def _get(self, url, query=None, fix_request=False, as_json=False, error_retries=3): + def _get( + self, + url, + query=None, + fix_request=False, + as_json=False, + error_retries=3, + allow_redirects=True, + ): payload = self._generatePayload(query) r = self._session.get( prefix_url(url), headers=self._header, params=payload, verify=self.ssl_verify, + allow_redirects=allow_redirects, ) if not fix_request: return r @@ -140,6 +149,7 @@ class Client(object): fix_request=fix_request, as_json=as_json, error_retries=error_retries - 1, + allow_redirects=allow_redirects, ) raise e @@ -176,20 +186,6 @@ class Client(object): ) raise e - def _cleanGet(self, url, query=None, allow_redirects=True): - return self._session.get( - prefix_url(url), - headers=self._header, - params=query, - verify=self.ssl_verify, - allow_redirects=allow_redirects, - ) - - def _cleanPost(self, url, query=None): - return self._session.post( - prefix_url(url), headers=self._header, data=query, verify=self.ssl_verify - ) - def _postFile( self, url, @@ -288,7 +284,7 @@ class Client(object): data["pass"] = password data["login"] = "Log In" - r = self._cleanPost("https://m.facebook.com/login.php?login_attempt=1", data) + r = self._post("https://m.facebook.com/login.php?login_attempt=1", data) # Usually, 'Checkpoint' will refer to 2FA if "checkpoint" in r.url and ('id="approvals_code"' in r.text.lower()): @@ -296,7 +292,7 @@ class Client(object): # Sometimes Facebook tries to show the user a "Save Device" dialog if "save-device" in r.url: - r = self._cleanGet("https://m.facebook.com/login/save-device/cancel/") + r = self._get("https://m.facebook.com/login/save-device/cancel/") if "home" in r.url: self._postLogin() @@ -317,7 +313,7 @@ class Client(object): data["codes_submitted"] = 0 log.info("Submitting 2FA code.") - r = self._cleanPost("https://m.facebook.com/login/checkpoint/", data) + r = self._post("https://m.facebook.com/login/checkpoint/", data) if "home" in r.url: return r @@ -331,7 +327,7 @@ class Client(object): log.info( "Saving browser." ) # At this stage, we have dtsg, nh, name_action_selected, submit[Continue] - r = self._cleanPost("https://m.facebook.com/login/checkpoint/", data) + r = self._post("https://m.facebook.com/login/checkpoint/", data) if "home" in r.url: return r @@ -340,7 +336,7 @@ class Client(object): log.info( "Starting Facebook checkup flow." ) # At this stage, we have dtsg, nh, submit[Continue] - r = self._cleanPost("https://m.facebook.com/login/checkpoint/", data) + r = self._post("https://m.facebook.com/login/checkpoint/", data) if "home" in r.url: return r @@ -350,7 +346,7 @@ class Client(object): log.info( "Verifying login attempt." ) # At this stage, we have dtsg, nh, submit[This was me] - r = self._cleanPost("https://m.facebook.com/login/checkpoint/", data) + r = self._post("https://m.facebook.com/login/checkpoint/", data) if "home" in r.url: return r @@ -361,7 +357,7 @@ class Client(object): log.info( "Saving device again." ) # At this stage, we have dtsg, nh, submit[Continue], name_action_selected - r = self._cleanPost("https://m.facebook.com/login/checkpoint/", data) + r = self._post("https://m.facebook.com/login/checkpoint/", data) return r def isLoggedIn(self): @@ -372,7 +368,7 @@ class Client(object): :rtype: bool """ # Send a request to the login url, to see if we're directed to the home page - r = self._cleanGet( + r = self._get( "https://m.facebook.com/login.php?login_attempt=1", allow_redirects=False ) return "Location" in r.headers and "home" in r.headers["Location"] From 94a0f6b3df974c3c02216b536a1515bf3c83d37c Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Wed, 26 Jun 2019 18:24:36 +0200 Subject: [PATCH 06/14] Move client session into State --- fbchat/_client.py | 17 ++++++++--------- fbchat/_state.py | 8 ++++++-- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/fbchat/_client.py b/fbchat/_client.py index a6718cb..c859489 100644 --- a/fbchat/_client.py +++ b/fbchat/_client.py @@ -130,7 +130,7 @@ class Client(object): allow_redirects=True, ): payload = self._generatePayload(query) - r = self._session.get( + r = self._state._session.get( prefix_url(url), headers=self._header, params=payload, @@ -163,7 +163,7 @@ class Client(object): error_retries=3, ): payload = self._generatePayload(query) - r = self._session.post( + r = self._state._session.post( prefix_url(url), headers=self._header, data=payload, verify=self.ssl_verify ) if not fix_request: @@ -200,7 +200,7 @@ class Client(object): headers = dict( (i, self._header[i]) for i in self._header if i != "Content-Type" ) - r = self._session.post( + r = self._state._session.post( prefix_url(url), headers=headers, data=payload, @@ -259,19 +259,18 @@ class Client(object): def _resetValues(self): self._state = State() - self._session = requests.session() self._seq = "0" self._uid = None self._client_id = hex(int(random() * 2147483648))[2:] def _postLogin(self): - self._uid = self._session.cookies.get_dict().get("c_user") + self._uid = self._state._session.cookies.get_dict().get("c_user") if self._uid is None: raise FBchatException("Could not find c_user cookie") self._uid = str(self._uid) r = self._get("/") - self._state = State.from_base_request(r.text) + self._state = State.from_base_request(self._state._session, r.text) def _login(self, email, password): soup = bs(self._get("https://m.facebook.com/").text, "html.parser") @@ -379,7 +378,7 @@ class Client(object): :return: A dictionay containing session cookies :rtype: dict """ - return self._session.cookies.get_dict() + return self._state._session.cookies.get_dict() def setSession(self, session_cookies): """Loads session cookies @@ -395,8 +394,8 @@ class Client(object): try: # Load cookies into current session - self._session.cookies = requests.cookies.merge_cookies( - self._session.cookies, session_cookies + self._state._session.cookies = requests.cookies.merge_cookies( + self._state._session.cookies, session_cookies ) self._postLogin() except Exception as e: diff --git a/fbchat/_state.py b/fbchat/_state.py index 7b734a6..6822f95 100644 --- a/fbchat/_state.py +++ b/fbchat/_state.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import attr import bs4 import re +import requests from . import _util @@ -14,6 +15,7 @@ FB_DTSG_REGEX = re.compile(r'name="fb_dtsg" value="(.*?)"') class State(object): """Stores and manages state required for most Facebook requests.""" + _session = attr.ib(factory=requests.session) fb_dtsg = attr.ib(None) _revision = attr.ib(None) _counter = attr.ib(0) @@ -35,7 +37,7 @@ class State(object): } @classmethod - def from_base_request(cls, content): + def from_base_request(cls, session, content): soup = bs4.BeautifulSoup(content, "html.parser") fb_dtsg_element = soup.find("input", {"name": "fb_dtsg"}) @@ -50,4 +52,6 @@ class State(object): logout_h_element = soup.find("input", {"name": "h"}) logout_h = logout_h_element["value"] if logout_h_element else None - return cls(fb_dtsg=fb_dtsg, revision=revision, logout_h=logout_h) + return cls( + session=session, fb_dtsg=fb_dtsg, revision=revision, logout_h=logout_h + ) From b01b371c66d7d87a30606fa32544552a6df58d02 Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Wed, 26 Jun 2019 20:03:38 +0200 Subject: [PATCH 07/14] Refactor session cookie handling into State --- fbchat/_client.py | 32 ++++++++++++++------------------ fbchat/_state.py | 25 +++++++++++++++++++++---- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/fbchat/_client.py b/fbchat/_client.py index c859489..cc31737 100644 --- a/fbchat/_client.py +++ b/fbchat/_client.py @@ -115,8 +115,8 @@ class Client(object): It may be a bad idea to do this in an exception handler, if you have a better method, please suggest it! """ if error_code == "1357004": - log.warning("Got error #1357004. Doing a _postLogin, and resending request") - self._postLogin() + log.warning("Got error #1357004. Refreshing state and resending request") + self._state = State.from_session(session=self._state._session) return True return False @@ -263,15 +263,6 @@ class Client(object): self._uid = None self._client_id = hex(int(random() * 2147483648))[2:] - def _postLogin(self): - self._uid = self._state._session.cookies.get_dict().get("c_user") - if self._uid is None: - raise FBchatException("Could not find c_user cookie") - self._uid = str(self._uid) - - r = self._get("/") - self._state = State.from_base_request(self._state._session, r.text) - def _login(self, email, password): soup = bs(self._get("https://m.facebook.com/").text, "html.parser") data = dict( @@ -294,7 +285,10 @@ class Client(object): r = self._get("https://m.facebook.com/login/save-device/cancel/") if "home" in r.url: - self._postLogin() + self._state = State.from_session(session=self._state._session) + self._uid = self._state.get_user_id() + if self._uid is None: + raise FBchatException("Could not find c_user cookie") return True, r.url else: return False, r.url @@ -378,7 +372,7 @@ class Client(object): :return: A dictionay containing session cookies :rtype: dict """ - return self._state._session.cookies.get_dict() + return self._state.get_cookies() def setSession(self, session_cookies): """Loads session cookies @@ -394,14 +388,16 @@ class Client(object): try: # Load cookies into current session - self._state._session.cookies = requests.cookies.merge_cookies( - self._state._session.cookies, session_cookies - ) - self._postLogin() + self._state = State.from_cookies(session_cookies) except Exception as e: log.exception("Failed loading session") - self._resetValues() + self._state = State() return False + uid = self._state.get_user_id() + if uid is None: + log.warning("Could not find c_user cookie") + return False + self._uid = uid return True def login(self, email, password, max_tries=5): diff --git a/fbchat/_state.py b/fbchat/_state.py index 6822f95..a6f321d 100644 --- a/fbchat/_state.py +++ b/fbchat/_state.py @@ -21,6 +21,12 @@ class State(object): _counter = attr.ib(0) _logout_h = attr.ib(None) + def get_user_id(self): + rtn = self.get_cookies().get("c_user") + if rtn is None: + return None + return str(rtn) + @property def logout_h(self): return self._logout_h @@ -37,17 +43,19 @@ class State(object): } @classmethod - def from_base_request(cls, session, content): - soup = bs4.BeautifulSoup(content, "html.parser") + def from_session(cls, session): + r = session.get(_util.prefix_url("/")) + + soup = bs4.BeautifulSoup(r.text, "html.parser") fb_dtsg_element = soup.find("input", {"name": "fb_dtsg"}) if fb_dtsg_element: fb_dtsg = fb_dtsg_element["value"] else: # Fall back to searching with a regex - fb_dtsg = FB_DTSG_REGEX.search(content).group(1) + fb_dtsg = FB_DTSG_REGEX.search(r.text).group(1) - revision = int(content.split('"client_revision":', 1)[1].split(",", 1)[0]) + revision = int(r.text.split('"client_revision":', 1)[1].split(",", 1)[0]) logout_h_element = soup.find("input", {"name": "h"}) logout_h = logout_h_element["value"] if logout_h_element else None @@ -55,3 +63,12 @@ class State(object): return cls( session=session, fb_dtsg=fb_dtsg, revision=revision, logout_h=logout_h ) + + def get_cookies(self): + return self._session.cookies.get_dict() + + @classmethod + def from_cookies(cls, cookies): + session = requests.session() + session.cookies = requests.cookies.merge_cookies(session.cookies, cookies) + return cls.from_session(session=session) From bcc8b44bb5af7b9a5e5c08795102dfc4ec767492 Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Wed, 26 Jun 2019 20:19:34 +0200 Subject: [PATCH 08/14] Handle ssl verification in State --- fbchat/_client.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/fbchat/_client.py b/fbchat/_client.py index cc31737..fc4502a 100644 --- a/fbchat/_client.py +++ b/fbchat/_client.py @@ -34,11 +34,19 @@ class Client(object): See https://fbchat.readthedocs.io for complete documentation of the API. """ - ssl_verify = True - """Verify ssl certificate, set to False to allow debugging with a proxy""" listening = False """Whether the client is listening. Used when creating an external event loop to determine when to stop listening""" + @property + def ssl_verify(self): + """Verify ssl certificate, set to False to allow debugging with a proxy.""" + # TODO: Deprecate this + return self._state._session.verify + + @ssl_verify.setter + def ssl_verify(self, value): + self._state._session.verify = value + @property def uid(self): """The ID of the client. @@ -134,7 +142,6 @@ class Client(object): prefix_url(url), headers=self._header, params=payload, - verify=self.ssl_verify, allow_redirects=allow_redirects, ) if not fix_request: @@ -164,7 +171,7 @@ class Client(object): ): payload = self._generatePayload(query) r = self._state._session.post( - prefix_url(url), headers=self._header, data=payload, verify=self.ssl_verify + prefix_url(url), headers=self._header, data=payload ) if not fix_request: return r @@ -201,11 +208,7 @@ class Client(object): (i, self._header[i]) for i in self._header if i != "Content-Type" ) r = self._state._session.post( - prefix_url(url), - headers=headers, - data=payload, - files=files, - verify=self.ssl_verify, + prefix_url(url), headers=headers, data=payload, files=files ) if not fix_request: return r From 1ba21e03c6f9405695277fefba9ee6784e7c565f Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Wed, 26 Jun 2019 20:27:58 +0200 Subject: [PATCH 09/14] Handle headers in State --- fbchat/_client.py | 46 ++++++++++++---------------------------------- fbchat/_state.py | 19 ++++++++++++++++--- 2 files changed, 28 insertions(+), 37 deletions(-) diff --git a/fbchat/_client.py b/fbchat/_client.py index fc4502a..fc72352 100644 --- a/fbchat/_client.py +++ b/fbchat/_client.py @@ -78,24 +78,16 @@ class Client(object): :raises: FBchatException on failed login """ self._sticky, self._pool = (None, None) - self._resetValues() + self._state = State.with_user_agent(user_agent=user_agent) + self._seq = "0" + self._uid = None + self._client_id = hex(int(random() * 2 ** 31))[2:] self._default_thread_id = None self._default_thread_type = None self._pull_channel = 0 self._markAlive = True self._buddylist = dict() - if not user_agent: - user_agent = choice(USER_AGENTS) - - self._header = { - "Content-Type": "application/x-www-form-urlencoded", - "Referer": "https://www.facebook.com", - "Origin": "https://www.facebook.com", - "User-Agent": user_agent, - "Connection": "keep-alive", - } - handler.setLevel(logging_level) # If session cookies aren't set, not properly loaded or gives us an invalid session, then do the login @@ -139,10 +131,7 @@ class Client(object): ): payload = self._generatePayload(query) r = self._state._session.get( - prefix_url(url), - headers=self._header, - params=payload, - allow_redirects=allow_redirects, + prefix_url(url), params=payload, allow_redirects=allow_redirects ) if not fix_request: return r @@ -170,9 +159,7 @@ class Client(object): error_retries=3, ): payload = self._generatePayload(query) - r = self._state._session.post( - prefix_url(url), headers=self._header, data=payload - ) + r = self._state._session.post(prefix_url(url), data=payload) if not fix_request: return r try: @@ -203,13 +190,7 @@ class Client(object): error_retries=3, ): payload = self._generatePayload(query) - # Removes 'Content-Type' from the header - headers = dict( - (i, self._header[i]) for i in self._header if i != "Content-Type" - ) - r = self._state._session.post( - prefix_url(url), headers=headers, data=payload, files=files - ) + r = self._state._session.post(prefix_url(url), data=payload, files=files) if not fix_request: return r try: @@ -260,12 +241,6 @@ class Client(object): LOGIN METHODS """ - def _resetValues(self): - self._state = State() - self._seq = "0" - self._uid = None - self._client_id = hex(int(random() * 2147483648))[2:] - def _login(self, email, password): soup = bs(self._get("https://m.facebook.com/").text, "html.parser") data = dict( @@ -391,7 +366,9 @@ class Client(object): try: # Load cookies into current session - self._state = State.from_cookies(session_cookies) + self._state = State.from_cookies( + session_cookies, user_agent=self._state._session.headers["User-Agent"] + ) except Exception as e: log.exception("Failed loading session") self._state = State() @@ -455,7 +432,8 @@ class Client(object): data = {"ref": "mb", "h": logout_h} r = self._get("/logout.php", data) - self._resetValues() + self._state = None + self._uid = None return r.ok """ diff --git a/fbchat/_state.py b/fbchat/_state.py index a6f321d..194cd3a 100644 --- a/fbchat/_state.py +++ b/fbchat/_state.py @@ -5,17 +5,26 @@ import attr import bs4 import re import requests +import random from . import _util FB_DTSG_REGEX = re.compile(r'name="fb_dtsg" value="(.*?)"') +def session_factory(user_agent=None): + session = requests.session() + session.headers["Referer"] = "https://www.facebook.com" + # TODO: Deprecate setting the user agent manually + session.headers["User-Agent"] = user_agent or random.choice(_util.USER_AGENTS) + return session + + @attr.s(slots=True, kw_only=True) class State(object): """Stores and manages state required for most Facebook requests.""" - _session = attr.ib(factory=requests.session) + _session = attr.ib(factory=session_factory) fb_dtsg = attr.ib(None) _revision = attr.ib(None) _counter = attr.ib(0) @@ -68,7 +77,11 @@ class State(object): return self._session.cookies.get_dict() @classmethod - def from_cookies(cls, cookies): - session = requests.session() + def from_cookies(cls, cookies, user_agent=None): + session = session_factory(user_agent=user_agent) session.cookies = requests.cookies.merge_cookies(session.cookies, cookies) return cls.from_session(session=session) + + @classmethod + def with_user_agent(cls, user_agent=None, **kwargs): + return cls(session=session_factory(user_agent=user_agent), **kwargs) From d0e9a7f6935ebaadb01eff0af782d441cd1acda3 Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Wed, 26 Jun 2019 23:02:10 +0200 Subject: [PATCH 10/14] Move login/2fa code to State --- fbchat/_client.py | 118 +++++----------------------------------------- fbchat/_state.py | 94 +++++++++++++++++++++++++++++++++++- 2 files changed, 106 insertions(+), 106 deletions(-) diff --git a/fbchat/_client.py b/fbchat/_client.py index fc72352..002b125 100644 --- a/fbchat/_client.py +++ b/fbchat/_client.py @@ -96,7 +96,7 @@ class Client(object): or not self.setSession(session_cookies) or not self.isLoggedIn() ): - self.login(email, password, max_tries) + self.login(email, password, max_tries, user_agent=user_agent) """ INTERNAL REQUEST METHODS @@ -241,96 +241,6 @@ class Client(object): LOGIN METHODS """ - def _login(self, email, password): - soup = bs(self._get("https://m.facebook.com/").text, "html.parser") - data = dict( - (elem["name"], elem["value"]) - for elem in soup.findAll("input") - if elem.has_attr("value") and elem.has_attr("name") - ) - data["email"] = email - data["pass"] = password - data["login"] = "Log In" - - r = self._post("https://m.facebook.com/login.php?login_attempt=1", data) - - # Usually, 'Checkpoint' will refer to 2FA - if "checkpoint" in r.url and ('id="approvals_code"' in r.text.lower()): - r = self._2FA(r) - - # Sometimes Facebook tries to show the user a "Save Device" dialog - if "save-device" in r.url: - r = self._get("https://m.facebook.com/login/save-device/cancel/") - - if "home" in r.url: - self._state = State.from_session(session=self._state._session) - self._uid = self._state.get_user_id() - if self._uid is None: - raise FBchatException("Could not find c_user cookie") - return True, r.url - else: - return False, r.url - - def _2FA(self, r): - soup = bs(r.text, "html.parser") - data = dict() - - s = self.on2FACode() - - data["approvals_code"] = s - data["fb_dtsg"] = soup.find("input", {"name": "fb_dtsg"})["value"] - data["nh"] = soup.find("input", {"name": "nh"})["value"] - data["submit[Submit Code]"] = "Submit Code" - data["codes_submitted"] = 0 - log.info("Submitting 2FA code.") - - r = self._post("https://m.facebook.com/login/checkpoint/", data) - - if "home" in r.url: - return r - - del data["approvals_code"] - del data["submit[Submit Code]"] - del data["codes_submitted"] - - data["name_action_selected"] = "save_device" - data["submit[Continue]"] = "Continue" - log.info( - "Saving browser." - ) # At this stage, we have dtsg, nh, name_action_selected, submit[Continue] - r = self._post("https://m.facebook.com/login/checkpoint/", data) - - if "home" in r.url: - return r - - del data["name_action_selected"] - log.info( - "Starting Facebook checkup flow." - ) # At this stage, we have dtsg, nh, submit[Continue] - r = self._post("https://m.facebook.com/login/checkpoint/", data) - - if "home" in r.url: - return r - - del data["submit[Continue]"] - data["submit[This was me]"] = "This Was Me" - log.info( - "Verifying login attempt." - ) # At this stage, we have dtsg, nh, submit[This was me] - r = self._post("https://m.facebook.com/login/checkpoint/", data) - - if "home" in r.url: - return r - - del data["submit[This was me]"] - data["submit[Continue]"] = "Continue" - data["name_action_selected"] = "save_device" - log.info( - "Saving device again." - ) # At this stage, we have dtsg, nh, submit[Continue], name_action_selected - r = self._post("https://m.facebook.com/login/checkpoint/", data) - return r - def isLoggedIn(self): """ Sends a request to Facebook to check the login status @@ -380,7 +290,7 @@ class Client(object): self._uid = uid return True - def login(self, email, password, max_tries=5): + def login(self, email, password, max_tries=5, user_agent=None): """ Uses `email` and `password` to login the user (If the user is already logged in, this will do a re-login) @@ -399,23 +309,21 @@ class Client(object): raise FBchatUserError("Email and password not set") for i in range(1, max_tries + 1): - login_successful, login_url = self._login(email, password) - if not login_successful: - log.warning( - "Attempt #{} failed{}".format( - i, {True: ", retrying"}.get(i < max_tries, "") - ) - ) + try: + state = State.login(email, password, user_agent=user_agent) + uid = state.get_user_id() + if uid is None: + raise FBchatException("Could not find user id") + except Exception: + if i >= max_tries: + raise + log.exception("Attempt #{} failed, retrying".format(i)) time.sleep(1) - continue else: + self._state = state + self._uid = uid self.onLoggedIn(email=email) break - else: - raise FBchatUserError( - "Login failed. Check email/password. " - "(Failed on url: {})".format(login_url) - ) def logout(self): """ diff --git a/fbchat/_state.py b/fbchat/_state.py index 194cd3a..8741f25 100644 --- a/fbchat/_state.py +++ b/fbchat/_state.py @@ -7,7 +7,7 @@ import re import requests import random -from . import _util +from . import _util, _exception FB_DTSG_REGEX = re.compile(r'name="fb_dtsg" value="(.*?)"') @@ -20,6 +20,63 @@ def session_factory(user_agent=None): return session +def _2fa_helper(session, code, r): + soup = bs4.BeautifulSoup(r.text, "html.parser") + data = dict() + + url = "https://m.facebook.com/login/checkpoint/" + + data["approvals_code"] = code + data["fb_dtsg"] = soup.find("input", {"name": "fb_dtsg"})["value"] + data["nh"] = soup.find("input", {"name": "nh"})["value"] + data["submit[Submit Code]"] = "Submit Code" + data["codes_submitted"] = 0 + _util.log.info("Submitting 2FA code.") + + r = session.post(url, data=data) + + if "home" in r.url: + return r + + del data["approvals_code"] + del data["submit[Submit Code]"] + del data["codes_submitted"] + + data["name_action_selected"] = "save_device" + data["submit[Continue]"] = "Continue" + _util.log.info("Saving browser.") + # At this stage, we have dtsg, nh, name_action_selected, submit[Continue] + r = session.post(url, data=data) + + if "home" in r.url: + return r + + del data["name_action_selected"] + _util.log.info("Starting Facebook checkup flow.") + # At this stage, we have dtsg, nh, submit[Continue] + r = session.post(url, data=data) + + if "home" in r.url: + return r + + del data["submit[Continue]"] + data["submit[This was me]"] = "This Was Me" + _util.log.info("Verifying login attempt.") + # At this stage, we have dtsg, nh, submit[This was me] + r = session.post(url, data=data) + + if "home" in r.url: + return r + + del data["submit[This was me]"] + data["submit[Continue]"] = "Continue" + data["name_action_selected"] = "save_device" + _util.log.info("Saving device again.") + # At this stage, we have dtsg, nh, submit[Continue], name_action_selected + r = session.post(url, data=data) + return r + + @attr.s(slots=True, kw_only=True) class State(object): """Stores and manages state required for most Facebook requests.""" @@ -51,6 +108,41 @@ class State(object): "fb_dtsg": self.fb_dtsg, } + @classmethod + def login(cls, email, password, user_agent=None): + session = session_factory(user_agent=user_agent) + + soup = bs4.BeautifulSoup( + session.get("https://m.facebook.com/").text, "html.parser" + ) + data = dict( + (elem["name"], elem["value"]) + for elem in soup.findAll("input") + if elem.has_attr("value") and elem.has_attr("name") + ) + data["email"] = email + data["pass"] = password + data["login"] = "Log In" + + r = session.post("https://m.facebook.com/login.php?login_attempt=1", data=data) + + # Usually, 'Checkpoint' will refer to 2FA + if "checkpoint" in r.url and ('id="approvals_code"' in r.text.lower()): + code = on_2fa_callback() + r = _2fa_helper(session, code, r) + + # Sometimes Facebook tries to show the user a "Save Device" dialog + if "save-device" in r.url: + r = session.get("https://m.facebook.com/login/save-device/cancel/") + + if "home" in r.url: + return cls.from_session(session=session) + else: + raise _exception.FBchatUserError( + "Login failed. Check email/password. " + "(Failed on url: {})".format(r.url) + ) + @classmethod def from_session(cls, session): r = session.get(_util.prefix_url("/")) From dc12e01fc7dd906fa31b9af41517549db038ee02 Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Wed, 26 Jun 2019 23:02:48 +0200 Subject: [PATCH 11/14] Move logout code to State --- fbchat/_client.py | 16 +++++----------- fbchat/_state.py | 14 ++++++++++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/fbchat/_client.py b/fbchat/_client.py index 002b125..bff06db 100644 --- a/fbchat/_client.py +++ b/fbchat/_client.py @@ -332,17 +332,11 @@ class Client(object): :return: True if the action was successful :rtype: bool """ - logout_h = self._state.logout_h - if not logout_h: - h_r = self._post("/bluebar/modern_settings_menu/", {"pmid": "4"}) - logout_h = re.search(r'name=\\"h\\" value=\\"(.*?)\\"', h_r.text).group(1) - - data = {"ref": "mb", "h": logout_h} - - r = self._get("/logout.php", data) - self._state = None - self._uid = None - return r.ok + if self._state.logout(): + self._state = None + self._uid = None + return True + return False """ END LOGIN METHODS diff --git a/fbchat/_state.py b/fbchat/_state.py index 8741f25..03f9970 100644 --- a/fbchat/_state.py +++ b/fbchat/_state.py @@ -93,10 +93,6 @@ class State(object): return None return str(rtn) - @property - def logout_h(self): - return self._logout_h - def get_params(self): if self.fb_dtsg is None: return {} @@ -143,6 +139,16 @@ class State(object): "(Failed on url: {})".format(r.url) ) + def logout(self): + logout_h = self._logout_h + if not logout_h: + url = _util.prefix_url("/bluebar/modern_settings_menu/") + h_r = self._session.post(url, data={"pmid": "4"}) + logout_h = re.search(r'name=\\"h\\" value=\\"(.*?)\\"', h_r.text).group(1) + + url = _util.prefix_url("/logout.php") + return self._session.get(url, params={"ref": "mb", "h": logout_h}).ok + @classmethod def from_session(cls, session): r = session.get(_util.prefix_url("/")) From 466f27a8c5c7d38c167cd60992f901b8ee717211 Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Wed, 26 Jun 2019 23:44:14 +0200 Subject: [PATCH 12/14] Move login check code into State --- fbchat/_client.py | 21 +++------------------ fbchat/_state.py | 6 ++++++ 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/fbchat/_client.py b/fbchat/_client.py index bff06db..6ff03a7 100644 --- a/fbchat/_client.py +++ b/fbchat/_client.py @@ -120,19 +120,9 @@ class Client(object): return True return False - def _get( - self, - url, - query=None, - fix_request=False, - as_json=False, - error_retries=3, - allow_redirects=True, - ): + def _get(self, url, query=None, fix_request=False, as_json=False, error_retries=3): payload = self._generatePayload(query) - r = self._state._session.get( - prefix_url(url), params=payload, allow_redirects=allow_redirects - ) + r = self._state._session.get(prefix_url(url), params=payload) if not fix_request: return r try: @@ -145,7 +135,6 @@ class Client(object): fix_request=fix_request, as_json=as_json, error_retries=error_retries - 1, - allow_redirects=allow_redirects, ) raise e @@ -248,11 +237,7 @@ class Client(object): :return: True if the client is still logged in :rtype: bool """ - # Send a request to the login url, to see if we're directed to the home page - r = self._get( - "https://m.facebook.com/login.php?login_attempt=1", allow_redirects=False - ) - return "Location" in r.headers and "home" in r.headers["Location"] + return self._state.is_logged_in() def getSession(self): """Retrieves session cookies diff --git a/fbchat/_state.py b/fbchat/_state.py index 03f9970..75b9dff 100644 --- a/fbchat/_state.py +++ b/fbchat/_state.py @@ -139,6 +139,12 @@ class State(object): "(Failed on url: {})".format(r.url) ) + def is_logged_in(self): + # Send a request to the login url, to see if we're directed to the home page + url = "https://m.facebook.com/login.php?login_attempt=1" + r = self._session.get(url, allow_redirects=False) + return "Location" in r.headers and "home" in r.headers["Location"] + def logout(self): logout_h = self._logout_h if not logout_h: From 230c849b600857e40e4a150299247664f001684d Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Wed, 26 Jun 2019 23:04:07 +0200 Subject: [PATCH 13/14] Always create the State object in a valid state --- fbchat/_client.py | 18 +++++------------- fbchat/_state.py | 10 ++-------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/fbchat/_client.py b/fbchat/_client.py index 6ff03a7..9c99d81 100644 --- a/fbchat/_client.py +++ b/fbchat/_client.py @@ -78,9 +78,7 @@ class Client(object): :raises: FBchatException on failed login """ self._sticky, self._pool = (None, None) - self._state = State.with_user_agent(user_agent=user_agent) self._seq = "0" - self._uid = None self._client_id = hex(int(random() * 2 ** 31))[2:] self._default_thread_id = None self._default_thread_type = None @@ -93,7 +91,7 @@ class Client(object): # If session cookies aren't set, not properly loaded or gives us an invalid session, then do the login if ( not session_cookies - or not self.setSession(session_cookies) + or not self.setSession(session_cookies, user_agent=user_agent) or not self.isLoggedIn() ): self.login(email, password, max_tries, user_agent=user_agent) @@ -247,7 +245,7 @@ class Client(object): """ return self._state.get_cookies() - def setSession(self, session_cookies): + def setSession(self, session_cookies, user_agent=None): """Loads session cookies :param session_cookies: A dictionay containing session cookies @@ -255,23 +253,17 @@ class Client(object): :return: False if `session_cookies` does not contain proper cookies :rtype: bool """ - # Quick check to see if session_cookies is formatted properly - if not session_cookies or "c_user" not in session_cookies: - return False - try: # Load cookies into current session - self._state = State.from_cookies( - session_cookies, user_agent=self._state._session.headers["User-Agent"] - ) + state = State.from_cookies(session_cookies, user_agent=user_agent) except Exception as e: log.exception("Failed loading session") - self._state = State() return False - uid = self._state.get_user_id() + uid = state.get_user_id() if uid is None: log.warning("Could not find c_user cookie") return False + self._state = state self._uid = uid return True diff --git a/fbchat/_state.py b/fbchat/_state.py index 75b9dff..d758156 100644 --- a/fbchat/_state.py +++ b/fbchat/_state.py @@ -82,8 +82,8 @@ class State(object): """Stores and manages state required for most Facebook requests.""" _session = attr.ib(factory=session_factory) - fb_dtsg = attr.ib(None) - _revision = attr.ib(None) + fb_dtsg = attr.ib() + _revision = attr.ib() _counter = attr.ib(0) _logout_h = attr.ib(None) @@ -94,8 +94,6 @@ class State(object): return str(rtn) def get_params(self): - if self.fb_dtsg is None: - return {} self._counter += 1 # TODO: Make this operation atomic / thread-safe return { "__a": 1, @@ -185,7 +183,3 @@ class State(object): session = session_factory(user_agent=user_agent) session.cookies = requests.cookies.merge_cookies(session.cookies, cookies) return cls.from_session(session=session) - - @classmethod - def with_user_agent(cls, user_agent=None, **kwargs): - return cls(session=session_factory(user_agent=user_agent), **kwargs) From 144e81bd46dbffd28264ca40309eb708232786c5 Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Mon, 1 Jul 2019 13:40:15 +0200 Subject: [PATCH 14/14] Add Python 2 support --- fbchat/_state.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fbchat/_state.py b/fbchat/_state.py index d758156..574015c 100644 --- a/fbchat/_state.py +++ b/fbchat/_state.py @@ -77,13 +77,13 @@ def _2fa_helper(session, code, r): return r -@attr.s(slots=True, kw_only=True) +@attr.s(slots=True) # TODO i Python 3: Add kw_only=True class State(object): """Stores and manages state required for most Facebook requests.""" - _session = attr.ib(factory=session_factory) fb_dtsg = attr.ib() _revision = attr.ib() + _session = attr.ib(factory=session_factory) _counter = attr.ib(0) _logout_h = attr.ib(None) @@ -172,7 +172,7 @@ class State(object): logout_h = logout_h_element["value"] if logout_h_element else None return cls( - session=session, fb_dtsg=fb_dtsg, revision=revision, logout_h=logout_h + fb_dtsg=fb_dtsg, revision=revision, session=session, logout_h=logout_h ) def get_cookies(self):