v1.0.4: Update Cloudflare bypass + fix some issues

Author: BenSouchet

README.md

@@ -31,23 +31,23 @@ Now you have two options: the **script with a User Interface** *(like a true sof
 
 ### Starting the Script with a User Interface
 1. In this case you need to make sure that **tkinter** is available on your computer ([check here](https://github.com/BenSouchet/cw-wizard/blob/main/REQUIREMENTS.md#optional-needed-for-the-user-interface-script)).
-2. Open on your favorite browser one Cardmarket tab (just one and don't log-in).
+2. Open on your favorite browser one Cardmarket tab (just one).
 3. Execute this command in your terminal:
 ```shell
 > python3 cw-wizard-gui.py
 ```
 4. Then simply follow the steps in the interface that popped up.
 
 ### Starting the Command Line Script
-1. Open, on your favorite browser, one Cardmarket tab (just one, and don't log-in)
+1. Open, on your favorite browser, one Cardmarket tab (just one)
 2. Check the script help (to see available parameters):
 ```shell
 > python3 cw-wizard.py -h
 ```
 ```text
 usage: CW Wizard [-h] [-v] -b BROWSER_NAME -u WANTLIST_URLS [WANTLIST_URLS ...] [-m MAX_SELLERS] [-w] [-c]
 
-CW Wizard v1.0.3, Find the best bundles for the cards in your wantlist(s).
+CW Wizard v1.0.4, Find the best bundles for the cards in your wantlist(s).
 
 options:
   -h, --help            show this help message and exit
@@ -77,6 +77,10 @@ Example with multiple wantlists:
 4. If it's the first time you start the script your Cardmarket credentials will be asked to create a `credentials.json` file at the root of the project directory. For info on why credentials are required [read this section](https://github.com/BenSouchet/cw-wizard/edit/main/README.md#security-and-authentification).
 <img width="598" alt="credentials_dialog" src="https://user-images.githubusercontent.com/17025808/159022712-b95ef3f5-0da6-4547-8f94-d49c0a4582ee.png">
 
+5. With your favorite browser search for "my user agent" on Google and copy the value (Should start with: `Mozilla/5.0`)
+
+6. When requested paste the value and press enter.
+
 5. It's all, if everything goes well a result HTML page will open on your default web browser, otherwise check the terminal to see the error message(s).
 
 ### Script Arguments
@@ -93,9 +97,13 @@ With the command line version of the script you can use the following arguments:
 | `-c` *OR* `--articles-comment` |If specified the script will retrieve and add sellers comments to the result page. | No |
 
 ## Version
-Current version is `1.0.3`, you can download this latest release on the Releases category (on the sidebar), from [this page](https://github.com/BenSouchet/cw-wizard/releases) or `git clone` the `main` branch of the repository.
+Current version is `1.0.4`, you can download this latest release on the Releases category (on the sidebar), from [this page](https://github.com/BenSouchet/cw-wizard/releases) or `git clone` the `main` branch of the repository.
 
 ## Changelog
+### 1.0.4
+- Update Cloudflare bypass, now require to specify your browser User-Agent to properly bypass retrictions.
+- Fix price retrieve due to recent modification in the Cardmarket interface.
+- Fix minor issues
 
 ### 1.0.3
 - Bypassing newly added Cloudflare protection by using user cookies (thanks to package **Browser Cookie 3**).

core.py

@@ -18,7 +18,7 @@
 # Global variables
 SCRIPT_NAME = 'CW Wizard'
 
-VERSION = '1.0.3'
+VERSION = '1.0.4'
 
 EXIT_ERROR_MSG = 'The Wizard encountered issue(s) please check previous logs.\n'
 EXIT_SUCCESS_MSG = 'The Wizard has finish is work, have a great day!\n'
@@ -58,27 +58,10 @@
 
 CREDENTIALS_PATH = Path.cwd().joinpath('credentials.json')
 
-USER_AGENT_HEADER = {
-    "Darwin": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36",
-    "Windows": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36",
-    "Linux": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36",
-}
-
 REQUEST_HEADERS = {
-    "authority": "cardmarket.com",
-    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
-    "accept-language": "fr,en-US;q=0.9,en;q=0.8",
-    "cache-control": "max-age=0",
-    "dnt": "1",
-    "sec-ch-ua": '"Chromium";v="112", "Google Chrome";v="112", "Not:A-Brand";v="99"',
-    "sec-ch-ua-mobile": "?0",
-    "sec-ch-ua-platform": '"macOS"',
-    "sec-fetch-dest": "document",
-    "sec-fetch-mode": "navigate",
-    "sec-fetch-site": "none",
-    "sec-fetch-user": "?1",
-    "Upgrade-Insecure-Requests": "1",
-    "User-Agent": USER_AGENT_HEADER[platform.system()]
+    "accept": "*/*",
+    "accept-language": "*",
+    "dnt": "1"
 }
 
 
@@ -442,7 +425,7 @@ def load_more_articles(session, funct_result, soup, card, articles_table):
     while active:
         # Step 3.A: Get the price of the last card currently displayed
         last_article = articles_table.contents[-1]
-        last_article_price_str = last_article.find('div', class_='price-container').find('span', class_='text-right').contents[0]
+        last_article_price_str = last_article.find('div', class_='price-container').find('span', class_='text-nowrap').contents[0]
         last_article_price = Decimal(last_article_price_str.split(' ')[0].replace('.','').replace(',','.'))
 
         # Step 3.B: Check if we need to load more articles, if yes send a new request to get more articles.
@@ -538,7 +521,7 @@ def populate_sellers_dict(session, sellers, wantlist, articles_comment=False, co
 
                 seller_profile_url = CARDMARKET_BASE_URL + seller_name_tag['href']
 
-                price_str = article_row.find('div', class_='price-container').find('span', class_='text-right').contents[0]
+                price_str = article_row.find('div', class_='price-container').find('span', class_='text-nowrap').contents[0]
                 price = Decimal(price_str.split(' ')[0].replace('.','').replace(',','.'))
                 # Step 4.E: Check if price isn't above maxPrice
                 if isinstance(card['maxPrice'], Decimal) and price > card['maxPrice']:
@@ -718,7 +701,7 @@ def get_cardmarket_cookies(browser_name):
     return browser_cookie3.load(domain_name=CARDMARKET_TOP_DOMAIN)
 
 
-def cardmarket_wantlist_wizard(browser_name, credentials, wantlist_urls, continue_on_warning, max_sellers, articles_comment=False):
+def cardmarket_wantlist_wizard(browser_name, user_agent, credentials, wantlist_urls, continue_on_warning, max_sellers, articles_comment=False):
     funct_result = FunctResult()
     LOG.debug('------- Calling the Wizard...\r\n')
 
@@ -730,7 +713,11 @@ def cardmarket_wantlist_wizard(browser_name, credentials, wantlist_urls, continu
     with requests.Session() as session:
         # Setting cookies and headers to bypass / skip Cloudflare protection
         session.cookies = get_cardmarket_cookies(browser_name)
-        session.headers.update(REQUEST_HEADERS)
+        headers = REQUEST_HEADERS.copy()
+        headers["user-agent"] = str(user_agent)
+        print(headers)
+        print(session.cookies)
+        session.headers.update(headers)
 
         # Step 2: Log-in to Cardmarket
         funct_result = cardmarket_log_in(session, credentials)
@@ -893,14 +880,16 @@ def create_credentials_file(credentials, silently=False):
     return True
 
 
-def check_credentials_validity(browser_name, credentials, silently=False):
+def check_credentials_validity(browser_name, user_agent, credentials, silently=False):
     funct_result = FunctResult()
 
      # Step 1: Create a web session
     with requests.Session() as session:
         # Setting cookies and headers to bypass / skip Cloudflare protection
         session.cookies = get_cardmarket_cookies(browser_name)
-        session.headers.update(REQUEST_HEADERS)
+        headers = REQUEST_HEADERS.copy()
+        headers["user-agent"] = str(user_agent)
+        session.headers.update(headers)
 
         # Step 2: Log-in to Cardmarket
         funct_result = cardmarket_log_in(session, credentials, silently=silently)

cw-wizard-gui.py

@@ -141,19 +141,19 @@ def close_window(window):
     window.root.destroy()
 
 
-def wizard_wrapper(credentials, wantlist_urls, max_sellers):
+def wizard_wrapper(browser_name, user_agent, credentials, wantlist_urls, max_sellers):
     # Step 1: Call the Wizard
-    result = cardmarket_wantlist_wizard(credentials, wantlist_urls, continue_on_warning=True, max_sellers=max_sellers)
+    result = cardmarket_wantlist_wizard(browser_name, user_agent, credentials, wantlist_urls, continue_on_warning=True, max_sellers=max_sellers)
     result.logMessages()
 
     # Step 2: Push result into the thread queue
     global THREAD_QUEUE
     THREAD_QUEUE.put(result)
 
 
-def credentials_validity_wrapper(browser_name, credentials):
+def credentials_validity_wrapper(browser_name, user_agent, credentials):
     # Step 1: Check the validity
-    result = check_credentials_validity(browser_name, credentials, silently=True)
+    result = check_credentials_validity(browser_name, user_agent, credentials, silently=True)
     result.logMessages()
 
     # Step 2: If valid create the file to avoid the user to enter them again
@@ -206,7 +206,7 @@ def credentials_validity_has_finished(params):
 
 
 def next_step(window, param1, param2):
-    if window.step == 'request_browser_name':
+    if window.step == 'request_browser_name_user_agent':
         # Step 1: Check browser name validity
         result = get_formatted_browser_name(param1.get())
         result.logMessages()
@@ -217,6 +217,13 @@ def next_step(window, param1, param2):
         # Set the reformatted browser_name
         window.browser_name = result.getResult()
 
+        # Step 1.B: Handling User-Agent
+        user_agent = param2.get()
+        if not user_agent:
+            set_window_description(window, 'Error: User-Agent cannot be empty.', is_error=True)
+            return True
+        window.user_agent = user_agent
+
         # Step 2: Retrieve credentials and check validity
         result = get_credentials_from_file()
         result.logMessages()
@@ -227,7 +234,7 @@ def next_step(window, param1, param2):
             if 'skip-check' not in credentials:
                 # Step 2.B: Cardmarket connexion test in a thread
                 window_wait_screen(window)
-                thread = threading.Thread(target=credentials_validity_wrapper, args=(window.browser_name, credentials,), daemon=True)
+                thread = threading.Thread(target=credentials_validity_wrapper, args=(window.browser_name, window.user_agent, credentials,), daemon=True)
                 thread.start()
 
                 # Step 2.C: Callback to move to the next step
@@ -254,7 +261,7 @@ def next_step(window, param1, param2):
 
         # Step 2.B: Cardmarket connexion test in a thread
         window_wait_screen(window)
-        thread = threading.Thread(target=credentials_validity_wrapper, args=(window.browser_name, credentials,), daemon=True)
+        thread = threading.Thread(target=credentials_validity_wrapper, args=(window.browser_name, window.user_agent, credentials,), daemon=True)
         thread.start()
 
         # Step 3: Callback to move to the next step
@@ -285,7 +292,7 @@ def next_step(window, param1, param2):
         # Step 3: All seems good, display the waiting screen and call the wizard !
         window.step = 'wizard_in_progress'
         window_wait_screen(window)
-        thread = threading.Thread(target=wizard_wrapper, args=(window.credentials, wantlist_urls, max_sellers,), daemon=True)
+        thread = threading.Thread(target=wizard_wrapper, args=(window.browser_name, window.user_agent, window.credentials, wantlist_urls, max_sellers,), daemon=True)
         thread.start()
 
         # Step 4: Callback to display the final screen when the wizard has finished
@@ -336,9 +343,9 @@ def set_window_description(window, message, is_error=False, is_warn=False):
     window.description.set(message)
 
 
-def window_request_browser_name(window):
+def window_request_browser_name_user_agent(window):
     # Step 1: Set current prog step
-    window.step = 'request_browser_name'
+    window.step = 'request_browser_name_user_agent'
 
     # Step 2: Set title and description
     window.title.set('Open a Cardmarket tab in your favorite browser')
@@ -357,15 +364,21 @@ def window_request_browser_name(window):
 
     # Step 3.C: Create the variable that will store the browser name
     browser_name = tkinter.StringVar()
+    user_agent = tkinter.StringVar()
 
     # Step 3.D: Create the two row
     label_browser_name = create_label(frame_parent, 'Browser Name', 'content', window.content_bg_color)
     label_browser_name.grid(row=0, column=0, pady=(0, 4), sticky=tkinter.E)
     entry_browser_name = create_entry(frame_parent, browser_name, window.content_bg_color)
     entry_browser_name.grid(row=0, column=1, ipadx=4, padx=(4, 0), pady=(0, 4), sticky=tkinter.NSEW)
 
+    label_user_agent = create_label(frame_parent, 'User-Agent', 'content', window.content_bg_color)
+    label_user_agent.grid(row=1, column=0, sticky=tkinter.E)
+    entry_user_agent = create_entry(frame_parent, user_agent, window.content_bg_color)
+    entry_user_agent.grid(row=1, column=1, padx=(4, 0), sticky=tkinter.NSEW)
+
     # Step 4: Add disclaimer message
-    text_disclaimer = "Disclaimer: Ensure only one Cardmarket tab is open and you aren't logged-in."
+    text_disclaimer = "Disclaimer: Ensure only one Cardmarket tab is open and you aren't logged-in.\nFor User-Agent search \"my user agent\" on Google."
     label_disclaimer = create_label(window.content, text_disclaimer, 'description', window.content_bg_color)
     label_disclaimer.grid(row=1, column=0, pady=(0, 14), sticky=tkinter.S)
 
@@ -376,7 +389,7 @@ def window_request_browser_name(window):
     # Step 6: Update Next button
     window.button_next_text.set('NEXT STEP')
     window.button_next.configure(state=tkinter.NORMAL)
-    window.button_next.configure(command=partial(next_step, window, browser_name, None))
+    window.button_next.configure(command=partial(next_step, window, browser_name, user_agent))
 
 
 def window_request_credentials(window, error_msg=None):
@@ -608,7 +621,7 @@ def main():
     window = create_window()
 
     # Step 2: Request the browser name used
-    window_request_browser_name(window)
+    window_request_browser_name_user_agent(window)
 
     # Step 4: Put back the alpha of the window to full opaque
     window.root.attributes('-alpha', 1.0)

cw-wizard.py

@@ -33,17 +33,26 @@ def get_credentials_user_inputs():
         return funct_result
 
     # Step 2: Ask for the login
-    credentials['login'] = input('Enter your Cardmarket login: ')
+    credentials['login'] = input('\nEnter your Cardmarket login: ')
 
     # Step 2: Ask for the login
-    credentials['password'] = getpass.getpass('Enter your Cardmarket password: ')
+    credentials['password'] = getpass.getpass('\nEnter your Cardmarket password: ')
+
+    print("\n")
 
     create_credentials_file(credentials)
 
     funct_result.addResult(credentials)
 
     return funct_result
 
+def get_user_agent_from_inputs():
+    funct_result = FunctResult()
+    LOG.info("To properly bypass Cloudflare your user agent is require.")
+    funct_result.addResult(input('\nEnter your browser User-Agent (search on Google "my user agent"):\n'))
+    print("\n")
+    return funct_result
+
 def main(browser_name, wantlist_urls, continue_on_warning, max_sellers, articles_comment):
     """Entry point of the CW Wizard script"""
 
@@ -68,28 +77,32 @@ def main(browser_name, wantlist_urls, continue_on_warning, max_sellers, articles
 
     credentials = None
     if result.isValid():
-        # Check the credentials are valid
         credentials = result.getResult()
+
+    result = get_user_agent_from_inputs()
+    user_agent = result.getResult()
+
+    if credentials:
+        # Check the credentials are valid
         if 'skip-check' not in credentials:
-            result = check_credentials_validity(browser_name, credentials, silently=True)
+            result = check_credentials_validity(browser_name, user_agent, credentials, silently=True)
             result.logMessages()
-        # Since we have maybe check the validity, perform a second check on the result object
-        if result.isValid():
-            credentials = result.getResult()
-
-    if not credentials:
+            if result.isValid():
+                credentials = result.getResult()
+            else:
+                return LOG.error(EXIT_ERROR_MSG)
+    else:
         # File not found, ask the user if he want to create the file
         result = get_credentials_user_inputs()
         result.logMessages()
 
         if result.isValid():
             credentials = result.getResult()
+        else:
+            return LOG.error(EXIT_ERROR_MSG)
 
-    if not credentials:
-        return LOG.error(EXIT_ERROR_MSG)
-
-    # Step 3: Call the Wizard
-    result = cardmarket_wantlist_wizard(browser_name, credentials, wantlist_urls, continue_on_warning=continue_on_warning, max_sellers=max_sellers, articles_comment=articles_comment)
+    # Step 3: Call the Wizards
+    result = cardmarket_wantlist_wizard(browser_name, user_agent, credentials, wantlist_urls, continue_on_warning=continue_on_warning, max_sellers=max_sellers, articles_comment=articles_comment)
     result.logMessages()
 
     if not result.isValid():