ching/chh-crawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactor(crawler): use webdriver-manager and add debug logging

Browse Source
This commit is contained in:
root 2025-12-04 11:35:17 +08:00
parent 6b5c4a3a1b
commit 7af1c6acbe
1 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
crawler.py
View File

@ -11,7 +11,9 @@ from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from webdriver_manager.chrome import ChromeDriverManager
import time
from mastodon import Mastodon
@ -70,7 +72,8 @@ def setup_driver():
chrome_options.add_argument(f'user-agent={user_agent}')
chrome_options.add_argument('--window-size=1920,1080')
driver = webdriver.Chrome(options=chrome_options)
service = Service(ChromeDriverManager().install())
driver = webdriver.Chrome(service=service, options=chrome_options)
return driver
def crawler():
@ -85,6 +88,13 @@ def crawler():
# Wait for page to load
time.sleep(3)
# Debug: log page info
logger.info(f"Page title: {driver.title}")
logger.info(f"Current URL: {driver.current_url}")
page_source = driver.page_source
logger.info(f"Page source length: {len(page_source)}")
logger.info(f"Page source preview (first 2000 chars):\n{page_source[:2000]}")
# Find all chiphell_box elements and get the last one
boxes = driver.find_elements(By.CLASS_NAME, 'chiphell_box')
if not boxes: