Вопрос Парсинг динамического сайта

[abnonebuispus]

Мой код парсит только 25 первых фильмов, мне нужно парсить все 100, как мне это сделать?

Вот код на питон: import requests

from bs4 import BeautifulSoup





def parse_movies(url):

headers = {

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'

}



response = requests.get(url, headers=headers)



if response.status_code != 200:

print(f"Ошибка при запросе страницы: {response.status_code}&#34

return []



soup = BeautifulSoup(response.text, 'html.parser&#39

movies_info = []



for movie in soup.find_all('li', class_='ipc-metadata-list-summary-item sc-4929eaf6-0 DLYcv cli-parent&#39:

title_element = movie.find('h3', class_='ipc-title__text&#39



metadata_elements = movie.find_all('span', class_='sc-300a8231-7 eaXxft cli-title-metadata-item&#39



if title_element and metadata_elements:

title = title_element.text.strip()

year = metadata_elements[0].text.strip() if len(metadata_elements) > 0 else 'N/A'

duration = metadata_elements[1].text.strip() if len(metadata_elements) > 1 else 'N/A'



rating_element = movie.find('span', class_='ipc-rating-star--rating&#39

rating = rating_element.text.strip() if rating_element else 'N/A'



age_rating = metadata_elements[-1].text.strip() if metadata_elements else 'N/A'



movies_info.append({

'title': title,

'year': year,

'duration': duration,

'rating': rating,

'age_rating': age_rating

})



return movies_info



url = 'https://www.imdb.com/chart/moviemeter/'

movies = parse_movies(url)



for movie in movies:

print(

f"Название: {movie['title']}, Год: {movie['year']}, Длительность: {movie['duration']}, Рейтинг: {movie['rating']}, Возрастное ограничение: {movie['age_rating']}&#34

 

[фндрей]

в 64 бит режиме запускай

 

[Sergey]

Держи, проверил у себя все ок, скажи если будут проблемы.

import requests
import json
from bs4 import BeautifulSoup


def safe_get(data, *keys, default='N/A&#39:
current = data
for key in keys:
if isinstance(current, dict):
current = current.get(key)
else:
return default
if current is None:
return default
return current if current is not None else default


def parse_movies(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}

response = requests.get(url, headers=headers)
if response.status_code != 200:
print(f"Ошибка при запросе страницы: {response.status_code}&#34
return []

soup = BeautifulSoup(response.text, 'html.parser&#39
script = soup.find('script', {'id': '__NEXT_DATA__', 'type': 'application/json'})

if not script:
print("Не удалось найти JSON-данные&#34
return []

data = json.loads(script.string)
movies = data.get('props', {}).get('pageProps', {}).get('pageData', {}).get('chartTitles', {}).get('edges', [])

movies_info = []
for movie in movies:
node = movie.get('node', {})

title = safe_get(node, 'titleText', 'text&#39
year = safe_get(node, 'releaseYear', 'year&#39

runtime_seconds = safe_get(node, 'runtime', 'seconds', default=0)
if runtime_seconds and runtime_seconds != 'N/A':
hours = runtime_seconds // 3600
minutes = (runtime_seconds % 3600) // 60
runtime = f"{hours}h {minutes}m" if hours else f"{minutes}m"
else:
runtime = safe_get(node, 'runtime', 'displayableProperty', 'value', 'plainText&#39

rating = safe_get(node, 'ratingsSummary', 'aggregateRating&#39
age_rating = safe_get(node, 'certificate', 'rating&#39

movies_info.append({
'title': title,
'year': year,
'duration': runtime,
'rating': rating,
'age_rating': age_rating
})

return movies_info


url = 'https://www.imdb.com/chart/moviemeter/'
movies = parse_movies(url)

for idx, movie in enumerate(movies, 1):
print(
f"{idx}. Название: {movie['title']}, Год: {movie['year']}, Длительность: {movie['duration']}, Рейтинг: {movie['rating']}, Возрастное ограничение: {movie['age_rating']}&#34

 

[вася_вася]

В дополнение: напрямую через селениум:

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC


def get_movie_titles(url, driver_path):
options = webdriver.ChromeOptions()
service = Service(driver_path)
driver = webdriver.Chrome(service=service, options=options)
driver.maximize_window()

try:
driver.get(url=url)
print("Страница успешно загружена&#34

WebDriverWait(driver, 10).until(
EC.presence_of_all_elements_located((By.CLASS_NAME, 'ipc-metadata-list-summary-item&#39)
)

movie_items = driver.find_elements(By.CLASS_NAME, 'ipc-metadata-list-summary-item&#39

for item in movie_items:
try:
title_element = item.find_element(By.CLASS_NAME, 'ipc-title__text&#39
title = title_element.text
except:
title = 'Название не найдено'

print(f"Название: {title}&#34
except Exception as ex:
print(f"Произошла ошибка: {ex}&#34
finally:
print("Нажмите Enter, чтобы закрыть браузер...&#34
input()
driver.quit()
print("Браузер закрыт&#34


def main():
driver_path = 'C:/chromedriver_win32/chromedriver.exe'
get_movie_titles('https://www.imdb.com/chart/moviemeter/', driver_path)


if __name__ == '__main__':
main()

 

[Fang2]

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from bs4 import BeautifulSoup
import time

# Настройка Selenium
chrome_options = Options()
chrome_options.add_argument("--headless&#34 # Запуск в фоновом режиме
service = Service('path/to/chromedriver&#39 # Укажите путь к chromedriver
driver = webdriver.Chrome(service=service, options=chrome_options)

# Открываем страницу
url = 'https://www.imdb.com/chart/moviemeter/'
driver.get(url)

# Ждем загрузки страницы
time.sleep(5) # Можно заменить на явное ожидание элементов

# Получаем HTML-код страницы
html = driver.page_source
driver.quit()

# Парсим данные
soup = BeautifulSoup(html, 'html.parser&#39
movies_info = []

for movie in soup.find_all('li', class_='ipc-metadata-list-summary-item&#39:
title_element = movie.find('h3', class_='ipc-title__text&#39
metadata_elements = movie.find_all('span', class_='sc-300a8231-7&#39

if title_element and metadata_elements:
title = title_element.text.strip()
year = metadata_elements[0].text.strip() if len(metadata_elements) > 0 else 'N/A'
duration = metadata_elements[1].text.strip() if len(metadata_elements) > 1 else 'N/A'

rating_element = movie.find('span', class_='ipc-rating-star--rating&#39
rating = rating_element.text.strip() if rating_element else 'N/A'

age_rating = metadata_elements[-1].text.strip() if metadata_elements else 'N/A'

movies_info.append({
'title': title,
'year': year,
'duration': duration,
'rating': rating,
'age_rating': age_rating
})

# Выводим результаты
for movie in movies_info:
print(
f"Название: {movie['title']}, Год: {movie['year']}, Длительность: {movie['duration']}, Рейтинг: {movie['rating']}, Возрастное ограничение: {movie['age_rating']}"
)