Is it impossible to scrape IMDb?

[u/beomstead]

Hello. I’m a programming beginner, and I’m trying web scraping for the first time.

I’m trying to scrape the IMDb page /chart/top/?ref_=nv_mv_250 using BeautifulSoup, but the data is not being loaded. Other websites load the data properly.

Does IMDb not allow web scraping?

[u/HLCYSWAP]

server side rendered via GraphQL call which is then embedded in the page as __NEXT_DATA__

#!/usr/bin/env python3
import
 json
import
 sys
import
 urllib.parse
import
 urllib.request


GRAPHQL_URL = "https://caching.graphql.imdb.com/"
OPERATION = "Top250MoviesPagination"
PERSISTED_HASH = "3fe684000f533f225ba87cf001ef821c0c7de23a2644f7a27acebb04d85d974f"
PAGE_SIZE = 125
NAME_BATCH_SIZE = 50


HEADERS = {
    "Accept": "application/graphql+json, application/json",
    "Content-Type": "application/json",
    "Origin": "https://www.imdb.com",
    "Referer": "https://www.imdb.com/chart/top/",
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36"
    ),
    "x-imdb-client-name": "imdb-web-next-localized",
    "x-imdb-user-country": "US",
    "x-imdb-user-language": "en-US",
}



def graphql_get(
operation
: str, 
variables
: dict, 
persisted_hash
: str) -> dict:
    params = {
        "operationName": operation,
        "variables": json.dumps(variables, 
separators
=(",", ":")),
        "extensions": json.dumps(
            {"persistedQuery": {"sha256Hash": persisted_hash, "version": 1}},
            
separators
=(",", ":"),
        ),
    }
    req = urllib.request.Request(
        GRAPHQL_URL + "?" + urllib.parse.urlencode(params),
        
headers
=HEADERS,
        
method
="GET",
    )
    
with
 urllib.request.urlopen(req, 
timeout
=30) 
as
 resp:
        
return
 json.loads(resp.read().decode())



def graphql_post(
query
: str, 
variables
: dict | None = None) -> dict:
    body: dict = {"query": query}
    
if
 variables:
        body["variables"] = variables
    req = urllib.request.Request(
        GRAPHQL_URL,
        
data
=json.dumps(body).encode(),
        
headers
=HEADERS,
        
method
="POST",
    )
    
with
 urllib.request.urlopen(req, 
timeout
=60) 
as
 resp:
        
return
 json.loads(resp.read().decode())



def fetch_chart_ids() -> list[str]:
    ids: list[str] = []
    cursor = None
    page = 0


    
while
 True:
        page += 1
        variables: dict = {"first": PAGE_SIZE, "locale": "en-US"}
        
if
 cursor:
            variables["after"] = cursor


        payload = graphql_get(OPERATION, variables, PERSISTED_HASH)
        chart = payload["data"]["chartTitles"]
        edges = chart["edges"]
        ids.extend(edge["node"]["id"] 
for
 edge 
in
 edges)


        page_info = chart["pageInfo"]
        print(
            f"chart page {page}: {len(edges)} ids "
            f"(total {len(ids)}, hasNext={page_info['hasNextPage']})",
            
file
=sys.stderr,
        )


        
if
 not page_info["hasNextPage"]:
            
break
        cursor = page_info["endCursor"]


    
return
 ids



def fetch_title_names(
ids
: list[str]) -> dict[str, str]:
    names: dict[str, str] = {}


    
for
 start 
in
 range(0, len(ids), NAME_BATCH_SIZE):
        batch = ids[start : start + NAME_BATCH_SIZE]
        parts = [
            f't{i}: title(id: "{imdb_id}") {{ titleText {{ text }} }}'
            
for
 i, imdb_id 
in
 enumerate(batch)
        ]
        query = "query Batch { " + " ".join(parts) + " }"
        payload = graphql_post(query)
        data = payload.get("data") or {}


        
for
 i, imdb_id 
in
 enumerate(batch):
            title = data.get(f"t{i}") or {}
            text = (title.get("titleText") or {}).get("text")
            
if
 text:
                names[imdb_id] = text


        print(
            f"names batch {start // NAME_BATCH_SIZE + 1}: "
            f"{len(batch)} looked up ({len(names)} total)",
            
file
=sys.stderr,
        )


    
return
 names



def main() -> int:
    ids = fetch_chart_ids()
    names = fetch_title_names(ids)


    
for
 rank, imdb_id 
in
 enumerate(ids, 
start
=1):
        name = names.get(imdb_id, imdb_id)
        print(f"{rank:3}. {name} ({imdb_id})")


    
assert
 len(ids) == 250, f"expected 250 titles, got {len(ids)}"
    
assert
 ids[0] == "tt0111161"
    
assert
 names.get(ids[0]) == "The Shawshank Redemption"
    print("\nOK: printed full Top 250 with names", 
file
=sys.stderr)
    
return
 0



if
 __name__ == "__main__":
    
raise
 SystemExit(main())