Scrape Google Play Games with Python

What will be scraped

📌Note: Google Play gives different results for logged in and not logged in users.

Using Google Play Games Store API from SerpApi

This section is to show the comparison between the DIY solution and our solution.

The main difference is that it's a quicker approach. Google Play Games Store API will bypass blocks from search engines and you don't have to create the parser from scratch and maintain it.

First, we need to install google-search-results:

pip install google-search-results

Import the necessary libraries for work:

from serpapi import GoogleSearch
import json

Next, we write a search query and the necessary parameters for making a request:

params = {
    'api_key': '...',	        # https://serpapi.com/manage-api-key
    'engine': 'google_play',    # SerpApi search engine
    'store': 'games'		    # Google Play Games
}

We then create a search object where the data is retrieved from the SerpApi backend. In the result_dict dictionary we get data from JSON:

search = GoogleSearch(params)
result_dict = search.get_dict()

The data is retrieved quite simply, we just need to turn to the 'organic_results' key.

google_play_games = result_dict['organic_results']

Example code to integrate:

from serpapi import GoogleSearch
import json

params = {
    'api_key': '...',	        # https://serpapi.com/manage-api-key
    'engine': 'google_play',    # SerpApi search engine
    'store': 'games'		    # Google Play Games
}

search = GoogleSearch(params)   # where data extraction happens on the SerpApi backend
result_dict = search.get_dict() # JSON -> Python dict

google_play_games = result_dict['organic_results']

print(json.dumps(google_play_games, indent=2, ensure_ascii=False))

Output:

[
  {
    "title": "In everyone's hands",
    "subtitle": "Download these games",
    "items": [
      {
        "title": "Roblox",
        "link": "https://play.google.com/store/apps/details?id=com.roblox.client",
        "product_id": "com.roblox.client",
        "serpapi_link": "https://serpapi.com/search.json?engine=google_play_product&gl=us&hl=en&product_id=com.roblox.client&store=apps",
        "rating": 4.4,
        "category": "Adventure",
        "video": "https://play.google.com/video/lava/web/player/yt:movie:LRI4BIHsfoQ?autoplay=1&embed=play",
        "thumbnail": "https://play-lh.googleusercontent.com/WNWZaxi9RdJKe2GQM3vqXIAkk69mnIl4Cc8EyZcir2SKlVOxeUv9tZGfNTmNaLC717Ht=s64-rw"
      },
      ... other items
    ]
  },
  ... other sections
  {
    "title": "Puzzle games",
    "items": [
      {
        "title": "Bubble Pop Origin! Puzzle Game",
        "link": "https://play.google.com/store/apps/details?id=com.puzzle1studio.go.bubblepoporiginpuzzlegame",
        "product_id": "com.puzzle1studio.go.bubblepoporiginpuzzlegame",
        "serpapi_link": "https://serpapi.com/search.json?engine=google_play_product&gl=us&hl=en&product_id=com.puzzle1studio.go.bubblepoporiginpuzzlegame&store=apps",
        "rating": 4.5,
        "category": "Puzzle",
        "video": "https://play.google.com/video/lava/web/player/yt:movie:Z17ENUc74_g?autoplay=1&embed=play",
        "thumbnail": "https://play-lh.googleusercontent.com/Lj79cXNkdJPS62d1C8mjQmyy2K5vuLq_ojH5wRrYk8fHqrZqqaMk9KNLVU64K-PDXdrQ=s64-rw"
      },
      ... other items
    ]
  }
]

DIY Code

If you don't need explanation, have a look at full code example in the online IDE.

import json
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from parsel import Selector

google_play_games = {
    'Top charts': {
        'Top free': [],
        'Top grossing': [],
        'Top paid': []
    },
}


def scroll_page(url):
    service = Service(ChromeDriverManager().install())

    options = webdriver.ChromeOptions()
    options.add_argument("--headless")
    options.add_argument("--lang=en")
    options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36")

    driver = webdriver.Chrome(service=service, options=options)
    driver.get(url)

    while True:
        try:
            scroll_button = driver.find_element(By.CSS_SELECTOR, '.snByac')
            driver.execute_script("arguments[0].click();", scroll_button)
            WebDriverWait(driver, 10000).until(EC.visibility_of_element_located((By.TAG_NAME, 'body')))
            break
        except:
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            WebDriverWait(driver, 10000).until(EC.visibility_of_element_located((By.TAG_NAME, 'body')))

    scrape_top_charts(driver=driver, chart='Top free', button_selector='#ct\|apps_topselling_free .ypTNYd')
    scrape_top_charts(driver=driver, chart='Top grossing', button_selector='#ct\|apps_topgrossing .ypTNYd')
    scrape_top_charts(driver=driver, chart='Top paid', button_selector='#ct\|apps_topselling_paid .ypTNYd')
    
    selector = Selector(driver.page_source)
    driver.quit()

    return selector
	

def scrape_top_charts(driver, chart, button_selector):
    button = driver.find_element(By.CSS_SELECTOR, button_selector)
    driver.execute_script("arguments[0].click();", button)
    WebDriverWait(driver, 10000).until(EC.visibility_of_element_located((By.TAG_NAME, 'body')))
    selector = Selector(driver.page_source)

    for result in selector.css('.itIJzb'):
        title = result.css('.OnEJge::text').get()
        link = 'https://play.google.com' + result.css('::attr(href)').get()
        category = result.css('.ubGTjb .sT93pb.w2kbF:not(.K4Wkre)::text').get()
        rating = float(result.css('.CKzsaf .w2kbF::text').get())
        thumbnail = result.css('.stzEZd::attr(srcset)').get().replace(' 2x', '')

        google_play_games['Top charts'][chart].append({
            'title': title,
            'link': link,
            'category': category,
            'rating': rating,
            'thumbnail': thumbnail,
        })


def scrape_all_sections(selector):	
    for section in selector.css('.Ubi8Z section'):
        section_title = section.css('.kcen6d span::text').get()
        google_play_games[section_title] = []

        for game in section.css('.TAQqTe'):
            title = game.css('.OnEJge::text').get()
            link = 'https://play.google.com' + game.css('::attr(href)').get()
            category = game.css('.ubGTjb .sT93pb.w2kbF:not(.K4Wkre)::text').get()
            rating = game.css('.CKzsaf .w2kbF::text').get()
            rating = float(rating) if rating else None
            thumbnail = game.css('.stzEZd::attr(srcset)').get().replace(' 2x', '')
            screenshot_image = game.css('.Vc0mnc img::attr(src), .jpDEN::attr(src)').get()

            google_play_games[section_title].append({
                'title': title,
                'link': link,
                'category': category,
                'rating': rating,
                'thumbnail': thumbnail,
                'screenshot_image': screenshot_image
            })
    
    print(json.dumps(google_play_games, indent=2, ensure_ascii=False))
	

def scrape_google_play_games():
    params = {
        'device': 'phone',  
        'hl': 'en_GB',		  # language 
        'gl': 'US',			    # country of the search
    }

    URL = f"https://play.google.com/store/games?device={params['device']}&hl={params['hl']}&gl={params['gl']}"

    result = scroll_page(URL)
    scrape_all_sections(result)


if __name__ == "__main__":
    scrape_google_play_games()

Preparation

Install libraries:

pip install parsel selenium webdriver webdriver_manager

Reduce the chance of being blocked

Make sure you're using request headers user-agent to act as a "real" user visit. Because default requests user-agent is python-requests and websites understand that it's most likely a script that sends a request. Check what's your user-agent.

There's a how to reduce the chance of being blocked while web scraping blog post that can get you familiar with basic and more advanced approaches.

Code Explanation

Import libraries:

import json
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from parsel import Selector

Define dictionary structure:

google_play_games = {
    'Top charts': {
        'Top free': [],
        'Top grossing': [],
        'Top paid': []
    },
}

Top-level code environment

At the beginning of the function, parameters are defined for generating the URL. If you want to pass other parameters to the URL, you can do so using the params dictionary.

Next, the URL is passed to the scroll_page(URL) function to scroll the page and get all data. The result that this function returns is passed to the scrape_all_sections(result) function to extract the necessary data. The explanation of these functions will be in the corresponding headings below.

This code uses the generally accepted rule of using the __name__ == "__main__" construct:

def scrape_google_play_games():
    params = {
        'device': 'phone',  
        'hl': 'en_GB',		  # language 
        'gl': 'US',			    # country of the search
    }

    URL = f"https://play.google.com/store/games?device={params['device']}&hl={params['hl']}&gl={params['gl']}"

    result = scroll_page(URL)
    scrape_all_categories(result)


if __name__ == "__main__":
    scrape_google_play_games()

This check will only be performed if the user has run this file. If the user imports this file into another, then the check will not work.

You can watch the video Python Tutorial: if name == 'main' for more details.

Scroll page

The function takes the URL and returns a full HTML structure.

First, let's understand how pagination works on the Google Play Games page. Data does not load immediately. If the user needs more data, they will simply scroll the page and site download a small package of data.

Accordingly, to get all the data, you need to scroll to the end of the page. But we will face the problem that on the last scroll the SHOW MORE button appears. By clicking on it, you will get the last piece of data. A page scroll demo is shown below:

In this case, selenium library is used, which allows you to simulate user actions in the browser. For selenium to work, you need to use ChromeDriver, which can be downloaded manually or using code. In our case, the second method is used. To control the start and stop of ChromeDriver, you need to use Service which will install browser binaries under the hood:

service = Service(ChromeDriverManager().install())

You should also add options to work correctly:

options = webdriver.ChromeOptions()
options.add_argument("--headless")
options.add_argument('--lang=en')
options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36")

Now we can start webdriver and pass the url to the get() method.

driver = webdriver.Chrome(service=service, options=options)
driver.get(url)

How does the page scrolling algorithm work? In each iteration of the loop, the program looks for the button selector. If the button is present, then the program is clicking on it and the data is loaded, after which the loop ends. Otherwise, the page scrolls down and the data loads.

Scrolling a page and clicking on a button is done by pasting the JavaScript code into the execute_script() method.

while True:
    try:
        scroll_button = driver.find_element(By.CSS_SELECTOR, '.snByac')
        driver.execute_script("arguments[0].click();", scroll_button)
        WebDriverWait(driver, 10000).until(EC.visibility_of_element_located((By.TAG_NAME, 'body')))
        break
    except:
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        WebDriverWait(driver, 10000).until(EC.visibility_of_element_located((By.TAG_NAME, 'body')))

Sometimes it is difficult to calculate how long it will take to load a page, it all depends on the speed of the Internet, the power of the computer and other factors. The method described below is much better than using a delay in seconds since the wait occurs exactly until the moment when the page is fully loaded:

WebDriverWait(driver, 10000).until(EC.visibility_of_element_located((By.TAG_NAME, 'body')))

📌Note: In this case, we give 10 seconds for the page to load, if it loads earlier then the wait will end.

After all the data has been loaded, you need to pass them to the scrape_top_charts function. This function will be described in the relevant section below. It is important to extract the data before we stop the driver.

scrape_top_charts(driver=driver, chart='Top free', button_selector='#ct\|apps_topselling_free .ypTNYd')
scrape_top_charts(driver=driver, chart='Top grossing', button_selector='#ct\|apps_topgrossing .ypTNYd')
scrape_top_charts(driver=driver, chart='Top paid', button_selector='#ct\|apps_topselling_paid .ypTNYd')

Now we need to process HTML using from Parsel package, in which we pass the HTML structure with all the data that was received after scrolling the page. This is necessary to successfully retrieve data in the next function. After all the operations are done, stop the driver:

selector = Selector(driver.page_source)
driver.quit()

The function looks like this:

def scroll_page(url):
    service = Service(ChromeDriverManager().install())

    options = webdriver.ChromeOptions()
    options.add_argument("--headless")
    options.add_argument("--lang=en")
    options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36")

    driver = webdriver.Chrome(service=service, options=options)
    driver.get(url)

    while True:
        try:
            scroll_button = driver.find_element(By.CSS_SELECTOR, '.snByac')
            driver.execute_script("arguments[0].click();", scroll_button)
            WebDriverWait(driver, 10000).until(EC.visibility_of_element_located((By.TAG_NAME, 'body')))
            break
        except:
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            WebDriverWait(driver, 10000).until(EC.visibility_of_element_located((By.TAG_NAME, 'body')))

    scrape_top_charts(driver=driver, chart='Top free', button_selector='#ct\|apps_topselling_free .ypTNYd')
    scrape_top_charts(driver=driver, chart='Top grossing', button_selector='#ct\|apps_topgrossing .ypTNYd')
    scrape_top_charts(driver=driver, chart='Top paid', button_selector='#ct\|apps_topselling_paid .ypTNYd')
    
    selector = Selector(driver.page_source)
    driver.quit()

    return selector

Scrape top charts

The function takes 3 parameters:

• driver - the full HTML structure.
• chart - the chart from which the data will be retrieved.
• button_selector - the selector of the button to go to the corresponding chart.

On the GIF, I show how the top charts work:

The corresponding data is loaded by pressing the corresponding buttons. The process of clicking the button and creating the object to retrieve data is the same as we did before in the scroll_page function:

button = driver.find_element(By.CSS_SELECTOR, button_selector)
driver.execute_script("arguments[0].click();", button)
time.sleep(2)
selector = Selector(driver.page_source)

To extract the necessary data, you need to find the selector where they are located. In our case, this is the .itIJzb selector, which contains all games. You need to iterate each game in the loop:

for result in selector.css('.itIJzb'):
    # data extraction will be here

For each game, data such as title, link, category, rating and thumbnail are extracted. You need to find the matching selector and get the text or attribute value. I want to additionally note that the thumbnail is retrieved from the srcset attribute, where it is of better quality:

title = result.css('.OnEJge::text').get()
link = 'https://play.google.com' + result.css('::attr(href)').get()
category = result.css('.ubGTjb .sT93pb.w2kbF:not(.K4Wkre)::text').get()
rating = float(result.css('.CKzsaf .w2kbF::text').get())
thumbnail = result.css('.stzEZd::attr(srcset)').get().replace(' 2x', '')

After extracting the data, they are added to the google_play_games dictionary by key. The key is the value of the сhart argument:

google_play_games['Top charts'][chart].append({
    'title': title,
    'link': link,
    'category': category,
    'rating': rating,
    'thumbnail': thumbnail,
})

The complete function to scrape top charts would look like this:

def scrape_top_charts(driver, chart, button_selector):
    button = driver.find_element(By.CSS_SELECTOR, button_selector)
    driver.execute_script("arguments[0].click();", button)
    WebDriverWait(driver, 10000).until(EC.visibility_of_element_located((By.TAG_NAME, 'body')))
    selector = Selector(driver.page_source)

    for result in selector.css('.itIJzb'):
        title = result.css('.OnEJge::text').get()
        link = 'https://play.google.com' + result.css('::attr(href)').get()
        category = result.css('.ubGTjb .sT93pb.w2kbF:not(.K4Wkre)::text').get()
        rating = float(result.css('.CKzsaf .w2kbF::text').get())
        thumbnail = result.css('.stzEZd::attr(srcset)').get().replace(' 2x', '')

        google_play_games['Top charts'][chart].append({
            'title': title,
            'link': link,
            'category': category,
            'rating': rating,
            'thumbnail': thumbnail,
        })

In the scroll_page function, we called this function 3 times to retrieve data from Top free, Top grossing and Top paid sections.

Scrape all sections

This function takes a full HTML structure and prints all results in JSON format.

To retrieve data from all sections, you need to find the selector of the section. This is the .Ubi8Z section selector. You need to iterate each section in the loop:

for section in selector.css('.Ubi8Z section'):
    # data extraction will be here

It is necessary to extract the section_title and make it a key in the google_play_games dictionary, where a list of extracted data for each game will be added later:

section_title = section.css('.kcen6d span::text').get()
google_play_games[section_title] = []

Each section has a certain number of games that should also need to iterate in another loop using the .TAQqTe selector:

for game in section.css('.TAQqTe'):
    # data extraction will be here

The data is retrieved here in the same way as in the scrape_top_charts function. But there is a slight difference in rating extraction. The fact is that for some games the rating is not displayed. In this case, a ternary expression is used, which extracts the numerical values for the data, if any:

rating = float(rating) if rating else None

Also, the screenshot_image is additionally extracted:

screenshot_image = game.css('.Vc0mnc img::attr(src), .jpDEN::attr(src)').get()

The complete function to scrape all sections would look like this:

def scrape_all_sections(selector):	
    for section in selector.css('.Ubi8Z section'):
        section_title = section.css('.kcen6d span::text').get()
        google_play_games[section_title] = []

        for game in section.css('.TAQqTe'):
            title = game.css('.OnEJge::text').get()
            link = 'https://play.google.com' + game.css('::attr(href)').get()
            category = game.css('.ubGTjb .sT93pb.w2kbF:not(.K4Wkre)::text').get()
            rating = game.css('.CKzsaf .w2kbF::text').get()
            rating = float(rating) if rating else None
            thumbnail = game.css('.stzEZd::attr(srcset)').get().replace(' 2x', '')
            screenshot_image = game.css('.Vc0mnc img::attr(src), .jpDEN::attr(src)').get()

            google_play_games[section_title].append({
                'title': title,
                'link': link,
                'category': category,
                'rating': rating,
                'thumbnail': thumbnail,
                'screenshot_image': screenshot_image
            })
            
    print(json.dumps(google_play_games, indent=2, ensure_ascii=False))

Output:

{
  "Top charts": {
    "Top free": [
      {
        "title": "Survivor.io",
        "link": "https://play.google.com/store/apps/details?id=com.dxx.firenow",
        "category": "Adventure",
        "rating": 4.7,
        "thumbnail": "https://play-lh.googleusercontent.com/jUJoxIWUU_MJ1ntLVOu_-YwtXRSzP2K87FYeFBpGTEMWAzdt20KjcF4UTsd1NGPeyg=s128-rw"
      },
      ... other games
    ],
    "Top grossing": [
      {
        "title": "Candy Crush Saga",
        "link": "https://play.google.com/store/apps/details?id=com.king.candycrushsaga",
        "category": "Casual",
        "rating": 4.6,
        "thumbnail": "https://play-lh.googleusercontent.com/TLUeelx8wcpEzf3hoqeLxPs3ai1tdGtAZTIFkNqy3gbDp1NPpNFTOzSFJDvZ9narFS0=s128-rw"
      },
      ... other games
    ],
    "Top paid": [
      {
        "title": "Minecraft",
        "link": "https://play.google.com/store/apps/details?id=com.mojang.minecraftpe",
        "category": "Arcade",
        "rating": 4.6,
        "thumbnail": "https://play-lh.googleusercontent.com/VSwHQjcAttxsLE47RuS4PqpC4LT7lCoSjE7Hx5AW_yCxtDvcnsHHvm5CTuL5BPN-uRTP=s128-rw"
      },
      ... other games
    ]
  },
  "In everyone's hands": [
    {
      "title": "Bloons TD 6",
      "link": "https://play.google.com/store/apps/details?id=com.ninjakiwi.bloonstd6",
      "category": "Strategy",
      "rating": 4.8,
      "thumbnail": "https://play-lh.googleusercontent.com/a4S3knhv7RGKTuKNbgTelxBFS9xOYypcpKDJ-KsXlyhbt9Pv9hZyvnSKs6_u9tozrYp6=s128-rw",
      "screenshot_image": "https://i.ytimg.com/vi/mZSY1PSIJ-E/hqdefault.jpg"
    },
    ... other games
  ],
  ... other sections
  "Indie Corner": [
    {
      "title": "quadline",
      "link": "https://play.google.com/store/apps/details?id=com.IvanKovalov.quadline.android",
      "category": "Puzzle",
      "rating": 4.6,
      "thumbnail": "https://play-lh.googleusercontent.com/SynBtvIRYxpqN5UwPnPdPa67uPZ3-rv5ntuFyEp3PC5bDuzabMgl4PXITrEkxY7VGXM=s128-rw",
      "screenshot_image": "https://i.ytimg.com/vi/epnaCIV9qX8/hqdefault.jpg"
    },
    ... other games
  ]
}

Links

• Code in the online IDE
• Google Play Games Store API

Join us on Twitter | YouTube

Add a Feature Request💫 or a Bug🐞