Scrape Google Search Results using a Custom Search Engine

This Python script is for automating web searches and data extraction, Using the Google Custom Search JSON API to execute search queries and save them into an excel file. It is particularly useful for SEO analysis.

Setting up the script
Replace 'your_api_key' and 'your_cse_id' with your actual API key and Custom Search Engine ID. You can get an API key from here: https://developers.google.com/custom-search/v1/introduction

This script will create an Excel file named search_results.xlsx with a sheet titled "Search Results" containing the query, titles, links, and snippets of the search results.

import requests
import json
from openpyxl import Workbook

def google_search(query, api_key, cse_id, start_page, num_results):
    url = "https://www.googleapis.com/customsearch/v1"
    params = {
        'q': query,
        'cx': cse_id,
        'key': api_key,
        'start': start_page,
        'num': num_results,
        'cr': 'countryUK'  # Restrict search to UK
    }
    response = requests.get(url, params=params)
    return response.json()

def main():
    api_key = 'your_api_key'  # Replace with your actual API key
    cse_id = 'your_cse_id'  # Replace with your Custom Search Engine ID

    queries = ["seo", "seo service"]  # List of queries

    # Create a workbook and select active worksheet
    wb = Workbook()
    ws = wb.active
    ws.title = "Search Results"
    ws.append(["Query", "Title", "Link", "Snippet"])  # Header row

    for query in queries:
        print(f"\nResults for '{query}':\n")
        total_results = 100
        start = 1

        while total_results > 0:
            num = min(10, total_results)
            results = google_search(query, api_key, cse_id, start, num)

            for item in results.get('items', []):
                title = item.get('title')
                link = item.get('link')
                snippet = item.get('snippet')
                print(f"Title: {title}\nLink: {link}\nSnippet: {snippet}\n")
                # Append row to the Excel sheet
                ws.append([query, title, link, snippet])

            total_results -= num
            start += num

    # Save the workbook
    wb.save("search_results.xlsx")

if __name__ == '__main__':
    main()