#!/usr/bin/env python3
"""index-check.py: check (and re-check) which sitemap URLs Google has indexed, using
Search Console's official URL Inspection API. Free; from https://indexkeeper.tuper.app/

Setup (once, about 10 minutes):
  1. Google Cloud console: create a project, enable "Google Search Console API".
  2. IAM & Admin > Service accounts > Create; then Keys > Add key > JSON. Save it as key.json.
  3. Search Console > Settings > Users and permissions > Add user: the service account's
     email (xxx@yyy.iam.gserviceaccount.com), permission "Restricted" is enough.
  4. pip install google-auth requests

Run:
  python3 index-check.py --key key.json --property sc-domain:example.com \
      --sitemap https://example.com/sitemap.xml [--limit 500]
  (--property is exactly as Search Console names it: "sc-domain:example.com" for a domain
   property or "https://example.com/" for a URL-prefix property.)

It writes index-<date>.csv and, when an earlier CSV exists in the folder, prints every URL whose
state changed since then (e.g. "Submitted and indexed" -> "Crawled - currently not indexed").
Run it from cron once a day or once a week for a simple index monitor.
To compare any two of its CSVs (or two Search Console Pages-report exports) as a table with a
CSV of the changes, use https://indexkeeper.tuper.app/compare-index-exports/ (runs in your browser).
To list the sitemap URLs that are not indexed (and indexed pages missing from the sitemap), drop
one of its CSVs with your sitemap into https://indexkeeper.tuper.app/sitemap-urls-not-indexed/ .
Google's quota: 2,000 inspections per property per day, 600 per minute.
"""
import argparse
import csv
import datetime as dt
import glob
import sys
import time
import xml.etree.ElementTree as ET

import requests
from google.auth.transport.requests import AuthorizedSession
from google.oauth2 import service_account

API = "https://searchconsole.googleapis.com/v1/urlInspection/index:inspect"
SCOPE = ["https://www.googleapis.com/auth/webmasters.readonly"]
NS = "{http://www.sitemaps.org/schemas/sitemap/0.9}"


def sitemap_urls(url, seen=None):
    """All <loc> URLs of a sitemap, following sitemap indexes."""
    seen = seen if seen is not None else set()
    if url in seen:
        return []
    seen.add(url)
    root = ET.fromstring(requests.get(url, timeout=30).content)
    locs = [e.text.strip() for e in root.iter(NS + "loc") if e.text]
    if root.tag == NS + "sitemapindex":
        out = []
        for child in locs:
            out += sitemap_urls(child, seen)
        return out
    return locs


def inspect(session, prop, url):
    for attempt in range(3):
        r = session.post(API, json={"inspectionUrl": url, "siteUrl": prop}, timeout=60)
        if r.status_code == 429:  # per-minute quota: wait and retry
            time.sleep(60)
            continue
        if r.status_code != 200:
            return {"verdict": "ERROR", "coverageState": f"HTTP {r.status_code}: {r.text[:120]}"}
        return r.json().get("inspectionResult", {}).get("indexStatusResult", {})
    return {"verdict": "ERROR", "coverageState": "quota (429) after 3 tries"}


def last_csv(today_file):
    files = sorted(f for f in glob.glob("index-*.csv") if f != today_file)
    if not files:
        return {}
    with open(files[-1], newline="") as fh:
        return {row["url"]: row["coverageState"] for row in csv.DictReader(fh)}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--key", required=True, help="service account JSON key")
    ap.add_argument("--property", required=True, help="sc-domain:example.com or https://example.com/")
    ap.add_argument("--sitemap", required=True)
    ap.add_argument("--limit", type=int, default=2000, help="max URLs this run (quota is 2,000/day)")
    a = ap.parse_args()

    creds = service_account.Credentials.from_service_account_file(a.key, scopes=SCOPE)
    session = AuthorizedSession(creds)
    urls = list(dict.fromkeys(sitemap_urls(a.sitemap)))[: a.limit]
    out = f"index-{dt.date.today().isoformat()}.csv"
    before = last_csv(out)
    fields = ["url", "verdict", "coverageState", "lastCrawlTime", "googleCanonical", "robotsTxtState"]
    changed = 0
    with open(out, "w", newline="") as fh:
        w = csv.DictWriter(fh, fieldnames=fields)
        w.writeheader()
        for i, url in enumerate(urls, 1):
            s = inspect(session, a.property, url)
            row = {"url": url, **{k: s.get(k, "") for k in fields[1:]}}
            w.writerow(row)
            old = before.get(url)
            if old is not None and old != row["coverageState"]:
                changed += 1
                print(f"CHANGED {url}: {old} -> {row['coverageState']}")
            print(f"{i}/{len(urls)} {row['verdict']:7} {row['coverageState']}  {url}", file=sys.stderr)
            time.sleep(0.12)  # stays under 600 per minute
    print(f"{len(urls)} URLs checked, {changed} changed since the last run; results in {out}")


if __name__ == "__main__":
    main()