Hour 8: Retrieving all content

May 17, 2025

In my last entry, I shared that I'd finally managed to figure out the API powering the Eyad Qunaibi app. Today, I'll retrieve all the content, and save it locally.

We'll then figure out a way to use AI to map that to questions people are seeking answers to via Google, and then create those answers based on the entire corpus of materials. This'll be fun!

Retrieving all the info locally

This bit was fairly straightforward. I simply fed the MITMproxy logs into the o3 model (GPT-4o couldn't handle the format, but o3 could for some reason) and asked it to write the code for me.

The code it came up with was partially correct, and I had to fix a few minor bugs with Cursor.

import argparse, json, os, sys, time
from pathlib import Path
import requests
import shutil # Added for potential future use, e.g. sanitizing filenames

try:
    from tqdm import tqdm
except ImportError:
    tqdm = None

# ----------------- CONFIG -----------------
BASE   = os.getenv("FORQAN_BASE", "https://www.forqan-cast.com")
API    = f"{BASE}/api/References"
KEY    = os.getenv("FORQAN_KEY")
HEADERS = {"User-Agent": "Dart/3.0", "app-key": KEY}

_DEF_NAMES = {
    "Essay":           {"false": "مقالات",         "true": "سلاسل مقالات"},
    "TranslatedEssay": {"false": "مقالات مترجمة"},
    "Tweet":           {"false": "تغريدات"},
    "Video":           {"false": "مرئيات",         "true": "سلاسل مرئية"},
    "ImageComment":    {"false": "صور وتعليقات"},
    "Book":            {"false": "كتب"},
    "TranslatedBook":  {"false": "كتب مترجمة"},
    "Brochure":        {"false": "مطويات"},
    "NewsComment":     {"false": "تعليقات إخبارية"}
}
NAMES = json.loads(os.getenv("FORQAN_NAMES", json.dumps(_DEF_NAMES)))
TYPES = list(NAMES)

OUT = Path("data"); OUT.mkdir(exist_ok=True)
MEDIA_OUT = Path("media_files"); # MEDIA_OUT.mkdir(exist_ok=True) will be in main

# -------------- HELPERS -------------------
def out_path(typ, series=False):
    return OUT / f"{typ.lower()}s{'_series' if series else ''}.jsonl"

def write_jsonl(path, obj):
    with open(path, "a", encoding="utf-8") as fh:
        fh.write(json.dumps(obj, ensure_ascii=False) + "\n")

def ref_request(session, params):
    r = session.get(API, params=params, timeout=15)
    if r.status_code == 401:
        raise SystemExit("401 Unauthorized – bad or missing FORQAN_KEY")
    r.raise_for_status()
    return r.json()

def download_media(session, media_id, target_path, item_title="item"):
    """Downloads media file from /api/Media/{media_id}."""
    media_url = f"{BASE}/api/Media/{media_id}"
    
    progress_bar = None
    try:
        r = session.get(media_url, stream=True, timeout=60) # Increased timeout for media
        if r.status_code == 401:
            print(f"Warning: 401 Unauthorized for media ID {media_id}. Skipping.", file=sys.stderr)
            return False # Indicate failure
        r.raise_for_status()

        file_size = int(r.headers.get('content-length', 0))
        
        # Setup tqdm progress bar for download
        if tqdm:
            progress_bar = tqdm(
                total=file_size, 
                unit='B', 
                unit_scale=True, 
                desc=f"DL {item_title[:30]}...", # Truncate long titles
                leave=False
            )

        with open(target_path, "wb") as f:
            for chunk in r.iter_content(chunk_size=8192):
                f.write(chunk)
                if progress_bar:
                    progress_bar.update(len(chunk))
        
        if progress_bar:
            progress_bar.close()
        return True # Indicate success
            
    except requests.exceptions.RequestException as e:
        print(f"Error downloading media ID {media_id} to {target_path}: {e}", file=sys.stderr)
        if target_path.exists(): # Clean up partial file
            try:
                os.remove(target_path)
            except OSError:
                pass # Ignore if removal fails
        return False # Indicate failure
    finally:
        if progress_bar:
            progress_bar.close()

# ----------- FETCH LOGIC ------------------
def page_iter(session, typ=None, is_series=None, start=0, parent=None, delay=0.2):
    """Yield slice by slice until empty.
    If `parent` is given, `childrenOfId` is used to fetch child items."""
    count = 20
    while True:
        if parent:
            params = {"childrenOfId": str(parent)}
        else:
            params = {
                "CategoryType": typ,
                "CategoryName": NAMES[typ]["true" if is_series else "false"],
                "IsSeries": str(is_series).lower(),
                "count": str(count),
                "startfrom": str(start)
            }
        data = ref_request(session, params)
        if not data:
            break
        yield from data
        start += count
        time.sleep(delay)

# ---------------- MAIN PROCESSING FUNCTIONS -------------------- 

def fetch_data_from_api(args, sess):
    seen = set(); total = 0
    print("Fetching data from API...")

    for typ in TYPES:
        print(f"\n### {typ}")
        
        tqdm_common_kwargs = {"unit": "item", "leave": True}

        raw_singles_iter = page_iter(sess, typ, False, delay=args.sleep)
        if tqdm:
            singles_iter_with_progress = tqdm(raw_singles_iter, desc=f"{typ} singles (data)", **tqdm_common_kwargs)
        else:
            singles_iter_with_progress = raw_singles_iter

        for obj in singles_iter_with_progress:
            if obj["Id"] in seen:
                if tqdm and isinstance(singles_iter_with_progress, tqdm): singles_iter_with_progress.set_postfix_str("seen, skipped")
                continue

            write_jsonl(out_path(typ), obj)
            seen.add(obj["Id"]); total += 1
            if tqdm and isinstance(singles_iter_with_progress, tqdm): singles_iter_with_progress.set_postfix_str(f"wrote {obj['Id']}")
            
        if "true" in NAMES[typ]:
            parent_ids_and_titles = []
            
            raw_series_iter = page_iter(sess, typ, True, delay=args.sleep)
            if tqdm:
                series_iter_with_progress = tqdm(raw_series_iter, desc=f"{typ} series (data)", **tqdm_common_kwargs)
            else:
                series_iter_with_progress = raw_series_iter

            for series_obj in series_iter_with_progress:
                pid = series_obj["Id"]
                series_title = series_obj.get("Title", f"series_{pid}")

                if pid not in seen:
                    write_jsonl(out_path(typ, True), series_obj)
                    if tqdm and isinstance(series_iter_with_progress, tqdm): series_iter_with_progress.set_postfix_str(f"wrote {pid}")
                    seen.add(pid)
                    total +=1 

                parent_ids_and_titles.append((pid, series_title))

            for pid, _ in parent_ids_and_titles: # series_title_for_path not needed here, just pid
                raw_ep_iter = page_iter(sess, parent=pid, delay=args.sleep)
                ep_desc = f"{typ}:{pid} eps (data)"
                if tqdm:
                    ep_iter_with_progress = tqdm(raw_ep_iter, desc=ep_desc, **tqdm_common_kwargs)
                else:
                    ep_iter_with_progress = raw_ep_iter
                
                for ep in ep_iter_with_progress:
                    if ep["Id"] in seen:
                        if tqdm and isinstance(ep_iter_with_progress, tqdm): ep_iter_with_progress.set_postfix_str("seen, skipped")
                        continue
                    write_jsonl(out_path(typ), ep)
                    seen.add(ep["Id"]); total += 1
                    if tqdm and isinstance(ep_iter_with_progress, tqdm): ep_iter_with_progress.set_postfix_str(f"wrote {ep['Id']}")

    print(f"\nFinished fetching data. {total} objects written.")

def download_media_from_local_files(args, sess):
    MEDIA_OUT.mkdir(exist_ok=True)
    print(f"Downloading media based on local data files. Media will be saved to: {MEDIA_OUT.resolve()}")

    parent_id_to_series_title_map = {}
    media_types_for_download = frozenset(["Book", "TranslatedBook", "Brochure"])

    print("Scanning existing series data for titles...")
    # Scan all types for series, as a series of one type might link to an episode that is a book/brochure - though unlikely
    for typ_scan in TYPES:
        series_file_path = OUT / f"{typ_scan.lower()}s_series.jsonl"
        if series_file_path.exists() and "true" in NAMES.get(typ_scan, {}):
            try:
                with open(series_file_path, "r", encoding="utf-8") as fh_series:
                    for line in fh_series:
                        try:
                            series_obj = json.loads(line)
                            if "Id" in series_obj: # Title is optional, Id is key
                                parent_id_to_series_title_map[str(series_obj["Id"])] = series_obj.get("Title", f"series_{series_obj['Id']}")
                        except json.JSONDecodeError:
                            # Optional: log this error if needed
                            pass 
            except IOError as e:
                print(f"Warning: Could not read series file {series_file_path}: {e}", file=sys.stderr)
    print(f"Found {len(parent_id_to_series_title_map)} series titles from existing data.")

    total_downloaded_media = 0
    tqdm_item_kwargs = {"unit": "item", "leave": True}

    for typ_process_name in media_types_for_download:
        data_file_path = OUT / f"{typ_process_name.lower()}s.jsonl"
        print(f"\n### Processing for media: {data_file_path}")

        if not data_file_path.exists():
            print(f"Data file not found, skipping: {data_file_path}")
            continue

        num_lines = 0
        try:
            with open(data_file_path, "r", encoding="utf-8") as f_count:
                num_lines = sum(1 for _ in f_count)
        except IOError: pass

        try:
            with open(data_file_path, "r", encoding="utf-8") as fh_data:
                file_iterator = fh_data
                if tqdm:
                    file_iterator = tqdm(fh_data, total=num_lines, desc=f"Scanning {typ_process_name} data", **tqdm_item_kwargs)
                
                for line_num, line in enumerate(file_iterator):
                    try:
                        item = json.loads(line)
                    except json.JSONDecodeError:
                        if tqdm and isinstance(file_iterator, tqdm): file_iterator.set_postfix_str(f"JSON err L{line_num+1}")
                        # print(f"Warning: JSON decode error in {data_file_path} line {line_num+1}. Skipping.", file=sys.stderr)
                        continue

                    item_id = item.get("Id")
                    # file_name = item.get("FileName") # No longer strictly needed for filename
                    item_title_for_pbar = item.get("Title", f"Item {item_id}") # Use ID if title missing for pbar

                    if not item_id: # Only skip if ID is missing
                        if tqdm and isinstance(file_iterator, tqdm): file_iterator.set_postfix_str("no Id")
                        continue

                    # Construct filename from ID
                    target_file_name = f"{item_id}.pdf"

                    parent_ref_id = item.get("ParentReferenceId")
                    item_type_for_path = typ_process_name.lower()

                    if parent_ref_id:
                        series_title = parent_id_to_series_title_map.get(str(parent_ref_id), str(parent_ref_id))
                        sane_series_folder_name = "".join(c if c.isalnum() or c in (' ', '_', '-') else '_' for c in series_title).strip().replace(' ', '_')
                        if not sane_series_folder_name: sane_series_folder_name = str(parent_ref_id)
                        media_item_dir = MEDIA_OUT / item_type_for_path / sane_series_folder_name
                    else:
                        media_item_dir = MEDIA_OUT / item_type_for_path
                    
                    media_item_dir.mkdir(parents=True, exist_ok=True)
                    target_file_path = media_item_dir / target_file_name # Use constructed filename

                    current_status = "exists"
                    if not target_file_path.exists():
                        if tqdm and isinstance(file_iterator, tqdm): file_iterator.set_postfix_str(f"DLing {target_file_name}...")
                        if download_media(sess, str(item_id), target_file_path, item_title=item_title_for_pbar):
                            total_downloaded_media += 1
                            current_status = f"OK: {target_file_name}"
                            time.sleep(args.sleep) 
                        else:
                            current_status = f"FAIL: {target_file_name}"
                    
                    if tqdm and isinstance(file_iterator, tqdm): file_iterator.set_postfix_str(current_status)

        except IOError as e:
            print(f"Error reading data file {data_file_path}: {e}", file=sys.stderr)
            continue
    
    print(f"\nFinished media download attempt. {total_downloaded_media} new media files downloaded.")


# ---------------- MAIN --------------------
def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--sleep", type=float, default=0.2)
    ap.add_argument(
        "--download-type", 
        choices=["data", "media"], 
        required=True,
        help=(
            "Specify the operation mode: "
            "\'data\' to fetch/update metadata from the API and save to data/ directory. "
            "\'media\' to download actual files (e.g., PDFs for Books, Brochures) based on existing .jsonl files in data/ directory, saving to media_files/."
        )
    )
    args = ap.parse_args()
    sess = requests.Session(); sess.headers.update(HEADERS)

    if args.download_type == "data":
        fetch_data_from_api(args, sess)
    elif args.download_type == "media":
        download_media_from_local_files(args, sess)

    # Original main logic moved to fetch_data_from_api or download_media_from_local_files
    # The old loop for typ in TYPES and its sub-logics for singles/series/episodes 
    # specific to --download-type media has been removed, as media download is now 
    # handled entirely by download_media_from_local_files.

if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        sys.exit()

The thing works like a charm!

I could now download all the materials in JSON, as well as the PDFs (which will come in handy later when converting to text, and vectorizing later too!)

An hour well spent!