import json, os, time, urllib.request, ssl, sys

BASE = "https://gis.dnp.gov.co/server/rest/services"
DUMP = "C:/Users/Squir/Desktop/COLOMBIA/DUMP 2_25_2026/arcgis-dnp"

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

def fetch(url, retries=2):
    for attempt in range(retries + 1):
        try:
            req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
            with urllib.request.urlopen(req, timeout=30, context=ctx) as resp:
                return resp.read().decode("utf-8", errors="replace")
        except Exception as e:
            if attempt < retries:
                time.sleep(2)
            else:
                return None

def fetch_json(url):
    raw = fetch(url + "?f=json")
    if raw:
        try:
            return json.loads(raw)
        except:
            return None
    return None

def save_json(path, data):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "w", encoding="utf-8") as f:
        json.dump(data, f, indent=2, ensure_ascii=False)

root = fetch_json(BASE)
if not root:
    print("FATAL: Cannot reach server")
    sys.exit(1)

folders = root.get("folders", [])
all_services = list(root.get("services", []))

for folder in folders:
    print(f"Folder: {folder}")
    fd = fetch_json(f"{BASE}/{folder}")
    if fd:
        save_json(f"{DUMP}/folders/{folder}/_catalog.json", fd)
        all_services.extend(fd.get("services", []))
    time.sleep(0.5)

print(f"\nTotal services: {len(all_services)}\n")

for i, svc in enumerate(all_services):
    name = svc.get("name", "?")
    stype = svc.get("type", "MapServer")
    safe = name.replace("/", "_")
    url = f"{BASE}/{name}/{stype}"
    print(f"[{i+1}/{len(all_services)}] {name}/{stype}")
    
    info = fetch_json(url)
    if info:
        sdir = f"{DUMP}/services/{safe}"
        os.makedirs(sdir, exist_ok=True)
        save_json(f"{sdir}/_info.json", info)
        
        layers = info.get("layers", []) + info.get("tables", [])
        for layer in layers:
            lid = layer.get("id", 0)
            lname = layer.get("name", f"layer_{lid}").replace("/","_").replace("\\","_").replace(":","_")
            
            layer_info = fetch_json(f"{url}/{lid}")
            if layer_info:
                save_json(f"{sdir}/{lname}_info.json", layer_info)
                
                qurl = f"{url}/{lid}/query?f=json&where=1%3D1&outFields=*&returnGeometry=true&resultRecordCount=5000"
                raw = fetch(qurl)
                if raw:
                    try:
                        fdata = json.loads(raw)
                        fc = len(fdata.get("features", []))
                        exceeded = fdata.get("exceededTransferLimit", False)
                        save_json(f"{sdir}/{lname}_data.json", fdata)
                        mark = " [MORE]" if exceeded else ""
                        print(f"  L{lid} '{lname}': {fc} features{mark}")
                        if exceeded:
                            cr = fetch(f"{url}/{lid}/query?f=json&where=1%3D1&returnCountOnly=true")
                            if cr:
                                try:
                                    cd = json.loads(cr)
                                    print(f"    Total: {cd.get('count','?')}")
                                    save_json(f"{sdir}/{lname}_count.json", cd)
                                except: pass
                    except:
                        pass
            time.sleep(0.3)
    time.sleep(0.5)

print("\nDNP DUMP COMPLETE")