import requests
import json
import os
import time
import urllib3
urllib3.disable_warnings()

session = requests.Session()
session.verify = False
session.headers.update({"User-Agent": "Mozilla/5.0"})

def safe_name(name):
    return name.replace("/", "_").replace("\\", "_").replace(" ", "_")

def fetch_json(url, params=None):
    try:
        if params is None:
            params = {}
        params["f"] = "json"
        r = session.get(url, params=params, timeout=30)
        data = r.json()
        if "error" in data:
            print(f"  API Error: {data['error'].get('message','unknown')}")
            return None
        return data
    except Exception as e:
        print(f"  ERROR fetching {url}: {e}")
        return None

def dump_features(url, layer_name, output_dir, max_records=5000):
    count_data = fetch_json(f"{url}/query", {"where": "1=1", "returnCountOnly": "true"})
    total = count_data.get("count", 0) if count_data else 0
    print(f"    Layer: {layer_name} - {total} features")
    if total == 0:
        return
    
    all_features = []
    offset = 0
    while True:
        data = fetch_json(f"{url}/query", {
            "where": "1=1", "outFields": "*",
            "resultOffset": str(offset), "resultRecordCount": str(max_records),
            "returnGeometry": "true"
        })
        if not data or "features" not in data:
            break
        features = data["features"]
        if not features:
            break
        all_features.extend(features)
        offset += len(features)
        if not data.get("exceededTransferLimit", False):
            break
        time.sleep(0.3)
    
    if all_features:
        out_file = os.path.join(output_dir, f"{safe_name(layer_name)}_features.json")
        with open(out_file, "w", encoding="utf-8") as f:
            json.dump({"count": len(all_features), "features": all_features}, f, ensure_ascii=False)
        print(f"    Saved {len(all_features)} features")

def process_service(base_url, svc_name, svc_type, output_dir):
    url = f"{base_url}/{svc_name}/{svc_type}"
    svc_dir = os.path.join(output_dir, safe_name(svc_name))
    os.makedirs(svc_dir, exist_ok=True)
    
    print(f"\nProcessing: {svc_name} ({svc_type})")
    info = fetch_json(url)
    if not info:
        return
    
    with open(os.path.join(svc_dir, "service_info.json"), "w", encoding="utf-8") as f:
        json.dump(info, f, ensure_ascii=False, indent=2)
    
    layers = info.get("layers", []) + info.get("tables", [])
    for layer in layers:
        lid = layer["id"]
        lname = layer.get("name", f"layer_{lid}")
        dump_features(f"{url}/{lid}", lname, svc_dir)
        time.sleep(0.2)

BASE_DIR = os.path.dirname(os.path.abspath(__file__))

# === CONTRALORIA ===
print("=" * 60)
print("CONTRALORIA - gis.contraloria.gov.co")
print("=" * 60)
contraloria_dir = os.path.join(BASE_DIR, "arcgis-contraloria")
os.makedirs(contraloria_dir, exist_ok=True)
contraloria_base = "https://gis.contraloria.gov.co/server/rest/services"

# Try all folders
for folder in ["Hosted", "GEOPORTALPROYECTOS", "GOB", "Integracion", "INVENTARIO", 
                "VISOR_IMAGENES", "005_PROYECTO_MINDEPORTE", "006_SEGUIMIENTO_PUENTES",
                "007_SEGUIMIENTO_PUENTES_ANI"]:
    print(f"\n--- Folder: {folder} ---")
    folder_data = fetch_json(f"{contraloria_base}/{folder}")
    if not folder_data:
        continue
    
    with open(os.path.join(contraloria_dir, f"{folder}_catalog.json"), "w", encoding="utf-8") as f:
        json.dump(folder_data, f, ensure_ascii=False, indent=2)
    
    for svc in folder_data.get("services", []):
        svc_type = svc["type"]
        if svc_type in ("MapServer", "FeatureServer"):
            process_service(contraloria_base, svc["name"], svc_type, contraloria_dir)
            time.sleep(0.5)

# === PARQUES NACIONALES ===
print("\n" + "=" * 60)
print("PARQUES NACIONALES - mapas.parquesnacionales.gov.co")
print("=" * 60)
parques_dir = os.path.join(BASE_DIR, "arcgis-parques")
os.makedirs(parques_dir, exist_ok=True)
parques_base = "https://mapas.parquesnacionales.gov.co/arcgis/rest/services"

for svc in [
    ("pnn/Coberturas_Antropicas_25K_Oficial", "MapServer"),
    ("pnn/coberturas", "MapServer"),
    ("pnn/Limites_oficiales_Poligono", "MapServer"),
    ("pnn/Limites_oficiales_Linea", "MapServer"),
    ("pnn/Limites_oficiales_Punto", "MapServer"),
    ("pnn/runap", "MapServer"),
    ("pnn/zonificacion", "MapServer"),
    ("IGAC/nuevas_areas", "MapServer"),
]:
    process_service(parques_base, svc[0], svc[1], parques_dir)
    time.sleep(0.5)

print("\n\nDONE!")