#!/usr/bin/env python3 """Pull a visual image for EVERY gallery item. 1. thumbnail/. — the gallery card thumbnail for every item that has one 2. webmap_render/.png — for each Web Map, render via a print service composition """ import json, time, threading, urllib.request, urllib.parse, urllib.error from pathlib import Path from concurrent.futures import ThreadPoolExecutor, as_completed ROOT = Path(__file__).parent INDEX = ROOT / "items_index.json" THUMB = ROOT / "thumbnails" WMR = ROOT / "webmap_render" LOG = ROOT / "logs" / "thumbs.log" STAT = ROOT / "logs" / "thumbs_status.json" BASE = "https://ergit.presidencia.gov.co/arcpre/sharing/rest" UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36" WORKERS = 12 TIMEOUT = 60 _lock = threading.Lock() _stats = {"thumbs":0, "webmap_pngs":0, "no_thumb":0, "err":0, "start":time.time()} def log(m): line=f"[{time.strftime('%H:%M:%S')}] {m}" with _lock: print(line, flush=True) with open(LOG,"a",encoding="utf-8") as f: f.write(line+"\n") def fetch(url, binary=False, retries=2): for i in range(retries+1): try: req=urllib.request.Request(url, headers={"User-Agent":UA}) with urllib.request.urlopen(req, timeout=TIMEOUT) as r: return r.read() if binary else r.read().decode("utf-8",errors="replace") except urllib.error.HTTPError as e: if e.code in (404,401,403): return None if i==retries: return None except Exception: if i==retries: return None time.sleep(1+i) return None def pull_thumb(it): iid = it["id"] thumb_field = it.get("thumbnail") if not thumb_field: with _lock: _stats["no_thumb"] += 1 return # extension from field ext = thumb_field.rsplit(".",1)[-1].lower() if ext not in ("png","jpg","jpeg","gif","webp","bmp"): ext = "png" out = THUMB / f"{iid}.{ext}" if out.exists(): return # thumbnail URL: /content/items//info/ url = f"{BASE}/content/items/{iid}/info/{urllib.parse.quote(thumb_field)}" blob = fetch(url, binary=True) if blob and len(blob)>0: out.write_bytes(blob) with _lock: _stats["thumbs"] += 1 else: with _lock: _stats["err"] += 1 def render_webmap(it): """For a Web Map, fetch its JSON (already in data/), then ask the print service to render it. If no print service is available, fall back to rendering each operational layer's MapServer /export separately.""" iid = it["id"] out = WMR / f"{iid}.png" if out.exists(): return # Pull webmap json wj = fetch(f"{BASE}/content/items/{iid}/data?f=json") if not wj: return try: w = json.loads(wj) except: return # Use the webmap's initialState/operationalLayers extent ext = (w.get("initialState",{}).get("viewpoint",{}).get("targetGeometry") or w.get("extent")) if not ext: # try first operational layer's url+full extent ops = w.get("operationalLayers",[]) if not ops: return layer_url = ops[0].get("url","").rstrip("/") if "/MapServer" in layer_url or "/FeatureServer" in layer_url: base = layer_url.split("/MapServer")[0]+"/MapServer" if "/MapServer" in layer_url else layer_url.split("/FeatureServer")[0]+"/FeatureServer" sj = fetch(f"{base}?f=json") if not sj: return try: meta = json.loads(sj) except: return ext = meta.get("fullExtent") or meta.get("initialExtent") if not ext: return if isinstance(ext, dict): try: xmin = ext.get("xmin"); ymin = ext.get("ymin"); xmax = ext.get("xmax"); ymax = ext.get("ymax") sr = ext.get("spatialReference",{}).get("wkid", 102100) if None in (xmin,ymin,xmax,ymax): return except: return elif isinstance(ext, list) and len(ext)==4: xmin,ymin,xmax,ymax = ext; sr=102100 else: return bbox = f"{xmin},{ymin},{xmax},{ymax}" # try each operational layer's MapServer /export and stitch by alpha-blending # Simpler: just use the FIRST operational layer that has a MapServer URL ops = w.get("operationalLayers", []) for op in ops: url = op.get("url","").rstrip("/") if not url: continue # if it's /FeatureServer/, swap to /MapServer/ msurl = url if "/FeatureServer" in msurl: msurl = msurl.replace("/FeatureServer", "/MapServer") if "/MapServer" not in msurl: continue # strip layer id suffix for suffix in [f"/{i}" for i in range(50)]: if msurl.endswith(suffix): msurl = msurl[:-len(suffix)] break eurl = (f"{msurl}/export?bbox={bbox}&bboxSR={sr}&imageSR={sr}" f"&size=2048,2048&format=png32&transparent=true&f=image") img = fetch(eurl, binary=True) if img and len(img) > 2000: out.write_bytes(img) with _lock: _stats["webmap_pngs"] += 1 return def writer(): while True: time.sleep(15) with _lock: s=dict(_stats) s["elapsed"]=int(time.time()-s["start"]) STAT.write_text(json.dumps(s,indent=2)) def main(): THUMB.mkdir(exist_ok=True); WMR.mkdir(exist_ok=True); LOG.parent.mkdir(exist_ok=True) items = json.load(open(INDEX, encoding="utf-8")) threading.Thread(target=writer, daemon=True).start() log(f"phase1: pulling {len(items)} thumbnails") with ThreadPoolExecutor(max_workers=WORKERS) as ex: futs=[ex.submit(pull_thumb,it) for it in items] for i,f in enumerate(as_completed(futs)): if (i+1)%100==0: with _lock: s=dict(_stats) log(f" thumb {i+1}/{len(items)} thumbs={s['thumbs']} nothumb={s['no_thumb']} err={s['err']}") webmaps = [i for i in items if i.get("type")=="Web Map"] log(f"phase2: rendering {len(webmaps)} Web Maps") with ThreadPoolExecutor(max_workers=WORKERS) as ex: futs=[ex.submit(render_webmap,it) for it in webmaps] for i,f in enumerate(as_completed(futs)): if (i+1)%20==0: with _lock: s=dict(_stats) log(f" webmap {i+1}/{len(webmaps)} pngs={s['webmap_pngs']}") log(f"DONE {_stats}") if __name__=="__main__": main()