commit acf5948fdb089f0548696f9bf75d80914f42dbd2 Author: Fabio Micheluz Date: Sun Feb 8 15:56:14 2026 +0100 first commit diff --git a/f1.sh b/f1.sh new file mode 100755 index 0000000..083f818 --- /dev/null +++ b/f1.sh @@ -0,0 +1,7 @@ +#!/bin/bash +date +echo "79350" +#wc -l manifest_dem.txt +#find terrain_rgb -type f \( -name "*.tif" -o -name "*.tiff" \) | wc -l +cat resume/dem_resume.txt +du -sh terrain_rgb diff --git a/f2.sh b/f2.sh new file mode 100755 index 0000000..bf01925 --- /dev/null +++ b/f2.sh @@ -0,0 +1,8 @@ +#!/bin/bash +date +echo "26450" +#wc -l manifest_dem.txt +#find terrain_rgb -type f \( -name "*.tif" -o -name "*.tiff" \) | wc -l +find resume/tiles -type f -name "*.done" | wc -l +#cat resume/dem_resume.txt +du -sh tiles diff --git a/make_tiles_italy.sh b/make_tiles_italy.sh new file mode 100755 index 0000000..ccdcc77 --- /dev/null +++ b/make_tiles_italy.sh @@ -0,0 +1,93 @@ +#!/bin/zsh + +############################################### +# CONFIG +############################################### + +PYTHON="/usr/bin/python3" + +TERRAIN_ROOT="/Volumes/terrain/tiles" +TILES_ROOT="/Volumes/SSD4T/tiles_italy_new" +LOG_DIR="/Volumes/SSD4T/logs_new" +RESUME_DIR="/Volumes/SSD4T/resume_new" + +Z_MIN=7 +Z_MAX=14 + +mkdir -p "$TILES_ROOT" "$LOG_DIR" "$RESUME_DIR" + + +############################################### +# CHECK PYTHON +############################################### + +if [ ! -x "$PYTHON" ]; then + echo "[ERRORE] Python non trovato: $PYTHON" + exit 1 +fi + + +############################################### +# LOOP SU TUTTI I DEM ITALIANI +############################################### + +DEM_LIST=$(find "$TERRAIN_ROOT" -maxdepth 1 -type d -name "Copernicus_DSM_COG_10_*_DEM" \ + | grep -E "N(3[6-9]|4[0-7])_00_E(00[6-9]|0(1[0-9]))_00" \ + | sort) + +echo "[INFO] Trovati $(echo "$DEM_LIST" | wc -l | tr -d ' ') DEM italiani" + +for DEM_DIR in $DEM_LIST; do + BASENAME=$(basename "$DEM_DIR") + TIF="$DEM_DIR/${BASENAME}_terrain_rgb.tif" + + if [ ! -f "$TIF" ]; then + echo "[WARN] Nessun TIFF TerrainRGB in $DEM_DIR" + continue + fi + + OUT_DIR="$TILES_ROOT/$BASENAME" + mkdir -p "$OUT_DIR" + + RESUME_FILE="$RESUME_DIR/${BASENAME}.done" + LOGFILE="$LOG_DIR/${BASENAME}.log" + + if [ -f "$RESUME_FILE" ]; then + echo "[SKIP] Tiles già generati per $BASENAME" + continue + fi + + echo "[INFO] Genero tiles per $BASENAME" + + { + echo "----------------------------------------" + echo "DEM: $BASENAME" + echo "Input: $TIF" + echo "Output: $OUT_DIR" + echo "Zoom: $Z_MIN-$Z_MAX" + echo "Start: $(date)" + echo "----------------------------------------" + + CORES=$(sysctl -n hw.ncpu 2>/dev/null || echo 4) + + $PYTHON terrain_rgb_to_tiles.py "$TIF" "$OUT_DIR" "$Z_MIN" "$Z_MAX" "$CORES" + STATUS=$? + + echo "End: $(date)" + echo "Status: $STATUS" + echo "----------------------------------------" + + } >> "$LOGFILE" 2>&1 + + if [ "$STATUS" -ne 0 ]; then + echo "[ERRORE] Tiles falliti per $BASENAME" + continue + fi + + touch "$RESUME_FILE" + echo "[OK] Tiles creati per $BASENAME" + +done + +echo "[INFO] Completato!" + diff --git a/merge_tiles.sh b/merge_tiles.sh new file mode 100755 index 0000000..0b786e1 --- /dev/null +++ b/merge_tiles.sh @@ -0,0 +1,35 @@ +#!/bin/zsh + +SRC_ROOT="tiles_italy_new" +DST_ROOT="merged_tiles" + +mkdir -p "$DST_ROOT" + +echo "[INFO] Merge tiles da $SRC_ROOT → $DST_ROOT" + +# Scorri tutte le tile z/x/y.png in tutte le sottocartelle +find "$SRC_ROOT" -type f -name '*.png' | while read -r TILE; do + # TILE = tiles_italy_new////.png + REL="${TILE#$SRC_ROOT/}" # ///.png + DEM="${REL%%/*}" # + REST="${REL#*/}" # //.png + + Z="$(echo "$REST" | cut -d'/' -f1)" + X="$(echo "$REST" | cut -d'/' -f2)" + Y_FILE="$(echo "$REST" | cut -d'/' -f3)" + + OUT_DIR="$DST_ROOT/$Z/$X" + OUT_PATH="$OUT_DIR/$Y_FILE" + + mkdir -p "$OUT_DIR" + + if [ -f "$OUT_PATH" ]; then + echo "[WARN] Tile già esistente, lascio la prima: +$OUT_PATH" + continue + fi + + cp "$TILE" "$OUT_PATH" +done + +echo "[INFO] Merge completato in $DST_ROOT" diff --git a/ph1.sh b/ph1.sh new file mode 100755 index 0000000..5ac6ca4 --- /dev/null +++ b/ph1.sh @@ -0,0 +1,113 @@ +#!/bin/zsh + +############################################### +# CONFIG +############################################### + +RIO="/opt/local/Library/Frameworks/Python.framework/Versions/3.11/bin/rio" +MANIFEST="manifest_dem.txt" + +OUT_ROOT="terrain_rgb" +LOG_DIR="logs/dem" +RESUME_FILE="resume/dem_resume.txt" + +mkdir -p "$OUT_ROOT" "$LOG_DIR" "resume" + + +############################################### +# CHECK BINARI +############################################### + +check_bin() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "[ERRORE] Binario non trovato: $1" + exit 1 + fi +} + +check_bin gdalinfo +check_bin gdal_translate + +if [ ! -x "$RIO" ]; then + echo "[ERRORE] rio non trovato: $RIO" + exit 1 +fi + + +############################################### +# RESUME +############################################### + +LAST_DONE=0 +if [ -f "$RESUME_FILE" ]; then + LAST_DONE=$(cat "$RESUME_FILE") +fi + +DEM_COUNT=0 + + +############################################### +# LOOP DEM +############################################### + +while IFS= read -r DEM; do + + DEM=$(echo "$DEM" | tr -d '\r' | xargs) + [ -z "$DEM" ] && continue + + if [ ! -f "$DEM" ]; then + echo "[WARN] DEM non trovato: $DEM" + continue + fi + + DEM_COUNT=$((DEM_COUNT + 1)) + + if [ "$DEM_COUNT" -le "$LAST_DONE" ]; then + echo "[SKIP] DEM #$DEM_COUNT già processato" + continue + fi + + BASENAME=$(basename "$DEM" .tif) + WORKDIR="$OUT_ROOT/$BASENAME" + mkdir -p "$WORKDIR" + + LOGFILE="$LOG_DIR/${BASENAME}.log" + + echo "[INFO] DEM #$DEM_COUNT → $DEM" + echo "[INFO] Log: $LOGFILE" + + OUT_TERRAIN="$WORKDIR/${BASENAME}_terrain_rgb.tif" + + { + echo "----------------------------------------" + echo "DEM: $DEM" + echo "Output: $OUT_TERRAIN" + echo "Start: $(date)" + echo "----------------------------------------" + + "$RIO" rgbify \ + --min-z 0 \ + --max-z 9000 \ + --format png \ + "$DEM" \ + "$OUT_TERRAIN" + + STATUS=$? + + echo "End: $(date)" + echo "Status: $STATUS" + echo "----------------------------------------" + + } >> "$LOGFILE" 2>&1 + + if [ "$STATUS" -ne 0 ]; then + echo "[ERRORE] rgbify fallito per $DEM" + continue + fi + + echo "$DEM_COUNT" > "$RESUME_FILE" + echo "[OK] Terrain‑RGB generato per $BASENAME" + +done < "$MANIFEST" + +echo "[INFO] Fase 1 completata" diff --git a/ph2.sh b/ph2.sh new file mode 100755 index 0000000..3a0ca18 --- /dev/null +++ b/ph2.sh @@ -0,0 +1,238 @@ + +#!/bin/zsh + +############################################### +# CONFIG +############################################### + +PYTHON="/opt/local/Library/Frameworks/Python.framework/Versions/3.11/bin/python3" + +TERRAIN_ROOT="terrain_rgb" +TILES_ROOT="tiles" +LOG_DIR="logs/tiles" +RESUME_DIR="resume/tiles" + +Z_MIN=5 +Z_MAX=14 + +mkdir -p "$TILES_ROOT" "$LOG_DIR" "$RESUME_DIR" + + +############################################### +# CHECK PYTHON +############################################### + +if [ ! -x "$PYTHON" ]; then + echo "[ERRORE] Python non trovato: $PYTHON" + exit 1 +fi + + +############################################### +# PYTHON SCRIPT INLINE (NUOVO E CORRETTO) +############################################### + +PY_SCRIPT="$(mktemp)" +cat > "$PY_SCRIPT" << 'EOF' +import os, sys, math +from concurrent.futures import ProcessPoolExecutor +import rasterio +from rasterio.warp import transform_bounds +from rasterio.enums import Resampling +from rasterio.windows import Window +from PIL import Image + + +# ---------- Tile math ---------- +def latlon_to_tile(lat, lon, z): + lat_rad = math.radians(lat) + n = 2 ** z + xtile = int((lon + 180) / 360 * n) + ytile = int((1 - math.log(math.tan(lat_rad) + 1 / math.cos(lat_rad)) / math.pi) / 2 * n) + return xtile, ytile + +def tile_bounds_wgs84(z, x, y): + n = 2 ** z + lon_min = x / n * 360 - 180 + lon_max = (x + 1) / n * 360 - 180 + lat_min = math.degrees(math.atan(math.sinh(math.pi * (1 - 2 * (y + 1) / n)))) + lat_max = math.degrees(math.atan(math.sinh(math.pi * (1 - 2 * y / n)))) + return lon_min, lat_min, lon_max, lat_max + + +# ---------- Clamp window ---------- +def clamp_window(window, src): + row_off = max(0, int(window.row_off)) + col_off = max(0, int(window.col_off)) + height = max(0, int(min(window.height, src.height - row_off))) + width = max(0, int(min(window.width, src.width - col_off))) + if width <= 0 or height <= 0: + return None + return Window(col_off, row_off, width, height) + + +# ---------- Tile job ---------- +def process_tile(args): + tif_path, tiles_root, z, x, y = args + try: + with rasterio.open(tif_path) as src: + # CRS check + if src.crs is None: + return False + + # Tile bounds in WGS84 + lon_min, lat_min, lon_max, lat_max = tile_bounds_wgs84(z, x, y) + + # Transform tile bounds raster CRS + left, bottom, right, top = transform_bounds( + "EPSG:4326", src.crs, lon_min, lat_min, lon_max, lat_max, densify_pts=0 + ) + + # Convert to pixel coords + r0, c0 = src.index(left, top) + r1, c1 = src.index(right, bottom) + + row_min, row_max = sorted((r0, r1)) + col_min, col_max = sorted((c0, c1)) + + window = clamp_window( + Window(col_min, row_min, col_max - col_min, row_max - row_min), + src + ) + if window is None: + return False + + # Read terrain RGB + data = src.read( + indexes=(1,2,3), + out_shape=(3, 256, 256), + window=window, + resampling=Resampling.nearest + ) + + if data.size == 0: + return False + + # Save tile + out_dir = os.path.join(tiles_root, str(z), str(x)) + os.makedirs(out_dir, exist_ok=True) + out_path = os.path.join(out_dir, f"{y}.png") + + Image.fromarray(data.transpose(1,2,0).astype("uint8"), "RGB").save(out_path) + + return True + + except Exception: + return False + + +# ---------- Main ---------- +def main(): + tif_path = sys.argv[1] + tiles_root = sys.argv[2] + z_min = int(sys.argv[3]) + z_max = int(sys.argv[4]) + max_workers = int(sys.argv[5]) + + with rasterio.open(tif_path) as src: + if src.crs is None: + print("[WARN] CRS mancante:", tif_path) + return + + left, bottom, right, top = transform_bounds(src.crs, "EPSG:4326", *src.bounds) + + jobs = [] + for z in range(z_min, z_max + 1): + x_min, y_max = latlon_to_tile(bottom, left, z) + x_max, y_min = latlon_to_tile(top, right, z) + + x0, x1 = sorted((x_min, x_max)) + y0, y1 = sorted((y_min, y_max)) + + for x in range(x0, x1 + 1): + for y in range(y0, y1 + 1): + jobs.append((tif_path, tiles_root, z, x, y)) + + if not jobs: + print("[WARN] Nessuna tile da generare per", tif_path) + return + + with ProcessPoolExecutor(max_workers=max_workers) as ex: + for _ in ex.map(process_tile, jobs): + pass + + +if __name__ == "__main__": + main() +EOF + + +############################################### +# LOOP SU TUTTI I TERRAINRGB (SOLO DEM) +############################################### + +for DEM_DIR in "$TERRAIN_ROOT"/*; do + [ -d "$DEM_DIR" ] || continue + + BASENAME=$(basename "$DEM_DIR") + + # Processa SOLO DEM (salta EDM e FLM) + case "$BASENAME" in + *DEM) ;; + *) echo "[SKIP] Non DEM: $BASENAME"; continue ;; + esac + + TIF="$DEM_DIR/${BASENAME}_terrain_rgb.tif" + + if [ ! -f "$TIF" ]; then + echo "[WARN] Nessun TIFF TerrainRGB in $DEM_DIR" + continue + fi + + OUT_DIR="$TILES_ROOT/$BASENAME" + mkdir -p "$OUT_DIR" + + RESUME_FILE="$RESUME_DIR/${BASENAME}.done" + LOGFILE="$LOG_DIR/${BASENAME}.log" + + if [ -f "$RESUME_FILE" ]; then + echo "[SKIP] Tiles gi generati per $BASENAME" + continue + fi + + echo "[INFO] Genero tiles per $BASENAME" + + { + echo "----------------------------------------" + echo "DEM: $BASENAME" + echo "Input: $TIF" + echo "Output: $OUT_DIR" + echo "Zoom: $Z_MIN$Z_MAX" + echo "Start: $(date)" + echo "----------------------------------------" + + CORES=$(sysctl -n hw.ncpu 2>/dev/null || echo 4) + + "$PYTHON" "$PY_SCRIPT" "$TIF" "$OUT_DIR" "$Z_MIN" "$Z_MAX" "$CORES" + STATUS=$? + + echo "End: $(date)" + echo "Status: $STATUS" + echo "----------------------------------------" + + } >> "$LOGFILE" 2>&1 + + + if [ "$STATUS" -ne 0 ]; then + echo "[ERRORE] Tiles falliti per $BASENAME" + continue + fi + + touch "$RESUME_FILE" + echo "[OK] Tiles creati per $BASENAME" + +done + +rm -f "$PY_SCRIPT" +echo "[INFO] Fase 2 completata" + diff --git a/terrain.txt b/terrain.txt new file mode 100644 index 0000000..9012afa --- /dev/null +++ b/terrain.txt @@ -0,0 +1,48 @@ +🚀 Scaricare i dati da mirror Copernicus GLO‑30 + +aws s3 sync s3://copernicus-dem-30m ./glo30 --no-sign-request + +Crea glo30 dir con tutto il mondo + +1️⃣ Crea un terrain_rgb partendo da glo30 + +Utilizza ph1.sh + +Puoi interrompere quando vuoi poi riparte tralasciano quello fatto (ci mette molto 1/2 gg) + +Circa 600GB + +Crea una dir terrain_rgb + + +2️⃣ Crea i tiles partendo da terrain_rgb + +Utilizza ph2.sh + +Puoi interrompere quando vuoi poi riparte tralasciano quello fatto (ci mette moltissimo 1/2 weeks) + +Circa 1.6TB + +Crea una dir tiles + +3️⃣ Estrarre soltanto una parte(opzionale) + +Utilizza make_tiles_italy.sh + +partendo da tiles ci mette solo i tiles dell'Italia + +crea una di /Volumes/SSD4T/tiles_italy_new + +4️⃣ Trasforma la dir tiles_italy_new in modo che sia utilizzabile damb-util + +Utilizza merge_tiles.sh + +crea una dir merged_tiles che può essere utilizzata con mb-util + +5️⃣ + + +6️⃣ + +🎯 Risultato +Hai un terrain globale ad alta qualità, basato su Copernicus GLO‑30, pronto per TileServer‑GL.