gc-wing/scripts/prepare-subcables.mjs

import fs from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

const OUT_DIR = path.resolve(__dirname, "..", "apps", "web", "public", "data", "subcables");
const GEO_URL = "https://www.submarinecablemap.com/api/v3/cable/cable-geo.json";
const DETAILS_URL_BASE = "https://www.submarinecablemap.com/api/v3/cable/";

const CONCURRENCY = Math.max(1, Math.min(24, Number(process.env.CONCURRENCY || 12)));
const TIMEOUT_MS = Math.max(5_000, Math.min(60_000, Number(process.env.TIMEOUT_MS || 20_000)));

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

async function fetchText(url, { timeoutMs = TIMEOUT_MS } = {}) {
  const controller = new AbortController();
  const timeout = setTimeout(() => controller.abort(), timeoutMs);
  try {
    const res = await fetch(url, {
      signal: controller.signal,
      headers: {
        accept: "application/json",
      },
    });
    const text = await res.text();
    const contentType = res.headers.get("content-type") || "";
    if (!res.ok) {
      throw new Error(`HTTP ${res.status} (${res.statusText})`);
    }
    return { text, contentType };
  } finally {
    clearTimeout(timeout);
  }
}

async function fetchJson(url) {
  const { text, contentType } = await fetchText(url);
  if (!contentType.toLowerCase().includes("application/json")) {
    const snippet = text.slice(0, 200).replace(/\s+/g, " ").trim();
    throw new Error(`Unexpected content-type (${contentType || "unknown"}): ${snippet || "<empty>"}`);
  }
  try {
    return JSON.parse(text);
  } catch (e) {
    throw new Error(`Invalid JSON: ${e instanceof Error ? e.message : String(e)}`);
  }
}

async function fetchJsonWithRetry(url, attempts = 2) {
  let lastErr = null;
  for (let i = 0; i < attempts; i += 1) {
    try {
      return await fetchJson(url);
    } catch (e) {
      lastErr = e;
      if (i < attempts - 1) {
        await sleep(250 * (i + 1));
      }
    }
  }
  throw lastErr;
}

function pickCableDetails(raw) {
  const obj = raw && typeof raw === "object" ? raw : {};
  const landingPoints = Array.isArray(obj.landing_points) ? obj.landing_points : [];
  return {
    id: String(obj.id || ""),
    name: String(obj.name || ""),
    length: obj.length == null ? null : String(obj.length),
    rfs: obj.rfs == null ? null : String(obj.rfs),
    rfs_year: typeof obj.rfs_year === "number" ? obj.rfs_year : null,
    is_planned: Boolean(obj.is_planned),
    owners: obj.owners == null ? null : String(obj.owners),
    suppliers: obj.suppliers == null ? null : String(obj.suppliers),
    landing_points: landingPoints.map((lp) => {
      const p = lp && typeof lp === "object" ? lp : {};
      return {
        id: String(p.id || ""),
        name: String(p.name || ""),
        country: String(p.country || ""),
        is_tbd: p.is_tbd === true,
      };
    }),
    notes: obj.notes == null ? null : String(obj.notes),
    url: obj.url == null ? null : String(obj.url),
  };
}

async function main() {
  await fs.mkdir(OUT_DIR, { recursive: true });

  console.log(`[subcables] fetching geojson: ${GEO_URL}`);
  const geo = await fetchJsonWithRetry(GEO_URL, 3);
  const geoPath = path.join(OUT_DIR, "cable-geo.json");
  await fs.writeFile(geoPath, JSON.stringify(geo));

  const features = Array.isArray(geo?.features) ? geo.features : [];
  const ids = Array.from(
    new Set(
      features
        .map((f) => f?.properties?.id)
        .filter((v) => typeof v === "string" && v.trim().length > 0)
        .map((v) => v.trim()),
    ),
  ).sort();

  console.log(`[subcables] cables: ${ids.length} (concurrency=${CONCURRENCY}, timeoutMs=${TIMEOUT_MS})`);

  const byId = {};
  const failures = [];
  let cursor = 0;
  let completed = 0;
  const startedAt = Date.now();

  const worker = async () => {
    for (;;) {
      const idx = cursor;
      cursor += 1;
      if (idx >= ids.length) return;
      const id = ids[idx];
      const url = new URL(`${id}.json`, DETAILS_URL_BASE).toString();
      try {
        const raw = await fetchJsonWithRetry(url, 2);
        const picked = pickCableDetails(raw);
        if (!picked.id) {
          throw new Error("Missing id in details response");
        }
        byId[id] = picked;
      } catch (e) {
        failures.push({ id, error: e instanceof Error ? e.message : String(e) });
      } finally {
        completed += 1;
        if (completed % 25 === 0 || completed === ids.length) {
          const sec = Math.max(1, Math.round((Date.now() - startedAt) / 1000));
          const rate = (completed / sec).toFixed(1);
          console.log(`[subcables] ${completed}/${ids.length} (${rate}/s)`);
        }
      }
    }
  };

  await Promise.all(Array.from({ length: CONCURRENCY }, () => worker()));

  const detailsOut = {
    version: 1,
    generated_at: new Date().toISOString(),
    by_id: byId,
  };
  const detailsPath = path.join(OUT_DIR, "cable-details.min.json");
  await fs.writeFile(detailsPath, JSON.stringify(detailsOut));

  if (failures.length > 0) {
    console.error(`[subcables] failures: ${failures.length}`);
    for (const f of failures.slice(0, 30)) {
      console.error(`- ${f.id}: ${f.error}`);
    }
    if (failures.length > 30) {
      console.error(`- ... +${failures.length - 30} more`);
    }
    process.exitCode = 1;
  }

  console.log(`[subcables] wrote: ${geoPath}`);
  console.log(`[subcables] wrote: ${detailsPath}`);
}

main().catch((e) => {
  console.error(`[subcables] fatal: ${e instanceof Error ? e.stack || e.message : String(e)}`);
  process.exitCode = 1;
});