first commit

scripts/merge-data.js

@@ -0,0 +1,70 @@
+import fs from "node:fs/promises";
+import { asyncMap } from "modern-async";
+import { getDistance } from "ol/sphere.js";
+
+function removeDuplicatesByUrl(points) {
+  const uniquePoints = [];
+  const urlSet = new Set();
+
+  points.forEach((points) => {
+    let isDuplicate = false;
+    const link = points.link;
+
+    if (urlSet.has(link)) {
+      isDuplicate = true;
+    }
+
+    if (!isDuplicate) {
+      uniquePoints.push(points);
+      urlSet.add(link);
+    }   
+  });
+
+  return uniquePoints;
+}
+
+function removeDuplicatesByRadius(points, radius) {
+    const uniquePoints = [];
+    const coordinatesSet = new Set();
+  
+    points.forEach((point) => {
+      const lonLat = point.coordinates;
+      let isDuplicate = false;
+  
+      coordinatesSet.forEach((setCoordinates) => {
+        if (getDistance(setCoordinates, lonLat) <= radius) {
+          isDuplicate = true;
+        }
+      });
+  
+      if (!isDuplicate) {
+        uniquePoints.push(point);
+        coordinatesSet.add(lonLat);
+      }
+    });
+  
+    return uniquePoints;
+  }
+
+function convert(data) {
+  return data.points.map((p) => ({
+    ...p,
+    source: data.source,
+    provider: data.name,
+  }));
+}
+
+async function main() {
+  const dataDir = await fs.readdir("./data");
+  let data = await asyncMap(dataDir, async (filename) => {
+    const fileContent = await fs.readFile(`./data/${filename}`);
+    const data = JSON.parse(fileContent);
+    return convert(data);
+  });
+  data = data.flatMap((v) => v);
+  data = removeDuplicatesByUrl(data);
+  data = removeDuplicatesByRadius(data, 10);
+  await fs.writeFile("merged-data.json", JSON.stringify(data, undefined, 2));
+}
+
+main();

scripts/update.js

@@ -0,0 +1,89 @@
+import fs from "node:fs/promises";
+import { asyncMap } from "modern-async";
+import { JSDOM } from "jsdom";
+import { getFinalURL } from "./utils.js";
+
+async function woyag() {
+  const apiResponse = await fetch("https://login.woyag.ru/ajax/pvz-list");
+  const json = await apiResponse.json();
+
+  let points = json.filter((point) => !!point.link);
+
+  points = await asyncMap(points, async (point) => {
+    const link = await getFinalURL(point.link).then(u => {
+        const final = new URL(u);
+        final.search = '';
+        return final.toString();
+    })
+
+    return {
+      coordinates: [parseFloat(point["geo_lng"]), parseFloat(point["geo_lat"])],
+      name: point.name,
+      address: point.fullname,
+      link,
+      operationTime: "пн-вс с 9:00 до 17:45",
+    };
+  });
+
+  await fs.writeFile(
+    "data/01_woyag.json",
+    JSON.stringify(
+      {
+        name: "WOЯЖ",
+        source: "https://login.woyag.ru/map",
+        points,
+      },
+      undefined,
+      4
+    )
+  );
+}
+
+const linkRegexp = new RegExp(/https:\/\/ozon\.ru\/point\/\d+/);
+
+async function sevenDostavka() {
+  const res = await fetch("https://dostavka.7telecom.ru");
+  const htmlText = await res.text();
+
+  const dom = new JSDOM(htmlText);
+  const document = dom.window.document;
+  const scripts = document.querySelectorAll("script");
+
+  for (let script of scripts) {
+    if (script.textContent.includes("ДОБАВИТЬ ПУНКТ ВЫДАЧИ В ПРИЛОЖЕНИЕ")) {
+      let x = script.textContent.split("[\n{\n").pop().split("\n},\n]")[0];
+
+      if (x) {
+        const points = new Function(`return [{${x}}]`)();
+
+        fs.writeFile(
+          "data/99_sevenDostavka.json",
+          JSON.stringify(
+            {
+              name: "7dostavka",
+              source: "https://dostavka.7telecom.ru",
+              points: points.map((point) => ({
+                coordinates: [
+                  parseFloat(point["lng"]),
+                  parseFloat(point["lat"]),
+                ],
+                link: linkRegexp.exec(point.descr)[0],
+                name: point.title,
+                operationTime: "неизвестно",
+              })),
+            },
+            undefined,
+            4
+          )
+        );
+      }
+    }
+  }
+}
+
+async function main() {
+  await woyag();
+  await sevenDostavka();
+}
+
+main();

scripts/utils.js

@@ -0,0 +1,23 @@
+export async function getFinalURL(url) {
+    let response = await fetch(url, {
+      method: "HEAD",
+      redirect: "manual",
+    });
+    while (
+      response.status >= 300 &&
+      response.status < 400 &&
+      response.headers.get("location") &&
+      new URL(url).hostname !== 'ozon.ru'
+    ) {
+      url = response.headers.get("location");
+      if (!url.startsWith("http")) {
+        const baseUrl = new URL(response.url);
+        url = `${baseUrl.protocol}//${baseUrl.host}${url}`;
+      }
+      response = await fetch(url, {
+        method: "HEAD",
+        redirect: "manual",
+      });
+    }
+    return url;
+  }