Limit concurrency

BlinkTagInc · Jun 19, 2024 · 240d036 · 240d036
1 parent 0f2c5e5
commit 240d036
Show file tree

Hide file tree

Showing 4 changed files with 126 additions and 81 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Progress Bar
 
+## Updated
+
+- Limit concurrency
+
 ## [3.7.0] - 2024-06-18
 
 ## Updated

diff --git a/lib/gtfs-to-geojson.js b/lib/gtfs-to-geojson.js
@@ -4,6 +4,7 @@ import { rm, mkdir, writeFile } from 'node:fs/promises';
 
 import { clone, omit, uniqBy } from 'lodash-es';
 import { getRoutes, getTrips, openDb, importGtfs } from 'gtfs';
+import pLimit from 'p-limit';
 import Timer from 'timer-machine';
 import sqlString from 'sqlstring-sqlite';
 import sanitize from 'sanitize-filename';
@@ -28,6 +29,8 @@ import stops from './formats/stops.js';
 import stopsBuffer from './formats/stops-buffer.js';
 import stopsDissolved from './formats/stops-dissolved.js';
 
+const limit = pLimit(20);
+
 const { version } = JSON.parse(
   readFileSync(new URL('../package.json', import.meta.url)),
 );
@@ -136,23 +139,25 @@ const buildGeoJSON = async (agencyKey, config, outputStats) => {
     );
 
     await Promise.all(
-      shapes.map(async (shape) => {
-        const geojson = getGeoJSONByFormat(config, {
-          shape_id: shape.shape_id,
-          service_id: serviceIds,
-        });
-
-        if (!geojson) {
-          return;
-        }
-
-        outputStats.files += 1;
-        outputStats.shapes += 1;
-        const fileName = sanitize(`${shape.shape_id}.geojson`);
-        const filePath = path.join(getExportPath(agencyKey), fileName);
-        await writeFile(filePath, JSON.stringify(geojson));
-        bar.increment();
-      }),
+      shapes.map(async (shape) =>
+        limit(async () => {
+          const geojson = getGeoJSONByFormat(config, {
+            shape_id: shape.shape_id,
+            service_id: serviceIds,
+          });
+
+          if (!geojson) {
+            return;
+          }
+
+          outputStats.files += 1;
+          outputStats.shapes += 1;
+          const fileName = sanitize(`${shape.shape_id}.geojson`);
+          const filePath = path.join(getExportPath(agencyKey), fileName);
+          await writeFile(filePath, JSON.stringify(geojson));
+          bar.increment();
+        }),
+      ),
     );
   } else if (config.outputType === 'route') {
     const routes = getRoutes({
@@ -166,65 +171,67 @@ const buildGeoJSON = async (agencyKey, config, outputStats) => {
     );
 
     await Promise.all(
-      routes.map(async (route) => {
-        outputStats.routes += 1;
+      routes.map(async (route) =>
+        limit(async () => {
+          outputStats.routes += 1;
 
-        const trips = getTrips(
-          {
-            route_id: route.route_id,
-            service_id: serviceIds,
-          },
-          ['trip_headsign', 'direction_id'],
-        );
-
-        const directions = uniqBy(trips, (trip) => trip.trip_headsign);
-        await Promise.all(
-          directions.map(async (direction) => {
-            const geojson = getGeoJSONByFormat(config, {
+          const trips = getTrips(
+            {
               route_id: route.route_id,
-              direction_id: direction.direction_id,
               service_id: serviceIds,
-            });
-
-            if (!geojson) {
-              return;
-            }
-
-            outputStats.files += 1;
-            let fileName = '';
-
-            if (route.agency_id !== undefined) {
-              fileName += `${route.agency_id}_`;
-            }
-
-            fileName += `${getRouteName(route)}_`;
-
-            if (direction.direction_id !== undefined) {
-              fileName += direction.direction_id;
-            }
-
-            // Check if file name will be unique, if not append route_id to filename
-            const identicalRoutes = routes.filter(
-              (r) =>
-                r.agency_id === route.agency_id &&
-                r.route_short_name === route.route_short_name,
-            );
-            if (identicalRoutes.length > 1) {
-              fileName += `_${route.route_id}`;
-            }
-
-            fileName += '.geojson';
-
-            const filePath = path.join(
-              getExportPath(agencyKey),
-              sanitize(fileName),
-            );
-            await writeFile(filePath, JSON.stringify(geojson));
-          }),
-        );
-
-        bar.increment();
-      }),
+            },
+            ['trip_headsign', 'direction_id'],
+          );
+
+          const directions = uniqBy(trips, (trip) => trip.trip_headsign);
+          await Promise.all(
+            directions.map(async (direction) => {
+              const geojson = getGeoJSONByFormat(config, {
+                route_id: route.route_id,
+                direction_id: direction.direction_id,
+                service_id: serviceIds,
+              });
+
+              if (!geojson) {
+                return;
+              }
+
+              outputStats.files += 1;
+              let fileName = '';
+
+              if (route.agency_id !== undefined) {
+                fileName += `${route.agency_id}_`;
+              }
+
+              fileName += `${getRouteName(route)}_`;
+
+              if (direction.direction_id !== undefined) {
+                fileName += direction.direction_id;
+              }
+
+              // Check if file name will be unique, if not append route_id to filename
+              const identicalRoutes = routes.filter(
+                (r) =>
+                  r.agency_id === route.agency_id &&
+                  r.route_short_name === route.route_short_name,
+              );
+              if (identicalRoutes.length > 1) {
+                fileName += `_${route.route_id}`;
+              }
+
+              fileName += '.geojson';
+
+              const filePath = path.join(
+                getExportPath(agencyKey),
+                sanitize(fileName),
+              );
+              await writeFile(filePath, JSON.stringify(geojson));
+            }),
+          );
+
+          bar.increment();
+        }),
+      ),
     );
   } else if (config.outputType === 'agency') {
     config.log(`${agencyKey}: Generating geoJSON`);

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -39,7 +39,7 @@
     "cli-table": "^0.3.11",
     "gtfs": "^4.12.0",
     "lodash-es": "^4.17.21",
-    "pluralize": "^8.0.0",
+    "p-limit": "^5.0.0",
     "pretty-error": "^4.0.0",
     "sanitize-filename": "^1.6.3",
     "sqlstring-sqlite": "^0.1.1",