Merge branch 'release/1.17.1'

Author: maurizi

CHANGELOG.md

@@ -5,11 +5,20 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## Unreleased
 ### Added
+
 ### Changed
+
 ### Fixed
 
+
+## [1.17.1]
+### Changed
+ - Improved PlanScore error logging & double timeout [#1204](https://github.com/PublicMapping/districtbuilder/pull/1204)
+ - Revert to using JSON for data serialization [#1206](https://github.com/PublicMapping/districtbuilder/pull/1206)
+
+
 ## [1.17.0]
 
 ### Added
@@ -490,7 +499,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Initial release.
 
-[unreleased]: https://github.com/publicmapping/districtbuilder/compare/1.17.0...HEAD
+[unreleased]: https://github.com/publicmapping/districtbuilder/compare/1.17.1...HEAD
+[1.17.1]: https://github.com/publicmapping/districtbuilder/compare/1.17.0...1.17.1
 [1.17.0]: https://github.com/publicmapping/districtbuilder/compare/1.16.1...1.17.0
 [1.16.1]: https://github.com/publicmapping/districtbuilder/compare/1.16.0...1.16.1
 [1.16.0]: https://github.com/publicmapping/districtbuilder/compare/1.15.1...1.16.0

scripts/load-full-state-data

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+set -e
+
+if [[ -n "${DB_DEBUG}" ]]; then
+    set -x
+fi
+
+S3_URIS=(
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/AK/2021-08-13T09:23:10.238Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/AL/2021-08-13T02:30:40.369Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/AR/2021-08-13T06:43:09.147Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/AZ/2021-08-12T21:42:38.860Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/CA/2022-04-26T14:12:02.125Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/CO/2021-10-26T03:41:25.119Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/CT/2021-10-26T00:33:03.521Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/DC/2021-08-13T09:30:57.826Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/DE/2021-08-13T09:33:21.756Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/FL/2021-08-12T23:20:51.764Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/GA/2021-08-13T00:54:52.804Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/HI/2021-08-13T08:14:36.412Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/IA/2021-08-13T07:04:13.080Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/ID/2021-08-12T22:37:52.328Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/IL/2021-08-12T21:17:43.961Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/IN/2021-09-25T22:14:17.196Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/KS/2021-08-13T07:25:22.917Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/KY/2021-08-13T10:53:12.868Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/LA/2021-08-13T06:19:58.665Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/MA/2021-08-26T04:34:52.224Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/MD/2021-10-26T04:11:55.734Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/ME/2021-11-09T05:45:39.150Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/MI/2021-08-12T22:15:23.087Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/MN/2021-08-13T05:36:28.223Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/MO/2021-08-26T06:00:34.143Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/MS/2021-08-16T22:06:02.598Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/MT/2021-08-12T23:47:43.719Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/NC/2021-08-13T02:01:34.979Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/ND/2021-11-09T05:17:57.283Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/NE/2021-08-13T08:11:04.020Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/NH/2021-08-13T08:31:17.998Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/NJ/2021-09-28T15:16:51.806Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/NM/2021-08-13T12:27:10.872Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/NV/2021-10-26T01:02:17.677Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/NY/2021-10-24T19:42:40.981Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/OH/2021-08-13T00:20:02.287Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/OK/2021-08-12T20:02:31.231Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/OR/2021-09-25T21:13:46.552Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/PA/2021-08-13T13:08:34.609Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/PR/2022-01-07T22:22:27.542Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/RI/2021-08-13T08:34:27.788Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/SC/2021-08-13T05:57:28.817Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/SD/2021-08-13T13:20:27.234Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/TN/2021-08-26T07:08:17.785Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/TX/2022-04-26T14:34:19.894Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/UT/2022-01-26T21:06:31.562Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/VA/2021-11-09T06:55:41.244Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/VT/2021-08-13T09:51:55.169Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/WA/2021-10-26T02:40:59.376Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/WI/2021-08-13T01:25:55.432Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/WV/2021-08-16T22:53:12.903Z/"
+    "s3://global-districtbuilder-dev-us-east-1/regions/US/WY/2021-08-13T10:07:40.461Z/"
+);
+
+function usage() {
+    echo -n \
+        "Usage: $(basename "$0")
+Load region configs for all 50 states + DC + PR
+"
+}
+
+if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
+    if [ "${1:-}" = "--help" ]; then
+        usage
+    else
+        # Bring up PostgreSQL and NestJS in a way that respects
+        # configured service health checks.
+        docker-compose \
+            -f docker-compose.yml \
+            up -d database server
+
+        for S3_URI in "${S3_URIS[@]}"; do
+          docker-compose \
+              exec database psql -U districtbuilder districtbuilder -c "
+                INSERT INTO region_config
+                VALUES (
+                  DEFAULT,
+                  '${S3_URI:53:2}',
+                  'US',
+                  '${S3_URI:53:2}',
+                  '$S3_URI'
+                ) ON CONFLICT DO NOTHING"
+        done
+        ./scripts/yarn server run set-topology-size
+    fi
+fi

scripts/manage

@@ -7,5 +7,6 @@ if [[ -n "${DB_DEBUG}" ]]; then
 fi
 
 if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
-    docker-compose run --rm manage ./bin/run "$@"
+    docker-compose up -d database
+    docker-compose run --no-deps --rm manage ./bin/run "$@"
 fi

scripts/yarn

@@ -24,8 +24,9 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
                 run --rm "${1}" \
                 "${@:2}"
         else
+            docker-compose up -d database
             docker-compose \
-                run --rm "${1}" \
+                run --rm --no-deps "${1}" \
                 "${@:2}"
         fi
     else

src/manage/package.json

@@ -31,6 +31,7 @@
     "recursive-readdir": "2.2.2",
     "rxjs": "7.4.0",
     "streamifier": "0.1.1",
+    "topobuf": "^1.1.0",
     "topojson-client": "3.1.0",
     "topojson-server": "3.0.1",
     "topojson-simplify": "3.0.3",

src/manage/src/commands/create-random-projects.ts

@@ -8,6 +8,7 @@ import { connectionOptions } from "../lib/dbUtils";
 import { RegionConfig } from "../../../server/src/region-configs/entities/region-config.entity";
 import { Project } from "../../../server/src/projects/entities/project.entity";
 import { TopologyService } from "../../../server/src/districts/services/topology.service";
+import { WorkerPoolService } from "../../../server/src/districts/services/worker-pool.service";
 import { User } from "../../../server/src/users/entities/user.entity";
 
 const PERCENT_COMPLETE = 0.25;
@@ -36,7 +37,8 @@ export default class CreateRandomProjects extends Command {
     const regionConfigRepo = connection.getRepository(RegionConfig);
     const projectRepo = connection.getRepository(Project);
     const userRepo = connection.getRepository(User);
-    const topologyService = new TopologyService(regionConfigRepo);
+    const topologyService = new TopologyService(regionConfigRepo, new WorkerPoolService());
+    topologyService.loadLayers();
 
     const regions = await regionConfigRepo.find(
       args.region === "all"

src/manage/src/commands/process-geojson.ts

@@ -21,7 +21,6 @@ import { feature as topo2feature, mergeArcs, quantize } from "topojson-client";
 import { topology } from "topojson-server";
 import { planarTriangleArea, presimplify, simplify } from "topojson-simplify";
 import { GeometryCollection, GeometryObject, Objects, Topology } from "topojson-specification";
-import { serialize, deserialize } from "v8";
 
 import {
   TypedArray,
@@ -504,36 +503,14 @@ it when necessary (file sizes ~1GB+).
     const bucket = uriComponents[2];
     const keyPrefix = uriComponents.slice(3).join("/");
 
-    console.log(bucket);
-    const bufFileExists = await s3
-      .headObject({
-        Bucket: bucket,
-        Key: `${keyPrefix}topo.buf`
-      })
-      .promise()
-      .then(
-        () => true,
-        err => {
-          if (err.code === "NotFound") {
-            return false;
-          }
-          throw err;
-        }
-      );
-
-    // Use binary format if it exists, but fallback to text format otherwise
-    const key = bufFileExists ? `${keyPrefix}topo.buf` : `${keyPrefix}topo.json`;
-
     const response: any = await s3
       .getObject({
         Bucket: bucket,
-        Key: key
+        Key: `${keyPrefix}topo.json`
       })
       .promise();
 
-    return bufFileExists
-      ? deserialize(response.Body as Buffer)
-      : JSON.parse(response.Body?.toString("utf8"));
+    return JSON.parse(response.Body?.toString("utf8"));
   }
 
   // Write TopoJSON file to disk
@@ -547,7 +524,7 @@ it when necessary (file sizes ~1GB+).
     this.log("Writing topojson file");
     const path = join(dir, "topo.buf");
     const output = createWriteStream(path, { encoding: "binary" });
-    output.write(serialize(filteredTopojson));
+    output.write(JSON.stringify(filteredTopojson));
     output.close();
   }
 

src/manage/src/commands/publish-region.ts

@@ -8,7 +8,6 @@ import { join } from "path";
 import readDir from "recursive-readdir";
 import { Topology } from "topojson-specification";
 import { createConnection } from "typeorm";
-import { deserialize } from "v8";
 
 import { RegionConfig } from "../../../server/src/region-configs/entities/region-config.entity";
 import { getTopologyLayerSize } from "../../../server/src/common/functions";
@@ -79,7 +78,9 @@ export default class PublishRegion extends Command {
     cli.action.stop();
     this.log(`Received ${responses.length} responses`);
 
-    const topology = deserialize(await readFile(join(args.staticDataDir, "topo.buf"))) as Topology;
+    const topology = JSON.parse(
+      await readFile(join(args.staticDataDir, "topo.json"), { encoding: "utf-8" })
+    ) as Topology;
 
     this.log("Saving region config to database");
     const regionConfig = new RegionConfig();