Merge pull request #40 from HSLdevcom/bug-dump-naming

geometry matcher and dump processing enhancements
HSLdevcom · Feb 17, 2020 · b7bf905 · b7bf905
2 parents d64b663 + b5b35b0
commit b7bf905
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 10 deletions.
diff --git a/src/constants.js b/src/constants.js
@@ -44,8 +44,8 @@ export const ADMIN_PASSWORD = secretsEnv.ADMIN_PASSWORD || "password";
 export const PATH_PREFIX = secretsEnv.PATH_PREFIX || "/";
 export const PYTHON_CMD = process.env.PYTHON_CMD || "python";
 export const PBF_DOWNLOAD_URL =
-  "http://download.geofabrik.de/europe/finland-latest.osm.pbf";
-export const PBF_FILENAME = "finland-latest.osm.pbf";
+  "https://karttapalvelu.storage.hsldev.com/hsl.osm/hsl.osm.pbf";
+export const PBF_FILENAME = "hsl.osm.pbf";
 export const SCHEMA = "jore";
 export const INTERMEDIATE_SCHEMA = "jore_new";
 export const AZURE_UPLOAD_CONTAINER = secretsEnv.AZURE_UPLOAD_CONTAINER || "joredumps";

diff --git a/src/geometryMatcher.js b/src/geometryMatcher.js
@@ -12,6 +12,8 @@ import { download } from "./utils/download";
 
 const cwd = process.cwd();
 const downloadDir = path.join(cwd, "downloads");
+const ONE_HOUR = 60 * 60 * 1000; /*/ ms */
+const PBF_UPDATE_INTERVAL = 24 * ONE_HOUR;
 
 export const runGeometryMatcher = async (schema = SCHEMA) => {
   return new Promise(async (resolve, reject) => {
@@ -23,9 +25,21 @@ export const runGeometryMatcher = async (schema = SCHEMA) => {
     const filePath = path.join(downloadDir, PBF_FILENAME);
     const fileExists = await fs.pathExists(filePath);
 
-    if (!fileExists) {
-      console.log("Downloading PBF data...");
-      await download(PBF_DOWNLOAD_URL, filePath).catch((err) => console.log('Downloading PBF data failed...' + err));
+    const modifiedLessThanDurationAgo = (durationMs) => {
+      const lastModified = fs.statSync(filePath, (err) => {
+        if (err) throw err;
+      }).mtimeMs;
+      return new Date().getTime() - durationMs > lastModified;
+    };
+
+    if (!fileExists || modifiedLessThanDurationAgo(PBF_UPDATE_INTERVAL)) {
+      console.log(
+        `PBF data does not exist or is older than ${PBF_UPDATE_INTERVAL /
+          3600000} hours. Downloading new PBF data...`,
+      );
+      await download(PBF_DOWNLOAD_URL, filePath).catch((err) =>
+        console.log(`Downloading PBF data failed...${err}`),
+      );
     } else {
       console.log("PBF file exists...");
     }

diff --git a/src/import.js b/src/import.js
@@ -128,8 +128,13 @@ export async function importFile(filePath) {
 
     // Disallow dump and upload by unsetting AZURE_STORAGE_ACCOUNT
     if (AZURE_STORAGE_ACCOUNT) {
-      const dumpFilePath = await createDbDump();
-      await uploadDbDump(dumpFilePath);
+      try {
+        const dumpFilePath = await createDbDump();
+        await uploadDbDump(dumpFilePath);
+      } catch (err) {
+        console.log(err.message || "DB upload failed.");
+        console.log(err);
+      }
     }
 
     const [execDuration] = process.hrtime(execStart);

diff --git a/src/utils/createDbDump.js b/src/utils/createDbDump.js
@@ -5,7 +5,6 @@ import fs from "fs-extra";
 import format from "date-fns/format";
 import { parse } from "pg-connection-string";
 
-const currentDate = format(new Date(), "YYYY-MM-DD");
 const cwd = process.cwd();
 const dumpsDir = path.join(cwd, "dumps");
 
@@ -17,6 +16,7 @@ export const createDbDump = async () => {
     let lastError = null;
 
     await fs.ensureDir(dumpsDir);
+    const currentDate = format(new Date(), "YYYY-MM-DD");
     const currentDateFilename = `jore_dump_${currentDate}`;
     const filePath = path.join(dumpsDir, currentDateFilename);
     const fileExists = await fs.pathExists(filePath);

diff --git a/src/utils/download.js b/src/utils/download.js
@@ -1,10 +1,10 @@
-import http from "http";
+import https from "https";
 import fs from "fs-extra";
 
 export const download = (url, dest) => {
   return new Promise((resolve, reject) => {
     const file = fs.createWriteStream(dest);
-    http
+    https
       .get(url, (response) => {
         response.pipe(file);
         file.on("finish", () => {