Allow handling of local GeoJSON files #1324 (#1326)

* handle local geojson files in styles and rendered tiles
- use 'file://' as indicator for local files
- add  directory as default directory
- serve local files at
- add documentation for static file serving
- add some minor fixes (icon directory, directory checking, decodeURIComponent, extend error message)

* Update .gitignore

---------

Co-authored-by: Miko <miko@home-laptop.fritz.box>
Co-authored-by: Andrew Calcutt <acalcutt@techidiots.net>

.dockerignore

@@ -5,3 +5,4 @@
 !package.json
 !package-lock.json
 !docker-entrypoint.sh
+**.gitignore

.github/workflows/ct.yml

@@ -46,7 +46,7 @@ jobs:
           https://github.com/maptiler/tileserver-gl/releases/download/v1.3.0/test_data.zip
 
       - name: Prepare test data 📦
-        run: unzip -q test_data.zip -d test_data
+        run: unzip -q test_data.zip
 
       - name: Run tests 🧪
         run: xvfb-run --server-args="-screen 0 1024x768x24" npm test

.gitignore

@@ -1,8 +1,11 @@
 docs/_build
 node_modules
 test_data
+test_data.zip
 data
 light
 plugins
 config.json
 *.mbtiles
+styles
+fonts

docs/config.rst

@@ -17,7 +17,8 @@ Example:
         "icons": "icons",
         "styles": "styles",
         "mbtiles": "data",
-        "pmtiles": "data"
+        "pmtiles": "data",
+        "files": "public/files"
       },
       "domains": [
         "localhost:8080",

docs/endpoints.rst

@@ -100,6 +100,18 @@ Source data
 
   * TileJSON at ``/data/{id}.json``
 
+Static files
+===========
+* Static files are served at ``/files/{filename}``
+
+  * The source folder can be configured (``options.paths.files``), default is ``public/files``
+
+  * This feature can be used to serve ``geojson`` files for styles and rendered tiles.
+
+    * Keep in mind, that each rendered tile loads the whole geojson file, if performance matters a conversion to a tiled format (e.g. with https://github.com/felt/tippecanoe)may be a better approch.
+
+    * Use ``file://{filename}`` to have matching paths for both endoints
+
 TileJSON arrays
 ===============
 Array of all TileJSONs is at ``[/{tileSize}]/index.json`` (``[/{tileSize}]/rendered.json``; ``/data.json``)

src/main.js

@@ -109,6 +109,8 @@ const startWithInputFile = async (inputFile) => {
     '../node_modules/tileserver-gl-styles/',
   );
 
+  const filesDir = path.resolve(__dirname, '../public/files');
+
   const config = {
     options: {
       paths: {
@@ -117,6 +119,7 @@ const startWithInputFile = async (inputFile) => {
         styles: 'styles',
         mbtiles: inputFilePath,
         pmtiles: inputFilePath,
+        files: filesDir,
       },
     },
     styles: {},

src/serve_rendered.js

@@ -41,7 +41,7 @@ import {
 } from './pmtiles_adapter.js';
 import { renderOverlay, renderWatermark, renderAttribution } from './render.js';
 import fsp from 'node:fs/promises';
-import { gunzipP } from './promises.js';
+import { existsP, gunzipP } from './promises.js';
 import { openMbTilesWrapper } from './mbtiles_wrapper.js';
 
 const FLOAT_PATTERN = '[+-]?(?:\\d+|\\d+.?\\d+)';
@@ -893,13 +893,15 @@ export const serve_rendered = {
             // console.log('Handling request:', req);
             if (protocol === 'sprites') {
               const dir = options.paths[protocol];
-              const file = unescape(req.url).substring(protocol.length + 3);
+              const file = decodeURIComponent(req.url).substring(
+                protocol.length + 3,
+              );
               fs.readFile(path.join(dir, file), (err, data) => {
                 callback(err, { data: data });
               });
             } else if (protocol === 'fonts') {
               const parts = req.url.split('/');
-              const fontstack = unescape(parts[2]);
+              const fontstack = decodeURIComponent(parts[2]);
               const range = parts[3].split('.')[0];
 
               try {
@@ -1039,6 +1041,25 @@ export const serve_rendered = {
                 const format = extensionToFormat[extension] || '';
                 createEmptyResponse(format, '', callback);
               }
+            } else if (protocol === 'file') {
+              const name = decodeURI(req.url).substring(protocol.length + 3);
+              const file = path.join(options.paths['files'], name);
+              if (await existsP(file)) {
+                const inputFileStats = await fsp.stat(file);
+                if (!inputFileStats.isFile() || inputFileStats.size === 0) {
+                  throw Error(
+                    `File is not valid: "${req.url}" - resolved to "${file}"`,
+                  );
+                }
+
+                fs.readFile(file, (err, data) => {
+                  callback(err, { data: data });
+                });
+              } else {
+                throw Error(
+                  `File does not exist: "${req.url}" - resolved to "${file}"`,
+                );
+              }
             }
           },
         });

src/serve_style.js

@@ -26,6 +26,9 @@ export const serve_style = {
       for (const name of Object.keys(styleJSON_.sources)) {
         const source = styleJSON_.sources[name];
         source.url = fixUrl(req, source.url, item.publicUrl);
+        if (typeof source.data == 'string') {
+          source.data = fixUrl(req, source.data, item.publicUrl);
+        }
       }
       // mapbox-gl-js viewer cannot handle sprite urls with query
       if (styleJSON_.sprite) {
@@ -89,7 +92,7 @@ export const serve_style = {
     try {
       styleFileData = fs.readFileSync(styleFile); // TODO: could be made async if this function was
     } catch (e) {
-      console.log('Error reading style file');
+      console.log(`Error reading style file "${params.style}"`);
       return false;
     }
 
@@ -128,6 +131,16 @@ export const serve_style = {
         }
         source.url = `local://data/${identifier}.json`;
       }
+
+      let data = source.data;
+      if (data && typeof data == 'string' && data.startsWith('file://')) {
+        source.data =
+          'local://files' +
+          path.resolve(
+            '/',
+            data.replace('file://', '').replace(options.paths.files, ''),
+          );
+      }
     }
 
     for (const obj of styleJSON.layers) {

src/server.js

@@ -94,24 +94,22 @@ function start(opts) {
   paths.sprites = path.resolve(paths.root, paths.sprites || '');
   paths.mbtiles = path.resolve(paths.root, paths.mbtiles || '');
   paths.pmtiles = path.resolve(paths.root, paths.pmtiles || '');
-  paths.icons = path.resolve(paths.root, paths.icons || '');
+  paths.icons = path.resolve(
+    paths.root,
+    paths.icons || 'public/resources/images',
+  );
+  paths.files = path.resolve(paths.root, paths.files || 'public/files');
 
   const startupPromises = [];
 
-  const checkPath = (type) => {
+  for (const type of Object.keys(paths)) {
     if (!fs.existsSync(paths[type])) {
       console.error(
         `The specified path for "${type}" does not exist (${paths[type]}).`,
       );
       process.exit(1);
     }
-  };
+  }
-  checkPath('styles');
-  checkPath('fonts');
-  checkPath('sprites');
-  checkPath('mbtiles');
-  checkPath('pmtiles');
-  checkPath('icons');
 
   /**
    * Recursively get all files within a directory.
@@ -161,6 +159,7 @@ function start(opts) {
   }
 
   app.use('/data/', serve_data.init(options, serving.data));
+  app.use('/files/', express.static(paths.files));
   app.use('/styles/', serve_style.init(options, serving.styles));
   if (!isLight) {
     startupPromises.push(

test/setup.js

@@ -9,7 +9,6 @@ global.supertest = supertest;
 
 before(function () {
   console.log('global setup');
-  process.chdir('test_data');
   const running = server({
     configPath: 'config.json',
     port: 8888,