Refactor style/rendered/data serving to allow for dynamic adding/removing of items

src/serve_data.js

@@ -12,144 +12,160 @@ const VectorTile = require('@mapbox/vector-tile').VectorTile;
 
 const utils = require('./utils');
 
-module.exports = (options, repo, params, id, styles, publicUrl) => {
+module.exports = {
-  const app = express().disable('x-powered-by');
+  init: (options, repo) => {
+    const app = express().disable('x-powered-by');
 
-  const mbtilesFile = path.resolve(options.paths.mbtiles, params.mbtiles);
+    app.get('/:id/:z(\\d+)/:x(\\d+)/:y(\\d+).:format([\\w.]+)', (req, res, next) => {
-  let tileJSON = {
+      const item = repo[req.params.id];
-    'tiles': params.domains || options.domains
+      if (!item) {
-  };
+        return res.sendStatus(404);
-
-  repo[id] = tileJSON;
-
-  const mbtilesFileStats = fs.statSync(mbtilesFile);
-  if (!mbtilesFileStats.isFile() || mbtilesFileStats.size === 0) {
-    throw Error(`Not valid MBTiles file: ${mbtilesFile}`);
-  }
-  let source;
-  const sourceInfoPromise = new Promise((resolve, reject) => {
-    source = new MBTiles(mbtilesFile, err => {
-      if (err) {
-        reject(err);
-        return;
       }
-      source.getInfo((err, info) => {
+      let tileJSONFormat = item.tileJSON.format;
+      const z = req.params.z | 0;
+      const x = req.params.x | 0;
+      const y = req.params.y | 0;
+      let format = req.params.format;
+      if (format === options.pbfAlias) {
+        format = 'pbf';
+      }
+      if (format !== tileJSONFormat &&
+          !(format === 'geojson' && tileJSONFormat === 'pbf')) {
+        return res.status(404).send('Invalid format');
+      }
+      if (z < item.tileJSON.minzoom || 0 || x < 0 || y < 0 ||
+          z > item.tileJSON.maxzoom ||
+          x >= Math.pow(2, z) || y >= Math.pow(2, z)) {
+        return res.status(404).send('Out of bounds');
+      }
+      item.source.getTile(z, x, y, (err, data, headers) => {
+        let isGzipped;
         if (err) {
-          reject(err);
+          if (/does not exist/.test(err.message)) {
-          return;
+            return res.status(204).send();
-        }
+          } else {
-        tileJSON['name'] = id;
+            return res.status(500).send(err.message);
-        tileJSON['format'] = 'pbf';
+          }
-
-        Object.assign(tileJSON, info);
-
-        tileJSON['tilejson'] = '2.0.0';
-        delete tileJSON['filesize'];
-        delete tileJSON['mtime'];
-        delete tileJSON['scheme'];
-
-        Object.assign(tileJSON, params.tilejson || {});
-        utils.fixTileJSONCenter(tileJSON);
-
-        if (options.dataDecoratorFunc) {
-          tileJSON = options.dataDecoratorFunc(id, 'tilejson', tileJSON);
-        }
-        resolve();
-      });
-    });
-  });
-
-  const tilePattern = `/${id}/:z(\\d+)/:x(\\d+)/:y(\\d+).:format([\\w.]+)`;
-
-  app.get(tilePattern, (req, res, next) => {
-    const z = req.params.z | 0;
-    const x = req.params.x | 0;
-    const y = req.params.y | 0;
-    let format = req.params.format;
-    if (format === options.pbfAlias) {
-      format = 'pbf';
-    }
-    if (format !== tileJSON.format &&
-        !(format === 'geojson' && tileJSON.format === 'pbf')) {
-      return res.status(404).send('Invalid format');
-    }
-    if (z < tileJSON.minzoom || 0 || x < 0 || y < 0 ||
-        z > tileJSON.maxzoom ||
-        x >= Math.pow(2, z) || y >= Math.pow(2, z)) {
-      return res.status(404).send('Out of bounds');
-    }
-    source.getTile(z, x, y, (err, data, headers) => {
-      let isGzipped;
-      if (err) {
-        if (/does not exist/.test(err.message)) {
-          return res.status(204).send();
         } else {
-          return res.status(500).send(err.message);
+          if (data == null) {
-        }
+            return res.status(404).send('Not found');
-      } else {
+          } else {
-        if (data == null) {
+            if (tileJSONFormat === 'pbf') {
-          return res.status(404).send('Not found');
+              isGzipped = data.slice(0, 2).indexOf(
-        } else {
+                Buffer.from([0x1f, 0x8b])) === 0;
-          if (tileJSON['format'] === 'pbf') {
+              if (options.dataDecoratorFunc) {
-            isGzipped = data.slice(0, 2).indexOf(
+                if (isGzipped) {
-              Buffer.from([0x1f, 0x8b])) === 0;
+                  data = zlib.unzipSync(data);
-            if (options.dataDecoratorFunc) {
+                  isGzipped = false;
+                }
+                data = options.dataDecoratorFunc(id, 'data', data, z, x, y);
+              }
+            }
+            if (format === 'pbf') {
+              headers['Content-Type'] = 'application/x-protobuf';
+            } else if (format === 'geojson') {
+              headers['Content-Type'] = 'application/json';
+
               if (isGzipped) {
                 data = zlib.unzipSync(data);
                 isGzipped = false;
               }
-              data = options.dataDecoratorFunc(id, 'data', data, z, x, y);
-            }
-          }
-          if (format === 'pbf') {
-            headers['Content-Type'] = 'application/x-protobuf';
-          } else if (format === 'geojson') {
-            headers['Content-Type'] = 'application/json';
 
-            if (isGzipped) {
+              const tile = new VectorTile(new Pbf(data));
-              data = zlib.unzipSync(data);
+              const geojson = {
-              isGzipped = false;
+                "type": "FeatureCollection",
-            }
+                "features": []
-
+              };
-            const tile = new VectorTile(new Pbf(data));
+              for (let layerName in tile.layers) {
-            const geojson = {
+                const layer = tile.layers[layerName];
-              "type": "FeatureCollection",
+                for (let i = 0; i < layer.length; i++) {
-              "features": []
+                  const feature = layer.feature(i);
-            };
+                  const featureGeoJSON = feature.toGeoJSON(x, y, z);
-            for (let layerName in tile.layers) {
+                  featureGeoJSON.properties.layer = layerName;
-              const layer = tile.layers[layerName];
+                  geojson.features.push(featureGeoJSON);
-              for (let i = 0; i < layer.length; i++) {
+                }
-                const feature = layer.feature(i);
-                const featureGeoJSON = feature.toGeoJSON(x, y, z);
-                featureGeoJSON.properties.layer = layerName;
-                geojson.features.push(featureGeoJSON);
               }
+              data = JSON.stringify(geojson);
             }
-            data = JSON.stringify(geojson);
+            delete headers['ETag']; // do not trust the tile ETag -- regenerate
-          }
+            headers['Content-Encoding'] = 'gzip';
-          delete headers['ETag']; // do not trust the tile ETag -- regenerate
+            res.set(headers);
-          headers['Content-Encoding'] = 'gzip';
-          res.set(headers);
 
-          if (!isGzipped) {
+            if (!isGzipped) {
-            data = zlib.gzipSync(data);
+              data = zlib.gzipSync(data);
-            isGzipped = true;
+              isGzipped = true;
-          }
+            }
 
-          return res.status(200).send(data);
+            return res.status(200).send(data);
+          }
         }
+      });
+    });
+
+    app.get('/:id.json', (req, res, next) => {
+      const item = repo[req.params.id];
+      if (!item) {
+        return res.sendStatus(404);
+      }
+      const info = clone(item.tileJSON);
+      info.tiles = utils.getTileUrls(req, info.tiles,
+                                     `data/${req.params.id}`, info.format, item.publicUrl, {
+                                       'pbf': options.pbfAlias
+                                     });
+      return res.send(info);
+    });
+
+    return app;
+  },
+  add: (options, repo, params, id, publicUrl) => {
+    const mbtilesFile = path.resolve(options.paths.mbtiles, params.mbtiles);
+    let tileJSON = {
+      'tiles': params.domains || options.domains
+    };
+
+    const mbtilesFileStats = fs.statSync(mbtilesFile);
+    if (!mbtilesFileStats.isFile() || mbtilesFileStats.size === 0) {
+      throw Error(`Not valid MBTiles file: ${mbtilesFile}`);
+    }
+    let source;
+    const sourceInfoPromise = new Promise((resolve, reject) => {
+      source = new MBTiles(mbtilesFile, err => {
+        if (err) {
+          reject(err);
+          return;
+        }
+        source.getInfo((err, info) => {
+          if (err) {
+            reject(err);
+            return;
+          }
+          tileJSON['name'] = id;
+          tileJSON['format'] = 'pbf';
+
+          Object.assign(tileJSON, info);
+
+          tileJSON['tilejson'] = '2.0.0';
+          delete tileJSON['filesize'];
+          delete tileJSON['mtime'];
+          delete tileJSON['scheme'];
+
+          Object.assign(tileJSON, params.tilejson || {});
+          utils.fixTileJSONCenter(tileJSON);
+
+          if (options.dataDecoratorFunc) {
+            tileJSON = options.dataDecoratorFunc(id, 'tilejson', tileJSON);
+          }
+          resolve();
+        });
+      });
+    });
+
+    return sourceInfoPromise.then(() => {
+      repo[id] = {
+        tileJSON,
+        publicUrl,
+        source
       }
     });
-  });
+  }
-
-  app.get(`/${id}.json`, (req, res, next) => {
-    const info = clone(tileJSON);
-    info.tiles = utils.getTileUrls(req, info.tiles,
-                                   `data/${id}`, info.format, publicUrl, {
-                                     'pbf': options.pbfAlias
-                                   });
-    return res.send(info);
-  });
-
-  return sourceInfoPromise.then(() => app);
 };

src/serve_style.js

@@ -8,112 +8,125 @@ const express = require('express');
 
 const utils = require('./utils');
 
-module.exports = (options, repo, params, id, publicUrl, reportTiles, reportFont) => {
+const httpTester = /^(http(s)?:)?\/\//;
-  const app = express().disable('x-powered-by');
 
-  const styleFile = path.resolve(options.paths.styles, params.style);
+const fixUrl = (req, url, publicUrl, opt_nokey) => {
+  if (!url || (typeof url !== 'string') || url.indexOf('local://') !== 0) {
+    return url;
+  }
+  const queryParams = [];
+  if (!opt_nokey && req.query.key) {
+    queryParams.unshift(`key=${req.query.key}`);
+  }
+  let query = '';
+  if (queryParams.length) {
+    query = `?${queryParams.join('&')}`;
+  }
+  return url.replace(
+    'local://', utils.getPublicUrl(publicUrl, req)) + query;
+};
 
-  const styleJSON = clone(require(styleFile));
+module.exports = {
-  for (const name of Object.keys(styleJSON.sources)) {
+  init: (options, repo) => {
-    const source = styleJSON.sources[name];
+    const app = express().disable('x-powered-by');
-    const url = source.url;
-    if (url && url.lastIndexOf('mbtiles:', 0) === 0) {
-      let mbtilesFile = url.substring('mbtiles://'.length);
-      const fromData = mbtilesFile[0] === '{' &&
-        mbtilesFile[mbtilesFile.length - 1] === '}';
 
-      if (fromData) {
+    app.get('/:id/style.json', (req, res, next) => {
-        mbtilesFile = mbtilesFile.substr(1, mbtilesFile.length - 2);
+      const item = repo[req.params.id];
-        const mapsTo = (params.mapping || {})[mbtilesFile];
+      if (!item) {
-        if (mapsTo) {
+        return res.sendStatus(404);
-          mbtilesFile = mapsTo;
+      }
+      const styleJSON_ = clone(item.styleJSON);
+      for (const name of Object.keys(styleJSON_.sources)) {
+        const source = styleJSON_.sources[name];
+        source.url = fixUrl(req, source.url, item.publicUrl);
+      }
+      // mapbox-gl-js viewer cannot handle sprite urls with query
+      if (styleJSON_.sprite) {
+        styleJSON_.sprite = fixUrl(req, styleJSON_.sprite, item.publicUrl, true);
+      }
+      if (styleJSON_.glyphs) {
+        styleJSON_.glyphs = fixUrl(req, styleJSON_.glyphs, item.publicUrl, false);
+      }
+      return res.send(styleJSON_);
+    });
+
+    app.get('/:id/sprite:scale(@[23]x)?.:format([\\w]+)', (req, res, next) => {
+      const item = repo[req.params.id];
+      if (!item || !item.spritePath) {
+        return res.sendStatus(404);
+      }
+      const scale = req.params.scale,
+        format = req.params.format;
+      const filename = `${item.spritePath + (scale || '')}.${format}`;
+      return fs.readFile(filename, (err, data) => {
+        if (err) {
+          console.log('Sprite load error:', filename);
+          return res.sendStatus(404);
+        } else {
+          if (format === 'json') res.header('Content-type', 'application/json');
+          if (format === 'png') res.header('Content-type', 'image/png');
+          return res.send(data);
+        }
+      });
+    });
+
+    return app;
+  },
+  add: (options, repo, params, id, publicUrl, reportTiles, reportFont) => {
+    const styleFile = path.resolve(options.paths.styles, params.style);
+
+    const styleJSON = clone(require(styleFile));
+    for (const name of Object.keys(styleJSON.sources)) {
+      const source = styleJSON.sources[name];
+      const url = source.url;
+      if (url && url.lastIndexOf('mbtiles:', 0) === 0) {
+        let mbtilesFile = url.substring('mbtiles://'.length);
+        const fromData = mbtilesFile[0] === '{' &&
+          mbtilesFile[mbtilesFile.length - 1] === '}';
+
+        if (fromData) {
+          mbtilesFile = mbtilesFile.substr(1, mbtilesFile.length - 2);
+          const mapsTo = (params.mapping || {})[mbtilesFile];
+          if (mapsTo) {
+            mbtilesFile = mapsTo;
+          }
+        }
+        const identifier = reportTiles(mbtilesFile, fromData);
+        source.url = `local://data/${identifier}.json`;
+      }
+    }
+
+    for (let obj of styleJSON.layers) {
+      if (obj['type'] === 'symbol') {
+        const fonts = (obj['layout'] || {})['text-font'];
+        if (fonts && fonts.length) {
+          fonts.forEach(reportFont);
+        } else {
+          reportFont('Open Sans Regular');
+          reportFont('Arial Unicode MS Regular');
         }
       }
-      const identifier = reportTiles(mbtilesFile, fromData);
-      source.url = `local://data/${identifier}.json`;
     }
-  }
 
-  for(let obj of styleJSON.layers) {
+    let spritePath;
-    if (obj['type'] === 'symbol') {
-      const fonts = (obj['layout'] || {})['text-font'];
-      if (fonts && fonts.length) {
-        fonts.forEach(reportFont);
-      } else {
-        reportFont('Open Sans Regular');
-        reportFont('Arial Unicode MS Regular');
-      }
-    }
-  }
 
-  let spritePath;
+    if (styleJSON.sprite && !httpTester.test(styleJSON.sprite)) {
-
+      spritePath = path.join(options.paths.sprites,
-  const httpTester = /^(http(s)?:)?\/\//;
-  if (styleJSON.sprite && !httpTester.test(styleJSON.sprite)) {
-    spritePath = path.join(options.paths.sprites,
         styleJSON.sprite
-            .replace('{style}', path.basename(styleFile, '.json'))
+          .replace('{style}', path.basename(styleFile, '.json'))
-            .replace('{styleJsonFolder}', path.relative(options.paths.sprites, path.dirname(styleFile)))
+          .replace('{styleJsonFolder}', path.relative(options.paths.sprites, path.dirname(styleFile)))
-            );
+      );
-    styleJSON.sprite = `local://styles/${id}/sprite`;
+      styleJSON.sprite = `local://styles/${id}/sprite`;
-  }
+    }
-  if (styleJSON.glyphs && !httpTester.test(styleJSON.glyphs)) {
+    if (styleJSON.glyphs && !httpTester.test(styleJSON.glyphs)) {
-    styleJSON.glyphs = 'local://fonts/{fontstack}/{range}.pbf';
+      styleJSON.glyphs = 'local://fonts/{fontstack}/{range}.pbf';
-  }
+    }
 
-  repo[id] = styleJSON;
+    repo[id] = {
-
+      styleJSON,
-  app.get(`/${id}/style.json`, (req, res, next) => {
+      spritePath,
-    const fixUrl = (url, opt_nokey) => {
+      publicUrl,
-      if (!url || (typeof url !== 'string') || url.indexOf('local://') !== 0) {
+      name: styleJSON.name
-        return url;
-      }
-      const queryParams = [];
-      if (!opt_nokey && req.query.key) {
-        queryParams.unshift(`key=${req.query.key}`);
-      }
-      let query = '';
-      if (queryParams.length) {
-        query = `?${queryParams.join('&')}`;
-      }
-      return url.replace(
-        'local://', utils.getPublicUrl(publicUrl, req)) + query;
     };
-
+  }
-    const styleJSON_ = clone(styleJSON);
-    for (const name of Object.keys(styleJSON_.sources)) {
-      const source = styleJSON_.sources[name];
-      source.url = fixUrl(source.url);
-    }
-    // mapbox-gl-js viewer cannot handle sprite urls with query
-    if (styleJSON_.sprite) {
-      styleJSON_.sprite = fixUrl(styleJSON_.sprite, true);
-    }
-    if (styleJSON_.glyphs) {
-      styleJSON_.glyphs = fixUrl(styleJSON_.glyphs, false);
-    }
-    return res.send(styleJSON_);
-  });
-
-  app.get(`/${id}/sprite:scale(@[23]x)?.:format([\\w]+)`,
-    (req, res, next) => {
-    if (!spritePath) {
-      return res.status(404).send('File not found');
-    }
-        const scale = req.params.scale,
-          format = req.params.format;
-        const filename = `${spritePath + (scale || '')}.${format}`;
-        return fs.readFile(filename, (err, data) => {
-      if (err) {
-        console.log('Sprite load error:', filename);
-        return res.status(404).send('File not found');
-      } else {
-        if (format === 'json') res.header('Content-type', 'application/json');
-        if (format === 'png') res.header('Content-type', 'image/png');
-        return res.send(data);
-      }
-    });
-  });
-
-  return Promise.resolve(app);
 };

src/server.js

@@ -103,6 +103,17 @@ function start(opts) {
     app.use(cors());
   }
 
+  app.use('/data/', serve_data.init(options, serving.data));
+  app.use('/styles/', serve_style.init(options, serving.styles));
+  if (serve_rendered) {
+    startupPromises.push(
+      serve_rendered.init(options, serving.rendered)
+        .then(sub => {
+          app.use('/styles/', sub);
+        })
+    );
+  }
+
   for (const id of Object.keys(config.styles || {})) {
     const item = config.styles[id];
     if (!item.style || item.style.length === 0) {
@@ -111,7 +122,7 @@ function start(opts) {
     }
 
     if (item.serve_data !== false) {
-      startupPromises.push(serve_style(options, serving.styles, item, id, opts.publicUrl,
+      serve_style.add(options, serving.styles, item, id, opts.publicUrl,
         (mbtiles, fromData) => {
           let dataItemId;
           for (const id of Object.keys(data)) {
@@ -140,27 +151,21 @@ function start(opts) {
           }
         }, font => {
           serving.fonts[font] = true;
-        }).then(sub => {
+        });
-          app.use('/styles/', sub);
-        }));
     }
     if (item.serve_rendered !== false) {
       if (serve_rendered) {
-        startupPromises.push(
+        startupPromises.push(serve_rendered.add(options, serving.rendered, item, id, opts.publicUrl,
-          serve_rendered(options, serving.rendered, item, id, opts.publicUrl,
+          mbtiles => {
-            mbtiles => {
+            let mbtilesFile;
-              let mbtilesFile;
+            for (const id of Object.keys(data)) {
-              for (const id of Object.keys(data)) {
+              if (id === mbtiles) {
-                if (id === mbtiles) {
+                mbtilesFile = data[id].mbtiles;
-                  mbtilesFile = data[id].mbtiles;
-                }
               }
-              return mbtilesFile;
             }
-          ).then(sub => {
+            return mbtilesFile;
-            app.use('/styles/', sub);
+          }
-          })
+        ));
-        );
       } else {
         item.serve_rendered = false;
       }
@@ -181,9 +186,7 @@ function start(opts) {
     }
 
     startupPromises.push(
-      serve_data(options, serving.data, item, id, serving.styles, opts.publicUrl).then(sub => {
+      serve_data.add(options, serving.data, item, id, opts.publicUrl)
-        app.use('/data/', sub);
-      })
     );
   }
 
@@ -191,7 +194,7 @@ function start(opts) {
     const result = [];
     const query = req.query.key ? (`?key=${req.query.key}`) : '';
     for (const id of Object.keys(serving.styles)) {
-      const styleJSON = serving.styles[id];
+      const styleJSON = serving.styles[id].styleJSON;
       result.push({
         version: styleJSON.version,
         name: styleJSON.name,
@@ -204,9 +207,10 @@ function start(opts) {
 
   const addTileJSONs = (arr, req, type) => {
     for (const id of Object.keys(serving[type])) {
-      const info = clone(serving[type][id]);
+      let info = clone(serving[type][id]);
       let path = '';
       if (type === 'rendered') {
+        info = info.tileJSON;
         path = `styles/${id}`;
       } else {
         path = `${type}/${id}`;
@@ -283,7 +287,7 @@ function start(opts) {
       style.serving_data = serving.styles[id];
       style.serving_rendered = serving.rendered[id];
       if (style.serving_rendered) {
-        const center = style.serving_rendered.center;
+        const center = style.serving_rendered.tileJSON.center;
         if (center) {
           style.viewer_hash = `#${center[2]}/${center[1].toFixed(5)}/${center[0].toFixed(5)}`;
 
@@ -292,8 +296,8 @@ function start(opts) {
         }
 
         style.xyz_link = utils.getTileUrls(
-          req, style.serving_rendered.tiles,
+          req, style.serving_rendered.tileJSON.tiles,
-          `styles/${id}`, style.serving_rendered.format, opts.publicUrl)[0];
+          `styles/${id}`, style.serving_rendered.tileJSON.format, opts.publicUrl)[0];
       }
     }
     const data = clone(serving.data || {});